hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[FEAT MERGE] system resource optimization

Browse Source 
Co-authored-by: tushicheng <18829573815@163.com>
Co-authored-by: HaHaJeff <jeffzhouhhh@gmail.com>
Co-authored-by: dimstars <liangjinrongcm@gmail.com>
This commit is contained in:
obdev 2023-08-29 10:41:01 +00:00 committed by ob-robot
parent 7678a4a676
commit 81b8923b14
191 changed files with 2817 additions and 2464 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
deps/easy/src/io/easy_baseth_pool.c vendored
View File

@ -25,8 +25,8 @@ static void easy_baseth_pool_invoke_debug(struct ev_loop *loop);
static int easy_monitor_interval = 100;
static const int64_t easy_monitor_signal = 34;
int ob_pthread_create(pthread_t *thread, const pthread_attr_t *attr,
void *(*start_routine) (void *), void *arg);
int ob_pthread_create(void **ptr, void *(*start_routine) (void *), void *arg);
pthread_t ob_pthread_get_pth(void *ptr);
void ob_set_thread_name(const char* type);
int64_t ob_update_loop_ts();
void ob_usleep(const useconds_t v);
@ -274,7 +274,7 @@ static void *easy_baseth_pool_monitor_func(void *args)
if (EASY_REACH_TIME_INTERVAL(1 * 1000 * 1000)) {
easy_warn_log("EASY SLOW: thread: %lx, lastrun: %f cost: %f loop:%d, slowcnt: %ld",
th->tid, last, now - last, id, slowcnts[th->idx]);
ob_pthread_get_pth(th->tid), last, now - last, id, slowcnts[th->idx]);
}
}
}
@ -325,12 +325,12 @@ void easy_baseth_pool_monitor(easy_thread_pool_t *tp)
sigemptyset(&sa.sa_mask);
rc = sigaction(easy_monitor_signal, &sa, NULL);
err = ob_pthread_create(&tp->monitor_tid, NULL, easy_baseth_pool_monitor_func, tp);
err = ob_pthread_create(&tp->monitor_tid, easy_baseth_pool_monitor_func, tp);
if (err != 0) {
tp->monitor_tid = 0;
easy_error_log("sigaction: %d, monitor_thread: 0x%lx, err:%d, errno:%d", rc, tp->monitor_tid, err, errno);
tp->monitor_tid = NULL;
easy_error_log("sigaction: %d, monitor_tid: %p, err:%d, errno:%d", rc, tp->monitor_tid, err, errno);
} else {
easy_info_log("monitor thread created, tp=0x%lx tid=%lx\n", tp, tp->monitor_tid);
easy_info_log("monitor thread created, tp=0x%lx tid=%p\n", tp, tp->monitor_tid);
}
}
}

27
deps/easy/src/io/easy_io.c vendored
View File

@ -47,8 +47,8 @@ static void easy_io_print_status(easy_io_t *eio);
static void easy_signal_handler(int sig);
static void easy_listen_close(easy_listen_t *l);
int ob_pthread_create(pthread_t *thread, const pthread_attr_t *attr,
void *(*start_routine) (void *), void *arg);
int ob_pthread_create(void **ptr, void *(*start_routine) (void *), void *arg);
void ob_pthread_join(void *ptr);
/**
* 初始化easy_io
*/
@ -275,9 +275,9 @@ int easy_eio_start(easy_io_t *eio)
easy_list_for_each_entry(tp, &eio->thread_pool_list, list_node) {
easy_thread_pool_for_each(th, tp, 0) {
int err = 0;
if ((err = ob_pthread_create(&(th->tid), NULL, th->on_start, (void *)th))) {
if ((err = ob_pthread_create(&(th->tid), th->on_start, (void *)th))) {
ret = EASY_ERROR;
th->tid = 0;
th->tid = NULL;
easy_error_log("easy_io_start, pthread_create error: %d(%d), idx: %d", err, errno, th->idx);
}
}
@ -302,27 +302,18 @@ int easy_eio_wait(easy_io_t *eio)
{
easy_baseth_t *th;
easy_thread_pool_t *tp;
// 等待thread
easy_spin_lock(&eio->lock);
easy_list_for_each_entry(tp, &eio->thread_pool_list, list_node) {
easy_spin_unlock(&eio->lock);
easy_thread_pool_for_each(th, tp, 0) {
if (th->tid && pthread_join(th->tid, NULL) == EDEADLK) {
easy_fatal_log("easy_io_wait fatal, eio=%p, tid=%lx\n", eio, th->tid);
abort();
}
th->tid = 0;
ob_pthread_join(th->tid);
th->tid = NULL;
}
easy_spin_lock(&eio->lock);
easy_info_log("easy_io_wait join monitor, tp=0x%lx tid=%lx\n", tp, tp->monitor_tid);
if (tp->monitor_tid && pthread_join(tp->monitor_tid, NULL) == EDEADLK) {
easy_fatal_log("easy_io_wait fatal, eio=%p, tid=%lx\n", eio, tp->monitor_tid);
abort();
}
tp->monitor_tid = 0;
easy_info_log("easy_io_wait join monitor, tp=0x%lx tid=%p\n", tp, tp->monitor_tid);
ob_pthread_join(tp->monitor_tid);
tp->monitor_tid = NULL;
}
easy_spin_unlock(&eio->lock);

4
deps/easy/src/io/easy_io_struct.h vendored
View File

@ -148,7 +148,7 @@ enum {
// async + spinlock
#define EASY_BASETH_DEFINE \
easy_baseth_on_start_pt *on_start; \
pthread_t tid; \
void *tid; \
int idx, iot; \
struct ev_loop *loop; \
ev_tstamp lastrun; \
@ -690,7 +690,7 @@ struct easy_thread_pool_t {
easy_list_t list_node;
easy_thread_pool_t *next;
char *last;
pthread_t monitor_tid;
void *monitor_tid;
easy_io_thread_t *ratelimit_thread;
char data[0];
};

11
deps/easy/src/io/easy_maccept.c vendored
View File

@ -30,7 +30,7 @@ typedef struct easy_ma_t {
int flags;
int efd;
int lfd;
pthread_t th;
void *th;
volatile int stop;
int g_count;
int g_start[MAX_GROUP_COUNT];
@ -39,8 +39,8 @@ typedef struct easy_ma_t {
} easy_ma_t;
easy_ma_t g_ma;
int ob_pthread_create(pthread_t *thread, const pthread_attr_t *attr,
void *(*start_routine) (void *), void *arg);
int ob_pthread_create(void **ptr, void *(*start_routine) (void *), void *arg);
void ob_pthread_join(void *ptr);
int ob_epoll_wait(int __epfd, struct epoll_event *__events,
int __maxevents, int __timeout);
void easy_ma_init(int port)
@ -219,7 +219,7 @@ int easy_ma_start()
g_ma.efd = efd;
g_ma.lfd = lfd;
g_ma.stop = 0;
if (0 != ob_pthread_create(&g_ma.th, NULL, (void*)easy_ma_thread_func, (void*)&g_ma)) {
if (0 != ob_pthread_create(&g_ma.th, (void*)easy_ma_thread_func, (void*)&g_ma)) {
goto error_exit;
}
@ -236,7 +236,8 @@ void easy_ma_stop()
{
if (!g_ma.stop) {
g_ma.stop = 1;
pthread_join(g_ma.th, NULL);
ob_pthread_join(g_ma.th);
g_ma.th = NULL;
}
}

7
deps/easy/src/io/easy_request.c vendored
View File

@ -24,6 +24,7 @@ static void easy_request_dosess(easy_request_thread_t *th, easy_list_t *session_
static void easy_request_cleanup(easy_buf_t *b, void *args);
static const int64_t REQUEST_ITEM_COST_RT = 100 * 1000;
pthread_t ob_pthread_get_pth(void *ptr);
/**
* 对request回复响应
*
@ -37,9 +38,9 @@ int easy_request_do_reply(easy_request_t *r)
// encode
m = (easy_message_t *)r->ms;
c = m->c;
if (c->ioth->tid != pthread_self()) {
easy_fatal_log("not run at other thread: %lx <> %lx\n", r, pthread_self(), c->ioth->tid);
pthread_t pth = ob_pthread_get_pth(c->ioth->tid);
if (pth != pthread_self()) {
easy_fatal_log("not run at other thread: %lx <> %lx\n", r, pthread_self(), pth);
return EASY_ERROR;
}

51
deps/oblib/src/lib/alloc/alloc_struct.cpp vendored
View File

@ -17,13 +17,19 @@
#include "lib/oblog/ob_log.h"
#include "lib/utility/ob_fast_convert.h"
using namespace oceanbase;
using namespace lib;
namespace oceanbase
{
using namespace common;
namespace lib
{
thread_local ObMemAttr ObMallocHookAttrGuard::tl_mem_attr(OB_SERVER_TENANT_ID,
"glibc_malloc",
ObCtxIds::GLIBC);
static int64_t g_divisive_mem_size[OB_MAX_CPU_NUM];
static thread_local bool g_is_ob_mem_mgr_path = false;
ObMallocHookAttrGuard::ObMallocHookAttrGuard(const ObMemAttr& attr)
: old_attr_(tl_mem_attr)
@ -86,4 +92,43 @@ void Label::fmt(char *buf, int64_t buf_len, int64_t &pos, const char *str)
pos = buf_len;
}
}
}
}
int64_t get_divisive_mem_size()
{
int64_t total_size = 0;
for (int64_t i = 0; i < OB_MAX_CPU_NUM; i++) {
total_size += g_divisive_mem_size[i];
}
return total_size;
}
void inc_divisive_mem_size(const int64_t size)
{
const int64_t idx = ob_gettid() % OB_MAX_CPU_NUM;
__sync_fetch_and_add(&(g_divisive_mem_size[idx]), size);
}
void dec_divisive_mem_size(const int64_t size)
{
const int64_t idx = ob_gettid() % OB_MAX_CPU_NUM;
__sync_fetch_and_add(&(g_divisive_mem_size[idx]), 0 - size);
}
void set_ob_mem_mgr_path()
{
g_is_ob_mem_mgr_path = true;
}
void unset_ob_mem_mgr_path()
{
g_is_ob_mem_mgr_path = false;
}
bool is_ob_mem_mgr_path()
{
return g_is_ob_mem_mgr_path;
}
} // end of namespace lib
} // end of namespace oceanbase

10
deps/oblib/src/lib/alloc/alloc_struct.h vendored
View File

@ -52,7 +52,7 @@ static const int64_t ALLOC_ABLOCK_CONCURRENCY = 4;
static ssize_t get_page_size()
{
static ssize_t ps = sysconf(_SC_PAGESIZE);
static ssize_t ps = getpagesize();
return ps;
}
@ -604,6 +604,14 @@ private:
ObMemAttr old_attr_;
};
extern void inc_divisive_mem_size(const int64_t size);
extern void dec_divisive_mem_size(const int64_t size);
extern int64_t get_divisive_mem_size();
extern void set_ob_mem_mgr_path();
extern void unset_ob_mem_mgr_path();
extern bool is_ob_mem_mgr_path();
#define FORCE_EXPLICT_500_MALLOC() \
OB_UNLIKELY(oceanbase::lib::ObMallocAllocator::get_instance()->force_explict_500_malloc_)

47
deps/oblib/src/lib/alloc/malloc_hook.cpp vendored
View File

@ -19,7 +19,7 @@ using namespace oceanbase;
using namespace oceanbase::common;
using namespace oceanbase::lib;
bool g_malloc_hook_inited = false;
static bool g_malloc_hook_inited = false;
void init_malloc_hook()
{
@ -74,6 +74,25 @@ void *ob_malloc_retry(size_t size)
return ptr;
}
static inline void *ob_mmap(void *addr, size_t length, int prot, int flags, int fd, loff_t offset)
{
void *ptr = (void*)syscall(SYS_mmap, addr, length, prot, flags, fd, offset);
if (OB_UNLIKELY(!is_ob_mem_mgr_path()) && OB_LIKELY(MAP_FAILED != ptr)) {
const int64_t page_size = get_page_size();
inc_divisive_mem_size(upper_align(length, page_size));
}
return ptr;
}
static inline int ob_munmap(void *addr, size_t length)
{
if (OB_UNLIKELY(!is_ob_mem_mgr_path())) {
const int64_t page_size = get_page_size();
dec_divisive_mem_size(upper_align(length, page_size));
}
return syscall(SYS_munmap, addr, length);
}
void *ob_malloc_hook(size_t size, const void *)
{
void *ptr = nullptr;
@ -81,7 +100,7 @@ void *ob_malloc_hook(size_t size, const void *)
void *tmp_ptr = nullptr;
bool from_mmap = false;
if (OB_UNLIKELY(!g_malloc_hook_inited || in_hook())) {
if (MAP_FAILED == (tmp_ptr = ::mmap(nullptr, real_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0))) {
if (MAP_FAILED == (tmp_ptr = ob_mmap(nullptr, real_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0))) {
tmp_ptr = nullptr;
}
from_mmap = true;
@ -107,7 +126,7 @@ void ob_free_hook(void *ptr, const void *)
header->mark_unused();
void *orig_ptr = (char*)header - header->offset_;
if (OB_UNLIKELY(header->from_mmap_)) {
::munmap(orig_ptr, header->data_size_ + Header::SIZE + header->offset_);
ob_munmap(orig_ptr, header->data_size_ + Header::SIZE + header->offset_);
} else {
bool in_hook_bak = in_hook();
in_hook()= true;
@ -124,7 +143,7 @@ void *ob_realloc_hook(void *ptr, size_t size, const void *caller)
void *tmp_ptr = nullptr;
bool from_mmap = false;
if (OB_UNLIKELY(!g_malloc_hook_inited || in_hook())) {
if (MAP_FAILED == (tmp_ptr = ::mmap(nullptr, real_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0))) {
if (MAP_FAILED == (tmp_ptr = ob_mmap(nullptr, real_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0))) {
tmp_ptr = nullptr;
}
from_mmap = true;
@ -163,7 +182,7 @@ void *ob_memalign_hook(size_t alignment, size_t size, const void *)
void *tmp_ptr = nullptr;
bool from_mmap = false;
if (OB_UNLIKELY(!g_malloc_hook_inited || in_hook())) {
if (MAP_FAILED == (tmp_ptr = ::mmap(nullptr, real_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0))) {
if (MAP_FAILED == (tmp_ptr = ob_mmap(nullptr, real_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0))) {
tmp_ptr = nullptr;
}
from_mmap = true;
@ -187,6 +206,20 @@ void *ob_memalign_hook(size_t alignment, size_t size, const void *)
return ptr;
}
EXTERN_C_BEGIN
void *ob_mmap_hook(void *addr, size_t length, int prot, int flags, int fd, loff_t offset)
{
return ob_mmap(addr, length, prot, flags, fd, offset);
}
int ob_munmap_hook(void *addr, size_t length)
{
return ob_munmap(addr, length);
}
EXTERN_C_END
#if !defined(__MALLOC_HOOK_VOLATILE)
#define MALLOC_HOOK_MAYBE_VOLATILE /**/
#else
@ -200,6 +233,10 @@ __attribute__((visibility("default"))) void (*MALLOC_HOOK_MAYBE_VOLATILE __free_
__attribute__((visibility("default"))) void *(*MALLOC_HOOK_MAYBE_VOLATILE __realloc_hook)(void *, size_t, const void *) = ob_realloc_hook;
__attribute__((visibility("default"))) void *(*MALLOC_HOOK_MAYBE_VOLATILE __memalign_hook)(size_t, size_t, const void *) = ob_memalign_hook;
__attribute__((visibility("default"))) void *mmap(void *addr, size_t, int, int, int, loff_t) __attribute__((weak,alias("ob_mmap_hook")));
__attribute__((visibility("default"))) void *mmap64(void *addr, size_t, int, int, int, loff_t) __attribute__((weak,alias("ob_mmap_hook")));
__attribute__((visibility("default"))) int munmap(void *addr, size_t length) __attribute__((weak,alias("ob_munmap_hook")));
size_t malloc_usable_size(void *ptr)
{
size_t ret = 0;

4
deps/oblib/src/lib/alloc/ob_malloc_sample_struct.h vendored
View File

@ -116,6 +116,7 @@ inline bool ObMallocSampleLimiter::malloc_sample_allowed(const int64_t size, con
inline void ObMallocSampleLimiter::set_interval(int32_t max_interval, int32_t min_interval)
{
#if defined(__x86_64__)
if (min_interval < 1 || max_interval > INTERVAL_UPPER_LIMIT
|| max_interval < min_interval) {
_OB_LOG_RET(WARN, common::OB_INVALID_ARGUMENT, "set the min or max malloc times between two samples unexpected,"
@ -126,6 +127,9 @@ inline void ObMallocSampleLimiter::set_interval(int32_t max_interval, int32_t mi
_OB_LOG_RET(INFO, common::OB_SUCCESS, "set the min or max malloc times between two samples succeed,"
"max_interval=%d, min_interval=%d", max_interval, min_interval);
}
#else
UNUSEDx(max_interval, min_interval);
#endif
}
inline int64_t ObMallocSampleKey::hash() const

3
deps/oblib/src/lib/allocator/ob_fixed_size_block_allocator.h vendored
View File

@ -61,8 +61,7 @@ private:
bool contains_internal(void* ptr) const;
public:
static const int64_t MAX_MEMORY_ALLOCATION = OB_MAX_SYS_BKGD_THREAD_NUM * 6
* OB_DEFAULT_MACRO_BLOCK_SIZE;
static const int64_t MAX_MEMORY_ALLOCATION = OB_MAX_SYS_BKGD_THREAD_NUM * 2 * OB_DEFAULT_MACRO_BLOCK_SIZE; //256MB
private:
// TODO(zhuixin.gsy) reduce 128*2M size to 64*2M, which is expanded temporarily for generating index blocks

6
deps/oblib/src/lib/geo/ob_srs_info.cpp vendored
View File

@ -84,7 +84,7 @@ int ObSrsUtils::check_is_wgs84(const ObGeographicRs *rs, bool &is_wgs84)
// todo@dazhi: compare the param_name and param_alias when epsg isnot comparable ?
int ObSrsUtils::get_simple_proj_params(const ObProjectionPrams &parsed_params,
ObVector<ObSimpleProjPram, common::ObArenaAllocator> &params)
ObVector<ObSimpleProjPram, common::ObFIFOAllocator> &params)
{
int ret = OB_SUCCESS;
FOREACH_X(parsed_param, parsed_params.vals, OB_SUCC(ret)) {
@ -307,7 +307,7 @@ int ObSpatialReferenceSystemBase::create_srs_internal(ObIAllocator* allocator, u
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("alloc projected srs failed", K(ret), K(srs_id));
} else {
tmp_srs_info = new(buf)SRS_T(static_cast<common::ObArenaAllocator*>(allocator));
tmp_srs_info = new(buf)SRS_T(static_cast<common::ObIAllocator*>(allocator));
if (OB_FAIL(tmp_srs_info->init(srs_id, rs))) {
LOG_WARN("srs info init failed", K(ret), KP(rs), K(srs_id));
} else {
@ -327,7 +327,7 @@ int ObSpatialReferenceSystemBase::create_geographic_srs(ObIAllocator* allocator,
return create_srs_internal<ObGeographicSrs, ObGeographicRs>(allocator, srs_id, rs, srs_info);
}
ObGeographicSrs::ObGeographicSrs(common::ObArenaAllocator* alloc)
ObGeographicSrs::ObGeographicSrs(common::ObIAllocator* alloc)
: semi_major_axis_(NAN), inverse_flattening_(NAN),
is_wgs84_(false), prime_meridian_(NAN), angular_factor_(NAN), bounds_info_(),
proj4text_()

93
deps/oblib/src/lib/geo/ob_srs_info.h vendored
View File

@ -18,6 +18,7 @@
#include "lib/container/ob_vector.h"
#include "lib/allocator/page_arena.h"
#include "lib/allocator/ob_allocator.h"
#include "lib/allocator/ob_fifo_allocator.h"
#include "lib/ob_errno.h"
#include "common/data_buffer.h"
@ -276,7 +277,7 @@ private:
class ObGeographicSrs : public ObSpatialReferenceSystemBase
{
public:
ObGeographicSrs(common::ObArenaAllocator* alloc);
ObGeographicSrs(common::ObIAllocator* alloc);
virtual ~ObGeographicSrs() {}
ObSrsType srs_type() const override { return ObSrsType::GEOGRAPHIC_SRS; }
virtual int init(uint32_t srid, const ObGeographicRs *rs);
@ -320,9 +321,9 @@ struct ObSimpleProjPram
class ObProjectedSrs : public ObSpatialReferenceSystemBase
{
public:
ObProjectedSrs(common::ObArenaAllocator *allocator) : geographic_srs_(allocator),
linear_unit_(NAN),
simple_proj_prams_(allocator)
ObProjectedSrs(common::ObIAllocator *allocator) : geographic_srs_(allocator),
linear_unit_(NAN),
simple_proj_prams_(static_cast<common::ObFIFOAllocator*>(allocator))
{
for (int i = 0; i < AXIS_DIRECTION_NUM; i++) {
axis_dir_[i] = ObAxisDirection::INIT;
@ -357,7 +358,7 @@ private:
ObAxisDirection axis_dir_[AXIS_DIRECTION_NUM]; // direction of x and y axis;
protected:
ObVector<ObSimpleProjPram, common::ObArenaAllocator> simple_proj_prams_; // should be filled by subclass
ObVector<ObSimpleProjPram, common::ObFIFOAllocator> simple_proj_prams_; // should be filled by subclass
};
#define OB_GEO_REG_PROJ_PARAMS(...) \
@ -374,7 +375,7 @@ for (int32_t i = 0; i < ARRAYSIZEOF(arr); ++i) { \
class ObUnknownProjectedSrs : public ObProjectedSrs
{
public:
ObUnknownProjectedSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObUnknownProjectedSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObUnknownProjectedSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::UNKNOWN; };
@ -387,7 +388,7 @@ public:
class ObPopularVisualPseudoMercatorSrs : public ObProjectedSrs
{
public:
ObPopularVisualPseudoMercatorSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObPopularVisualPseudoMercatorSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObPopularVisualPseudoMercatorSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::POPULAR_VISUAL_PSEUDO_MERCATOR; };
@ -400,7 +401,7 @@ public:
class ObLambertAzimuthalEqualAreaSphericalSrs: public ObProjectedSrs
{
public:
ObLambertAzimuthalEqualAreaSphericalSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObLambertAzimuthalEqualAreaSphericalSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObLambertAzimuthalEqualAreaSphericalSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::LAMBERT_AZIMUTHAL_EQUAL_AREA_SPHERICAL; };
@ -413,7 +414,7 @@ public:
class ObEquidistantCylindricalSrs : public ObProjectedSrs
{
public:
ObEquidistantCylindricalSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObEquidistantCylindricalSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObEquidistantCylindricalSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::EQUIDISTANT_CYLINDRICAL; };
@ -426,7 +427,7 @@ public:
class ObEquidistantCylindricalSphericalSrs : public ObProjectedSrs
{
public:
ObEquidistantCylindricalSphericalSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObEquidistantCylindricalSphericalSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObEquidistantCylindricalSphericalSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::EQUIDISTANT_CYLINDRICAL_SPHERICAL; };
@ -439,7 +440,7 @@ public:
class ObKrovakNorthOrientatedSrs : public ObProjectedSrs
{
public:
ObKrovakNorthOrientatedSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObKrovakNorthOrientatedSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObKrovakNorthOrientatedSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::KROVAK_NORTH_ORIENTATED; };
@ -452,7 +453,7 @@ public:
class ObKrovakModifiedSrs : public ObProjectedSrs
{
public:
ObKrovakModifiedSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObKrovakModifiedSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObKrovakModifiedSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::KROVAK_MODIFIED; };
@ -465,7 +466,7 @@ public:
class ObKrovakModifiedNorthOrientatedSrs : public ObProjectedSrs
{
public:
ObKrovakModifiedNorthOrientatedSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObKrovakModifiedNorthOrientatedSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObKrovakModifiedNorthOrientatedSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::KROVAK_MODIFIED_NORTH_ORIENTATED; };
@ -478,7 +479,7 @@ public:
class ObLambertConicConformal2SPMichiganSrs : public ObProjectedSrs
{
public:
ObLambertConicConformal2SPMichiganSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObLambertConicConformal2SPMichiganSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObLambertConicConformal2SPMichiganSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::LAMBERT_CONIC_CONFORMAL_2SP_MICHIGAN; };
@ -491,7 +492,7 @@ public:
class ObColombiaUrbanSrs : public ObProjectedSrs
{
public:
ObColombiaUrbanSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObColombiaUrbanSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObColombiaUrbanSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::COLOMBIA_URBAN; };
@ -504,7 +505,7 @@ public:
class ObLambertConicConformal1SPSrs : public ObProjectedSrs
{
public:
ObLambertConicConformal1SPSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObLambertConicConformal1SPSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObLambertConicConformal1SPSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::LAMBERT_CONIC_CONFORMAL_1SP; };
@ -517,7 +518,7 @@ public:
class ObLambertConicConformal2SPSrs : public ObProjectedSrs
{
public:
ObLambertConicConformal2SPSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObLambertConicConformal2SPSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObLambertConicConformal2SPSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::LAMBERT_CONIC_CONFORMAL_2SP; };
@ -530,7 +531,7 @@ public:
class ObLambertConicConformal2SPBelgiumSrs : public ObProjectedSrs
{
public:
ObLambertConicConformal2SPBelgiumSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObLambertConicConformal2SPBelgiumSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObLambertConicConformal2SPBelgiumSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::LAMBERT_CONIC_CONFORMAL_2SP_BELGIUM; };
@ -543,7 +544,7 @@ public:
class ObMercatorvariantASrs : public ObProjectedSrs
{
public:
ObMercatorvariantASrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObMercatorvariantASrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObMercatorvariantASrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::MERCATOR_VARIANT_A; };
@ -556,7 +557,7 @@ public:
class ObMercatorvariantBSrs : public ObProjectedSrs
{
public:
ObMercatorvariantBSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObMercatorvariantBSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObMercatorvariantBSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::MERCATOR_VARIANT_B; };
@ -569,7 +570,7 @@ public:
class ObCassiniSoldnerSrs : public ObProjectedSrs
{
public:
ObCassiniSoldnerSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObCassiniSoldnerSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObCassiniSoldnerSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::MERCATOR_VARIANT_B; };
@ -582,7 +583,7 @@ public:
class ObTransverseMercatorSrs : public ObProjectedSrs
{
public:
ObTransverseMercatorSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObTransverseMercatorSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObTransverseMercatorSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::TRANSVERSE_MERCATOR; };
@ -595,7 +596,7 @@ public:
class ObTransverseMercatorSouthOrientatedSrs : public ObProjectedSrs
{
public:
ObTransverseMercatorSouthOrientatedSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObTransverseMercatorSouthOrientatedSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObTransverseMercatorSouthOrientatedSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::TRANSVERSE_MERCATOR_SOUTH_ORIENTATED; };
@ -608,7 +609,7 @@ public:
class ObObliqueStereographicSrs : public ObProjectedSrs
{
public:
ObObliqueStereographicSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObObliqueStereographicSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObObliqueStereographicSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::OBLIQUE_STEREOGRAPHIC; };
@ -621,7 +622,7 @@ public:
class ObPolarStereographicVariantASrs : public ObProjectedSrs
{
public:
ObPolarStereographicVariantASrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObPolarStereographicVariantASrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObPolarStereographicVariantASrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::POLAR_STEREOGRAPHIC_VARIANT_A; };
@ -634,7 +635,7 @@ public:
class ObNewZealandMapGridSrs : public ObProjectedSrs
{
public:
ObNewZealandMapGridSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObNewZealandMapGridSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObNewZealandMapGridSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::NEW_ZEALAND_MAP_GRID; };
@ -647,7 +648,7 @@ public:
class ObHotineObliqueMercatorvariantASrs : public ObProjectedSrs
{
public:
ObHotineObliqueMercatorvariantASrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObHotineObliqueMercatorvariantASrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObHotineObliqueMercatorvariantASrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::HOTINE_OBLIQUE_MERCATOR_VARIANT_A; };
@ -660,7 +661,7 @@ public:
class ObLabordeObliqueMercatorSrs : public ObProjectedSrs
{
public:
ObLabordeObliqueMercatorSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObLabordeObliqueMercatorSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObLabordeObliqueMercatorSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::LABORDE_OBLIQUE_MERCATOR; };
@ -673,7 +674,7 @@ public:
class ObHotineObliqueMercatorVariantBSrs : public ObProjectedSrs
{
public:
ObHotineObliqueMercatorVariantBSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObHotineObliqueMercatorVariantBSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObHotineObliqueMercatorVariantBSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::HOTINE_OBLIQUE_MERCATOR_VARIANT_B; };
@ -686,7 +687,7 @@ public:
class ObTunisiaMiningGridSrs : public ObProjectedSrs
{
public:
ObTunisiaMiningGridSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObTunisiaMiningGridSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObTunisiaMiningGridSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::TUNISIA_MINING_GRID; };
@ -699,7 +700,7 @@ public:
class ObLambertConicNearConformalSrs : public ObProjectedSrs
{
public:
ObLambertConicNearConformalSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObLambertConicNearConformalSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObLambertConicNearConformalSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::LAMBERT_CONIC_NEAR_CONFORMAL; };
@ -712,7 +713,7 @@ public:
class ObAmericanPolyconicSrs : public ObProjectedSrs
{
public:
ObAmericanPolyconicSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObAmericanPolyconicSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObAmericanPolyconicSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::AMERICAN_POLYCONIC; };
@ -725,7 +726,7 @@ public:
class ObKrovakSrs : public ObProjectedSrs
{
public:
ObKrovakSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObKrovakSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObKrovakSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::KROVAK; };
@ -738,7 +739,7 @@ public:
class ObLambertAzimuthalEqualAreaSrs : public ObProjectedSrs
{
public:
ObLambertAzimuthalEqualAreaSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObLambertAzimuthalEqualAreaSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObLambertAzimuthalEqualAreaSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::LAMBERT_AZIMUTHAL_EQUAL_AREA; };
@ -751,7 +752,7 @@ public:
class ObAlbersEqualAreaSrs : public ObProjectedSrs
{
public:
ObAlbersEqualAreaSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObAlbersEqualAreaSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObAlbersEqualAreaSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::ALBERS_EQUAL_AREA; };
@ -764,7 +765,7 @@ public:
class ObTransverseMercatorZonedGridSystemSrs : public ObProjectedSrs
{
public:
ObTransverseMercatorZonedGridSystemSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObTransverseMercatorZonedGridSystemSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObTransverseMercatorZonedGridSystemSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::TRANSVERSE_MERCATOR_ZONED_GRID_SYSTEM; };
@ -777,7 +778,7 @@ public:
class ObLambertConicConformalWestOrientatedSrs : public ObProjectedSrs
{
public:
ObLambertConicConformalWestOrientatedSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObLambertConicConformalWestOrientatedSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObLambertConicConformalWestOrientatedSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::LAMBERT_CONIC_CONFORMAL_WEST_ORIENTATED; };
@ -790,7 +791,7 @@ public:
class ObBonneSouthOrientatedSrs : public ObProjectedSrs
{
public:
ObBonneSouthOrientatedSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObBonneSouthOrientatedSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObBonneSouthOrientatedSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::BONNE_SOUTH_ORIENTATED; };
@ -803,7 +804,7 @@ public:
class ObPolarStereographicVariantBSrs : public ObProjectedSrs
{
public:
ObPolarStereographicVariantBSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObPolarStereographicVariantBSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObPolarStereographicVariantBSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::POLAR_STEREOGRAPHIC_VARIANT_B; };
@ -816,7 +817,7 @@ public:
class ObPolarStereographicVariantCSrs : public ObProjectedSrs
{
public:
ObPolarStereographicVariantCSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObPolarStereographicVariantCSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObPolarStereographicVariantCSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::POLAR_STEREOGRAPHIC_VARIANT_C; };
@ -829,7 +830,7 @@ public:
class ObGuamProjectionSrs : public ObProjectedSrs
{
public:
ObGuamProjectionSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObGuamProjectionSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObGuamProjectionSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::GUAM_PROJECTION; };
@ -842,7 +843,7 @@ public:
class ObModifiedAzimuthalEquidistantSrs : public ObProjectedSrs
{
public:
ObModifiedAzimuthalEquidistantSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObModifiedAzimuthalEquidistantSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObModifiedAzimuthalEquidistantSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::MODIFIED_AZIMUTHAL_EQUIDISTANT; };
@ -855,7 +856,7 @@ public:
class ObHyperbolicCassiniSoldnerSrs : public ObProjectedSrs
{
public:
ObHyperbolicCassiniSoldnerSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObHyperbolicCassiniSoldnerSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObHyperbolicCassiniSoldnerSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::HYPERBOLIC_CASSINI_SOLDNER; };
@ -868,7 +869,7 @@ public:
class ObLambertCylindricalEqualAreaSphericalSrs : public ObProjectedSrs
{
public:
ObLambertCylindricalEqualAreaSphericalSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObLambertCylindricalEqualAreaSphericalSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObLambertCylindricalEqualAreaSphericalSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::LAMBERT_CYLINDRICAL_EQUAL_AREA_SPHERICAL; };
@ -881,7 +882,7 @@ public:
class ObLambertCylindricalEqualAreaSrs : public ObProjectedSrs
{
public:
ObLambertCylindricalEqualAreaSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {}
ObLambertCylindricalEqualAreaSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {}
virtual ~ObLambertCylindricalEqualAreaSrs() {}
ObProjectionType get_projection_type() const override { return ObProjectionType::LAMBERT_CYLINDRICAL_EQUAL_AREA; };
@ -932,7 +933,7 @@ public:
static int check_is_wgs84(const ObGeographicRs *rs, bool &is_wgs84);
static int get_simple_proj_params(const ObProjectionPrams &parsed_params,
ObVector<ObSimpleProjPram, common::ObArenaAllocator> &params);
ObVector<ObSimpleProjPram, common::ObFIFOAllocator> &params);
static int check_authority(const ObRsAuthority& auth, const char *target_auth_name, int target_auth_code, bool allow_invalid, bool &res);
constexpr static double WGS_SEMI_MAJOR_AXIS = 6378137.0;

9
deps/oblib/src/lib/hash/ob_hashset.h vendored
View File

@ -137,6 +137,15 @@ public:
{
clear();
}
_key_type *get(const _key_type &key) const
{
const _key_type *ret = NULL;
const pair_type *pair = NULL;
if (OB_SUCCESS == const_cast<hashtable &>(ht_).get_refactored(key, pair)) {
ret = &(pair->first);
}
return const_cast<_key_type*>(ret);
}
// return:
// OB_HASH_EXIST node exists
// OB_HASH_NOT_EXIST node not exists

42
deps/oblib/src/lib/hash/ob_hashutils.h vendored
View File

@ -1239,28 +1239,18 @@ struct SimpleAllocerBlock
Block *next;
};
template <bool B, class T>
template <class T, const int NODE_PAGE_SIZE=common::OB_MALLOC_NORMAL_BLOCK_SIZE>
struct NodeNumTraits
{
/*
24 : sizeof(SimpleAllocerBlock 's members except nodes)
32 : sizeof(SimpleAllocerNode's members except data)
128: for robust
*/
static const int32_t NODE_NUM = ((NODE_PAGE_SIZE - 24 - 128) < (32 + sizeof(T))) ? 1 : ((NODE_PAGE_SIZE -
24 - 128) /
(32 + sizeof(T)));
};
template <class T>
struct NodeNumTraits<false, T>
{
static const int32_t NODE_NUM = (common::OB_MALLOC_NORMAL_BLOCK_SIZE -
24/*=sizeof(SimpleAllocerBlock 's members except nodes)*/ - 128/*for robust*/) /
(32/*sizeof(SimpleAllocerNode's members except data)*/ + sizeof(T));
};
template <class T>
struct NodeNumTraits<true, T>
{
static const int32_t NODE_NUM = 1;
};
#define IS_BIG_OBJ(T) \
((common::OB_MALLOC_NORMAL_BLOCK_SIZE - 24 - 128) < (32 + sizeof(T)))
/*
block_free_list_:
Block C: node1->node2->node3...
@ -1284,7 +1274,7 @@ free:
4. neither
*/
template <class T,
int32_t NODE_NUM = NodeNumTraits<IS_BIG_OBJ(T), T>::NODE_NUM,
int32_t NODE_NUM = NodeNumTraits<T>::NODE_NUM,
class DefendMode = SpinMutexDefendMode,
class Allocer = DefaultSimpleAllocerAllocator>
class SimpleAllocer
@ -1297,15 +1287,20 @@ class SimpleAllocer
typedef typename DefendMode::lock_type lock_type;
typedef typename DefendMode::lock_initer lock_initer;
public:
SimpleAllocer() : block_remainder_(NULL), block_free_list_(NULL)
SimpleAllocer() : leak_check_(true), block_remainder_(NULL), block_free_list_(NULL)
{
lock_initer initer(lock_);
}
~SimpleAllocer()
{
OB_ASSERT(NULL == block_remainder_ &&
NULL == block_free_list_);
if (leak_check_) {
if (NULL != block_remainder_ ||
NULL != block_free_list_) {
HASH_WRITE_LOG_RET(HASH_FATAL, OB_ERR_UNEXPECTED, "SimpleAllocer memory leak");
}
}
}
void set_leak_check(const bool check) { leak_check_ = check; }
void set_attr(const ObMemAttr &attr) { allocer_.set_attr(attr); }
void set_label(const lib::ObLabel &label) { allocer_.set_label(label); }
template <class ... TYPES>
@ -1413,6 +1408,7 @@ public:
}
}
private:
bool leak_check_;
Block *block_remainder_;
Block *block_free_list_;
lock_type lock_;

52
deps/oblib/src/lib/lock/ob_latch.h vendored
View File

@ -42,13 +42,17 @@ extern bool USE_CO_LATCH;
if (lib::is_diagnose_info_enabled()) { \
ObDiagnoseTenantInfo *di = ObDiagnoseTenantInfo::get_local_diagnose_info(); \
if (NULL != di) { \
ObLatchStat &latch_stat = di->get_latch_stats().items_[latch_id]; \
if (OB_SUCC(ret)) { \
++latch_stat.immediate_gets_; \
} else { \
++latch_stat.immediate_misses_; \
} \
latch_stat.spin_gets_ += spin_cnt; \
ObLatchStat *p_latch_stat = di->get_latch_stats().get_or_create_item(latch_id); \
if (OB_ISNULL(p_latch_stat)) { \
} else { \
ObLatchStat &latch_stat = *p_latch_stat; \
if (OB_SUCC(ret)) { \
++latch_stat.immediate_gets_; \
} else { \
++latch_stat.immediate_misses_; \
} \
latch_stat.spin_gets_ += spin_cnt; \
} \
} \
} \
} while(0)
@ -58,21 +62,25 @@ extern bool USE_CO_LATCH;
if (lib::is_diagnose_info_enabled()) { \
ObDiagnoseTenantInfo *di = ObDiagnoseTenantInfo::get_local_diagnose_info(); \
if (NULL != di) { \
ObLatchStat &latch_stat = di->get_latch_stats().items_[latch_id]; \
++latch_stat.gets_; \
latch_stat.spin_gets_ += spin_cnt; \
latch_stat.sleeps_ += yield_cnt; \
if (OB_UNLIKELY(waited)) { \
++latch_stat.misses_; \
ObDiagnoseSessionInfo *dsi = ObDiagnoseSessionInfo::get_local_diagnose_info(); \
if (NULL != dsi) { \
latch_stat.wait_time_ += dsi->get_curr_wait().wait_time_; \
if (dsi->get_curr_wait().wait_time_ > 1000 * 1000) { \
COMMON_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "The Latch wait too much time, ", \
K(dsi->get_curr_wait()), KCSTRING(lbt())); \
} \
} \
} \
ObLatchStat *p_latch_stat = di->get_latch_stats().get_or_create_item(latch_id); \
if (OB_ISNULL(p_latch_stat)) { \
} else { \
ObLatchStat &latch_stat = *p_latch_stat; \
++latch_stat.gets_; \
latch_stat.spin_gets_ += spin_cnt; \
latch_stat.sleeps_ += yield_cnt; \
if (OB_UNLIKELY(waited)) { \
++latch_stat.misses_; \
ObDiagnoseSessionInfo *dsi = ObDiagnoseSessionInfo::get_local_diagnose_info(); \
if (NULL != dsi) { \
latch_stat.wait_time_ += dsi->get_curr_wait().wait_time_; \
if (dsi->get_curr_wait().wait_time_ > 1000 * 1000) { \
COMMON_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "The Latch wait too much time, ", \
K(dsi->get_curr_wait()), KCSTRING(lbt())); \
} \
} \
} \
} \
} \
} \
} while(0)

2
deps/oblib/src/lib/ob_define.h vendored
View File

@ -1595,7 +1595,7 @@ OB_INLINE bool is_bootstrap_resource_pool(const uint64_t resource_pool_id)
}
// ob_malloc & ob_tc_malloc
const int64_t OB_MALLOC_NORMAL_BLOCK_SIZE = (1LL << 13) - 128; // 8KB
const int64_t OB_MALLOC_NORMAL_BLOCK_SIZE = (1LL << 13) - 256; // 8KB
const int64_t OB_MALLOC_MIDDLE_BLOCK_SIZE = (1LL << 16) - 128; // 64KB
const int64_t OB_MALLOC_BIG_BLOCK_SIZE = (1LL << 21) - ACHUNK_PRESERVE_SIZE;// 2MB (-17KB)

3
deps/oblib/src/lib/objectpool/ob_server_object_pool.h vendored
View File

@ -177,7 +177,8 @@ private:
ObPoolArenaHead *array;
int32_t array_size;
};
static const int64_t MAX_POOL_NUM = 1L << 21;
// This structure is used to index all object pools. It is currently invalid and needs to be redesigned later
static const int64_t MAX_POOL_NUM = 1024;
static PoolPair pool_list_[MAX_POOL_NUM];
static int64_t pool_num_;
};

4
deps/oblib/src/lib/queue/ob_dedup_queue.h vendored
View File

@ -155,7 +155,7 @@ class ObDedupQueue : public lib::ThreadPool
public:
static const int64_t TOTAL_LIMIT = 1024L * 1024L * 1024L;
static const int64_t HOLD_LIMIT = 512L * 1024L * 1024L;
static const int64_t PAGE_SIZE = common::OB_MALLOC_BIG_BLOCK_SIZE;
static const int64_t PAGE_SIZE = common::OB_MALLOC_MIDDLE_BLOCK_SIZE;
static const int64_t TASK_MAP_SIZE = 20L * 1000;
static const int64_t TASK_QUEUE_SIZE = 20L * 1000;
public:
@ -186,7 +186,7 @@ private:
HashAllocator;
typedef hash::ObHashMap<const IObDedupTask *,
IObDedupTask *,
hash::MultiWriteDefendMode,
hash::LatchReadWriteDefendMode,
hash::hash_func<const IObDedupTask *>,
hash::equal_to<const IObDedupTask *>,
HashAllocator,

16
deps/oblib/src/lib/queue/ob_priority_queue.h vendored
View File

@ -77,14 +77,14 @@ public:
ret = OB_INVALID_ARGUMENT;
COMMON_LOG(ERROR, "timeout is invalid", K(ret), K(timeout_us));
} else {
auto key = sem_.get_key();
sem_.wait(key, timeout_us);
for(int i = 0; OB_ENTRY_NOT_EXIST == ret && i < PRIO_CNT; i++) {
if (OB_SUCCESS == queue_[i].pop(data)) {
ret = OB_SUCCESS;
}
}
if (OB_FAIL(ret)) {
auto key = sem_.get_key();
sem_.wait(key, timeout_us);
data = NULL;
} else {
(void)ATOMIC_FAA(&size_, -1);
@ -93,6 +93,18 @@ public:
return ret;
}
void destroy()
{
clear();
}
void clear()
{
ObLink* p = NULL;
while(OB_SUCCESS == pop(p, 0))
;
}
private:
SimpleCond sem_;
ObLinkQueue queue_[PRIO_CNT];

8
deps/oblib/src/lib/resource/achunk_mgr.cpp vendored
View File

@ -23,6 +23,7 @@
#include "lib/alloc/alloc_struct.h"
#include "lib/alloc/alloc_failed_reason.h"
#include "lib/alloc/memory_sanity.h"
#include "deps/oblib/src/lib/alloc/malloc_hook.h"
using namespace oceanbase::lib;
@ -143,9 +144,11 @@ void *AChunkMgr::low_alloc(const uint64_t size, const bool can_use_huge_page, bo
huge_flags = flags | MAP_HUGETLB;
}
#endif
const int fd = -1;
// for debug more efficiently
const int fd = -1234;
const int offset = 0;
const int large_page_type = ObLargePageHelper::get_type();
set_ob_mem_mgr_path();
if (SANITY_BOOL_EXPR(alloc_shadow)) {
int64_t new_addr = ATOMIC_FAA(&global_canonical_addr, size);
if (!SANITY_ADDR_IN_RANGE((void*)new_addr)) {
@ -188,11 +191,13 @@ void *AChunkMgr::low_alloc(const uint64_t size, const bool can_use_huge_page, bo
//SANITY_UNPOISON(ptr, size); // maybe no need?
}
}
unset_ob_mem_mgr_path();
return ptr;
}
void AChunkMgr::low_free(const void *ptr, const uint64_t size)
{
set_ob_mem_mgr_path();
if (SANITY_ADDR_IN_RANGE(ptr)) {
void *shad_ptr = SANITY_TO_SHADOW((void*)ptr);
ssize_t shad_size = SANITY_TO_SHADOW_SIZE(size);
@ -200,6 +205,7 @@ void AChunkMgr::low_free(const void *ptr, const uint64_t size)
::munmap(shad_ptr, shad_size);
}
::munmap((void*)ptr, size);
unset_ob_mem_mgr_path();
}
AChunk *AChunkMgr::alloc_chunk(const uint64_t size, bool high_prio)

16
deps/oblib/src/lib/stat/ob_di_cache.cpp vendored
View File

@ -37,10 +37,10 @@ void ObDISessionCollect::clean()
base_value_.reset();
}
ObDITenantCollect::ObDITenantCollect()
ObDITenantCollect::ObDITenantCollect(ObIAllocator *allocator)
: tenant_id_(0),
last_access_time_(0),
base_value_()
base_value_(allocator)
{
}
@ -78,7 +78,7 @@ int ObDISessionCache::get_node(uint64_t session_id, ObDISessionCollect *&session
{
int ret = OB_SUCCESS;
thread_local ObRandom random;
ObSessionBucket &bucket = di_map_[session_id % OB_MAX_SERVER_SESSION_CNT];
ObSessionBucket &bucket = di_map_[session_id % MAX_SESSION_COLLECT_NUM];
while (1) {
bucket.lock_.rdlock();
if (OB_SUCCESS == (ret = bucket.get_the_node(session_id, session_collect))) {
@ -91,14 +91,14 @@ int ObDISessionCache::get_node(uint64_t session_id, ObDISessionCollect *&session
bucket.lock_.unlock();
int64_t pos = 0;
while (1) {
pos = random.get(0, OB_MAX_SERVER_SESSION_CNT-1);
pos = random.get(0, MAX_SESSION_COLLECT_NUM - 1);
if (OB_SUCCESS == (ret = collects_[pos].lock_.try_wrlock())) {
break;
}
}
if (OB_SUCCESS == ret) {
if (0 != collects_[pos].session_id_) {
ObSessionBucket &des_bucket = di_map_[collects_[pos].session_id_ % OB_MAX_SERVER_SESSION_CNT];
ObSessionBucket &des_bucket = di_map_[collects_[pos].session_id_ % MAX_SESSION_COLLECT_NUM];
des_bucket.lock_.wrlock();
des_bucket.list_.remove(&collects_[pos]);
collects_[pos].clean();
@ -135,7 +135,7 @@ int ObDISessionCache::get_all_diag_info(ObIArray<std::pair<uint64_t, ObDISession
std::pair<uint64_t, ObDISessionCollect*> pair;
ObDISessionCollect *head = NULL;
ObDISessionCollect *node = NULL;
for (int64_t i = 0; OB_SUCC(ret) && i < OB_MAX_SERVER_SESSION_CNT; ++i) {
for (int64_t i = 0; OB_SUCC(ret) && i < MAX_SESSION_COLLECT_NUM; ++i) {
ObSessionBucket &bucket = di_map_[i];
bucket.lock_.rdlock();
head = bucket.list_.get_header();
@ -158,7 +158,7 @@ int ObDISessionCache::get_the_diag_info(
ObDISessionCollect *&diag_infos)
{
int ret = OB_SUCCESS;
ObSessionBucket &bucket = di_map_[session_id % OB_MAX_SERVER_SESSION_CNT];
ObSessionBucket &bucket = di_map_[session_id % MAX_SESSION_COLLECT_NUM];
bucket.lock_.rdlock();
ObDISessionCollect *collect = NULL;
if (OB_SUCCESS == (ret = bucket.get_the_node(session_id, collect))) {
@ -501,7 +501,7 @@ int ObDIGlobalTenantCache::get_tenant_stat(ObIAllocator &allocator,
if (NULL == (buf = allocator.alloc(sizeof(ObDITenantCollect)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
} else {
got_collect = new (buf) ObDITenantCollect();
got_collect = new (buf) ObDITenantCollect(&allocator);
got_collect->tenant_id_ = collect->tenant_id_;
bucket.list_.add_last(got_collect);
}

10
deps/oblib/src/lib/stat/ob_di_cache.h vendored
View File

@ -42,7 +42,7 @@ public:
class ObDITenantCollect : public ObDINode<ObDITenantCollect>
{
public:
ObDITenantCollect();
ObDITenantCollect(ObIAllocator *allocator = NULL);
virtual ~ObDITenantCollect();
void clean();
uint64_t tenant_id_;
@ -62,6 +62,10 @@ public:
uint64_t session_id,
ObDISessionCollect *&diag_infos);
int get_node(uint64_t session_id, ObDISessionCollect *&session_collect);
private:
static const int64_t MAX_SESSION_COLLECT_NUM = OB_MAX_SERVER_SESSION_CNT / 4;
STATIC_ASSERT((MAX_SESSION_COLLECT_NUM > 4 * 1024 && MAX_SESSION_COLLECT_NUM < 10 * 1024),
"unexpected MAX_SESSION_COLLECT_NUM");
private:
struct ObSessionBucket
{
@ -89,8 +93,8 @@ private:
};
ObDISessionCache();
virtual ~ObDISessionCache();
ObSessionBucket di_map_[OB_MAX_SERVER_SESSION_CNT];
ObDISessionCollect collects_[OB_MAX_SERVER_SESSION_CNT];
ObSessionBucket di_map_[MAX_SESSION_COLLECT_NUM];
ObDISessionCollect collects_[MAX_SESSION_COLLECT_NUM];
};
struct ObTenantBucket

72
deps/oblib/src/lib/stat/ob_diagnose_info.cpp vendored
View File

@ -75,16 +75,31 @@ void ObLatchStat::reset()
/**
* ----------------------------------------------------------ObLatchStatArray-----------------------------------------------------
*/
ObLatchStatArray::ObLatchStatArray()
: items_()
ObLatchStatArray::ObLatchStatArray(ObIAllocator *allocator)
: allocator_(allocator), items_()
{
}
ObLatchStatArray::~ObLatchStatArray()
{
for (int64_t i = 0; i < ObLatchIds::LATCH_END; ++i) {
if (OB_ISNULL(items_[i])) {
} else {
free_item(items_[i]);
items_[i] = NULL;
}
}
}
int ObLatchStatArray::add(const ObLatchStatArray &other)
{
int ret = OB_SUCCESS;
for (int64_t i = 0; i < ObLatchIds::LATCH_END && OB_SUCCESS == ret; ++i) {
ret = items_[i].add(other.items_[i]);
if (OB_ISNULL(other.get_item(i))) continue;
auto *item = get_or_create_item(i);
if (OB_NOT_NULL(item)) {
ret = item->add(*other.get_item(i));
}
}
return ret;
}
@ -92,7 +107,52 @@ int ObLatchStatArray::add(const ObLatchStatArray &other)
void ObLatchStatArray::reset()
{
for (int64_t i = 0; i < ObLatchIds::LATCH_END; ++i) {
items_[i].reset();
if (OB_ISNULL(items_[i])) {
} else {
items_[i]->reset();
}
}
}
static constexpr int NODE_NUM =
common::hash::NodeNumTraits<ObLatchStat, common::OB_MALLOC_MIDDLE_BLOCK_SIZE>::NODE_NUM;
using LatchStatAlloc = hash::SimpleAllocer<ObLatchStat, NODE_NUM>;
LatchStatAlloc &get_latch_stat_alloc()
{
struct Wrapper
{
Wrapper()
{
instance_.set_attr(SET_USE_500("LatchStat"));
instance_.set_leak_check(false);
}
LatchStatAlloc instance_;
};
static Wrapper w;
return w.instance_;
}
ObLatchStat *ObLatchStatArray::create_item()
{
ObLatchStat *stat = NULL;
lib::ObDisableDiagnoseGuard disable_diagnose_guard;
if (OB_ISNULL(allocator_)) {
stat = get_latch_stat_alloc().alloc();
} else {
stat = OB_NEWx(ObLatchStat, allocator_);
}
return stat;
}
void ObLatchStatArray::free_item(ObLatchStat *stat)
{
lib::ObDisableDiagnoseGuard disable_diagnose_guard;
if (OB_ISNULL(allocator_)) {
get_latch_stat_alloc().free(stat);
} else {
stat->~ObLatchStat();
allocator_->free(stat);
}
}
@ -567,11 +627,11 @@ ObDiagnoseSessionInfo *ObDiagnoseSessionInfo::get_local_diagnose_info()
return di;
}
ObDiagnoseTenantInfo::ObDiagnoseTenantInfo()
ObDiagnoseTenantInfo::ObDiagnoseTenantInfo(ObIAllocator *allocator)
: event_stats_(),
stat_add_stats_(),
stat_set_stats_(),
latch_stats_()
latch_stats_(allocator)
{
}

24
deps/oblib/src/lib/stat/ob_diagnose_info.h vendored
View File

@ -50,10 +50,28 @@ struct ObLatchStat
struct ObLatchStatArray
{
ObLatchStatArray();
public:
ObLatchStatArray(ObIAllocator *allocator = NULL);
~ObLatchStatArray();
int add(const ObLatchStatArray &other);
void reset();
ObLatchStat items_[ObLatchIds::LATCH_END];
ObLatchStat *get_item(int32_t idx) const
{
return items_[idx];
}
ObLatchStat *get_or_create_item(int32_t idx)
{
if (OB_ISNULL(items_[idx])) {
items_[idx] = create_item();
}
return items_[idx];
}
private:
ObLatchStat *create_item();
void free_item(ObLatchStat *stat);
private:
ObIAllocator *allocator_;
ObLatchStat *items_[ObLatchIds::LATCH_END] = {NULL};
};
class ObWaitEventHistoryIter
@ -140,7 +158,7 @@ private:
class ObDiagnoseTenantInfo final
{
public:
ObDiagnoseTenantInfo();
ObDiagnoseTenantInfo(ObIAllocator *allocator = NULL);
~ObDiagnoseTenantInfo();
void add(const ObDiagnoseTenantInfo &other);
void add_wait_event(const ObDiagnoseTenantInfo &other);

3
deps/oblib/src/lib/statistic_event/ob_stat_event.h vendored
View File

@ -531,6 +531,9 @@ STAT_EVENT_SET_DEF(IS_MINI_MODE, "is mini mode", ObStatClassIds::RESOURCE, 14001
STAT_EVENT_SET_DEF(MEMORY_HOLD_SIZE, "observer memory hold size", ObStatClassIds::RESOURCE, 140011, false, true)
STAT_EVENT_SET_DEF(WORKER_TIME, "worker time", ObStatClassIds::RESOURCE, 140012, false, true)
STAT_EVENT_SET_DEF(CPU_TIME, "cpu time", ObStatClassIds::RESOURCE, 140013, false, true)
STAT_EVENT_SET_DEF(MEMORY_LIMIT, "effective observer memory limit", ObStatClassIds::RESOURCE, 140014, false, true)
STAT_EVENT_SET_DEF(SYSTEM_MEMORY, "effective system memory", ObStatClassIds::RESOURCE, 140015, false, true)
STAT_EVENT_SET_DEF(HIDDEN_SYS_MEMORY, "effective hidden sys memory", ObStatClassIds::RESOURCE, 140016, false, true)
//CLOG

131
deps/oblib/src/lib/task/ob_timer.cpp vendored
View File

@ -104,7 +104,7 @@ void ObTimer::stop()
void ObTimer::wait()
{
ObMonitor<Mutex>::Lock guard(monitor_);
while (has_running_task_) {
while (running_task_ != NULL) {
static const int64_t WAIT_INTERVAL_US = 2000000; // 2s
(void)monitor_.timed_wait(ObSysTime(WAIT_INTERVAL_US));
}
@ -125,7 +125,6 @@ void ObTimer::destroy()
{
ObMonitor<Mutex>::Lock guard(monitor_);
for (int64_t i = 0; i < tasks_num_; ++i) {
tokens_[i].task->cancelCallBack();
ATOMIC_STORE(&(tokens_[i].task->timer_), nullptr);
}
tasks_num_ = 0;
@ -245,41 +244,112 @@ int ObTimer::insert_token(const Token &token)
return ret;
}
int ObTimer::cancel(const ObTimerTask &task)
int ObTimer::cancel_task(const ObTimerTask &task)
{
int ret = OB_SUCCESS;
ObMonitor<Mutex>::Lock guard(monitor_);
if (!is_inited_) {
ret = OB_NOT_INIT;
} else {
int64_t pos = -1;
for (int64_t i = 0; i < tasks_num_; ++i) {
if (&task == tokens_[i].task) {
pos = i;
break;
if (&task == uncanceled_task_) {
// repeat cancel, do-nothing
} else if (&task == running_task_) {
if (uncanceled_task_ != NULL) {
ret = OB_ERR_UNEXPECTED;
} else {
ATOMIC_STORE(&const_cast<ObTimerTask&>(task).timer_, nullptr);
uncanceled_task_ = &const_cast<ObTimerTask&>(task);
OB_LOG(INFO, "cancel task", KP(this), K_(thread_id), K(wakeup_time_), K(tasks_num_), K(task));
}
} else {
int64_t pos = -1;
for (int64_t i = 0; i < tasks_num_; ++i) {
if (&task == tokens_[i].task) {
pos = i;
break;
}
}
if (-1 == pos) {
// not found, do-nothing
} else {
ATOMIC_STORE(&const_cast<ObTimerTask&>(task).timer_, nullptr);
memmove(&tokens_[pos], &tokens_[pos + 1],
sizeof(tokens_[0]) * (tasks_num_ - pos - 1));
--tasks_num_;
OB_LOG(INFO, "cancel task", KP(this), K_(thread_id), K(wakeup_time_), K(tasks_num_), K(task));
}
}
if (pos != -1) {
tokens_[pos].task->cancelCallBack();
ATOMIC_STORE(&(tokens_[pos].task->timer_), nullptr);
memmove(&tokens_[pos], &tokens_[pos + 1],
sizeof(tokens_[0]) * (tasks_num_ - pos - 1));
--tasks_num_;
OB_LOG(INFO, "cancel task", KP(this), K_(thread_id), K(pos), K(wakeup_time_), K(tasks_num_), K(task));
}
}
return ret;
}
int ObTimer::wait_task(const ObTimerTask &task)
{
int ret = OB_SUCCESS;
ObMonitor<Mutex>::Lock guard(monitor_);
if (!is_inited_) {
ret = OB_NOT_INIT;
} else {
do {
bool exist = &task == running_task_;
if (!exist) exist = &task == uncanceled_task_;
for (int64_t i = 0; !exist && i < tasks_num_; ++i) {
if (&task == tokens_[i].task) {
exist = true;
break;
}
}
if (!exist) {
break;
} else {
(void)monitor_.timed_wait(ObSysTime(10 * 1000/*10ms*/));
}
} while (true);
}
return ret;
}
int ObTimer::cancel(const ObTimerTask &task)
{
int ret = OB_SUCCESS;
if (OB_FAIL(cancel_task(task))) {
OB_LOG(WARN, "failed to cancel_task", K(ret), K(task));
// @TODO: nijia.nj
//} else if (OB_FAIL(wait_task(task))) {
// OB_LOG(WARN, "failed to wait_task", K(ret), K(task));
}
return ret;
}
void ObTimer::cancel_all()
{
ObMonitor<Mutex>::Lock guard(monitor_);
for (int64_t i = 0; i < tasks_num_; ++i) {
tokens_[i].task->cancelCallBack();
ATOMIC_STORE(&(tokens_[i].task->timer_), nullptr);
}
tasks_num_ = 0;
OB_LOG(INFO, "cancel all", KP(this), K_(thread_id), K(wakeup_time_), K(tasks_num_));
int ret = OB_SUCCESS;
do {
ObMonitor<Mutex>::Lock guard(monitor_);
for (int64_t i = 0; i < tasks_num_; ++i) {
ATOMIC_STORE(&(tokens_[i].task->timer_), nullptr);
}
tasks_num_ = 0;
if (running_task_ != NULL) {
if (uncanceled_task_ == running_task_) {
// do-nothing
} else if (uncanceled_task_ != NULL) {
ret = OB_ERR_UNEXPECTED;
} else {
ATOMIC_STORE(&running_task_->timer_, nullptr);
uncanceled_task_ = running_task_;
}
if (OB_SUCC(ret)) {
(void)monitor_.timed_wait(ObSysTime(10 * 1000/*10ms*/));
}
} else {
if (uncanceled_task_ != NULL) {
ret = OB_ERR_UNEXPECTED;
}
break;
}
} while (OB_SUCC(ret));
OB_LOG(INFO, "cancel all", K(ret), KP(this), K_(thread_id), K(wakeup_time_), K(tasks_num_));
}
void ObTimer::run1()
@ -308,8 +378,9 @@ void ObTimer::run1()
if (is_destroyed_) {
break;
}
//add repeated task to tasks_ again
if (token.delay != 0 && token.task->need_retry() && !is_stopped_) {
if (token.delay != 0 && !is_stopped_) {
has_running_repeat_task_ = false;
token.scheduled_time = ObSysTime::now(ObSysTime::Monotonic).toMicroSeconds() + token.delay;
if (OB_SUCCESS != (tmp_ret = insert_token(
@ -317,7 +388,7 @@ void ObTimer::run1()
OB_LOG_RET(WARN, tmp_ret, "insert token error", K(tmp_ret), K(token));
}
}
has_running_task_ = false;
running_task_ = NULL;
if (is_stopped_) {
monitor_.notify_all();
}
@ -340,8 +411,8 @@ void ObTimer::run1()
while (tasks_num_ > 0 && !is_destroyed_ && !is_stopped_) {
const int64_t now = ObSysTime::now(ObSysTime::Monotonic).toMicroSeconds();
if (tokens_[0].scheduled_time <= now) {
has_running_task_ = true;
token = tokens_[0];
running_task_ = token.task;
memmove(tokens_, tokens_ + 1, (tasks_num_ - 1) * sizeof(tokens_[0]));
--tasks_num_;
ATOMIC_STORE(&(token.task->timer_), nullptr);
@ -374,13 +445,19 @@ void ObTimer::run1()
}
}
if (token.task != NULL && has_running_task_ && !is_destroyed_ && !is_stopped_) {
if (token.task != NULL && running_task_ != NULL && !is_destroyed_ && !is_stopped_) {
bool timeout_check = token.task->timeout_check();
const int64_t start_time = ::oceanbase::common::ObTimeUtility::current_time();
if (timeout_check) {
ObTimerMonitor::get_instance().start_task(thread_id_, start_time, token.delay, token.task);
}
token.task->runTimerTask();
ObMonitor<Mutex>::Lock guard(monitor_);
if (token.task == uncanceled_task_) {
uncanceled_task_ = NULL;
token.delay = 0;
monitor_.notify_all();
}
const int64_t end_time = ::oceanbase::common::ObTimeUtility::current_time();
const int64_t elapsed_time = end_time - start_time;

10
deps/oblib/src/lib/task/ob_timer.h vendored
View File

@ -41,7 +41,6 @@ class ObTimerTask
public:
ObTimerTask() : timeout_check_(true), timer_(nullptr) {}
virtual ~ObTimerTask() { abort_unless(OB_ISNULL(ATOMIC_LOAD(&timer_))); }
virtual void cancelCallBack() {}
virtual void runTimerTask() = 0;
virtual int64_t to_string(char *buf, const int64_t buf_len) const
{
@ -55,7 +54,6 @@ public:
inline void enable_timeout_check() { timeout_check_ = true; }
inline void disable_timeout_check() { timeout_check_ = false; }
inline bool timeout_check() { return timeout_check_; }
virtual bool need_retry() { return true; }
private:
bool timeout_check_;
ObTimer* timer_;
@ -67,7 +65,7 @@ class ObTimer
public:
friend class oceanbase::tests::blocksstable::FakeTabletManager;
ObTimer(int64_t max_task_num = 32): tasks_num_(0), max_task_num_(max_task_num), wakeup_time_(0), is_inited_(false), is_stopped_(false),
is_destroyed_(false), tokens_(nullptr), has_running_task_(false), has_running_repeat_task_(false),
is_destroyed_(false), tokens_(nullptr), running_task_(nullptr), uncanceled_task_(nullptr), has_running_repeat_task_(false),
thread_id_(-1), thread_name_(nullptr) {}
~ObTimer();
int init(const char* thread_name = nullptr,
@ -88,6 +86,8 @@ public:
return OB_SUCCESS;
}
int cancel(const ObTimerTask &task);
int cancel_task(const ObTimerTask &task);
int wait_task(const ObTimerTask &task);
void cancel_all();
int32_t get_tasks_num() const { return tasks_num_; }
void dump() const;
@ -100,6 +100,7 @@ private:
TO_STRING_KV(K(scheduled_time), K(delay), KP(task), KPC(task));
int64_t scheduled_time;
int64_t delay;
bool canceled_;
ObTimerTask *task;
};
int insert_token(const Token &token);
@ -116,7 +117,8 @@ private:
bool is_destroyed_;
obutil::ObMonitor<obutil::Mutex> monitor_;
Token* tokens_;
bool has_running_task_;
ObTimerTask *running_task_;
ObTimerTask *uncanceled_task_;
bool has_running_repeat_task_;
int64_t thread_id_;
const char* thread_name_;

4
deps/oblib/src/lib/thread/ob_async_task_queue.cpp vendored
View File

@ -35,7 +35,7 @@ ObAsyncTaskQueue::~ObAsyncTaskQueue()
}
}
int ObAsyncTaskQueue::init(const int64_t thread_cnt, const int64_t queue_size, const char *thread_name)
int ObAsyncTaskQueue::init(const int64_t thread_cnt, const int64_t queue_size, const char *thread_name, const int64_t page_size)
{
int ret = OB_SUCCESS;
if (is_inited_) {
@ -44,7 +44,7 @@ int ObAsyncTaskQueue::init(const int64_t thread_cnt, const int64_t queue_size, c
} else if (thread_cnt <= 0|| queue_size <= 0 || 0 != (queue_size & (queue_size - 1))) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(thread_cnt), K(queue_size), K(ret));
} else if (OB_FAIL(allocator_.init(TOTAL_LIMIT, HOLD_LIMIT, PAGE_SIZE))) {
} else if (OB_FAIL(allocator_.init(TOTAL_LIMIT, HOLD_LIMIT, page_size))) {
LOG_WARN("allocator init failed", "total limit", static_cast<int64_t>(TOTAL_LIMIT),
"hold limit", static_cast<int64_t>(HOLD_LIMIT),
"page size", static_cast<int64_t>(PAGE_SIZE), K(ret));

4
deps/oblib/src/lib/thread/ob_async_task_queue.h vendored
View File

@ -97,7 +97,7 @@ public:
virtual ~ObAsyncTaskQueue();
//attention queue_size should be 2^n
int init(const int64_t thread_cnt, const int64_t queue_size,
const char *thread_name = nullptr);
const char *thread_name = nullptr, const int64_t page_size = PAGE_SIZE);
int start();
void stop();
void wait();
@ -107,7 +107,7 @@ public:
protected:
static const int64_t TOTAL_LIMIT = 1024L * 1024L * 1024L;
static const int64_t HOLD_LIMIT = 512L * 1024L * 1024L;
static const int64_t PAGE_SIZE = common::OB_MALLOC_BIG_BLOCK_SIZE;
static const int64_t PAGE_SIZE = common::OB_MALLOC_MIDDLE_BLOCK_SIZE;
static const int64_t SLEEP_INTERVAL = 10000; //10ms
virtual void run2();
virtual int blocking_run() { BLOCKING_RUN_IMPLEMENT(); }

34
deps/oblib/src/lib/thread/ob_tenant_hook.cpp vendored
View File

@ -179,40 +179,6 @@ int futex_hook(uint32_t *uaddr, int futex_op, uint32_t val, const struct timespe
return ret;
}
struct PthreadCreateArgument
{
PthreadCreateArgument(void *(*start_routine)(void *), void *arg)
{
start_routine_ = start_routine;
arg_ = arg;
in_use_ = 1;
}
void *(*start_routine_)(void *);
void *arg_;
int in_use_; // TO avoid memory alloc, there is a sync wait for pthread_create.
};
void* run_func(void* arg)
{
struct PthreadCreateArgument* parg = (struct PthreadCreateArgument*)arg;
void *(*start_routine)(void *) = parg->start_routine_;
void *real_arg = parg->arg_;
ATOMIC_STORE(&(parg->in_use_), 0);
::oceanbase::lib::ObStackHeaderGuard stack_header_guard;
return start_routine(real_arg);
}
int ob_pthread_create(pthread_t *thread, const pthread_attr_t *attr,
void *(*start_routine) (void *), void *arg)
{
struct PthreadCreateArgument parg(start_routine, arg);
int ret = pthread_create(thread, attr, run_func, &parg);
while (ATOMIC_LOAD(&(parg.in_use_)) != 0) {
sched_yield();
}
return ret;
}
void ob_set_thread_name(const char* type)
{
::oceanbase::lib::set_thread_name(type);

1
deps/oblib/src/lib/thread/thread.h vendored
View File

@ -55,6 +55,7 @@ public:
void wait();
void destroy();
void dump_pth();
pthread_t get_pthread() { return pth_; }
/// \brief Get current thread object.
///

33
deps/oblib/src/lib/thread/thread_mgr.h vendored
View File

@ -190,6 +190,16 @@ public:
{
return common::OB_NOT_SUPPORTED;
}
virtual int cancel_task(const common::ObTimerTask &task)
{
UNUSED(task);
return common::OB_NOT_SUPPORTED;
}
virtual int wait_task(const common::ObTimerTask &task)
{
UNUSED(task);
return common::OB_NOT_SUPPORTED;
}
virtual void set_queue_size(const int64_t qsize)
{
UNUSED(qsize);
@ -829,6 +839,26 @@ public:
}
return ret;
}
int cancel_task(const common::ObTimerTask &task) override
{
int ret = common::OB_SUCCESS;
if (OB_ISNULL(timer_)) {
ret = common::OB_ERR_UNEXPECTED;
} else {
ret = timer_->cancel_task(task);
}
return ret;
}
int wait_task(const common::ObTimerTask &task) override
{
int ret = common::OB_SUCCESS;
if (OB_ISNULL(timer_)) {
ret = common::OB_ERR_UNEXPECTED;
} else {
ret = timer_->wait_task(task);
}
return ret;
}
void destroy()
{
if (timer_ != nullptr) {
@ -1048,6 +1078,7 @@ public:
TG_MGR.tgs_[tg_id]; \
if (OB_ISNULL(tg)) { \
ret = common::OB_ERR_UNEXPECTED; \
OB_LOG(WARN, "null tg", K(tg_id)); \
} else { \
auto f = [&]() { return tg->func(args); }; \
ret = lib::FWrap<decltype(f), std::is_void<decltype(f())>::value>()(f); \
@ -1084,6 +1115,8 @@ public:
#define TG_STOP(tg_id) do { int r = TG_INVOKE(tg_id, stop); UNUSED(r); } while (0)
#define TG_CANCEL_R(tg_id, args...) TG_INVOKE(tg_id, cancel, args)
#define TG_CANCEL(tg_id, args...) do { int r = TG_INVOKE(tg_id, cancel, args); UNUSED(r); } while (0)
#define TG_CANCEL_TASK(tg_id, args...) do { int r = TG_INVOKE(tg_id, cancel_task, args); UNUSED(r); } while (0)
#define TG_WAIT_TASK(tg_id, args...) do { int r = TG_INVOKE(tg_id, wait_task, args); UNUSED(r); } while (0)
#define TG_CANCEL_ALL(tg_id) TG_INVOKE(tg_id, cancel_all)
#define TG_TASK_EXIST(tg_id, args...) TG_INVOKE(tg_id, task_exist, args)
#define TG_SCHEDULE(tg_id, args...) TG_INVOKE(tg_id, schedule, args)

50
deps/oblib/src/lib/thread/threads.cpp vendored
View File

@ -19,7 +19,7 @@
#include "lib/oblog/ob_log.h"
#include "lib/signal/ob_signal_struct.h"
#include "lib/worker.h"
using namespace oceanbase;
using namespace oceanbase::lib;
using namespace oceanbase::common;
@ -271,3 +271,51 @@ void Threads::destroy()
threads_ = nullptr;
}
}
extern "C" {
int ob_pthread_create(void **ptr, void *(*start_routine) (void *), void *arg)
{
int ret = OB_SUCCESS;
ObPThread *thread = NULL;
// Temporarily set expect_run_wrapper to NULL for creating normal thread
IRunWrapper *expect_run_wrapper = Threads::get_expect_run_wrapper();
Threads::get_expect_run_wrapper() = NULL;
DEFER(Threads::get_expect_run_wrapper() = expect_run_wrapper);
OB_LOG(INFO, "ob_pthread_create start");
if (OB_ISNULL(thread = OB_NEW(ObPThread, SET_USE_500("PThread"), start_routine, arg))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
OB_LOG(WARN, "alloc memory failed", K(ret));
} else if (OB_FAIL(thread->start())) {
OB_LOG(WARN, "failed to start thread", K(ret));
}
if (OB_FAIL(ret)) {
if (OB_NOT_NULL(thread)) {
OB_DELETE(ObPThread, SET_USE_500("PThread"), thread);
}
} else {
*ptr = thread;
OB_LOG(INFO, "ob_pthread_create succeed", KP(thread));
}
return ret;
}
void ob_pthread_join(void *ptr)
{
if (OB_NOT_NULL(ptr)) {
ObPThread *thread = (ObPThread*) ptr;
thread->wait();
OB_LOG(INFO, "ob_pthread_join succeed", KP(thread));
OB_DELETE(ObPThread, SET_USE_500("PThread"), thread);
}
}
pthread_t ob_pthread_get_pth(void *ptr)
{
pthread_t pth = 0;
if (OB_NOT_NULL(ptr)) {
ObPThread *thread = (ObPThread*) ptr;
pth = thread->get_pthread(0);
}
return pth;
}
} /* extern "C" */

23
deps/oblib/src/lib/thread/threads.h vendored
View File

@ -93,6 +93,14 @@ public:
IGNORE_RETURN lib::Thread::update_loop_ts();
return stop_;
}
pthread_t get_pthread(int64_t idx)
{
pthread_t pth = 0;
if (idx < n_threads_) {
pth = threads_[idx]->get_pthread();
}
return pth;
}
protected:
int64_t get_thread_count() const { return n_threads_; }
uint64_t get_thread_idx() const { return thread_idx_; }
@ -121,6 +129,21 @@ private:
IRunWrapper *run_wrapper_;
};
class ObPThread : public Threads
{
public:
ObPThread(void *(*start_routine) (void *), void *arg)
: start_routine_(start_routine), arg_(arg)
{}
void run1() override
{
start_routine_(arg_);
}
private:
void *(*start_routine_)(void *);
void *arg_;
};
using ThreadPool = Threads;
} // lib

68
deps/oblib/src/lib/timezone/ob_timezone_info.cpp vendored
View File

@ -39680,6 +39680,38 @@ int ObTimeZoneInfoPos::timezone_to_str(char *buf, const int64_t buf_len, int64_t
return ret;
}
void ObTimeZoneInfoPos::set_tz_type_attr(const lib::ObMemAttr &attr)
{
tz_tran_types_[0].set_attr(attr);
tz_tran_types_[1].set_attr(attr);
tz_revt_types_[0].set_attr(attr);
tz_revt_types_[1].set_attr(attr);
}
bool ObTimeZoneInfoPos::operator==(const ObTimeZoneInfoPos &other) const
{
bool same = (tz_id_ == other.tz_id_
&& default_type_ == other.default_type_)
&& curr_idx_ == other.curr_idx_
&& (0 == curr_idx_ || 1 == curr_idx_)
&& 0 == STRCASECMP(tz_name_, other.tz_name_);
if (same) {
same = tz_tran_types_[curr_idx_].count() == other.tz_tran_types_[curr_idx_].count()
&& tz_revt_types_[curr_idx_].count() == other.tz_revt_types_[curr_idx_].count();
}
if (same) {
for (int64_t i = 0; i < tz_tran_types_[curr_idx_].count() && same; i++) {
same = tz_tran_types_[curr_idx_].at(i) == other.tz_tran_types_[curr_idx_].at(i);
}
}
if (same) {
for (int64_t i = 0; i < tz_revt_types_[curr_idx_].count() && same; i++) {
same = tz_revt_types_[curr_idx_].at(i) == other.tz_revt_types_[curr_idx_].at(i);
}
}
return same;
}
OB_DEF_SERIALIZE(ObTimeZoneInfoPos)
{
int ret = OB_SUCCESS;
@ -39752,8 +39784,6 @@ void ObTZNameIDAlloc::free_value(ObTZNameIDInfo *info)
{
op_free(info);
info = NULL;
}
ObTZNameHashNode* ObTZNameIDAlloc::alloc_node(ObTZNameIDInfo *value)
@ -39776,9 +39806,9 @@ int ObTZInfoMap::init(const lib::ObMemAttr &attr)
if (OB_UNLIKELY(inited_)) {
ret = OB_INIT_TWICE;
LOG_WARN("init twice", K(ret));
} else if (OB_FAIL(id_map_.init(attr))) {
} else if (OB_FAIL(id_map_buf_.init(attr))) {
LOG_WARN("fail to init id map", K(ret));
} else if (OB_FAIL(name_map_.init(attr))) {
} else if (OB_FAIL(name_map_buf_.init(attr))) {
LOG_WARN("fail to init name map", K(ret));
} else {
inited_ = true;
@ -39786,18 +39816,12 @@ int ObTZInfoMap::init(const lib::ObMemAttr &attr)
return ret;
}
int ObTZInfoMap::reset()
{
int ret = OB_SUCCESS;
id_map_.reset();
name_map_.reset();
return ret;
}
void ObTZInfoMap::destroy()
{
id_map_.destroy();
name_map_.destroy();
id_map_ = NULL;
name_map_ = NULL;
id_map_buf_.destroy();
name_map_buf_.destroy();
}
static bool print_tz_info(ObTZIDKey &key, ObTimeZoneInfoPos *tz_info)
@ -39816,7 +39840,7 @@ static bool print_tz_info(ObTZIDKey &key, ObTimeZoneInfoPos *tz_info)
int ObTZInfoMap::print_tz_info_map()
{
int ret = OB_SUCCESS;
if (OB_FAIL(id_map_.for_each(print_tz_info))) {
if (OB_FAIL(id_map_->for_each(print_tz_info))) {
LOG_WARN("fail to call for_each", K(ret));
}
return ret;
@ -39826,7 +39850,7 @@ int ObTZInfoMap::get_tz_info_by_id(const int64_t tz_id, ObTimeZoneInfoPos &tz_in
{
int ret = OB_SUCCESS;
ObTimeZoneInfoPos *tmp_tz_info = NULL;
if (OB_FAIL(id_map_.get(tz_id, tmp_tz_info))) {
if (OB_FAIL(id_map_->get(tz_id, tmp_tz_info))) {
LOG_WARN("fail to get tz_info_by_id, should not happened", K(tz_id), K(ret));
} else if (OB_FAIL(tz_info_by_id.assign(*tmp_tz_info))) {
LOG_WARN("assign time zone info pos failed", K(ret));
@ -39834,7 +39858,7 @@ int ObTZInfoMap::get_tz_info_by_id(const int64_t tz_id, ObTimeZoneInfoPos &tz_in
LOG_DEBUG("succ to get tz_info_by_id", K(tz_id), KPC(tmp_tz_info), K(ret));
}
if (NULL != tmp_tz_info) {
id_map_.revert(tmp_tz_info);
id_map_->revert(tmp_tz_info);
}
return ret;
}
@ -39843,7 +39867,7 @@ int ObTZInfoMap::get_tz_info_by_name(const ObString &tz_name, ObTimeZoneInfoPos
{
int ret = OB_SUCCESS;
ObTZNameIDInfo *name_id_info = NULL;
if (OB_FAIL(name_map_.get(ObTZNameKey(tz_name), name_id_info))) {
if (OB_FAIL(name_map_->get(ObTZNameKey(tz_name), name_id_info))) {
LOG_WARN("fail to get get_tz_info_by_name", K(tz_name), K(ret));
} else if (OB_FAIL(get_tz_info_by_id(name_id_info->tz_id_, tz_info_by_name))) {
LOG_WARN("fail to get get_tz_info_by_name", KPC(name_id_info), K(ret));
@ -39856,7 +39880,7 @@ int ObTZInfoMap::get_tz_info_by_name(const ObString &tz_name, ObTimeZoneInfoPos
}
if (NULL != name_id_info) {
name_map_.revert(name_id_info);
name_map_->revert(name_id_info);
name_id_info = NULL;
}
@ -39870,7 +39894,7 @@ int ObTZInfoMap::get_tz_info_by_id(const int64_t tz_id, ObTimeZoneInfoPos *&tz_i
if (OB_NOT_NULL(tz_info_by_id)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tz_info_by_id should be null here", K(ret));
} else if (OB_FAIL(id_map_.get(tz_id, tz_info_by_id))) {
} else if (OB_FAIL(id_map_->get(tz_id, tz_info_by_id))) {
LOG_WARN("fail to get tz_info_by_id, should not happened", K(tz_id), K(ret));
}
return ret;
@ -39883,7 +39907,7 @@ int ObTZInfoMap::get_tz_info_by_name(const ObString &tz_name, ObTimeZoneInfoPos
if (OB_NOT_NULL(tz_info_by_name)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("v should be null here", K(ret));
} else if (OB_FAIL(name_map_.get(ObTZNameKey(tz_name), name_id_info))) {
} else if (OB_FAIL(name_map_->get(ObTZNameKey(tz_name), name_id_info))) {
LOG_WARN("fail to get get_tz_info_by_name", K(tz_name), K(ret));
} else if (OB_FAIL(get_tz_info_by_id(name_id_info->tz_id_, tz_info_by_name))) {
LOG_WARN("fail to get get_tz_info_by_name", KPC(name_id_info), K(ret));
@ -39892,7 +39916,7 @@ int ObTZInfoMap::get_tz_info_by_name(const ObString &tz_name, ObTimeZoneInfoPos
ret = OB_ERR_UNKNOWN_TIME_ZONE;
}
if (NULL != name_id_info) {
name_map_.revert(name_id_info);
name_map_->revert(name_id_info);
name_id_info = NULL;
}

18
deps/oblib/src/lib/timezone/ob_timezone_info.h vendored
View File

@ -236,6 +236,7 @@ public:
{ }
const ObTZInfoMap *get_tz_map() const { return tz_info_map_; }
void set_tz_map(const common::ObTZInfoMap *tz_info_map);
TO_STRING_KV(KP_(tz_info_map));
private:
ObTZInfoMap *tz_info_map_;
};
@ -541,6 +542,8 @@ public:
common::ObSArray<ObTZTransitionTypeInfo> &get_next_tz_tran_types() { return tz_tran_types_[get_next_idx() % 2]; }
common::ObSArray<ObTZRevertTypeInfo> &get_next_tz_revt_types() { return tz_revt_types_[get_next_idx() % 2]; }
int calc_revt_types();
bool operator==(const ObTimeZoneInfoPos &other) const;
void set_tz_type_attr(const lib::ObMemAttr &attr);
virtual int timezone_to_str(char *buf, const int64_t len, int64_t &pos) const;
VIRTUAL_TO_STRING_KV("tz_name", common::ObString(common::OB_MAX_TZ_NAME_LEN, tz_name_),
"tz_id", tz_id_,
@ -594,7 +597,12 @@ public:
}
~ObTZNameIDInfo() {}
TO_STRING_KV(K_(tz_id), KCSTRING_(tz_name));
bool operator==(const ObTZNameIDInfo &other) const
{
return (tz_id_ == other.tz_id_
&& 0 == STRCASECMP(tz_name_, other.tz_name_));
}
public:
int64_t tz_id_;
char tz_name_[common::OB_MAX_TZ_NAME_LEN];
@ -630,10 +638,9 @@ typedef common::ObLinkHashMap<ObTZNameKey, ObTZNameIDInfo, ObTZNameIDAlloc> ObTZ
class ObTZInfoMap
{
public:
ObTZInfoMap() : inited_(false), id_map_(), name_map_() {}
ObTZInfoMap() : inited_(false), id_map_(&id_map_buf_), name_map_(&name_map_buf_) {}
~ObTZInfoMap() {}
int init(const lib::ObMemAttr &attr);
int reset();
void destroy();
int print_tz_info_map();
bool is_inited() { return inited_; }
@ -641,11 +648,12 @@ public:
int get_tz_info_by_name(const common::ObString &tz_name, ObTimeZoneInfoPos &tz_info_by_name);
int get_tz_info_by_id(const int64_t tz_id, ObTimeZoneInfoPos *&tz_info_by_id);
int get_tz_info_by_name(const common::ObString &tz_name, ObTimeZoneInfoPos *&tz_info_by_name);
void free_tz_info_pos(ObTimeZoneInfoPos *&tz_info) { id_map_.revert(tz_info); tz_info = NULL; }
public:
bool inited_;
ObTZInfoIDPosMap id_map_; // tz_id => ObTimeZoneInfoPos
ObTZInfoNameIDMap name_map_; // tz_name => tz_id
ObTZInfoIDPosMap *id_map_;
ObTZInfoNameIDMap *name_map_;
ObTZInfoIDPosMap id_map_buf_; // tz_id => ObTimeZoneInfoPos
ObTZInfoNameIDMap name_map_buf_; // tz_name => tz_id
private:
DISALLOW_COPY_AND_ASSIGN(ObTZInfoMap);

18
deps/oblib/src/rpc/frame/ob_req_queue_thread.cpp vendored
View File

@ -34,22 +34,22 @@ using namespace oceanbase::lib;
ObReqQueue::ObReqQueue(int capacity)
: wait_finish_(true),
push_worker_count_(0),
capacity_(capacity),
queue_(),
qhandler_(NULL),
host_()
{
queue_.set_limit(capacity);
}
ObReqQueue::~ObReqQueue()
{
LOG_INFO("begin to destroy queue", K(queue_.size()));
queue_.destroy();
}
int ObReqQueue::init(const int64_t tenant_id)
{
return queue_.init(capacity_, "ReqQueue", tenant_id);
UNUSED(tenant_id);
return OB_SUCCESS;
}
void ObReqQueue::set_qhandler(ObiReqQHandler *qhandler)
@ -74,18 +74,18 @@ bool ObReqQueue::push(ObRequest *req, int max_queue_len, bool block)
}
if (bret) {
bret = OB_LIKELY(OB_SUCCESS == queue_.push(req));
bret = OB_LIKELY(OB_SUCCESS == queue_.push(req, 0));
}
return bret;
}
oceanbase::rpc::ObRequest *ObReqQueue::pop()
{
void *task = NULL;
ObLink *task = NULL;
int64_t timeout = 0;
ObRequest *req = NULL;
if (queue_.size() > 0 && OB_LIKELY(OB_SUCCESS == queue_.pop(task, timeout)) && OB_NOT_NULL(task)) {
req = reinterpret_cast<ObRequest *>(task);
req = static_cast<ObRequest *>(task);
}
return req;
}
@ -95,7 +95,7 @@ void ObReqQueue::set_host(const ObAddr &host)
host_ = host;
}
int ObReqQueue::process_task(void *task)
int ObReqQueue::process_task(ObLink *task)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(task) || OB_ISNULL(qhandler_)) {
@ -106,7 +106,7 @@ int ObReqQueue::process_task(void *task)
param.set_mem_attr(common::OB_SERVER_TENANT_ID, ObModIds::OB_ROOT_CONTEXT, ObCtxIds::WORK_AREA)
.set_properties(USE_TL_PAGE_OPTIONAL);
CREATE_WITH_TEMP_CONTEXT(param) {
ObRequest *req = reinterpret_cast<ObRequest *>(task);
ObRequest *req = static_cast<ObRequest *>(task);
// init trace id
if (ObRequest::OB_RPC == req->get_type()) {
@ -164,7 +164,7 @@ void ObReqQueue::loop()
{
int ret = OB_SUCCESS;
int64_t timeout = 3000 * 1000;
void *task = NULL;
ObLink *task = NULL;
if (OB_ISNULL(qhandler_)) {
ret = OB_INVALID_ARGUMENT;
LOG_ERROR("invalid argument", K(qhandler_));

7
deps/oblib/src/rpc/frame/ob_req_queue_thread.h vendored
View File

@ -15,7 +15,7 @@
#include "lib/ob_define.h"
#include "lib/queue/ob_lighty_queue.h"
#include "lib/queue/ob_priority_queue.h"
#include "lib/profile/ob_trace_id.h"
#include "lib/net/ob_addr.h"
#include "rpc/frame/obi_req_qhandler.h"
@ -66,7 +66,7 @@ public:
}
private:
int process_task(void *task);
int process_task(ObLink *task);
DISALLOW_COPY_AND_ASSIGN(ObReqQueue);
@ -74,8 +74,7 @@ protected:
bool wait_finish_;
int push_worker_count_;
int64_t capacity_;
common::ObLightyQueue queue_;
common::ObPriorityQueue<1> queue_;
ObiReqQHandler *qhandler_;
static const int64_t MAX_PACKET_SIZE = 2 * 1024 * 1024L; // 2M

5
deps/oblib/src/rpc/pnio/interface/group.c vendored
View File

@ -240,7 +240,7 @@ PN_API int pn_provision(int listen_id, int gid, int thread_count)
pn_t* pn = pn_create(listen_id, gid, count);
if (NULL == pn) {
err = ENOMEM;
} else if (0 != (err = ob_pthread_create(&pn->pd, NULL, pn_thread_func, pn))) {
} else if (0 != (err = ob_pthread_create(&pn->pd, pn_thread_func, pn))) {
pn_destroy(pn);
} else {
pn->has_stopped_ = false;
@ -384,7 +384,8 @@ PN_API void pn_wait(uint64_t gid)
for (int tid = 0; tid < pgrp->count; tid++) {
pn_t *pn = get_pn_for_send(pgrp, tid);
if (!pn->has_stopped_) {
pthread_join(pn->pd, NULL);
ob_pthread_join(pn->pd);
pn->pd = NULL;
pn->has_stopped_ = true;
}
}

2
deps/oblib/src/rpc/pnio/interface/group.h vendored
View File

@ -40,7 +40,7 @@ typedef struct pn_grp_comm_t
#define PN_COMM \
bool is_stop_; \
bool has_stopped_; \
pthread_t pd; \
void *pd; \
int accept_qfd; \
int gid; \
int tid; \

9
deps/oblib/src/rpc/pnio/r0/log.h vendored
View File

@ -7,8 +7,8 @@
typedef void (*log_func_t)(int level, const char *file, int line, const char *function, const char *fmt, va_list ap);
extern void do_log(int level, const char* file, int line, const char* func, const char* format, ...) __attribute__((format(printf, 5, 6)));
extern void ob_set_thread_name(const char* type);
extern int ob_pthread_create(pthread_t *thread, const pthread_attr_t *attr,
void *(*start_routine) (void *), void *arg);
extern int ob_pthread_create(void **ptr, void *(*start_routine) (void *), void *arg);
extern void ob_pthread_join(void *ptr);
extern int64_t ob_update_loop_ts();
extern log_func_t g_log_func;
extern int g_log_level;
@ -22,11 +22,6 @@ void ob_set_thread_name(const char* type)
{
prctl(PR_SET_NAME, type);
}
int ob_pthread_create(pthread_t *thread, const pthread_attr_t *attr,
void *(*start_routine) (void *), void *arg)
{
return pthread_create(thread, attr, start_routine, arg);
}
int64_t ob_update_loop_ts()
{
return 0;

56
deps/oblib/unittest/lib/task/test_timer.cpp vendored
View File

@ -28,14 +28,17 @@ public:
void runTimerTask()
{
has_run_ = true;
running_ = true;
++task_run_count_;
::usleep(50000);//50ms
::usleep(exec_time_);
running_ = false;
}
volatile bool running_;
int64_t task_run_count_;
int64_t exec_time_ = 50000; // 50ms
bool has_run_ = false;
};
TEST(TestTimer, timer_task)
@ -120,6 +123,57 @@ TEST(TestTimer, start_stop)
timer.destroy();
}
TEST(TestTimer, task_cancel_wait)
{
TestTimerTask task;
ObTimer timer;
ASSERT_EQ(OB_SUCCESS, timer.init());
ASSERT_EQ(OB_SUCCESS, timer.start());
// cancel from non-running
{
ASSERT_EQ(OB_SUCCESS, timer.schedule(task, 1000000, true));
ASSERT_EQ(1, timer.get_tasks_num());
ASSERT_EQ(OB_SUCCESS, timer.cancel_task(task));
ASSERT_FALSE(timer.task_exist(task));
// repeat cancel
ASSERT_EQ(OB_SUCCESS, timer.cancel_task(task));
ASSERT_FALSE(timer.task_exist(task));
}
// cancel from running
{
task.exec_time_ = 1000000;
int64_t cur_time = ObTimeUtility::current_time();
ASSERT_EQ(OB_SUCCESS, timer.schedule(task, 0, true));
usleep(10000);
ASSERT_EQ(OB_SUCCESS, timer.cancel_task(task));
// repeat cancel
ASSERT_EQ(OB_SUCCESS, timer.cancel_task(task));
ASSERT_LT(ObTimeUtility::current_time() - cur_time, 1000000);
ASSERT_EQ(OB_SUCCESS, timer.wait_task(task));
ASSERT_GT(ObTimeUtility::current_time() - cur_time, 1000000);
ASSERT_FALSE(timer.task_exist(task));
// wait non-exist task
ASSERT_EQ(OB_SUCCESS, timer.wait_task(task));
ASSERT_TRUE(NULL == timer.running_task_);
ASSERT_TRUE(NULL == timer.uncanceled_task_);
}
// cancel all
{
TestTimerTask task2;
TestTimerTask task3;
task2.exec_time_ = 1000000;
ASSERT_EQ(OB_SUCCESS, timer.schedule(task2, 0, true));
ASSERT_EQ(OB_SUCCESS, timer.schedule(task3, 1000, true));
usleep(10000);
timer.cancel_all();
ASSERT_TRUE(task2.has_run_);
ASSERT_FALSE(task3.has_run_);
}
timer.stop();
timer.wait();
timer.destroy();
}
} // end namespace common
} // end namespace oceanbase

20
deps/oblib/unittest/lib/thread/test_threads.cpp vendored
View File

@ -96,7 +96,27 @@ TEST(TestThreads, DynamicThread)
ASSERT_EQ(4, starts);
ASSERT_EQ(4, exits);
}
extern "C" {
int ob_pthread_create(void **ptr, void *(*start_routine) (void *), void *arg);
void ob_pthread_join(void *ptr);
pthread_t ob_pthread_get_pth(void *ptr);
}
void *my_func(void *arg)
{
pthread_t *pth = (pthread_t*)arg;
*pth = pthread_self();
return NULL;
}
TEST(TestThreads, ObPthread)
{
pthread_t pth = 0;
void *tid = NULL;
ASSERT_EQ(0, ob_pthread_create(&tid, my_func, &pth));
sleep(1);
ASSERT_EQ(ob_pthread_get_pth(tid), pth);
ob_pthread_join(tid);
}
int main(int argc, char **argv)
{
::testing::InitGoogleTest(&argc, argv);

11
deps/ussl-hook/ussl-loop.c vendored
View File

@ -30,10 +30,10 @@ uloop_t global_ussl_loop_struct;
static int ussl_has_listened = 0;
static ussl_sf_t acceptfd_fty;
static ussl_sf_t clientfd_fty;
static pthread_t ussl_bg_thread_id;
static void *ussl_bg_thread_id;
int ob_pthread_create(pthread_t *thread, const pthread_attr_t *attr,
void *(*start_routine) (void *), void *arg);
int ob_pthread_create(void **ptr, void *(*start_routine) (void *), void *arg);
void ob_pthread_join(void *ptr);
void ob_set_thread_name(const char* type);
static int uloop_init(uloop_t *l)
@ -102,7 +102,7 @@ int ussl_init_bg_thread()
if (0 == ret) {
if (0 != uloop_init(&global_ussl_loop_struct)) {
ussl_log_error("initialize uloop failed.")
} else if (0 != ob_pthread_create(&ussl_bg_thread_id, NULL, bg_thread_func, NULL)) {
} else if (0 != ob_pthread_create(&ussl_bg_thread_id, bg_thread_func, NULL)) {
ret = EIO;
ussl_log_error("create background thread failed, errno:%d", errno);
} else {
@ -116,7 +116,8 @@ extern int is_ussl_bg_thread_started;
void ussl_wait_bg_thread()
{
if (ATOMIC_LOAD(&is_ussl_bg_thread_started)) {
pthread_join(ussl_bg_thread_id, NULL);
ob_pthread_join(ussl_bg_thread_id);
ussl_bg_thread_id = NULL;
ATOMIC_STORE(&is_ussl_bg_thread_started, 0);
}
}

41
mittest/logservice/test_ob_simple_log_disk_mgr.cpp
View File

@ -389,6 +389,47 @@ TEST_F(TestObSimpleLogDiskMgr, overshelling)
}
TEST_F(TestObSimpleLogDiskMgr, hidden_sys)
{
SET_CASE_LOG_FILE(TEST_NAME, "hidden_sys");
int server_idx = 0;
PalfEnv *palf_env = NULL;
int64_t leader_idx = 0;
int64_t id = ATOMIC_AAF(&palf_id_, 1);
int64_t total_used_size = 0, total_size = 0;
share::SCN create_scn = share::SCN::base_scn();
EXPECT_EQ(OB_SUCCESS, get_palf_env(0, palf_env));
{
PalfHandleImplGuard leader;
EXPECT_EQ(OB_SUCCESS, palf_env->get_stable_disk_usage(total_used_size, total_size));
EXPECT_EQ(0, total_used_size);
EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, create_scn, leader_idx, leader));
EXPECT_EQ(OB_SUCCESS, palf_env->get_stable_disk_usage(total_used_size, total_size));
EXPECT_NE(0, total_used_size);
EXPECT_EQ(OB_NOT_SUPPORTED, update_disk_options(0));
}
EXPECT_EQ(OB_SUCCESS, delete_paxos_group(id));
EXPECT_EQ(OB_SUCCESS, update_disk_options(0));
// tenant unit中记录的disk_opts直接生效
PalfDiskOptions disk_opts;
EXPECT_EQ(OB_SUCCESS, get_disk_options(0, disk_opts));
EXPECT_EQ(0, disk_opts.log_disk_usage_limit_size_);
EXPECT_EQ(PalfDiskOptionsWrapper::Status::SHRINKING_STATUS,
palf_env->palf_env_impl_.disk_options_wrapper_.status_);
usleep(palf::BlockGCTimerTask::BLOCK_GC_TIMER_INTERVAL_MS + 5*10000);
usleep(ObLooper::INTERVAL_US * 2);
EXPECT_EQ(PalfDiskOptionsWrapper::Status::NORMAL_STATUS,
palf_env->palf_env_impl_.disk_options_wrapper_.status_);
EXPECT_EQ(OB_SUCCESS, palf_env->get_stable_disk_usage(total_used_size, total_size));
EXPECT_EQ(0, total_used_size);
EXPECT_EQ(0, total_size);
EXPECT_EQ(OB_SUCCESS, update_disk_options(8));
PalfHandleImplGuard leader;
EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, create_scn, leader_idx, leader));
EXPECT_EQ(OB_SUCCESS, palf_env->get_stable_disk_usage(total_used_size, total_size));
EXPECT_NE(0, total_used_size);
}
} // namespace unittest
} // namespace oceanbase

2
mittest/mtlenv/CMakeLists.txt
View File

@ -1,7 +1,7 @@
sql_unittest(test_session_serde)
#ob_unittest(test_ob_election_priority)
storage_unittest(test_tx_data_table)
storage_unittest(test_multi_tenant test_multi_tenant.cpp)
#storage_unittest(test_multi_tenant test_multi_tenant.cpp)
storage_unittest(test_buffer_ctx_node test_buffer_ctx_node.cpp)
add_subdirectory(storage)

8
mittest/mtlenv/mock_tenant_module_env.h
View File

@ -32,6 +32,8 @@
#include "observer/omt/ob_tenant.h"
#include "observer/omt/ob_worker_processor.h"
#include "observer/omt/ob_tenant_meta.h"
#include "observer/omt/ob_multi_tenant.h"
#include "observer/omt/ob_tenant_srs.h"
#include "share/allocator/ob_tenant_mutil_allocator_mgr.h"
#include "share/ob_alive_server_tracer.h"
#include "share/ob_device_manager.h"
@ -600,7 +602,7 @@ int MockTenantModuleEnv::init_before_start_mtl()
STORAGE_LOG(WARN, "fail to init env", K(ret));
} else if (OB_FAIL(session_mgr_.init())) {
STORAGE_LOG(WARN, "fail to init env", K(ret));
} else if (OB_FAIL(ObVirtualTenantManager::get_instance().init(10))) {
} else if (OB_FAIL(ObVirtualTenantManager::get_instance().init())) {
STORAGE_LOG(WARN, "fail to init env", K(ret));
} else if (OB_FAIL(OB_SERVER_BLOCK_MGR.init(THE_IO_DEVICE, 2 * 1024 * 1024))) {
STORAGE_LOG(WARN, "fail to init env", K(ret));
@ -638,8 +640,6 @@ int MockTenantModuleEnv::init_before_start_mtl()
STORAGE_LOG(ERROR, "init timer fail", KR(ret));
} else if (OB_FAIL(TG_START(lib::TGDefIDs::MemDumpTimer))) {
STORAGE_LOG(ERROR, "init memory dump timer fail", KR(ret));
} else if (OB_FAIL(ObOptStatMonitorManager::get_instance().init(&sql_proxy_))) {
STORAGE_LOG(ERROR, "failed to init opt stat monitor manager", KR(ret));
} else {
obrpc::ObRpcNetHandler::CLUSTER_ID = 1;
oceanbase::palf::election::INIT_TS = 1;
@ -665,6 +665,7 @@ int MockTenantModuleEnv::init()
} else {
oceanbase::ObClusterVersion::get_instance().update_data_version(DATA_CURRENT_VERSION);
MTL_BIND(ObTenantIOManager::mtl_init, ObTenantIOManager::mtl_destroy);
MTL_BIND2(mtl_new_default, omt::ObSharedTimer::mtl_init, omt::ObSharedTimer::mtl_start, omt::ObSharedTimer::mtl_stop, omt::ObSharedTimer::mtl_wait, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObTenantSchemaService::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObStorageLogger::mtl_init, ObStorageLogger::mtl_start, ObStorageLogger::mtl_stop, ObStorageLogger::mtl_wait, mtl_destroy_default);
MTL_BIND2(ObTenantMetaMemMgr::mtl_new, mtl_init_default, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
@ -699,6 +700,7 @@ int MockTenantModuleEnv::init()
MTL_BIND2(server_obj_pool_mtl_new<ObTableScanIterator>, nullptr, nullptr, nullptr, nullptr, server_obj_pool_mtl_destroy<ObTableScanIterator>);
MTL_BIND(ObTenantSQLSessionMgr::mtl_init, ObTenantSQLSessionMgr::mtl_destroy);
MTL_BIND2(mtl_new_default, ObRebuildService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, omt::ObTenantSrs::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
}
if (OB_FAIL(ret)) {

1
mittest/multi_replica/env/ob_multi_replica_test_base.cpp vendored
View File

@ -507,7 +507,6 @@ int ObMultiReplicaTestBase::start()
GCONF.enable_perf_event = false;
GCONF.enable_sql_audit = true;
GCONF.enable_record_trace_log = false;
GMEMCONF.set_server_memory_limit(10 * 1024 * 1024 * 1024ul);
int32_t log_level;
bool change_log_level = false;

1
mittest/simple_server/env/ob_simple_cluster_test_base.cpp vendored
View File

@ -161,7 +161,6 @@ int ObSimpleClusterTestBase::start()
GCONF.enable_perf_event = false;
GCONF.enable_sql_audit = true;
GCONF.enable_record_trace_log = false;
GMEMCONF.set_server_memory_limit(10 * 1024 * 1024 * 1024ul);
int32_t log_level;
bool change_log_level = false;

99
mittest/simple_server/test_observer_expand_shrink.cpp
View File

@ -210,105 +210,6 @@ TEST_F(ObserverExpandShink, resize_tenant_log_disk)
// EXPECT_NE(0, GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_);
// SERVER_LOG(INFO, "create_tenant_after_restart trace", KPC(GCTX.log_block_mgr_));
//}
TEST_F(ObserverExpandShink, test_hidden_sys_tenant)
{
omt::ObMultiTenant *omt = GCTX.omt_;
bool remove_tenant_succ = false;
int64_t log_disk_size_in_use = GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_;
share::TenantUnits units;
EXPECT_EQ(OB_SUCCESS, omt->get_tenant_units(units, false));
EXPECT_EQ(false, units.empty());
int64_t origin_sys_log_disk_size = 0;
int64_t hidden_sys_log_disk_size = 0;
for (int i = 0; i < units.count(); i++) {
if (OB_SYS_TENANT_ID == units[i].tenant_id_) {
origin_sys_log_disk_size = units[i].config_.log_disk_size();
EXPECT_EQ(OB_SUCCESS, omt->convert_real_to_hidden_sys_tenant());
}
}
EXPECT_EQ(OB_SUCCESS, omt->get_tenant_units(units, false));
EXPECT_EQ(false, units.empty());
ObUnitInfoGetter::ObTenantConfig sys_unit_config;
for (int i = 0; i < units.count(); i++) {
if (OB_SYS_TENANT_ID == units[i].tenant_id_) {
hidden_sys_log_disk_size = units[i].config_.log_disk_size();
sys_unit_config = units[i];
sys_unit_config.config_.resource_.log_disk_size_ = origin_sys_log_disk_size + 512*1024*1024;
}
}
CLOG_LOG(INFO, "runlin trace convert_hidden_to_real_sys_tenant", K(log_disk_size_in_use), KPC(GCTX.log_block_mgr_), K(origin_sys_log_disk_size), K(hidden_sys_log_disk_size));
// 类型转换后，sys租户的unit规格可能会发生变化（隐藏sys租户的规格会被重新生成，具体逻辑参见gen_sys_tenant_unit_config）
EXPECT_EQ(log_disk_size_in_use-origin_sys_log_disk_size+hidden_sys_log_disk_size,
GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_);
log_disk_size_in_use = GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_;
EXPECT_EQ(OB_SUCCESS, omt->convert_hidden_to_real_sys_tenant(sys_unit_config));
EXPECT_EQ(log_disk_size_in_use-hidden_sys_log_disk_size+sys_unit_config.config_.log_disk_size(),
GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_);
log_disk_size_in_use = GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_;
CLOG_LOG(INFO, "runlin trace after convert_hidden_to_real_sys_tenant", K(log_disk_size_in_use), KPC(GCTX.log_block_mgr_),
K(origin_sys_log_disk_size));
int64_t new_sys_log_disk_size = sys_unit_config.config_.log_disk_size()+512*1024*1024;
omt::ObTenant *tenant = nullptr;
EXPECT_EQ(OB_SUCCESS, omt->get_tenant(OB_SYS_TENANT_ID, tenant));
share::ObUnitInfoGetter::ObTenantConfig unit_config = tenant->get_unit();
unit_config.config_.resource_.log_disk_size_ = new_sys_log_disk_size;
// 扩容直接成功
EXPECT_EQ(OB_SUCCESS, omt->update_tenant_unit(unit_config));
unit_config = tenant->get_unit();
EXPECT_EQ(unit_config.config_.log_disk_size(), new_sys_log_disk_size);
EXPECT_EQ(log_disk_size_in_use-sys_unit_config.config_.log_disk_size()+new_sys_log_disk_size,
GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_);
CLOG_LOG(INFO, "runlin trace after convert_real_to_hidden_sys_tenant", K(log_disk_size_in_use), KPC(GCTX.log_block_mgr_),
K(origin_sys_log_disk_size));
}
// don't has any case after this.
TEST_F(ObserverExpandShink, paralle_set)
{
omt::ObTenantNodeBalancer::get_instance().refresh_interval_ = 1000 * 1000 * 1000;
sleep(3);
LOG_INFO("start to test parallel_set");
share::ObTenantSwitchGuard tguard;
ASSERT_EQ(OB_SUCCESS, tguard.switch_to(1));
ObLogService *log_service = MTL(ObLogService*);
palf::PalfOptions opts;
ASSERT_NE(nullptr, log_service);
EXPECT_EQ(OB_SUCCESS, log_service->get_palf_options(opts));
EXPECT_EQ(OB_INVALID_ARGUMENT, log_service->update_log_disk_usage_limit_size(1000));
opts.disk_options_.log_disk_utilization_limit_threshold_ = 10;
opts.disk_options_.log_disk_utilization_threshold_ = 11;
EXPECT_EQ(OB_INVALID_ARGUMENT, log_service->palf_env_->update_options(opts));
{
const int64_t new_log_disk_size = opts.disk_options_.log_disk_usage_limit_size_*50/100;
EXPECT_EQ(OB_SUCCESS, log_service->update_log_disk_usage_limit_size(new_log_disk_size));
sleep(1);
EXPECT_EQ(log_service->palf_env_->palf_env_impl_.disk_options_wrapper_.disk_opts_for_stopping_writing_.log_disk_usage_limit_size_,
new_log_disk_size);
}
{
const int64_t count1 = 10000;
const int64_t count2 = 9000;
std::atomic<int> new_log_disk_size(1*1024*1024*1024);
auto update_size = [&log_service, &new_log_disk_size](int64_t count)
{
for (int64_t i = 0; i < count; i++) {
log_service->update_log_disk_usage_limit_size(++new_log_disk_size);
}
};
std::thread t1(update_size, count1);
std::thread t2(update_size, count2);
t1.join();
t2.join();
EXPECT_EQ(OB_SUCCESS, log_service->get_palf_options(opts));
EXPECT_EQ(opts.disk_options_.log_disk_usage_limit_size_, 1*1024*1024*1024+count1+count2);
}
}
} // end unittest
} // end oceanbase

27
mittest/simple_server/test_transfer_task_operator.cpp
View File

@ -191,16 +191,16 @@ TEST_F(TestTransferTaskOperator, test_basic_func)
// ObTrasnferTaskComment
ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::EMPTY_COMMENT), ""));
ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::WAIT_FOR_MEMBER_LIST), "WAIT FOR MEMBER LIST TO BE SAME"));
ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::TASK_COMPLETED_AS_NO_VALID_PARTITION), "TASK COMPLETED AS NO VALID PARTITION"));
ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::TASK_CANCELED), "TASK CANCELED"));
ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::MAX_COMMENT), "UNKNOW"));
ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::WAIT_FOR_MEMBER_LIST), "Wait for member list to be same"));
ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::TASK_COMPLETED_AS_NO_VALID_PARTITION), "Task completed as no valid partition"));
ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::TASK_CANCELED), "Task canceled"));
ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::MAX_COMMENT), "Unknow"));
ASSERT_TRUE(ObTransferTaskComment::WAIT_FOR_MEMBER_LIST == str_to_transfer_task_comment("WAIT FOR MEMBER LIST TO BE SAME"));
ASSERT_TRUE(ObTransferTaskComment::TASK_COMPLETED_AS_NO_VALID_PARTITION == str_to_transfer_task_comment("TASK COMPLETED AS NO VALID PARTITION"));
ASSERT_TRUE(ObTransferTaskComment::TASK_CANCELED == str_to_transfer_task_comment("TASK CANCELED"));
ASSERT_TRUE(ObTransferTaskComment::WAIT_FOR_MEMBER_LIST == str_to_transfer_task_comment("Wait for member list to be same"));
ASSERT_TRUE(ObTransferTaskComment::TASK_COMPLETED_AS_NO_VALID_PARTITION == str_to_transfer_task_comment("Task completed as no valid partition"));
ASSERT_TRUE(ObTransferTaskComment::TASK_CANCELED == str_to_transfer_task_comment("Task canceled"));
ASSERT_TRUE(ObTransferTaskComment::EMPTY_COMMENT == str_to_transfer_task_comment(""));
ASSERT_TRUE(ObTransferTaskComment::MAX_COMMENT == str_to_transfer_task_comment("UNKNOW"));
ASSERT_TRUE(ObTransferTaskComment::MAX_COMMENT == str_to_transfer_task_comment("Unknow"));
ASSERT_TRUE(ObTransferTaskComment::MAX_COMMENT == str_to_transfer_task_comment("XXXXX"));
}
@ -218,6 +218,11 @@ TEST_F(TestTransferTaskOperator, test_operator)
ASSERT_EQ(OB_ENTRY_NOT_EXIST, ObTransferTaskOperator::get_all_task_status(sql_proxy, tenant_id_, task_status));
ASSERT_TRUE(task_status.empty());
// get_max_task_id_from_history when history is empty
ObTransferTaskID max_task_id;
ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get_max_task_id_from_history(sql_proxy, tenant_id_, max_task_id));
ASSERT_TRUE(!max_task_id.is_valid());
// insert
ObTransferTask other_task;
ObTransferTaskID other_task_id(222);
@ -235,7 +240,7 @@ TEST_F(TestTransferTaskOperator, test_operator)
ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(sql_proxy, tenant_id_, task_id_, false, task));
ASSERT_TRUE(task.get_task_id() == task_id_);
ASSERT_TRUE(task.get_tablet_list().empty());
ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(task.get_comment()), "TASK CANCELED"));
ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(task.get_comment()), "Task canceled"));
LOG_INFO("get from table", K(task));
task.reset();
ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(sql_proxy, tenant_id_, task_id_, true, task));
@ -375,6 +380,10 @@ TEST_F(TestTransferTaskOperator, test_operator)
ASSERT_TRUE(0 == compare(lock_conflict_part_list_str, history_lock_conflict_part_list_str));
ASSERT_EQ(OB_ENTRY_NOT_EXIST, ObTransferTaskOperator::get_history_task(sql_proxy,
tenant_id_, ObTransferTaskID(555), history_task, create_time, finish_time));
max_task_id.reset();
ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get_max_task_id_from_history(sql_proxy, tenant_id_, max_task_id));
ASSERT_TRUE(max_task_id == task_id_);
}
} // namespace share

6
src/logservice/libobcdc/src/ob_log_timezone_info_getter.cpp
View File

@ -547,7 +547,7 @@ int ObLogTimeZoneInfoGetter::export_timezone_info_(common::ObTZInfoMap &tz_info_
return OB_SUCCESS == tmp_ret;
};
if (OB_FAIL(tz_info_map.id_map_.for_each(tz_info_op))) {
if (OB_FAIL(tz_info_map.id_map_->for_each(tz_info_op))) {
LOG_ERROR("generate ObRequestTZInfoResult failed", KR(ret));
} else if (0 >= tz_info_res.tz_array_.count()) {
// skip empty tz_info_result.
@ -599,7 +599,7 @@ int ObLogTimeZoneInfoGetter::import_timezone_info_(common::ObTZInfoMap &tz_info_
for (int idx= 0; OB_SUCC(ret) && idx < tz_info_cnt; idx++) {
ObTimeZoneInfoPos &new_tz_info = tz_info_res.tz_array_.at(idx);
if (OB_FAIL(tz_info_map.id_map_.get(new_tz_info.get_tz_id(), stored_tz_info))) {
if (OB_FAIL(tz_info_map.id_map_->get(new_tz_info.get_tz_id(), stored_tz_info))) {
if (OB_ENTRY_NOT_EXIST == ret) {
ret = OB_SUCCESS;
} else {
@ -619,7 +619,7 @@ int ObLogTimeZoneInfoGetter::import_timezone_info_(common::ObTZInfoMap &tz_info_
}
if (OB_NOT_NULL(stored_tz_info)) {
tz_info_map.id_map_.revert(stored_tz_info);
tz_info_map.id_map_->revert(stored_tz_info);
stored_tz_info = nullptr;
}
} // end for

9
src/logservice/ob_server_log_block_mgr.cpp
View File

@ -362,7 +362,7 @@ int ObServerLogBlockMgr::update_tenant(const int64_t old_log_disk_size,
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
CLOG_LOG(WARN, "ObServerLogBlockMGR is not inited", K(old_log_disk_size), K(new_log_disk_size), KPC(this));
} else if (old_log_disk_size <= 0 || new_log_disk_size <= 0 || OB_ISNULL(log_service)) {
} else if (old_log_disk_size < 0 || new_log_disk_size < 0 || OB_ISNULL(log_service)) {
ret = OB_INVALID_ARGUMENT;
CLOG_LOG(WARN, "invalid argument", K(old_log_disk_size), K(new_log_disk_size), KP(log_service), KPC(this));
} else if (OB_FAIL(log_service->get_palf_stable_disk_usage(used_log_disk_size, palf_log_disk_size))) {
@ -387,7 +387,12 @@ int ObServerLogBlockMgr::update_tenant(const int64_t old_log_disk_size,
// 2. if 'palf_log_disk_size' which get from palf is 50G, we think palf has been in normal status. and we will
// construct 'new_unit' with 80G because 'palf_log_disk_size' is smaller than new log disk size(80G), but udpate
// palf with 80G.
} else if (FALSE_IT(can_update_log_disk_size_with_expected_log_disk = (new_log_disk_size >= palf_log_disk_size))) {
//
// NB: when new_log_disk_size is zero(means convert real sys tenant to hidden sys tenant), need make allowed_new_log_disk_size
// to zero directlly, otherwise, there is no chance to update log disk size to zero because hidden sys tenant is invisible
// for ObTenantNodeBalancer, and min_log_disk_size_for_all_tenants_ can not be reduce.
} else if (FALSE_IT(can_update_log_disk_size_with_expected_log_disk =
(new_log_disk_size >= palf_log_disk_size || 0 == new_log_disk_size))) {
// For expanding log disk, we can update 'allowed_new_log_disk_size' to 'new_log_disk_size' directlly.
} else if (can_update_log_disk_size_with_expected_log_disk && FALSE_IT(allowed_new_log_disk_size = new_log_disk_size)) {
// For shrinking log disk, we still update log disk size of 'new_unit' to 'old_log_disk_size'.

8
src/logservice/palf/palf_env_impl.cpp
View File

@ -700,19 +700,19 @@ int PalfEnvImpl::try_recycle_blocks()
total_size_to_stop_write
* disk_opts_for_stopping_writing.log_disk_utilization_limit_threshold_ / 100LL;
const bool need_recycle =
usable_disk_size_to_recycle_blocks > total_used_size_byte ? false : true;
usable_disk_size_to_recycle_blocks >= total_used_size_byte ? false : true;
const bool is_shrinking = disk_options_wrapper_.is_shrinking();
// Assume that, recycle speed is higher than write speed, therefor, the abnormal case
// is that, after each 'recycle_blocks_', the 'total_used_size_byte' is one PALF_BLOCK_SIZE
// more than 'usable_disk_size'.
const bool curr_diskspace_enough =
usable_disk_limit_size_to_stop_writing > total_used_size_byte ? true : false;
usable_disk_limit_size_to_stop_writing >= total_used_size_byte ? true : false;
constexpr int64_t MB = 1024 * 1024LL;
const int64_t print_error_log_disk_size =
disk_opts_for_stopping_writing.log_disk_usage_limit_size_
* disk_opts_for_stopping_writing.log_disk_utilization_threshold_ / 100LL;
const bool need_print_error_log =
print_error_log_disk_size > total_used_size_byte ? false : true;
print_error_log_disk_size >= total_used_size_byte ? false : true;
// step1. change SHRINKING_STATUS to normal
// 1. when there is no possibility to stop writing,
@ -720,7 +720,7 @@ int PalfEnvImpl::try_recycle_blocks()
bool has_recycled = false;
int64_t oldest_palf_id = INVALID_PALF_ID;
if (OB_SUCC(ret) && PalfDiskOptionsWrapper::Status::SHRINKING_STATUS == status) {
if (total_used_size_byte < usable_disk_size_to_recycle_blocks) {
if (total_used_size_byte <= usable_disk_size_to_recycle_blocks) {
disk_options_wrapper_.change_to_normal(sequence);
PALF_LOG(INFO, "change_to_normal success", K(disk_options_wrapper_),
K(total_used_size_byte), K(usable_disk_size_to_recycle_blocks));

2
src/logservice/palf/palf_options.cpp
View File

@ -46,7 +46,7 @@ bool PalfDiskOptions::is_valid() const
{
const int64_t MIN_DURATION = 1 * 1000 * 1000L;
const int64_t MAX_DURATION = 3 * 24 * 60 * 60 * 1000 * 1000L;
return -1 != log_disk_usage_limit_size_ && log_disk_usage_limit_size_ >= 4 * PALF_PHY_BLOCK_SIZE
return 0 <= log_disk_usage_limit_size_
&& 1 <=log_disk_utilization_threshold_ && 100 >= log_disk_utilization_threshold_
&& 1 <=log_disk_utilization_limit_threshold_ && 100 >= log_disk_utilization_limit_threshold_
&& log_disk_utilization_limit_threshold_ > log_disk_utilization_threshold_

1
src/observer/CMakeLists.txt
View File

@ -118,7 +118,6 @@ ob_set_subtarget(ob_server omt
omt/ob_th_worker.cpp
omt/ob_worker_processor.cpp
omt/ob_multi_tenant_operator.cpp
omt/ob_tenant_srs_mgr.cpp
omt/ob_tenant_srs.cpp
)

16
src/observer/dbms_job/ob_dbms_job_master.cpp
View File

@ -41,7 +41,7 @@ using namespace obrpc;
namespace dbms_job
{
int ObDBMSJobTask::init(ObLightyQueue *ready_queue)
int ObDBMSJobTask::init(ObDBMSJobQueue *ready_queue)
{
int ret = OB_SUCCESS;
if (inited_) {
@ -110,7 +110,7 @@ void ObDBMSJobTask::runTimerTask()
|| OB_ISNULL(ready_queue_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("null ptr", K(ret), K(job_key_), K(ready_queue_));
} else if (OB_FAIL(ready_queue_->push(static_cast<void*>(job_key_)))) {
} else if (OB_FAIL(ready_queue_->push(job_key_, 0))) {
LOG_WARN("fail to push ready job to queue", K(ret), K(*job_key_));
} else {
job_key_ = NULL;
@ -198,7 +198,7 @@ int ObDBMSJobTask::immediately(ObDBMSJobKey *job_key)
LOG_WARN("NULL ptr", K(ret), K(job_key), K(ready_queue_));
} else {
ObSpinLockGuard guard(lock_);
if (OB_FAIL(ready_queue_->push(static_cast<void*>(job_key)))) {
if (OB_FAIL(ready_queue_->push(job_key, 0))) {
LOG_WARN("fail to push ready job to queue", K(ret), K(*job_key));
}
}
@ -240,10 +240,6 @@ int ObDBMSJobMaster::init(ObISQLClient *sql_client,
ObMultiVersionSchemaService *schema_service)
{
int ret = OB_SUCCESS;
uint64_t ready_queue_size = MAX_READY_JOBS_CAPACITY;
if (is_mini_mode()) {
ready_queue_size *= lib::mini_mode_resource_ratio();
}
if (inited_) {
ret = OB_INIT_TWICE;
LOG_WARN("dbms job master already inited", K(ret), K(inited_));
@ -253,8 +249,8 @@ int ObDBMSJobMaster::init(ObISQLClient *sql_client,
) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("null ptr", K(ret), K(sql_client), K(schema_service));
} else if (OB_FAIL(ready_queue_.init(ready_queue_size))) {
LOG_WARN("fail to init ready job queue for all jobs", K(ret));
} else if (FALSE_IT(ready_queue_.set_limit(MAX_READY_JOBS_CAPACITY))) {
// do-nothing
} else if (OB_FAIL(scheduler_task_.init(&ready_queue_))) {
LOG_WARN("fail to init ready queue", K(ret));
} else if (OB_FAIL(scheduler_thread_.init(1, 1))) {
@ -324,7 +320,7 @@ int ObDBMSJobMaster::scheduler()
LOG_INFO("NOTICE: DBMS Job master start running!", K(ret), K(running_));
lib::set_thread_name("DBMS_JOB_MASTER");
while (OB_SUCC(ret) && !stoped_) {
void* ptr = NULL;
ObLink* ptr = NULL;
int64_t timeout = MIN_SCHEDULER_INTERVAL;
ObDBMSJobKey *job_key = NULL;
if (OB_FAIL(ready_queue_.pop(ptr, timeout))) {

11
src/observer/dbms_job/ob_dbms_job_master.h
View File

@ -23,7 +23,6 @@
#include "lib/lock/ob_spin_lock.h"
#include "lib/thread/ob_simple_thread_pool.h"
#include "lib/task/ob_timer.h"
#include "lib/queue/ob_lighty_queue.h"
#include "lib/container/ob_iarray.h"
#include "share/schema/ob_schema_service.h"
@ -43,7 +42,7 @@ class ObDBMSJobThread : public ObSimpleThreadPool
};
class ObDBMSJobKey
class ObDBMSJobKey : public common::ObLink
{
public:
ObDBMSJobKey(
@ -52,7 +51,7 @@ public:
bool check_job, bool check_new, bool check_new_tenant)
: tenant_id_(tenant_id),
job_id_(job_id),
execute_at_(execute_at),
execute_at_(execute_at),
delay_(delay),
check_job_(check_job),
check_new_(check_new),
@ -120,7 +119,7 @@ public:
virtual ~ObDBMSJobTask() {}
int init(common::ObLightyQueue *ready_queue);
int init(ObDBMSJobQueue *ready_queue);
int start();
int stop();
int destroy();
@ -139,7 +138,7 @@ public:
private:
bool inited_;
ObDBMSJobKey *job_key_;
common::ObLightyQueue *ready_queue_;
ObDBMSJobQueue *ready_queue_;
WaitVector wait_vector_;
ObSpinLock lock_;
@ -217,7 +216,7 @@ private:
obrpc::ObDBMSJobRpcProxy *job_rpc_proxy_;
common::ObAddr self_addr_;
common::ObLightyQueue ready_queue_;
ObDBMSJobQueue ready_queue_;
ObDBMSJobTask scheduler_task_;
ObDBMSJobThread scheduler_thread_;
ObDBMSJobUtils job_utils_;

2
src/observer/dbms_job/ob_dbms_job_utils.h
View File

@ -18,6 +18,7 @@
#include "lib/utility/ob_print_utils.h"
#include "lib/mysqlclient/ob_isql_client.h"
#include "lib/container/ob_iarray.h"
#include "lib/queue/ob_priority_queue.h"
namespace oceanbase
{
@ -174,6 +175,7 @@ private:
common::ObISQLClient *sql_proxy_;
};
using ObDBMSJobQueue = common::ObPriorityQueue<1>;
}
}

15
src/observer/dbms_scheduler/ob_dbms_sched_job_master.cpp
View File

@ -56,7 +56,7 @@ int ObDBMSSchedJobTask::init()
return ret;
}
int ObDBMSSchedJobTask::start(ObLightyQueue *ready_queue)
int ObDBMSSchedJobTask::start(dbms_job::ObDBMSJobQueue *ready_queue)
{
int ret = OB_SUCCESS;
if (!inited_) {
@ -111,7 +111,7 @@ void ObDBMSSchedJobTask::runTimerTask()
|| OB_ISNULL(ready_queue_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("null ptr", K(ret), K(job_key_), K(ready_queue_));
} else if (OB_FAIL(ready_queue_->push(static_cast<void*>(job_key_)))) {
} else if (OB_FAIL(ready_queue_->push(job_key_, 0))) {
LOG_WARN("fail to push ready job to queue", K(ret), K(*job_key_));
} else {
job_key_ = NULL;
@ -199,7 +199,7 @@ int ObDBMSSchedJobTask::immediately(ObDBMSSchedJobKey *job_key)
LOG_WARN("NULL ptr", K(ret), K(job_key), K(ready_queue_));
} else {
ObSpinLockGuard guard(lock_);
if (OB_FAIL(ready_queue_->push(static_cast<void*>(job_key)))) {
if (OB_FAIL(ready_queue_->push(job_key, 0))) {
LOG_WARN("fail to push ready job to queue", K(ret), K(*job_key));
}
}
@ -258,8 +258,8 @@ int ObDBMSSchedJobMaster::init(ObUnitManager *unit_mgr,
) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("null ptr", K(ret), K(unit_mgr), K(sql_client), K(schema_service));
} else if (OB_FAIL(ready_queue_.init(MAX_READY_JOBS_CAPACITY))) {
LOG_WARN("fail to init ready job queue for all jobs", K(ret));
} else if (FALSE_IT(ready_queue_.set_limit(MAX_READY_JOBS_CAPACITY))) {
// do-nothing
} else if (OB_FAIL(scheduler_task_.init())) {
LOG_WARN("fail to init ready queue", K(ret));
} else if (OB_FAIL(scheduler_thread_.init(1, 1))) {
@ -332,7 +332,7 @@ int ObDBMSSchedJobMaster::scheduler()
LOG_INFO("NOTICE: DBMS Sched Job master start running!", K(ret), K(running_));
lib::set_thread_name("DBMS_SCHEDULER");
while (OB_SUCC(ret) && !stoped_) {
void* ptr = NULL;
ObLink* ptr = NULL;
int64_t timeout = MIN_SCHEDULER_INTERVAL;
ObDBMSSchedJobKey *job_key = NULL;
if (OB_FAIL(ready_queue_.pop(ptr, timeout))) {
@ -507,7 +507,7 @@ int ObDBMSSchedJobMaster::server_random_pick(int64_t tenant_id, ObString &pick_z
}
}
}
}
}
}
}
if (OB_SUCC(ret) && 0 == total_server.count()) {
@ -730,4 +730,3 @@ int ObDBMSSchedJobMaster::register_job(
} // end for namespace dbms_scheduler
} // end for namespace oceanbase

12
src/observer/dbms_scheduler/ob_dbms_sched_job_master.h
View File

@ -24,12 +24,12 @@
#include "lib/lock/ob_spin_lock.h"
#include "lib/thread/ob_simple_thread_pool.h"
#include "lib/task/ob_timer.h"
#include "lib/queue/ob_lighty_queue.h"
#include "lib/container/ob_iarray.h"
#include "share/schema/ob_schema_service.h"
#include "share/schema/ob_multi_version_schema_service.h"
#include "observer/dbms_job/ob_dbms_job_utils.h"
#include "rootserver/ob_ddl_service.h"
@ -47,7 +47,7 @@ private:
virtual void handle(void *task);
};
class ObDBMSSchedJobKey
class ObDBMSSchedJobKey : public common::ObLink
{
public:
ObDBMSSchedJobKey(
@ -57,7 +57,7 @@ public:
: tenant_id_(tenant_id),
is_oracle_tenant_(is_oracle_tenant),
job_id_(job_id),
execute_at_(execute_at),
execute_at_(execute_at),
delay_(delay),
check_job_(check_job),
check_new_(check_new),
@ -130,7 +130,7 @@ public:
virtual ~ObDBMSSchedJobTask() {}
int init();
int start(common::ObLightyQueue *ready_queue);
int start(dbms_job::ObDBMSJobQueue *ready_queue);
int stop();
int destroy();
@ -148,7 +148,7 @@ public:
private:
bool inited_;
ObDBMSSchedJobKey *job_key_;
common::ObLightyQueue *ready_queue_;
dbms_job::ObDBMSJobQueue *ready_queue_;
WaitVector wait_vector_;
ObSpinLock lock_;
@ -226,7 +226,7 @@ private:
obrpc::ObDBMSSchedJobRpcProxy *job_rpc_proxy_;
common::ObAddr self_addr_;
common::ObLightyQueue ready_queue_;
dbms_job::ObDBMSJobQueue ready_queue_;
ObDBMSSchedJobTask scheduler_task_;
ObDBMSSchedJobThread scheduler_thread_;
ObDBMSSchedTableOperator table_operator_;

4
src/observer/mysql/ob_eliminate_task.cpp
View File

@ -38,8 +38,8 @@ int ObEliminateTask::init(const ObMySQLRequestManager *request_manager)
} else {
request_manager_ = const_cast<ObMySQLRequestManager*>(request_manager);
// can't call ObMySQLRequestManager::get_mem_limit for now, tenant not inited
// set config_mem_limit_ to 64M
config_mem_limit_ = 64 * 1024 * 1024; // 64M
// set config_mem_limit_ to 16M
config_mem_limit_ = 16 * 1024 * 1024; // 16M
common::ObConcurrentFIFOAllocator *allocator = request_manager_->get_allocator();
if (OB_ISNULL(allocator)) {
ret = OB_NOT_INIT;

13
src/observer/ob_rpc_processor_simple.cpp
View File

@ -2152,10 +2152,15 @@ int ObHandlePartTransCtxP::process()
int ObFlushLocalOptStatMonitoringInfoP::process()
{
int ret = OB_SUCCESS;
ObOptStatMonitorManager &opt_stat_monitor_mgr = ObOptStatMonitorManager::get_instance();
if (OB_FAIL(opt_stat_monitor_mgr.update_opt_stat_monitoring_info(arg_))) {
LOG_WARN("failed to flush opt stat monitoring info", K(ret));
} else {/*do nothing*/}
MTL_SWITCH(arg_.tenant_id_) {
ObOptStatMonitorManager *optstat_monitor_mgr = NULL;
if (OB_ISNULL(optstat_monitor_mgr = MTL(ObOptStatMonitorManager*))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected null", K(ret), K(optstat_monitor_mgr));
} else if (OB_FAIL(optstat_monitor_mgr->update_opt_stat_monitoring_info(arg_))) {
LOG_WARN("failed to flush opt stat monitoring info", K(ret));
} else {/*do nothing*/}
}
return ret;
}

41
src/observer/ob_server.cpp
View File

@ -36,7 +36,7 @@
#include "observer/ob_server_utils.h"
#include "observer/ob_rpc_extra_payload.h"
#include "observer/omt/ob_tenant_timezone_mgr.h"
#include "observer/omt/ob_tenant_srs_mgr.h"
#include "observer/omt/ob_tenant_srs.h"
#include "observer/table/ob_table_rpc_processor.h"
#include "observer/mysql/ob_query_retry_ctrl.h"
#include "rpc/obrpc/ob_rpc_handler.h"
@ -166,7 +166,6 @@ ObServer::ObServer()
reload_config_(config_, gctx_), config_mgr_(config_, reload_config_),
tenant_config_mgr_(omt::ObTenantConfigMgr::get_instance()),
tenant_timezone_mgr_(omt::ObTenantTimezoneMgr::get_instance()),
tenant_srs_mgr_(omt::ObTenantSrsMgr::get_instance()),
schema_service_(share::schema::ObMultiVersionSchemaService::get_instance()),
lst_operator_(), tablet_operator_(),
server_tracer_(),
@ -291,8 +290,6 @@ int ObServer::init(const ObServerOptions &opts, const ObPLogWriterCfg &log_cfg)
ret = OB_ELECTION_ASYNC_LOG_WARN_INIT;
} else if (OB_FAIL(init_tz_info_mgr())) {
LOG_ERROR("init tz_info_mgr failed", KR(ret));
} else if (OB_FAIL(init_srs_mgr())) {
LOG_ERROR("init srs_mgr fail", K(ret));
} else if (OB_FAIL(ObSqlTaskFactory::get_instance().init())) {
LOG_ERROR("init sql task factory failed", KR(ret));
}
@ -434,8 +431,6 @@ int ObServer::init(const ObServerOptions &opts, const ObPLogWriterCfg &log_cfg)
} else if (OB_FAIL(ObOptStatManager::get_instance().init(
&sql_proxy_, &config_))) {
LOG_ERROR("init opt stat manager failed", KR(ret));
} else if (OB_FAIL(ObOptStatMonitorManager::get_instance().init(&sql_proxy_))) {
LOG_ERROR("init opt stat monitor manager failed", KR(ret));
} else if (OB_FAIL(lst_operator_.set_callback_for_obs(
rs_rpc_proxy_, srv_rpc_proxy_, rs_mgr_, sql_proxy_))) {
LOG_ERROR("set_use_rpc_table failed", KR(ret));
@ -720,10 +715,6 @@ void ObServer::destroy()
tenant_timezone_mgr_.destroy();
FLOG_INFO("tenant timezone manager destroyed");
FLOG_INFO("begin to destroy tenant srs manager");
tenant_srs_mgr_.destroy();
FLOG_INFO("tenant srs manager destroyed");
FLOG_INFO("begin to destroy ObMdsEventBuffer");
ObMdsEventBuffer::destroy();
FLOG_INFO("ObMdsEventBuffer destroyed");
@ -1045,11 +1036,6 @@ int ObServer::start()
}
FLOG_INFO("check if timezone usable", KR(ret), K(stop_), K(timezone_usable));
while (OB_SUCC(ret) && !stop_ && !tenant_srs_mgr_.is_sys_load_completed()) {
SLEEP(1);
}
LOG_INFO("[NOTICE] check if sys srs usable", K(ret), K(stop_));
// check log replay and user tenant schema refresh status
if (OB_SUCC(ret)) {
if (stop_) {
@ -1354,10 +1340,6 @@ int ObServer::stop()
//ObPartitionScheduler::get_instance().stop_merge();
//FLOG_INFO("partition scheduler stopped", KR(ret));
FLOG_INFO("begin to stop tenant srs manager");
tenant_srs_mgr_.stop();
FLOG_INFO("tenant srs manager stopped");
FLOG_INFO("begin to stop opt stat manager ");
ObOptStatManager::get_instance().stop();
FLOG_INFO("opt stat manager stopped");
@ -1647,10 +1629,6 @@ int ObServer::wait()
TG_WAIT(lib::TGDefIDs::DiskUseReport);
FLOG_INFO("wait disk usage report task success");
FLOG_INFO("begin to wait tenant srs manager");
tenant_srs_mgr_.wait();
FLOG_INFO("wait tenant srs manager success");
FLOG_INFO("begin to wait ob_server_block_mgr");
OB_SERVER_BLOCK_MGR.wait();
FLOG_INFO("wait ob_server_block_mgr success");
@ -1744,16 +1722,6 @@ int ObServer::init_tz_info_mgr()
return ret;
}
int ObServer::init_srs_mgr()
{
int ret = OB_SUCCESS;
if (OB_FAIL(tenant_srs_mgr_.init(&sql_proxy_, self_addr_, &schema_service_))) {
LOG_WARN("tenant_srs_mgr_ init failed", K_(self_addr), K(ret));
}
return ret;
}
int ObServer::init_config()
{
int ret = OB_SUCCESS;
@ -2005,6 +1973,8 @@ int ObServer::init_pre_setting()
int ret = OB_SUCCESS;
reset_mem_leak_checker_label(GCONF.leak_mod_to_check.str());
ObMallocSampleLimiter::set_interval(GCONF._max_malloc_sample_interval,
GCONF._min_malloc_sample_interval);
// oblog configuration
if (OB_SUCC(ret)) {
@ -2372,10 +2342,7 @@ int ObServer::init_global_kvcache()
{
int ret = OB_SUCCESS;
int64_t bucket_num = ObKVGlobalCache::get_instance().get_suitable_bucket_num();
int64_t max_cache_size = ObKVGlobalCache::DEFAULT_MAX_CACHE_SIZE;
if (is_mini_mode()) {
max_cache_size *= lib::mini_mode_resource_ratio();
}
const int64_t max_cache_size = MIN(sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE), ObKVGlobalCache::DEFAULT_MAX_CACHE_SIZE);
if (OB_FAIL(ObKVGlobalCache::get_instance().init(&ObTenantMemLimitGetter::get_instance(),
bucket_num,
max_cache_size))) {

4
src/observer/ob_server.h
View File

@ -85,7 +85,6 @@ namespace oceanbase
namespace omt
{
class ObTenantTimezoneMgr;
class ObTenantSrsMgr;
}
namespace share
{
@ -281,7 +280,6 @@ private:
int init_table_lock_rpc_client();
int start_log_mgr();
int stop_log_mgr();
int init_srs_mgr();
int reload_bandwidth_throttle_limit(int64_t network_speed);
int get_network_speed_from_sysfs(int64_t &network_speed);
int get_network_speed_from_config_file(int64_t &network_speed);
@ -361,8 +359,6 @@ private:
common::ObConfigManager config_mgr_;
omt::ObTenantConfigMgr &tenant_config_mgr_;
omt::ObTenantTimezoneMgr &tenant_timezone_mgr_;
// gis: srs mgr
omt::ObTenantSrsMgr &tenant_srs_mgr_;
// The Oceanbase schema relating to.
share::schema::ObMultiVersionSchemaService &schema_service_;

5
src/observer/ob_server_reload_config.cpp
View File

@ -157,10 +157,8 @@ int ObServerReloadConfig::operator()()
#ifdef OB_USE_ASAN
__MemoryContext__::set_enable_asan_allocator(GCONF.enable_asan_for_memory_context);
#endif
#if defined(__x86_64__)
ObMallocSampleLimiter::set_interval(GCONF._max_malloc_sample_interval,
GCONF._min_malloc_sample_interval);
#endif
if (!is_arbitration_mode) {
ObIOConfig io_config;
int64_t cpu_cnt = GCONF.cpu_count;
@ -199,6 +197,9 @@ int ObServerReloadConfig::operator()()
int64_t cache_size = GCONF.memory_chunk_cache_size;
if (0 == cache_size) {
cache_size = GMEMCONF.get_server_memory_limit();
if (cache_size >= (32L<<30)) {
cache_size -= (4L<<30);
}
}
int64_t large_cache_size = GCONF._memory_large_chunk_cache_size;
if (0 == large_cache_size) {

44
src/observer/ob_uniq_task_queue.h
View File

@ -15,6 +15,7 @@
#include "lib/container/ob_se_array.h"
#include "lib/hash/ob_hashmap.h"
#include "lib/hash/ob_hashset.h"
#include "lib/list/ob_dlink_node.h"
#include "lib/thread/ob_thread_name.h"
#include "lib/list/ob_dlist.h"
@ -94,8 +95,8 @@ public:
friend class TestBatchProcessQueue_test_update_process_Test;
friend class TestBatchProcessQueue_test_single_update_Test;
static const int64_t QUEUE_WAIT_INTERVAL_MS = 200; // 200ms
ObUniqTaskQueue() : inited_(false), queue_size_(0), thread_name_(nullptr), task_map_(),
task_count_(0), group_map_(), processing_task_map_(), cur_group_(NULL),
ObUniqTaskQueue() : inited_(false), queue_size_(0), thread_name_(nullptr), task_set_(),
task_count_(0), group_map_(), processing_task_set_(), cur_group_(NULL),
processing_thread_count_(0), barrier_task_count_(0),
updater_(NULL) {}
virtual ~ObUniqTaskQueue() { }
@ -146,14 +147,13 @@ private:
const char *thread_name_;
common::ObThreadCond cond_;
// FIXME baihua: half memory wasted here.
common::hash::ObHashMap<Task,
Task,
common::hash::ObHashSet<Task,
common::hash::NoPthreadDefendMode,
common::hash::hash_func<Task>,
common::hash::equal_to<Task>,
common::hash::SimpleAllocer<typename common::hash::HashMapTypes<Task, Task>::AllocType,
common::hash::NodeNumTraits<IS_BIG_OBJ(Task), Task>::NODE_NUM,
common::hash::NoPthreadDefendMode, ObHighPrioMemAllocator> > task_map_;
common::hash::SimpleAllocer<typename common::hash::HashSetTypes<Task>::AllocType,
common::hash::NodeNumTraits<Task>::NODE_NUM,
common::hash::NoPthreadDefendMode, ObHighPrioMemAllocator> > task_set_;
int64_t task_count_;
common::hash::ObHashMap<uint64_t, Group,
common::hash::NoPthreadDefendMode,
@ -161,16 +161,15 @@ private:
common::hash::equal_to<uint64_t>,
common::hash::SimpleAllocer<
typename common::hash::HashMapTypes<uint64_t, Group>::AllocType,
common::hash::NodeNumTraits<IS_BIG_OBJ(Group), Group>::NODE_NUM,
common::hash::NodeNumTraits<Group>::NODE_NUM,
common::hash::NoPthreadDefendMode, ObHighPrioMemAllocator> > group_map_;
common::hash::ObHashMap<Task,
int,
common::hash::ObHashSet<Task,
common::hash::NoPthreadDefendMode,
common::hash::hash_func<Task>,
CompareT<Task>,
common::hash::SimpleAllocer<typename common::hash::HashMapTypes<Task, int>::AllocType,
common::hash::NodeNumTraits<IS_BIG_OBJ(Task), Task>::NODE_NUM,
common::hash::NoPthreadDefendMode, ObHighPrioMemAllocator> > processing_task_map_;
common::hash::SimpleAllocer<typename common::hash::HashSetTypes<Task>::AllocType,
common::hash::NodeNumTraits<Task>::NODE_NUM,
common::hash::NoPthreadDefendMode, ObHighPrioMemAllocator> > processing_task_set_;
common::ObDList<Group> groups_;
Group *cur_group_;
int64_t processing_thread_count_;
@ -211,12 +210,13 @@ int ObUniqTaskQueue<Task, Process>::init_only(Process *updater, const int64_t th
SERVER_LOG(WARN, "invalid argument", K(thread_num), K(queue_size), K(updater));
} else if (OB_FAIL(cond_.init(common::ObWaitEventIds::PARTITION_TABLE_UPDATER_COND_WAIT))) {
SERVER_LOG(WARN, "fai to init condition, ", K(ret));
} else if (OB_FAIL(task_map_.create(queue_size, attr, attr))) {
} else if (OB_FAIL(task_set_.create(queue_size, attr, attr))) {
SERVER_LOG(WARN, "create hash map failed", K(ret), K(queue_size));
} else if (OB_FAIL(group_map_.create(group_count,
attr, attr))) {
SERVER_LOG(WARN, "create hash map failed", K(ret), K(group_count));
} else if (OB_FAIL(processing_task_map_.create(queue_size, attr, attr))) {
} else if (OB_FAIL(processing_task_set_.create(common::UNIQ_TASK_QUEUE_BATCH_EXECUTE_NUM * thread_num,
attr, attr))) {
SERVER_LOG(WARN, "create hash map failed", K(ret));
} else {
this->set_thread_count(static_cast<int32_t>(thread_num));
@ -259,7 +259,7 @@ int ObUniqTaskQueue<Task, Process>::check_exist(const Task &task, bool &exist)
exist = false;
common::ObThreadCondGuard guard(cond_);
const Task *stored_task = nullptr;
if (nullptr == (stored_task = task_map_.get(task))) {
if (nullptr == (stored_task = task_set_.get(task))) {
exist = false;
} else {
exist = true;
@ -288,10 +288,10 @@ int ObUniqTaskQueue<Task, Process>::add(const Task &task)
}
} else {
const Task *stored_task = NULL;
if (OB_FAIL(task_map_.set_refactored(task, task))) {
if (OB_FAIL(task_set_.set_refactored(task, 0))) {
if (common::OB_HASH_EXIST == ret) {
if (task.need_assign_when_equal()) {
if (NULL == (stored_task = task_map_.get(task))) {
if (NULL == (stored_task = task_set_.get(task))) {
ret = common::OB_ERR_SYS;
SERVER_LOG(WARN, "get inserted task failed", K(ret), K(task));
} else if (OB_FAIL(const_cast<Task *>(stored_task)->assign_when_equal(task))) {
@ -304,7 +304,7 @@ int ObUniqTaskQueue<Task, Process>::add(const Task &task)
} else {
SERVER_LOG(WARN, "insert into hash failed", K(ret), K(task));
}
} else if (NULL == (stored_task = task_map_.get(task))) {
} else if (NULL == (stored_task = task_set_.get(task))) {
ret = common::OB_ERR_SYS;
SERVER_LOG(WARN, "get inserted task failed", K(ret), K(task));
} else {
@ -381,7 +381,7 @@ void ObUniqTaskQueue<Task, Process>::run1()
} else if (OB_FAIL(tasks.push_back(*t))) {
SERVER_LOG(WARN, "push_back failed", K(ret));
} else {
if (OB_FAIL(task_map_.erase_refactored(*t))) {
if (OB_FAIL(task_set_.erase_refactored(*t))) {
SERVER_LOG(WARN, "erase task from task map failed", K(ret), "task", *t);
} else {
--task_count_;
@ -550,7 +550,7 @@ int ObUniqTaskQueue<Task, Process>::try_lock(const Task &task)
if (!task.is_valid()) {
ret = common::OB_INVALID_ARGUMENT;
SERVER_LOG(WARN, "get invalid task", K(ret), K(task));
} else if (OB_FAIL(processing_task_map_.set_refactored(task, 1))) {
} else if (OB_FAIL(processing_task_set_.set_refactored(task, 0))) {
if (common::OB_HASH_EXIST == ret) {
ret = common::OB_EAGAIN;
SERVER_LOG(TRACE, "same task exist", K(task));
@ -570,7 +570,7 @@ int ObUniqTaskQueue<Task, Process>::batch_unlock(const common::ObIArray<Task> &t
if (OB_ISNULL(task)) {
tmp_ret = common::OB_ERR_UNEXPECTED;
SERVER_LOG(WARN, "get invalid task", K(tmp_ret), K(task));
} else if (common::OB_SUCCESS != (tmp_ret = processing_task_map_.erase_refactored(*task))) {
} else if (common::OB_SUCCESS != (tmp_ret = processing_task_set_.erase_refactored(*task))) {
SERVER_LOG(ERROR, "fail to erase task", K(tmp_ret), K(*task));
}
if (common::OB_SUCCESS != tmp_ret && OB_SUCC(ret)) {

76
src/observer/omt/ob_multi_tenant.cpp
View File

@ -88,6 +88,7 @@
#include "share/io/ob_io_manager.h"
#include "rootserver/freeze/ob_major_freeze_service.h"
#include "observer/omt/ob_tenant_config_mgr.h"
#include "observer/omt/ob_tenant_srs.h"
#include "observer/report/ob_tenant_meta_checker.h"
#include "storage/high_availability/ob_storage_ha_service.h"
#include "rootserver/ob_tenant_info_loader.h"//ObTenantInfoLoader
@ -523,6 +524,9 @@ int ObMultiTenant::init(ObAddr myaddr,
MTL_BIND2(mtl_new_default, ObTenantErrsimEventMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
#endif
MTL_BIND2(mtl_new_default, ObSharedTimer::mtl_init, ObSharedTimer::mtl_start, ObSharedTimer::mtl_stop, ObSharedTimer::mtl_wait, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObOptStatMonitorManager::mtl_init, ObOptStatMonitorManager::mtl_start, ObOptStatMonitorManager::mtl_stop, ObOptStatMonitorManager::mtl_wait, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObTenantSrs::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
}
if (OB_SUCC(ret)) {
@ -768,7 +772,6 @@ int ObMultiTenant::convert_hidden_to_real_sys_tenant(const ObUnitInfoGetter::ObT
int64_t bucket_lock_idx = -1;
int64_t lock_timeout_ts = abs_timeout_us - 3000000; // reserve 3s for converting tenant
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
@ -1071,12 +1074,14 @@ int ObMultiTenant::update_tenant_unit_no_lock(const ObUnitInfoGetter::ObTenantCo
return ret;
}
int ObMultiTenant::update_tenant_memory(const ObUnitInfoGetter::ObTenantConfig &unit)
int ObMultiTenant::update_tenant_memory(const ObUnitInfoGetter::ObTenantConfig &unit,
const int64_t extra_memory /* = 0 */)
{
int ret = OB_SUCCESS;
ObTenant *tenant = nullptr;
const uint64_t tenant_id = unit.tenant_id_;
int64_t allowed_mem_limit = 0;
int64_t memory_size = unit.config_.memory_size() + extra_memory;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
@ -1085,9 +1090,9 @@ int ObMultiTenant::update_tenant_memory(const ObUnitInfoGetter::ObTenantConfig &
} else if (OB_ISNULL(tenant)) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("tenant is nullptr", K(tenant_id));
} else if (OB_FAIL(update_tenant_memory(tenant_id, unit.config_.memory_size(), allowed_mem_limit))) {
} else if (OB_FAIL(update_tenant_memory(tenant_id, memory_size, allowed_mem_limit))) {
LOG_WARN("fail to update tenant memory", K(ret), K(tenant_id));
} else if (OB_FAIL(update_tenant_freezer_mem_limit(tenant_id, unit.config_.memory_size(), allowed_mem_limit))) {
} else if (OB_FAIL(update_tenant_freezer_mem_limit(tenant_id, memory_size, allowed_mem_limit))) {
LOG_WARN("fail to update_tenant_freezer_mem_limit", K(ret), K(tenant_id));
} else if (FALSE_IT(tenant->set_unit_memory_size(allowed_mem_limit))) {
// unreachable
@ -1100,7 +1105,7 @@ int ObMultiTenant::construct_allowed_unit_config(const int64_t allowed_new_log_d
ObUnitInfoGetter::ObTenantConfig &allowed_new_unit)
{
int ret = OB_SUCCESS;
if (0 >= allowed_new_log_disk_size
if (0 > allowed_new_log_disk_size
|| !expected_unit_config.is_valid()) {
ret= OB_INVALID_ARGUMENT;
} else if (OB_FAIL(allowed_new_unit.assign(expected_unit_config))) {
@ -1639,15 +1644,6 @@ int ObMultiTenant::remove_tenant(const uint64_t tenant_id, bool &remove_tenant_s
allocator->try_purge();
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(ObOptStatMonitorManager::get_instance().erase_opt_stat_monitoring_info_map(tenant_id))) {
if (OB_HASH_NOT_EXIST == ret) {
ret = OB_SUCCESS;
} else {
LOG_WARN("failed to erase column usage map", K(ret));
}
}
}
if (OB_SUCC(ret)) {
if (is_virtual_tenant_id(tenant_id) &&
OB_FAIL(ObVirtualTenantManager::get_instance().del_tenant(tenant_id))) {
@ -2392,3 +2388,55 @@ int ObSrvNetworkFrame::reload_sql_thread_config()
}
return ret;
}
int ObSharedTimer::mtl_init(ObSharedTimer *&st)
{
int ret = common::OB_SUCCESS;
if (st != NULL) {
int &tg_id = st->tg_id_;
if (OB_FAIL(TG_CREATE_TENANT(lib::TGDefIDs::TntSharedTimer, tg_id))) {
LOG_WARN("init shared timer failed", K(ret));
}
}
return ret;
}
int ObSharedTimer::mtl_start(ObSharedTimer *&st)
{
int ret = common::OB_SUCCESS;
if (st != NULL) {
int &tg_id = st->tg_id_;
if (OB_FAIL(TG_START(tg_id))) {
LOG_WARN("init shared timer failed", K(ret), K(tg_id));
}
}
return ret;
}
void ObSharedTimer::mtl_stop(ObSharedTimer *&st)
{
if (st != NULL) {
int &tg_id = st->tg_id_;
if (tg_id > 0) {
TG_STOP(tg_id);
}
}
}
void ObSharedTimer::mtl_wait(ObSharedTimer *&st)
{
if (st != NULL) {
int &tg_id = st->tg_id_;
if (tg_id > 0) {
TG_WAIT(tg_id);
}
}
}
void ObSharedTimer::destroy()
{
if (tg_id_ > 0) {
TG_DESTROY(tg_id_);
tg_id_ = -1;
}
}

17
src/observer/omt/ob_multi_tenant.h
View File

@ -106,7 +106,8 @@ public:
int convert_real_to_hidden_sys_tenant();
int update_tenant_cpu(const uint64_t tenant_id, const double min_cpu, const double max_cpu);
int update_tenant_memory(const uint64_t tenant_id, const int64_t mem_limit, int64_t &allowed_mem_limit);
int update_tenant_memory(const share::ObUnitInfoGetter::ObTenantConfig &unit);
int update_tenant_memory(const share::ObUnitInfoGetter::ObTenantConfig &unit,
const int64_t extra_memory = 0);
int update_tenant_log_disk_size(const uint64_t tenant_id,
const int64_t old_log_disk_size,
const int64_t new_log_disk_size,
@ -256,6 +257,20 @@ bool ObMultiTenant::has_synced() const
return has_synced_;
}
class ObSharedTimer
{
public:
ObSharedTimer() : tg_id_(-1) {}
static int mtl_init(ObSharedTimer *&st);
static int mtl_start(ObSharedTimer *&st);
static void mtl_stop(ObSharedTimer *&st);
static void mtl_wait(ObSharedTimer *&st);
void destroy();
int get_tg_id() const { return tg_id_; }
private:
int tg_id_;
};
} // end of namespace omt
} // end of namespace oceanbase

1
src/observer/omt/ob_tenant_config.h
View File

@ -52,7 +52,6 @@ public:
virtual ~TenantConfigUpdateTask() {}
TenantConfigUpdateTask(const TenantConfigUpdateTask &) = delete;
TenantConfigUpdateTask &operator=(const TenantConfigUpdateTask &) = delete;
void cancelCallBack() override {}
void set_tenant_config(ObTenantConfig *config) { tenant_config_ = config; }
void runTimerTask(void) override;
ObTenantConfigMgr *config_mgr_;

17
src/observer/omt/ob_tenant_node_balancer.cpp
View File

@ -259,8 +259,8 @@ int ObTenantNodeBalancer::get_server_allocated_resource(ServerResource &server_r
server_resource.max_cpu_ += tenant_units.at(i).config_.max_cpu();
server_resource.min_cpu_ += tenant_units.at(i).config_.min_cpu();
}
server_resource.memory_size_ += max(ObMallocAllocator::get_instance()->get_tenant_limit(tenant_units.at(i).tenant_id_),
int64_t extra_memory = is_sys_tenant(tenant_units.at(i).tenant_id_) ? GMEMCONF.get_extra_memory() : 0;
server_resource.memory_size_ += max(ObMallocAllocator::get_instance()->get_tenant_limit(tenant_units.at(i).tenant_id_) - extra_memory,
tenant_units.at(i).config_.memory_size());
server_resource.log_disk_size_ += tenant_units.at(i).config_.log_disk_size();
}
@ -346,17 +346,19 @@ int ObTenantNodeBalancer::check_new_tenant(const ObUnitInfoGetter::ObTenantConfi
}
}
} else {
if (is_sys_tenant(tenant_id) && tenant->is_hidden()) {
if (OB_FAIL(omt_->convert_hidden_to_real_sys_tenant(unit, abs_timeout_us))) {
int64_t extra_memory = 0;
if (is_sys_tenant(tenant_id)) {
if (tenant->is_hidden() && OB_FAIL(omt_->convert_hidden_to_real_sys_tenant(unit, abs_timeout_us))) {
LOG_WARN("fail to create real sys tenant", K(unit));
}
extra_memory = GMEMCONF.get_extra_memory();
}
if (OB_SUCC(ret) && !(unit == tenant->get_unit())) {
if (OB_FAIL(omt_->update_tenant_unit(unit))) {
LOG_WARN("fail to update tenant unit", K(ret), K(tenant_id));
}
}
if (OB_SUCC(ret) && OB_FAIL(omt_->update_tenant_memory(unit))) {
if (OB_SUCC(ret) && OB_FAIL(omt_->update_tenant_memory(unit, extra_memory))) {
LOG_ERROR("fail to update tenant memory", K(ret), K(tenant_id));
}
}
@ -371,16 +373,13 @@ int ObTenantNodeBalancer::check_new_tenant(const ObUnitInfoGetter::ObTenantConfi
int ObTenantNodeBalancer::refresh_hidden_sys_memory()
{
int ret = OB_SUCCESS;
int64_t sys_tenant_memory = 0;
int64_t allowed_mem_limit = 0;
ObTenant *tenant = nullptr;
if (OB_FAIL(omt_->get_tenant(OB_SYS_TENANT_ID, tenant))) {
LOG_WARN("get sys tenant failed", K(ret));
} else if (OB_ISNULL(tenant) || !tenant->is_hidden()) {
// do nothing
} else if (OB_FAIL(ObUnitResource::get_sys_tenant_default_memory(sys_tenant_memory))) {
LOG_WARN("get hidden sys tenant default memory failed", K(ret));
} else if (OB_FAIL(omt_->update_tenant_memory(OB_SYS_TENANT_ID, sys_tenant_memory, allowed_mem_limit))) {
} else if (OB_FAIL(omt_->update_tenant_memory(OB_SYS_TENANT_ID, GMEMCONF.get_hidden_sys_memory(), allowed_mem_limit))) {
LOG_WARN("update hidden sys tenant memory failed", K(ret));
} else {
LOG_INFO("update hidden sys tenant memory succeed ", K(allowed_mem_limit));

190
src/observer/omt/ob_tenant_srs.cpp
View File

@ -12,15 +12,16 @@
#define USING_LOG_PREFIX SERVER_OMT
#include "ob_tenant_srs.h"
#include "ob_tenant_srs_mgr.h"
#include "ob_tenant_srs.h"
#include "lib/string/ob_sql_string.h"
#include "common/ob_smart_var.h"
#include "observer/ob_sql_client_decorator.h"
#include "lib/mysqlclient/ob_mysql_proxy.h"
#include "share/ob_thread_mgr.h"
#include "share/inner_table/ob_inner_table_schema.h"
#include "share/schema/ob_multi_version_schema_service.h"
#include "share/rc/ob_tenant_base.h"
#include "observer/omt/ob_multi_tenant.h"
#include "observer/ob_server_struct.h"
#include "lib/geo/ob_geo_utils.h"
using namespace oceanbase::share;
@ -32,21 +33,74 @@ namespace oceanbase
namespace omt
{
int ObTenantSrs::init(ObTenantSrsMgr *srs_mgr)
int ObTenantSrs::mtl_init(ObTenantSrs* &tenant_srs)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(srs_mgr)) {
ret = OB_ERR_NULL_VALUE;
LOG_WARN("unexpected null srs mgr", K(ret));
} else if (OB_ISNULL(srs_mgr->sql_proxy_)) {
ret = OB_ERR_NULL_VALUE;
LOG_WARN("unexpected null sql proxy_ in srs mgr", K(ret));
} else if (OB_FAIL(srs_update_periodic_task_.init(srs_mgr, this))) {
LOG_WARN("failed to init srs update task", K(ret));
if (OB_FAIL(tenant_srs->init())) {
LOG_WARN("fail to init tenant srs", K(ret));
}
return ret;
}
int ObTenantSrs::start()
{
int ret = OB_SUCCESS;
if (!inited_) {
ret = OB_NOT_INIT;
LOG_WARN("tenant srs isn't inited", K(ret));
} else if (OB_FAIL(TG_SCHEDULE(MTL(omt::ObSharedTimer*)->get_tg_id(), srs_update_periodic_task_, 0, false))) {
LOG_WARN("failed to schedule tenant srs update task", K(ret));
}
return ret;
}
void ObTenantSrs::stop()
{
if (OB_LIKELY(inited_)) {
TG_CANCEL_TASK(MTL(omt::ObSharedTimer*)->get_tg_id(), srs_update_periodic_task_);
}
}
void ObTenantSrs::wait()
{
if (OB_LIKELY(inited_)) {
TG_WAIT_TASK(MTL(omt::ObSharedTimer*)->get_tg_id(), srs_update_periodic_task_);
}
}
void ObTenantSrs::destroy()
{
if (OB_LIKELY(inited_)) {
cancle_update_task();
recycle_old_snapshots();
recycle_last_snapshots();
allocator_.~ObFIFOAllocator();
}
}
int ObTenantSrs::init()
{
int ret = OB_SUCCESS;
sql_proxy_ = GCTX.sql_proxy_;
lib::ObMemAttr mem_attr(MTL_ID(), "TenantSrs");
if (inited_) {
ret = OB_INIT_TWICE;
LOG_WARN("ObTenantSrs init twice.", K(ret));
} else if (FALSE_IT(alloc_.set_tenant_id(MTL_ID()))) {
} else if (OB_FAIL(allocator_.init(&alloc_, OB_MALLOC_MIDDLE_BLOCK_SIZE, mem_attr))) {
LOG_WARN("ObTenantSrs allocator init failed.", K(ret));
} else {
srs_mgr_ = srs_mgr;
if (OB_FAIL(TG_SCHEDULE(lib::TGDefIDs::SRS_MGR, srs_update_periodic_task_, 0, false))) {
LOG_WARN("failed to schedule tenant srs update task", K(ret));
page_allocator_.set_allocator(&allocator_);
page_allocator_.set_attr(mem_attr);
mode_arena_.init(DEFAULT_PAGE_SIZE, page_allocator_);
if (OB_FAIL(srs_update_periodic_task_.init(this))) {
LOG_WARN("failed to init srs update task", K(ret));
} else {
inited_ = true;
infinite_plane_.minX_ = INT32_MIN;
infinite_plane_.minY_ = INT32_MIN;
infinite_plane_.maxX_ = INT32_MAX;
infinite_plane_.maxY_ = INT32_MAX;
}
}
return ret;
@ -81,6 +135,42 @@ int ObSrsCacheGuard::get_srs_item(uint64_t srs_id, const ObSrsItem *&srs_item)
return ret;
}
int ObTenantSrs::get_tenant_srs_guard(ObSrsCacheGuard &srs_guard)
{
int ret = OB_SUCCESS;
if (OB_FAIL(try_get_last_snapshot(srs_guard))) {
if (ret == OB_ERR_EMPTY_QUERY) {
ret = OB_ERR_SRS_EMPTY;
LOG_WARN("srs table might be empty", K(ret), K(MTL_ID()));
LOG_USER_ERROR(OB_ERR_SRS_EMPTY);
} else {
LOG_WARN("failed to get tenant srs", K(ret), K(MTL_ID()));
}
}
return ret;
}
int ObTenantSrs::get_srs_bounds(uint64_t srid, const ObSrsItem *srs_item, const ObSrsBoundsItem *&bounds_item)
{
int ret = OB_SUCCESS;
if (srid == 0) {
bounds_item = &infinite_plane_;
} else if (OB_ISNULL(srs_item)) {
ret = OB_ERR_NULL_VALUE;
LOG_ERROR("srs item is null", K(ret));
} else {
const ObSrsBoundsItem *tmp_bounds = srs_item->get_bounds();
if (isnan(tmp_bounds->minX_) || isnan(tmp_bounds->minY_)
|| isnan(tmp_bounds->maxX_) || isnan(tmp_bounds->maxY_)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid bounds info", K(ret), K(srid), K(srs_item->get_srid()), K(*tmp_bounds));
} else {
bounds_item = tmp_bounds;
}
}
return ret;
}
int ObTenantSrs::get_last_sys_snapshot(ObSrsCacheSnapShot *&sys_cache)
{
int ret = OB_SUCCESS;
@ -151,7 +241,8 @@ int ObTenantSrs::refresh_srs(bool is_sys)
int ret = OB_SUCCESS;
ObSrsCacheSnapShot *srs = NULL;
uint64_t tenant_data_version = 0;
if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id_, tenant_data_version))) {
const uint64_t tenant_id = MTL_ID();
if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, tenant_data_version))) {
LOG_WARN("get tenant data version failed", K(ret));
} else if (tenant_data_version < DATA_VERSION_4_1_0_0) {
ret = OB_ERR_EMPTY_QUERY;
@ -170,15 +261,15 @@ int ObTenantSrs::refresh_srs(bool is_sys)
if (last_snapshot != NULL) {
if (last_snapshot->get_ref_count() > 0
&& OB_FAIL(srs_old_snapshots_.push_back(last_snapshot))) {
LOG_WARN("failed to push last_snapshot to recycle queue", K(ret), K(tenant_id_), K(is_sys));
LOG_WARN("failed to push last_snapshot to recycle queue", K(ret), K(tenant_id), K(is_sys));
} else {
OB_DELETE(ObSrsCacheSnapShot, ObModIds::OMT, last_snapshot);
allocator_.free(last_snapshot);
}
}
last_snapshot = srs;
local_version = srs->get_srs_version();
LOG_INFO("fetch srs cache snapshot success", K(local_version), K(remote_version),
K(srs->get_srs_count()), K(srs_old_snapshots_.size()), K(tenant_id_), K(is_sys));
K(srs->get_srs_count()), K(srs_old_snapshots_.size()), K(tenant_id), K(is_sys));
}
return ret;
}
@ -193,16 +284,14 @@ int ObTenantSrs::refresh_usr_srs()
return refresh_srs(false);
}
int ObTenantSrs::TenantSrsUpdatePeriodicTask::init(ObTenantSrsMgr *srs_mgr, ObTenantSrs *srs)
int ObTenantSrs::TenantSrsUpdatePeriodicTask::init(ObTenantSrs *srs)
{
tenant_srs_mgr_ = srs_mgr;
tenant_srs_ = srs;
return OB_SUCCESS;
}
int ObTenantSrs::TenantSrsUpdateTask::init(ObTenantSrsMgr *srs_mgr, ObTenantSrs *srs)
int ObTenantSrs::TenantSrsUpdateTask::init(ObTenantSrs *srs)
{
tenant_srs_mgr_ = srs_mgr;
tenant_srs_ = srs;
return OB_SUCCESS;
}
@ -212,12 +301,14 @@ void ObTenantSrs::recycle_last_snapshots()
TCWLockGuard guard(lock_);
if (OB_NOT_NULL(last_sys_snapshot_) &&
last_sys_snapshot_->get_ref_count() <= 0) {
OB_DELETE(ObSrsCacheSnapShot, ObModIds::OMT, last_sys_snapshot_);
last_sys_snapshot_->~ObSrsCacheSnapShot();
allocator_.free(last_sys_snapshot_);
last_sys_snapshot_ = NULL;
}
if (OB_NOT_NULL(last_user_snapshot_) &&
last_user_snapshot_->get_ref_count() <= 0) {
OB_DELETE(ObSrsCacheSnapShot, ObModIds::OMT, last_user_snapshot_);
last_user_snapshot_->~ObSrsCacheSnapShot();
allocator_.free(last_user_snapshot_);
last_user_snapshot_ = NULL;
}
}
@ -246,7 +337,8 @@ void ObTenantSrs::recycle_old_snapshots()
if (OB_FAIL(srs_old_snapshots_.remove(i))) {
LOG_WARN("failed to remove old snapshot", K(ret));
} else {
OB_DELETE(ObSrsCacheSnapShot, ObModIds::OMT, snap);
snap->~ObSrsCacheSnapShot();
allocator_.free(snap);
}
}
}
@ -269,12 +361,15 @@ void ObTenantSrs::TenantSrsUpdateTask::runTimerTask()
void ObTenantSrs::TenantSrsUpdatePeriodicTask::runTimerTask()
{
int ret = OB_SUCCESS;
ObMultiVersionSchemaService *schema_service = nullptr;
const uint64_t tenant_id = MTL_ID();
bool is_sys_overdue = false;
bool is_user_overdue = false;
uint32_t delay = SLEEP_USECONDS;
// check tenant schema whether is ready
if ((tenant_srs_->tenant_id_ == OB_SYS_TENANT_ID && !tenant_srs_mgr_->is_sys_schema_ready()) ||
(tenant_srs_->tenant_id_ != OB_SYS_TENANT_ID && !tenant_srs_mgr_->schema_service_->is_tenant_full_schema(tenant_srs_->tenant_id_))) {
if ((tenant_id == OB_SYS_TENANT_ID && !GSCHEMASERVICE.is_sys_full_schema()) ||
(tenant_id != OB_SYS_TENANT_ID && !GSCHEMASERVICE.is_tenant_full_schema(tenant_id))) {
delay = BOOTSTRAP_PERIOD;
} else {
uint32_t old_snapshot_size = 0;
@ -297,16 +392,12 @@ void ObTenantSrs::TenantSrsUpdatePeriodicTask::runTimerTask()
if (is_user_overdue) {
// to do:user srs refresh
}
if (OB_UNLIKELY(tenant_srs_->tenant_id_ == OB_SYS_TENANT_ID && !tenant_srs_mgr_->is_sys_load_completed())) {
LOG_INFO("sys_tenant init load completed");
tenant_srs_mgr_->set_sys_load_completed();
}
if (old_snapshot_size > 0) {
tenant_srs_->recycle_old_snapshots();
}
}
// timer task, ignore error code
if (OB_FAIL(TG_SCHEDULE(lib::TGDefIDs::SRS_MGR, *this, delay, false))) {
if (OB_FAIL(TG_SCHEDULE(MTL(omt::ObSharedTimer*)->get_tg_id(), *this, delay, false))) {
LOG_WARN("schedule srs update task failed", K(ret));
}
}
@ -315,11 +406,11 @@ int ObTenantSrs::cancle_update_task()
{
int ret = OB_SUCCESS;
bool is_exist = true;
if (OB_FAIL(TG_TASK_EXIST(lib::TGDefIDs::SRS_MGR, srs_update_periodic_task_, is_exist))) {
LOG_WARN("failed to check tenant srs update task", K(ret), K(tenant_id_));
if (OB_FAIL(TG_TASK_EXIST(MTL(omt::ObSharedTimer*)->get_tg_id(), srs_update_periodic_task_, is_exist))) {
LOG_WARN("failed to check tenant srs update task", K(ret));
} else if (is_exist) {
if (OB_FAIL(TG_CANCEL_R(lib::TGDefIDs::SRS_MGR, srs_update_periodic_task_))) {
LOG_WARN("failed to cancel tenant srs update task", K(ret), K(tenant_id_));
if (OB_FAIL(TG_CANCEL_R(MTL(omt::ObSharedTimer*)->get_tg_id(), srs_update_periodic_task_))) {
LOG_WARN("failed to cancel tenant srs update task", K(ret));
}
}
return ret;
@ -343,9 +434,10 @@ int ObTenantSrs::fetch_all_srs(ObSrsCacheSnapShot *&srs_snapshot, bool is_sys_sr
ObSrsCacheType snapshot_type;
ObSrsCacheSnapShot *snapshot = NULL;
uint32_t res_count = 0;
const uint64_t tenant_id = MTL_ID();
ObSqlString sql;
ObSQLClientRetryWeak sql_client_retry_weak(srs_mgr_->sql_proxy_, tenant_id_, OB_ALL_SPATIAL_REFERENCE_SYSTEMS_TID);
ObSQLClientRetryWeak sql_client_retry_weak(sql_proxy_, tenant_id, OB_ALL_SPATIAL_REFERENCE_SYSTEMS_TID);
SMART_VAR(ObMySQLProxy::MySQLResult, res) {
ObMySQLResult *result = NULL;
if (is_sys_srs) {
@ -359,8 +451,8 @@ int ObTenantSrs::fetch_all_srs(ObSrsCacheSnapShot *&srs_snapshot, bool is_sys_sr
}
if (OB_FAIL(ret)) {
LOG_WARN("append sql failed", K(ret));
} else if (OB_FAIL(sql_client_retry_weak.read(res, tenant_id_, sql.ptr()))) {
LOG_WARN("execute sql failed", K(sql), K(ret), K(tenant_id_));
} else if (OB_FAIL(sql_client_retry_weak.read(res, tenant_id, sql.ptr()))) {
LOG_WARN("execute sql failed", K(sql), K(ret), K(tenant_id));
} else if (OB_UNLIKELY(NULL == (result = res.get_result()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("fail to get result. ", K(ret));
@ -370,7 +462,7 @@ int ObTenantSrs::fetch_all_srs(ObSrsCacheSnapShot *&srs_snapshot, bool is_sys_sr
const ObSrsItem *tmp = NULL;
res_count++;
if (OB_ISNULL(snapshot)) {
snapshot = OB_NEW(ObSrsCacheSnapShot, ObModIds::OMT, snapshot_type);
snapshot = OB_NEWx(ObSrsCacheSnapShot, &allocator_, &allocator_, snapshot_type);
if (OB_ISNULL(snapshot)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to create ObSrsCacheSnapShot", K(ret));
@ -403,14 +495,14 @@ int ObTenantSrs::fetch_all_srs(ObSrsCacheSnapShot *&srs_snapshot, bool is_sys_sr
} else {
if (OB_FAIL(generate_pg_reserved_srs(snapshot))) {
LOG_WARN("failed to geneate pg reserved srs", K(ret));
OB_DELETE(ObSrsCacheSnapShot, ObModIds::OMT, snapshot);
allocator_.free(snapshot);
} else {
snapshot->set_srs_version(srs_version);
srs_snapshot = snapshot;
}
}
} else if (snapshot != NULL) {
OB_DELETE(ObSrsCacheSnapShot, ObModIds::OMT, snapshot);
allocator_.free(snapshot);
LOG_WARN("failed to get all srs item, iter quit", K(ret));
}
}
@ -478,16 +570,16 @@ int ObSrsCacheSnapShot::parse_srs_item(ObMySQLResult *result, const ObSrsItem *&
LOG_WARN("failed to extract maxx value", K(ret));
} else if (OB_FAIL(extract_bounds_numberic(result, "maxY", max_y))) {
LOG_WARN("failed to extract maxy value", K(ret));
} else if (OB_FAIL(ObSrsWktParser::parse_srs_wkt(allocator_, srs_id, definition, srs_info))) {
} else if (OB_FAIL(ObSrsWktParser::parse_srs_wkt(*allocator_, srs_id, definition, srs_info))) {
LOG_WARN("failed to parse srs wkt from definition", K(ret), K(definition));
} else {
ObSrsItem *new_srs_item = OB_NEWx(ObSrsItem, (&allocator_), srs_info);
ObSrsItem *new_srs_item = OB_NEWx(ObSrsItem, allocator_, srs_info);
if (OB_ISNULL(new_srs_item)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to alloc memory for srs item", K(ret));
} else if (!proj4text.empty()) {
srs_info->set_bounds(min_x, min_y, max_x, max_y);
if (OB_FAIL(srs_info->set_proj4text(allocator_, proj4text))) {
if (OB_FAIL(srs_info->set_proj4text(*allocator_, proj4text))) {
LOG_WARN("fail to set proj4text for srs item", K(ret), K(srs_id));
}
}
@ -505,14 +597,14 @@ int ObSrsCacheSnapShot::add_pg_reserved_srs_item(const ObString &pg_wkt, const u
ObSpatialReferenceSystemBase *srs_info = NULL;
lib::ObMallocHookAttrGuard malloc_guard(lib::ObMemAttr(common::OB_SERVER_TENANT_ID, "SRSWKTParser"));
if (OB_FAIL(ObSrsWktParser::parse_srs_wkt(allocator_, srs_id, pg_wkt, srs_info))) {
if (OB_FAIL(ObSrsWktParser::parse_srs_wkt(*allocator_, srs_id, pg_wkt, srs_info))) {
LOG_WARN("failed to parse pg reserved srs wkt", K(ret), K(srs_id), K(pg_wkt));
} else {
ObSrsItem *new_srs_item = OB_NEWx(ObSrsItem, (&allocator_), srs_info);
ObSrsItem *new_srs_item = OB_NEWx(ObSrsItem, allocator_, srs_info);
if (OB_ISNULL(new_srs_item)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to alloc memory for srs item", K(ret));
} else if (OB_FAIL(ObGeoTypeUtil::get_pg_reserved_prj4text(&allocator_, srs_id, proj4text))) {
} else if (OB_FAIL(ObGeoTypeUtil::get_pg_reserved_prj4text(allocator_, srs_id, proj4text))) {
LOG_WARN("fail to generate proj4text for pg srs item", K(ret));
} else if (OB_FAIL(add_srs_item(new_srs_item->get_srid(), new_srs_item))) {
LOG_WARN("failed to add pg srs item to snapshot", K(ret), K(new_srs_item->get_srid()));

54
src/observer/omt/ob_tenant_srs.h
View File

@ -14,6 +14,8 @@
#define OCEANBASE_TENANT_SRS_H_
#include "share/ob_define.h"
#include "share/rc/ob_tenant_base.h"
#include "lib/mysqlclient/ob_mysql_proxy.h"
#include "lib/hash/ob_pointer_hashmap.h"
#include "lib/container/ob_vector.h"
#include "lib/lock/ob_tc_rwlock.h"
@ -47,10 +49,10 @@ class ObSrsCacheSnapShot
{
public:
static const uint32_t SRS_ITEM_BUCKET_NUM = 6144;
explicit ObSrsCacheSnapShot(ObSrsCacheType srs_type)
: allocator_("SrsSnapShot"), srs_type_(srs_type), srs_version_(0), ref_count_(0) {}
explicit ObSrsCacheSnapShot(common::ObIAllocator *allocator, ObSrsCacheType srs_type)
: allocator_(allocator), srs_type_(srs_type), srs_version_(0), ref_count_(0) {}
virtual ~ObSrsCacheSnapShot() { srs_item_map_.destroy(); }
int init() { return srs_item_map_.create(SRS_ITEM_BUCKET_NUM, "SrsSnapShot", "SrsSnapShot"); }
int init() { return srs_item_map_.create(SRS_ITEM_BUCKET_NUM, "SrsSnapShot", "SrsSnapShot", MTL_ID()); }
int add_srs_item(uint64_t srid, const common::ObSrsItem* srs_item) { return srs_item_map_.set_refactored(srid, srs_item); }
int get_srs_item(uint64_t srid, const common::ObSrsItem *&srs_item);
void set_srs_version(uint64_t version) { srs_version_ = version; }
@ -64,7 +66,7 @@ public:
int add_pg_reserved_srs_item(const common::ObString &pg_wkt, const uint32_t srs_id);
private:
common::ObArenaAllocator allocator_;
common::ObIAllocator *allocator_;
ObSrsCacheType srs_type_;
uint64_t srs_version_;
volatile int64_t ref_count_;
@ -86,7 +88,6 @@ private:
ObSrsCacheSnapShot *srs_cache_;
};
class ObTenantSrsMgr;
class ObTenantSrs
{
@ -94,32 +95,28 @@ public:
class TenantSrsUpdatePeriodicTask : public common::ObTimerTask
{
public:
TenantSrsUpdatePeriodicTask() : tenant_srs_mgr_(nullptr),
tenant_srs_(nullptr) {}
TenantSrsUpdatePeriodicTask() : tenant_srs_(nullptr) {}
virtual ~TenantSrsUpdatePeriodicTask() {}
int init(ObTenantSrsMgr *srs_mgr, ObTenantSrs *srs);
int init(ObTenantSrs *srs);
TenantSrsUpdatePeriodicTask(const TenantSrsUpdatePeriodicTask &) = delete;
TenantSrsUpdatePeriodicTask &operator=(const TenantSrsUpdatePeriodicTask &) = delete;
void runTimerTask(void) override;
private:
static const uint64_t SLEEP_USECONDS = 5000000;
static const uint64_t BOOTSTRAP_PERIOD = 1000000;
ObTenantSrsMgr *tenant_srs_mgr_;
ObTenantSrs *tenant_srs_;
};
class TenantSrsUpdateTask : public common::ObTimerTask
{
public:
TenantSrsUpdateTask() : tenant_srs_mgr_(nullptr),
tenant_srs_(nullptr) {}
TenantSrsUpdateTask() : tenant_srs_(nullptr) {}
virtual ~TenantSrsUpdateTask() {}
int init(ObTenantSrsMgr *srs_mgr, ObTenantSrs *srs);
int init(ObTenantSrs *srs);
TenantSrsUpdateTask(const TenantSrsUpdateTask &) = delete;
TenantSrsUpdateTask &operator=(const TenantSrsUpdateTask &) = delete;
void runTimerTask(void) override;
private:
ObTenantSrsMgr *tenant_srs_mgr_;
ObTenantSrs *tenant_srs_;
};
@ -131,19 +128,17 @@ public:
static const uint32_t RETRY_TIMES = 45;
static const uint32_t RETRY_INTERVAL_US = 100000;
explicit ObTenantSrs(common::ObArenaAllocator *allocator, uint64_t tenant_id)
: allocator_(allocator),tenant_id_(tenant_id),
page_allocator_(*allocator, common::ObModIds::OB_MODULE_PAGE_ALLOCATOR),
mode_arena_(DEFAULT_PAGE_SIZE, page_allocator_),
explicit ObTenantSrs()
: alloc_("TenantSrs"), sql_proxy_(nullptr), inited_(false),
last_sys_snapshot_(nullptr), last_user_snapshot_(nullptr),
srs_old_snapshots_(&mode_arena_, common::ObModIds::OB_MODULE_PAGE_ALLOCATOR),
remote_sys_srs_version_(0), remote_user_srs_version_(0),
local_sys_srs_version_(0), local_user_srs_version_(0),
srs_mgr_(nullptr) {}
local_sys_srs_version_(0), local_user_srs_version_(0), infinite_plane_() {}
virtual ~ObTenantSrs() {};
int init(ObTenantSrsMgr *srs_mgr);
inline uint64_t tenant_id() { return tenant_id_; }
int init();
inline uint64_t tenant_id() { return MTL_ID(); }
int get_tenant_srs_guard(ObSrsCacheGuard &srs_guard);
int get_srs_bounds(uint64_t srid, const ObSrsItem *srs_item, const ObSrsBoundsItem *&bounds_item);
int get_last_snapshot(ObSrsCacheGuard &srs_guard);
TenantSrsUpdatePeriodicTask &get_update_srs_task() { return srs_update_periodic_task_; }
int try_get_last_snapshot(ObSrsCacheGuard &srs_guard);
@ -151,6 +146,11 @@ public:
void recycle_last_snapshots();
uint32_t get_snapshots_size();
int cancle_update_task();
static int mtl_init(ObTenantSrs* &tenant_srs);
int start();
void stop();
void wait();
void destroy();
private:
typedef common::PageArena<ObSrsCacheSnapShot*, common::ModulePageAllocator> ObCGeoModuleArena;
@ -165,9 +165,10 @@ private:
int get_last_user_snapshot(ObSrsCacheSnapShot *&user_cache);
int generate_pg_reserved_srs(ObSrsCacheSnapShot *&srs_snapshot);
common::ObArenaAllocator *allocator_;
uint64_t tenant_id_;
common::ObFIFOAllocator allocator_;
common::ObArenaAllocator alloc_;
common::ObMySQLProxy *sql_proxy_;
bool inited_;
common::ModulePageAllocator page_allocator_;
ObCGeoModuleArena mode_arena_;
common::TCRWLock lock_;
@ -187,10 +188,11 @@ private:
uint64_t local_user_srs_version_;
TenantSrsUpdatePeriodicTask srs_update_periodic_task_;
TenantSrsUpdateTask srs_update_task_;
ObTenantSrsMgr *srs_mgr_;
common::ObSrsBoundsItem infinite_plane_;
DISALLOW_COPY_AND_ASSIGN(ObTenantSrs);
};
#define OTSRS_MGR (MTL(omt::ObTenantSrs*))
} // namespace omt
} // namespace oceanbase

369
src/observer/omt/ob_tenant_srs_mgr.cpp
View File

@ -1,369 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SERVER_OMT
#include "ob_tenant_srs_mgr.h"
#include "share/ob_thread_mgr.h"
#include "lib/thread/thread_mgr.h"
#include "share/schema/ob_multi_version_schema_service.h"
#include "share/schema/ob_schema_getter_guard.h"
using namespace oceanbase::common;
namespace oceanbase
{
namespace omt
{
ObTenantSrsMgr::ObTenantSrsMgr()
: allocator_("TenantSrs"), sql_proxy_(nullptr), self_(),
rwlock_(ObLatchIds::SRS_LOCK), tenant_srs_map_(),
add_tenant_task_(this), del_tenant_task_(this),
is_inited_(false), is_sys_load_completed_(false),
schema_service_(nullptr), nonexist_tenant_srs_(allocator_), infinite_plane_()
{
}
ObTenantSrsMgr::~ObTenantSrsMgr()
{
}
ObTenantSrsMgr &ObTenantSrsMgr::get_instance()
{
static ObTenantSrsMgr ob_tenant_srs_mgr;
return ob_tenant_srs_mgr;
}
void ObTenantSrsMgr::AddTenantTask::runTimerTask()
{
int ret = OB_SUCCESS;
uint32_t delay = DEFAULT_PERIOD;
if (OB_ISNULL(tenant_srs_mgr_)) {
ret = OB_ERR_NULL_VALUE;
LOG_ERROR("failed to do add tenant srs task, tenant srs mgr is null", K(ret));
} else if (!tenant_srs_mgr_->is_sys_schema_ready()) {
delay = BOOTSTRAP_PERIOD;
} else if (OB_FAIL(tenant_srs_mgr_->try_to_add_new_tenants())) {
LOG_WARN("failed to update tenants srs", K(ret));
}
if (OB_FAIL(TG_SCHEDULE(lib::TGDefIDs::SRS_MGR, *this, delay, false))) {
LOG_ERROR("failed to schedule add tenant srs task", K(ret));
}
}
void ObTenantSrsMgr::DeleteTenantTask::runTimerTask()
{
int ret = OB_SUCCESS;
if (OB_ISNULL(tenant_srs_mgr_)) {
ret = OB_ERR_NULL_VALUE;
LOG_ERROR("failed to do add tenant srs task, tenant srs mgr is null", K(ret));
} else if (!tenant_srs_mgr_->is_sys_schema_ready()) {
// do nothing
} else if (OB_FAIL(tenant_srs_mgr_->remove_nonexist_tenants())) {
LOG_WARN("failed to add nonexist tenants to del list", K(ret));
} else if (OB_FAIL(tenant_srs_mgr_->delete_nonexist_tenants_srs())) {
LOG_WARN("failed to delete nonexist tenants", K(ret));
}
if (OB_FAIL(TG_SCHEDULE(lib::TGDefIDs::SRS_MGR, *this, DEFAULT_PERIOD, false))) {
LOG_ERROR("failed to schedule add tenant srs task", K(ret));
}
}
int ObTenantSrsMgr::init(ObMySQLProxy *sql_proxy, const ObAddr &server,
share::schema::ObMultiVersionSchemaService *schema_service)
{
int ret = OB_SUCCESS;
if (OB_FAIL(TG_START(lib::TGDefIDs::SRS_MGR))) {
LOG_WARN("fail to init timer", K(ret));
} else {
sql_proxy_ = sql_proxy;
self_ = server;
schema_service_ = schema_service;
is_inited_ = true;
infinite_plane_.minX_ = INT32_MIN;
infinite_plane_.minY_ = INT32_MIN;
infinite_plane_.maxX_ = INT32_MAX;
infinite_plane_.maxY_ = INT32_MAX;
if (OB_FAIL(add_tenant_srs(OB_SYS_TENANT_ID))) {
LOG_WARN("add sys tenant srs info failed", K(ret));
} else if (OB_FAIL(TG_SCHEDULE(lib::TGDefIDs::SRS_MGR, add_tenant_task_, 0, false))) {
LOG_WARN("schedule add tenant srs task failed", K(ret));
} else if (OB_FAIL(TG_SCHEDULE(lib::TGDefIDs::SRS_MGR, del_tenant_task_, 0, false))) {
LOG_WARN("schedule del tenant srs task failed", K(ret));
}
}
return ret;
}
int ObTenantSrsMgr::get_srs_bounds(uint64_t srid, const ObSrsItem *srs_item,
const ObSrsBoundsItem *&bounds_item)
{
int ret = OB_SUCCESS;
if (srid == 0) {
bounds_item = &infinite_plane_;
} else if (OB_ISNULL(srs_item)) {
ret = OB_ERR_NULL_VALUE;
LOG_ERROR("srs item is null", K(ret));
} else {
const ObSrsBoundsItem *tmp_bounds = srs_item->get_bounds();
if (isnan(tmp_bounds->minX_) || isnan(tmp_bounds->minY_)
|| isnan(tmp_bounds->maxX_) || isnan(tmp_bounds->maxY_)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid bounds info", K(ret), K(srid), K(srs_item->get_srid()), K(*tmp_bounds));
} else {
bounds_item = tmp_bounds;
}
}
return ret;
}
int ObTenantSrsMgr::remove_nonexist_tenants()
{
int ret = OB_SUCCESS;
ObArray<uint64_t> nonexist_tenants_id;
{
share::schema::ObSchemaGetterGuard sys_schema_guard;
if (OB_ISNULL(schema_service_)) {
ret = OB_ERR_NULL_VALUE;
LOG_WARN("not init", K(ret));
} else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID,
sys_schema_guard))) {
LOG_WARN("get sys tenant schema guard failed", K(ret));
}
DRWLock::RDLockGuard guard(rwlock_);
TenantSrsMap::const_iterator it = tenant_srs_map_.begin();
for (; OB_SUCC(ret) && it != tenant_srs_map_.end(); it++) {
bool is_dropped = false;
if (OB_FAIL(sys_schema_guard.check_if_tenant_has_been_dropped(it->first, is_dropped))) {
LOG_WARN("check if tenant has been dropped failed", K(ret));
} else if (is_dropped && OB_FAIL(nonexist_tenants_id.push_back(it->first))) {
LOG_WARN("push back failed", K(ret), K(it->first));
}
}
}
for (uint64_t i = 0; i < nonexist_tenants_id.count() && OB_SUCC(ret); i++) {
if (OB_FAIL(move_tenant_srs_to_nonexist_list(nonexist_tenants_id.at(i)))) {
LOG_WARN("failed to move tenant srs to nonexist list", K(ret), K(nonexist_tenants_id.at(i)));
}
}
return ret;
}
int ObTenantSrsMgr::delete_nonexist_tenants_srs()
{
int ret = OB_SUCCESS;
FOREACH_X(iter, nonexist_tenant_srs_, OB_SUCC(ret)) {
ObTenantSrs *tenant_srs = *iter;
if (OB_ISNULL(tenant_srs)) {
ret = OB_ERR_NULL_VALUE;
LOG_WARN("tenant srs in recycle list is null", K(ret));
} else {
if (OB_FAIL(tenant_srs->cancle_update_task())) {
LOG_WARN("failed to cancle tenant srs update task", K(ret), K(tenant_srs->tenant_id()));
} else {
tenant_srs->recycle_old_snapshots();
tenant_srs->recycle_last_snapshots();
if (tenant_srs->get_snapshots_size() == 0) {
if (OB_FAIL(nonexist_tenant_srs_.erase(iter))) {
LOG_WARN("failed to erase tenant srs from nonexist list", K(ret));
} else {
tenant_srs->~ObTenantSrs();
allocator_.free(tenant_srs);
}
}
}
}
}
return ret;
}
int ObTenantSrsMgr::move_tenant_srs_to_nonexist_list(uint64_t tenant_id)
{
int ret = OB_SUCCESS;
ObTenantSrs *tenant_srs = nullptr;
DRWLock::WRLockGuard guard(rwlock_);
if (is_virtual_tenant_id(tenant_id)) {
// do nothing
} else if (OB_FAIL(tenant_srs_map_.get_refactored(tenant_id, tenant_srs))) {
if (OB_HASH_NOT_EXIST == ret) {
ret = OB_SUCCESS;
} else {
LOG_WARN("get tenant tenant_srs failed", K(tenant_id), K(ret));
}
} else if (OB_ISNULL(tenant_srs)) {
ret = OB_ERR_NULL_VALUE;
LOG_WARN("tenant srs is null", K(ret), K(tenant_id));
} else if (OB_FAIL(tenant_srs_map_.erase_refactored(tenant_id))) {
LOG_WARN("erase tenant tenant_srs failed", K(ret), K(tenant_id));
} else if (OB_FAIL(nonexist_tenant_srs_.push_back(tenant_srs))) {
LOG_WARN("push back tenant_srs failed", K(ret));
} else {
LOG_INFO("drop tenant srs push back succeed", K(tenant_srs->tenant_id()));
}
return ret;
}
int ObTenantSrsMgr::try_to_add_new_tenants()
{
int ret = OB_SUCCESS;
ObArray<uint64_t> new_tenants_id;
{
share::schema::ObSchemaGetterGuard sys_schema_guard;
if (OB_ISNULL(schema_service_)) {
ret = OB_ERR_NULL_VALUE;
LOG_WARN("schema service is null", K(ret));
} else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID,
sys_schema_guard))) {
LOG_WARN("get sys tenant schema guard failed", K(ret));
} else if (OB_FAIL(sys_schema_guard.get_tenant_ids(new_tenants_id))) {
LOG_WARN("get tenant ids failed", K(ret));
}
}
if (OB_SUCC(ret) && OB_FAIL(add_tenants_srs(new_tenants_id))) {
LOG_WARN("failed to add tenants srs", K(ret), K(new_tenants_id));
}
return ret;
}
int ObTenantSrsMgr::add_tenants_srs(const common::ObIArray<uint64_t> &new_tenants_id)
{
int ret = OB_SUCCESS;
for (uint32_t i = 0; i < new_tenants_id.count() && OB_SUCC(ret); i++) {
uint64_t tenant_id = new_tenants_id.at(i);
if (OB_FAIL(add_tenant_srs(tenant_id))) {
LOG_WARN("failed to add tenant srs", K(ret), K(tenant_id));
}
}
return ret;
}
int ObTenantSrsMgr::add_tenant_srs(const uint64_t tenant_id)
{
int ret = OB_SUCCESS;
ObTenantSrs *const *old_tenant_srs = NULL;
DRWLock::WRLockGuard guard(rwlock_);
if (!is_inited_) {
ret = OB_NOT_INIT;
LOG_WARN("tenant srs mgr not inited", K(ret));
} else if (is_virtual_tenant_id(tenant_id)
|| OB_NOT_NULL(old_tenant_srs = tenant_srs_map_.get(tenant_id))) {
LOG_INFO("try to add exist tenant or virtual tenant", K(tenant_id));
} else {
void *buf = allocator_.alloc(sizeof(ObTenantSrs));
if (OB_ISNULL(buf)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to alloc memory for tenant srs", K(ret));
} else {
ObTenantSrs *new_tenant_srs = new (buf) ObTenantSrs(&allocator_, tenant_id);
if (OB_FAIL(new_tenant_srs->init(this))) {
LOG_WARN("failed to init new tenant srs", K(ret));
} else if (OB_FAIL(tenant_srs_map_.set_refactored(tenant_id, new_tenant_srs, 1))) {
LOG_WARN("failed to set new tenant srs", K(ret), K(tenant_id));
}
if (OB_FAIL(ret)) {
if (OB_FAIL(new_tenant_srs->cancle_update_task())) {
LOG_WARN("failed to cancle update srs task", K(ret), K(tenant_id));
} else {
new_tenant_srs->~ObTenantSrs();
allocator_.free(new_tenant_srs);
}
}
}
}
return OB_SUCCESS;
}
int ObTenantSrsMgr::refresh_tenant_srs(uint64_t tenant_id)
{
int ret = OB_SUCCESS;
bool is_exist = false;
if (OB_LIKELY(OB_INVALID_ID != tenant_id)) {
share::schema::ObSchemaGetterGuard sys_schema_guard;
if (OB_ISNULL(schema_service_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", K(ret));
} else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID,
sys_schema_guard))) {
LOG_WARN("get sys tenant schema guard failed", K(ret));
} else if (OB_FAIL(sys_schema_guard.check_tenant_exist(tenant_id, is_exist))) {
LOG_WARN("get tenant ids failed", K(ret));
}
}
if (OB_SUCC(ret) && is_exist && OB_FAIL(add_tenant_srs(tenant_id))) {
LOG_WARN("add tenant timezone failed", K(ret));
}
return ret;
}
int ObTenantSrsMgr::get_tenant_guard_inner(uint64_t tenant_id, ObSrsCacheGuard &srs_guard)
{
int ret = OB_SUCCESS;
ObTenantSrs *tenant_srs = NULL;
DRWLock::RDLockGuard guard(rwlock_);
if (OB_FAIL(tenant_srs_map_.get_refactored(tenant_id, tenant_srs))) {
if (ret != OB_HASH_NOT_EXIST) {
LOG_WARN("failed to get tenant srs", K(ret), K(tenant_id));
}
} else if (OB_FAIL(tenant_srs->try_get_last_snapshot(srs_guard))) {
LOG_WARN("failed to get last srs snapshot", K(ret), K(tenant_id));
}
return ret;
}
int ObTenantSrsMgr::get_tenant_srs_guard(uint64_t tenant_id, ObSrsCacheGuard &srs_guard)
{
int ret = OB_SUCCESS;
if (OB_FAIL(get_tenant_guard_inner(tenant_id, srs_guard))) {
if (ret == OB_HASH_NOT_EXIST) {
if (OB_FAIL(refresh_tenant_srs(tenant_id))) {
LOG_WARN("update srs tenant map failed", K(ret), K(tenant_id));
} else if (OB_FAIL(get_tenant_guard_inner(tenant_id, srs_guard))) {
LOG_WARN("failed to get tenant srs", K(ret), K(tenant_id));
}
} else if (ret == OB_ERR_EMPTY_QUERY) {
ret = OB_ERR_SRS_EMPTY;
LOG_WARN("srs table might be empty", K(ret), K(tenant_id));
LOG_USER_ERROR(OB_ERR_SRS_EMPTY);
} else {
LOG_WARN("failed to get tenant srs", K(ret), K(tenant_id));
}
}
return ret;
}
bool ObTenantSrsMgr::is_sys_schema_ready()
{
return schema_service_->is_sys_full_schema();
}
void ObTenantSrsMgr::stop()
{
TG_STOP(lib::TGDefIDs::SRS_MGR);
}
void ObTenantSrsMgr::wait()
{
TG_WAIT(lib::TGDefIDs::SRS_MGR);
}
void ObTenantSrsMgr::destroy()
{
TG_DESTROY(lib::TGDefIDs::SRS_MGR);
tenant_srs_map_.destroy();
}
} // namespace omt
} // namespace oceanbase

143
src/observer/omt/ob_tenant_srs_mgr.h
View File

@ -1,143 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_OBSERVER_OMT_OB_TENANT_SRS_MGR_H_
#define OCEANBASE_OBSERVER_OMT_OB_TENANT_SRS_MGR_H_
#include "ob_tenant_srs.h"
#include "share/ob_define.h"
#include "lib/hash/ob_pointer_hashmap.h"
#include "lib/container/ob_vector.h"
#include "lib/list/ob_list.h"
#include "lib/allocator/page_arena.h"
#include "lib/lock/ob_drw_lock.h"
#include "lib/task/ob_timer.h"
#include "lib/geo/ob_srs_wkt_parser.h"
#include "share/schema/ob_schema_struct.h"
namespace oceanbase
{
namespace share
{
namespace schema
{
class ObMultiVersionSchemaService;
}
}
namespace omt
{
class ObTenantSrsMgr;
class ObTenantSrsMgr
{
template <class Key, class Value, int num>
class ObSrsContainer
: public common::hash::ObHashMap<Key, Value *, common::hash::NoPthreadDefendMode>
{
public:
ObSrsContainer()
{
this->create(num, "TenantSrsMgr","TenantSrs");
}
virtual ~ObSrsContainer() {}
private:
DISALLOW_COPY_AND_ASSIGN(ObSrsContainer);
};
class AddTenantTask : public common::ObTimerTask
{
public:
AddTenantTask(ObTenantSrsMgr *tenant_srs_mgr)
: tenant_srs_mgr_(tenant_srs_mgr) {}
virtual ~AddTenantTask() {}
AddTenantTask(const AddTenantTask &) = delete;
AddTenantTask &operator=(const AddTenantTask &) = delete;
void runTimerTask(void) override;
int add_tenants_to_map(common::ObIArray<uint64_t> &latest_tenant_ids);
private:
static const uint64_t DEFAULT_PERIOD = 5000000;
static const uint64_t BOOTSTRAP_PERIOD = 500000;
ObTenantSrsMgr *tenant_srs_mgr_;
};
class DeleteTenantTask : public common::ObTimerTask
{
public:
DeleteTenantTask(ObTenantSrsMgr *tenant_srs_mgr)
: tenant_srs_mgr_(tenant_srs_mgr) {}
virtual ~DeleteTenantTask() {}
DeleteTenantTask(const DeleteTenantTask &) = delete;
DeleteTenantTask &operator=(const DeleteTenantTask &) = delete;
void runTimerTask(void) override;
private:
static const uint64_t DEFAULT_PERIOD = 5000000;
ObTenantSrsMgr *tenant_srs_mgr_;
};
friend ObTenantSrs;
public:
using TenantSrsMap = ObSrsContainer<uint64_t, ObTenantSrs, common::OB_MAX_SERVER_TENANT_CNT>;
virtual ~ObTenantSrsMgr();
static ObTenantSrsMgr &get_instance();
int init(common::ObMySQLProxy *sql_proxy, const common::ObAddr &server,
share::schema::ObMultiVersionSchemaService *schema_service);
int add_tenant_srs(const uint64_t tenant_id);
int get_tenant_srs_guard(uint64_t tenant_id, ObSrsCacheGuard &srs_guard);
void stop();
void wait();
void destroy();
bool is_sys_load_completed() { return is_sys_load_completed_; }
void set_sys_load_completed() { is_sys_load_completed_ = true; }
bool is_sys_schema_ready();
int get_srs_bounds(uint64_t srid, const common::ObSrsItem *srs_item,
const common::ObSrsBoundsItem *&bounds_item);
private:
ObTenantSrsMgr();
int refresh_tenant_srs(uint64_t tenant_id);
int get_tenant_guard_inner(uint64_t tenant_id, ObSrsCacheGuard &srs_guard);
int add_tenants_srs(const common::ObIArray<uint64_t> &new_tenants_id);
int try_to_add_new_tenants();
int remove_nonexist_tenants();
int move_tenant_srs_to_nonexist_list(uint64_t tenant_id);
int delete_nonexist_tenants_srs();
common::ObArenaAllocator allocator_;
common::ObMySQLProxy *sql_proxy_;
common::ObAddr self_;
// protect tenant_srs_map_
common::DRWLock rwlock_;
TenantSrsMap tenant_srs_map_;
AddTenantTask add_tenant_task_;
DeleteTenantTask del_tenant_task_;
bool is_inited_;
bool is_sys_load_completed_;
share::schema::ObMultiVersionSchemaService *schema_service_;
common::ObList<ObTenantSrs *, common::ObArenaAllocator> nonexist_tenant_srs_;
common::ObSrsBoundsItem infinite_plane_;
DISALLOW_COPY_AND_ASSIGN(ObTenantSrsMgr);
};
} // namespace omt
} // namespace oceanbase
#define OTSRS_MGR (::oceanbase::omt::ObTenantSrsMgr::get_instance())
#endif

40
src/observer/omt/ob_tenant_timezone.cpp
View File

@ -21,54 +21,34 @@
#include "observer/omt/ob_tenant_timezone_mgr.h"
#include "share/ob_thread_mgr.h"
#include "share/ob_time_zone_info_manager.h"
#include "observer/ob_server.h"
using namespace oceanbase::common;
namespace oceanbase {
namespace omt {
ObTenantTimezone::ObTenantTimezone() : tenant_id_(OB_INVALID_TENANT_ID)
{
}
ObTenantTimezone::ObTenantTimezone(uint64_t tenant_id)
: is_inited_(false), tenant_id_(tenant_id), tenant_tz_mgr_(nullptr),
tz_info_mgr_(nullptr), tz_info_map_(nullptr), update_task_not_exist_(false)
ObTenantTimezone::ObTenantTimezone(common::ObMySQLProxy &sql_proxy, uint64_t tenant_id)
: is_inited_(false), tenant_id_(tenant_id),
tz_info_mgr_(sql_proxy, tenant_id), update_task_not_exist_(false)
{
}
ObTenantTimezone::~ObTenantTimezone()
{
}
int ObTenantTimezone::init(ObTenantTimezoneMgr *tz_mgr)
int ObTenantTimezone::init()
{
int ret = OB_SUCCESS;
tenant_tz_mgr_ = tz_mgr;
is_inited_ = true;
tz_info_map_ = OB_NEW(ObTZInfoMap, SET_USE_500("TZInfoMap"));
tz_info_mgr_ = OB_NEW(ObTimeZoneInfoManager, SET_USE_500("TZInfoMgr"), OBSERVER.get_common_rpc_proxy(),
OBSERVER.get_mysql_proxy(), OBSERVER.get_root_service(),
*tz_info_map_, tenant_id_);
if (OB_ISNULL(tz_info_map_) || OB_ISNULL(tz_info_mgr_)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to allocate mem for tz_info", K(ret), K(tz_info_map_), K(tz_info_mgr_));
} else if (OB_FAIL(tz_info_map_->init(SET_USE_500("TZInfoMap")))) {
LOG_WARN("fail to init tz_info_map_", K(ret));
} else if (OB_FAIL(tz_info_mgr_->init())) {
if (OB_UNLIKELY(is_inited_)) {
ret = OB_INIT_TWICE;
LOG_WARN("init twice", K(ret), K(tenant_id_));
} else if (OB_FAIL(tz_info_mgr_.init())) {
LOG_WARN("fail to init tz_info_mgr_", K(ret));
} else {
LOG_INFO("tenant timezone init", K(tz_info_map_), K(tenant_id_));
is_inited_ = true;
}
LOG_INFO("tenant timezone init", K(ret), K(tenant_id_), K(sizeof(ObTimeZoneInfoManager)));
return ret;
}
void ObTenantTimezone::destroy()
{
if (NULL != tz_info_map_) {
tz_info_map_->destroy();
}
}
} // omt
} // oceanbase

18
src/observer/omt/ob_tenant_timezone.h
View File

@ -13,7 +13,7 @@
#ifndef OCEANBASE_TENANT_TIMEZONE_H_
#define OCEANBASE_TENANT_TIMEZONE_H_
#include "lib/timezone/ob_timezone_info.h"
#include "src/share/ob_time_zone_info_manager.h"
namespace oceanbase {
@ -25,34 +25,28 @@ class ObTenantTimezone
{
friend class ObTenantTimezoneMgr;
public:
ObTenantTimezone();
ObTenantTimezone(uint64_t tenant_id);
ObTenantTimezone(common::ObMySQLProxy &sql_proxy, uint64_t tenant_id);
virtual ~ObTenantTimezone();
ObTenantTimezone(const ObTenantTimezone &)=delete;
ObTenantTimezone &operator=(const ObTenantTimezone &)=delete;
int init(ObTenantTimezoneMgr *tz_mgr);
int init();
int update_timezone(int64_t tz_version);
bool is_inited() { return is_inited_; }
bool get_update_task_not_exist() { return update_task_not_exist_; }
int get_ref_count(int64_t &ref_count);
uint64_t get_tenant_id() const { return tenant_id_; }
common::ObTZInfoMap *get_tz_map() { return tz_info_map_; }
common::ObTimeZoneInfoManager *get_tz_mgr() { return tz_info_mgr_; }
common::ObTZInfoMap *get_tz_map() { return tz_info_mgr_.get_tz_info_map(); }
common::ObTimeZoneInfoManager &get_tz_mgr() { return tz_info_mgr_; }
void set_update_task_not_exist() { update_task_not_exist_ = true; }
void set_tz_mgr(common::ObTimeZoneInfoManager *tz_mgr) { tz_info_mgr_ = tz_mgr; }
void set_tenant_tz_mgr(ObTenantTimezoneMgr *tz_mgr) { tenant_tz_mgr_ = tz_mgr; };
void destroy();
VIRTUAL_TO_STRING_KV(K_(is_inited), K_(tenant_id));
private:
bool is_inited_;
uint64_t tenant_id_;
ObTenantTimezoneMgr *tenant_tz_mgr_;
common::ObTimeZoneInfoManager *tz_info_mgr_;
common::ObTZInfoMap *tz_info_map_;
common::ObTimeZoneInfoManager tz_info_mgr_;
bool update_task_not_exist_;
};

16
src/observer/omt/ob_tenant_timezone_mgr.cpp
View File

@ -18,6 +18,7 @@
#include "share/ob_time_zone_info_manager.h"
#include "share/schema/ob_multi_version_schema_service.h"
#include "lib/hash/ob_hashset.h"
#include "observer/ob_server.h"
using namespace oceanbase::common;
@ -51,7 +52,7 @@ int ObTenantTimezoneMgr::UpdateTenantTZOp::operator() (common::hash::HashMapPair
{
int ret = OB_SUCCESS;
ObTenantTimezone &tenant_tz = *entry.second;
if (OB_FAIL(tenant_tz.get_tz_mgr()->fetch_time_zone_info())) {
if (OB_FAIL(tenant_tz.get_tz_mgr().fetch_time_zone_info())) {
LOG_WARN("fail to update time zone info", K(ret));
}
return ret;
@ -157,16 +158,16 @@ int ObTenantTimezoneMgr::add_tenant_timezone(uint64_t tenant_id)
|| OB_NOT_NULL(timezone = timezone_map_.get(tenant_id))) {
} else {
ObTenantTimezone *new_timezone = nullptr;
new_timezone = OB_NEW(ObTenantTimezone, "TenantTZ", tenant_id);
new_timezone = OB_NEW(ObTenantTimezone, "TenantTZ", OBSERVER.get_mysql_proxy(), tenant_id);
if (OB_ISNULL(new_timezone)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("alloc new tenant timezone failed", K(ret));
} else if(OB_FAIL(new_timezone->init(this))) {
} else if(OB_FAIL(new_timezone->init())) {
LOG_WARN("new tenant timezone init failed", K(ret));
} else if (OB_FAIL(timezone_map_.set_refactored(tenant_id, new_timezone, 1))) {
LOG_WARN("add new tenant timezone failed", K(ret));
} else {
LOG_INFO("add tenant timezone success!", K(tenant_id));
LOG_INFO("add tenant timezone success!", K(tenant_id), K(sizeof(ObTenantTimezone)));
}
if (OB_FAIL(ret)) {
ob_delete(new_timezone);
@ -193,10 +194,7 @@ int ObTenantTimezoneMgr::del_tenant_timezone(uint64_t tenant_id)
} else if (OB_FAIL(timezone_map_.erase_refactored(tenant_id))) {
LOG_WARN("erase tenant timezone failed", K(ret), K(tenant_id));
} else {
ObTZInfoMap *tz_map = timezone->get_tz_map();
LOG_INFO("drop tenant tz push back succeed", K(timezone->get_tz_map()),
K(timezone->get_tenant_id()));
timezone->destroy();
LOG_INFO("drop tenant tz push back succeed", K(timezone->get_tenant_id()));
ob_delete(timezone);
}
return ret;
@ -218,7 +216,7 @@ int ObTenantTimezoneMgr::get_tenant_timezone_inner(const uint64_t tenant_id,
LOG_WARN("tenant tz is null", K(ret));
} else {
timezone_wrap.set_tz_map(timezone->get_tz_map());
tz_info_mgr = timezone->get_tz_mgr();
tz_info_mgr = &(timezone->get_tz_mgr());
}
return ret;
}

4
src/observer/report/ob_tablet_table_updater.h
View File

@ -220,8 +220,8 @@ private:
private:
const int64_t MINI_MODE_UPDATE_TASK_THREAD_CNT = 1;
const int64_t UPDATE_TASK_THREAD_CNT = 7;
const int64_t MINI_MODE_UPDATE_QUEUE_SIZE = 20 * 10000;
const int64_t UPDATE_QUEUE_SIZE = 100 * 10000;
const int64_t MINI_MODE_UPDATE_QUEUE_SIZE = 5 * 10000;
const int64_t UPDATE_QUEUE_SIZE = 10 * 10000;
bool inited_;
bool stopped_;
ObService *ob_service_;

2
src/observer/table_load/ob_table_load_store.cpp
View File

@ -311,7 +311,7 @@ int ObTableLoadStore::commit(ObTableLoadResultInfo &result_info)
LOG_WARN("fail to commit sql stats", KR(ret));
} else if (OB_FAIL(store_ctx_->merger_->collect_dml_stat(dml_stats))) {
LOG_WARN("fail to build dml stat", KR(ret));
} else if (OB_FAIL(ObOptStatMonitorManager::get_instance().update_dml_stat_info_from_direct_load(dml_stats.dml_stat_array_))) {
} else if (OB_FAIL(ObOptStatMonitorManager::update_dml_stat_info_from_direct_load(dml_stats.dml_stat_array_))) {
LOG_WARN("fail to update dml stat info", KR(ret));
} else if (OB_FAIL(store_ctx_->set_status_commit())) {
LOG_WARN("fail to set store status commit", KR(ret));

4
src/observer/virtual_table/ob_all_latch.cpp
View File

@ -131,7 +131,9 @@ int ObAllLatch::inner_get_next_row(ObNewRow *&row)
OB_SUCC(ret) && cell_idx < output_column_ids_.count();
++cell_idx) {
const uint64_t column_id = output_column_ids_.at(cell_idx);
const ObLatchStat& latch_stat = dipair.second->get_latch_stats().items_[latch_iter_];
ObLatchStat *p_latch_stat = dipair.second->get_latch_stats().get_item(latch_iter_);
if (OB_ISNULL(p_latch_stat)) continue;
const ObLatchStat& latch_stat = *p_latch_stat;
switch(column_id) {
case TENANT_ID: {
cells[cell_idx].set_int(dipair.first);

77
src/observer/virtual_table/ob_all_virtual_dml_stats.cpp
View File

@ -22,6 +22,24 @@ using namespace sql;
namespace observer
{
int ObAllVirtualDMmlStats::inner_open()
{
int ret = OB_SUCCESS;
// sys tenant show all tenant infos
if (is_sys_tenant(effective_tenant_id_)) {
if (OB_FAIL(GCTX.omt_->get_mtl_tenant_ids(tenant_ids_))) {
SERVER_LOG(WARN, "failed to add tenant id", K(ret));
}
} else {
// user tenant show self tenant infos
if (OB_FAIL(tenant_ids_.push_back(effective_tenant_id_))) {
SERVER_LOG(WARN, "failed to push back tenant id", KR(ret), K(effective_tenant_id_),
K(tenant_ids_));
}
}
return ret;
}
int ObOptDmlStatMapGetter::operator()(common::hash::HashMapPair<StatKey, ObOptDmlStat> &entry)
{
int ret = OB_SUCCESS;
@ -77,27 +95,8 @@ int ObOptDmlStatMapGetter::operator()(common::hash::HashMapPair<StatKey, ObOptDm
return ret;
}
int ObOptDmlStatMapsGetter::operator()(common::hash::HashMapPair<uint64_t, DmlStatMap *> &entry)
{
int ret = OB_SUCCESS;
uint64_t tenant_id = entry.first;
DmlStatMap *dml_stat_map = entry.second;
if (effective_tenant_id_ != tenant_id && !is_sys_tenant(effective_tenant_id_)) {
SERVER_LOG(TRACE, "not full fill tenant id", K(effective_tenant_id_), K(tenant_id));
} else if (OB_ISNULL(dml_stat_map)) {
ret = OB_ERR_UNEXPECTED;
SERVER_LOG(WARN, "get unexpected null", K(ret), K(dml_stat_map));
} else {
ObOptDmlStatMapGetter getter(scanner_, output_column_ids_, svr_ip_, port_, cur_row_, effective_tenant_id_);
if (OB_FAIL(dml_stat_map->foreach_refactored(getter))) {
SERVER_LOG(WARN, "fail to get dml stat", K(ret));
} else {/*do nothing*/}
}
return ret;
}
ObAllVirtualDMmlStats::ObAllVirtualDMmlStats()
: port_(0)
: port_(0), tenant_ids_(), tenant_idx_(0)
{
MEMSET(svr_ip_, 0, sizeof(svr_ip_));
}
@ -122,19 +121,27 @@ void ObAllVirtualDMmlStats::reset()
int ObAllVirtualDMmlStats::inner_get_next_row(ObNewRow *&row)
{
int ret = OB_SUCCESS;
ObObj *cells = cur_row_.cells_;
if (!start_to_read_) {
if (OB_FAIL(fill_scanner())) {
if (tenant_idx_ >= tenant_ids_.count()) {
ret = OB_ITER_END;
} else if (!start_to_read_) {
if (OB_FAIL(fill_scanner(tenant_ids_.at(tenant_idx_)))) {
SERVER_LOG(WARN, "fill scanner failed", K(ret));
} else {
start_to_read_ = true;
}
}
if (OB_SUCCESS == ret && start_to_read_) {
if (OB_SUCC(ret) && start_to_read_) {
if (OB_FAIL(scanner_it_.get_next_row(cur_row_))) {
if (OB_ITER_END != ret) {
SERVER_LOG(WARN, "fail to get next row", K(ret));
} else if (++ tenant_idx_ < tenant_ids_.count()) {//load a new tenant info
ret = OB_SUCCESS;
start_to_read_ = false;
if (OB_FAIL(SMART_CALL(inner_get_next_row(row)))) {
if (OB_ITER_END != ret) {
SERVER_LOG(WARN, "failed to inner get next row", K(ret));
}
}
}
} else {
row = &cur_row_;
@ -143,7 +150,7 @@ int ObAllVirtualDMmlStats::inner_get_next_row(ObNewRow *&row)
return ret;
}
int ObAllVirtualDMmlStats::fill_scanner()
int ObAllVirtualDMmlStats::fill_scanner(uint64_t tenant_id)
{
int ret = OB_SUCCESS;
ObObj *cells = NULL;
@ -156,12 +163,18 @@ int ObAllVirtualDMmlStats::fill_scanner()
ret = OB_ERR_UNEXPECTED;
} else {
port_ = addr.get_port();
ObOptDmlStatMapsGetter getter(scanner_, output_column_ids_, svr_ip_, port_, cur_row_, effective_tenant_id_);
if (OB_FAIL(ObOptStatMonitorManager::get_instance().generate_opt_stat_monitoring_info_rows(getter))) {
SERVER_LOG(WARN, "generate monitor info array failed", K(ret));
} else {
scanner_it_ = scanner_.begin();
start_to_read_ = true;
MTL_SWITCH(tenant_id) {
ObOptDmlStatMapGetter getter(scanner_, output_column_ids_, svr_ip_, port_, cur_row_, tenant_id);
ObOptStatMonitorManager *optstat_monitor_mgr = MTL(ObOptStatMonitorManager*);
if (OB_ISNULL(optstat_monitor_mgr)) {
ret = OB_ERR_UNEXPECTED;
SERVER_LOG(WARN, "optstat monitor mgr is NULL", K(ret), K(tenant_id), K(effective_tenant_id_));
} else if (OB_FAIL(optstat_monitor_mgr->generate_opt_stat_monitoring_info_rows(getter))) {
SERVER_LOG(WARN, "generate monitor info array failed", K(ret));
} else {
scanner_it_ = scanner_.begin();
start_to_read_ = true;
}
}
}
return ret;

33
src/observer/virtual_table/ob_all_virtual_dml_stats.h
View File

@ -54,34 +54,6 @@ private:
uint64_t effective_tenant_id_;
};
class ObOptDmlStatMapsGetter
{
public:
explicit ObOptDmlStatMapsGetter(common::ObScanner &scanner,
common::ObIArray<uint64_t> &output_column_ids,
char *svr_ip,
int32_t port,
common::ObNewRow &cur_row,
uint64_t effective_tenant_id)
: scanner_(scanner),
output_column_ids_(output_column_ids),
svr_ip_(svr_ip),
port_(port),
cur_row_(cur_row),
effective_tenant_id_(effective_tenant_id)
{}
virtual ~ObOptDmlStatMapsGetter() {};
int operator() (common::hash::HashMapPair<uint64_t, DmlStatMap *> &entry);
DISALLOW_COPY_AND_ASSIGN(ObOptDmlStatMapsGetter);
private:
common::ObScanner &scanner_;
common::ObIArray<uint64_t> &output_column_ids_;
char *svr_ip_;
int32_t port_;
common::ObNewRow &cur_row_;
uint64_t effective_tenant_id_;
};
class ObAllVirtualDMmlStats : public ObVirtualTableScannerIterator
{
friend class ObOptDmlStatMapGetter;
@ -91,6 +63,7 @@ public:
void destroy();
virtual void reset() override;
virtual int inner_get_next_row(common::ObNewRow *&row) override;
virtual int inner_open() override;
private:
enum COLUMNS
{
@ -105,7 +78,9 @@ private:
};
int32_t port_;
char svr_ip_[common::OB_IP_STR_BUFF];
int fill_scanner();
int fill_scanner(uint64_t tenant_id);
common::ObSEArray<uint64_t, 16> tenant_ids_;
int64_t tenant_idx_;
DISALLOW_COPY_AND_ASSIGN(ObAllVirtualDMmlStats);
};

6
src/observer/virtual_table/ob_all_virtual_sys_stat.cpp
View File

@ -193,6 +193,12 @@ int ObAllVirtualSysStat::update_all_stats_(const int64_t tenant_id, ObStatEventS
(OB_SYS_TENANT_ID == tenant_id) ? global_poc_server.get_ratelimit_rxbytes() : -1;
stat_events.get(ObStatEventIds::STANDBY_FETCH_LOG_BANDWIDTH_LIMIT - ObStatEventIds::STAT_EVENT_ADD_END -1)->stat_value_ =
(OB_SYS_TENANT_ID == tenant_id) ? global_poc_server.get_ratelimit() : -1;
stat_events.get(ObStatEventIds::MEMORY_LIMIT - ObStatEventIds::STAT_EVENT_ADD_END -1)->stat_value_ =
(OB_SYS_TENANT_ID == tenant_id) ? GMEMCONF.get_server_memory_limit() : 0;
stat_events.get(ObStatEventIds::SYSTEM_MEMORY - ObStatEventIds::STAT_EVENT_ADD_END -1)->stat_value_ =
(OB_SYS_TENANT_ID == tenant_id) ? GMEMCONF.get_reserved_server_memory() : 0;
stat_events.get(ObStatEventIds::HIDDEN_SYS_MEMORY - ObStatEventIds::STAT_EVENT_ADD_END -1)->stat_value_ =
(OB_SYS_TENANT_ID == tenant_id) ? GMEMCONF.get_hidden_sys_memory() : 0;
int ret_bk = ret;
if (NULL != GCTX.omt_) {

2
src/pl/ob_pl_package_state.h
View File

@ -114,7 +114,7 @@ public:
const ObPackageStateVersion &state_version,
bool serially_reusable)
: inner_allocator_(this),
cursor_allocator_(),
cursor_allocator_("PlPkgCursor", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()),
package_id_(package_id),
state_version_(state_version),
serially_reusable_(serially_reusable),

2
src/pl/ob_pl_router.h
View File

@ -29,7 +29,7 @@ public:
session_info_(session_info),
schema_guard_(schema_guard),
sql_proxy_(sql_proxy),
inner_allocator_(),
inner_allocator_(ObModIds::OB_PL_TEMP, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()),
expr_factory_(inner_allocator_) {}
virtual ~ObPLRouter() {}

2
src/pl/sys_package/ob_dbms_stats.cpp
View File

@ -4814,7 +4814,7 @@ int ObDbmsStats::process_not_size_manual_column(sql::ObExecContext &ctx,
if (OB_SUCC(ret) && !auto_columns.empty()) {
if (OB_FAIL(ObOptStatMonitorManager::flush_database_monitoring_info(ctx, true, false))) {
LOG_WARN("failed to do flush database monitoring info", K(ret));
} else if (OB_FAIL(ObOptStatMonitorManager::get_instance().get_column_usage_from_table(
} else if (OB_FAIL(ObOptStatMonitorManager::get_column_usage_from_table(
ctx, auto_columns, table_param.tenant_id_, table_param.table_id_))) {
LOG_WARN("failed to get column usage from table", K(ret));
} else {

8
src/rootserver/ob_disaster_recovery_task_mgr.h
View File

@ -107,7 +107,7 @@ public:
// @param [in] task_mgr, to execute over a task
int handle_not_in_progress_task(
ObDRTaskMgr &task_mgr);
// remove task from schedule_list and clean it
// @param [in] task, the task to finish schedule
int finish_schedule(
@ -195,7 +195,7 @@ private:
class ObDRTaskMgr : public ObRsReentrantThread
{
public:
const static int64_t TASK_QUEUE_LIMIT = 1L << 20;
const static int64_t TASK_QUEUE_LIMIT = 10000; //1w
const static int64_t ONCE_ADD_TASK_CNT = TASK_QUEUE_LIMIT / 2;
const static int64_t SAFE_DISCARD_TASK_INTERVAL = 200L * 1000L; // 200ms
const static int64_t DATA_IN_CLEAR_INTERVAL = 20L * 60L * 1000000L; // 20min
@ -298,7 +298,7 @@ public:
const ObDRTaskRetComment &ret_comment);
// operations of reach_concurrency_limit
void set_reach_concurrency_limit() {
void set_reach_concurrency_limit() {
concurrency_limited_ts_ = common::ObTimeUtility::current_time();
}
void clear_reach_concurrency_limit() {
@ -365,7 +365,7 @@ private:
const ObDRTask &task);
// try to log inmemory task infos according to balancer_log_interval
// @param [in] last_dump_ts, last time do logging
// @param [in] last_dump_ts, last time do logging
int try_dump_statistic_(
int64_t &last_dump_ts) const;
int inner_dump_statistic_() const;

4
src/rootserver/ob_disaster_recovery_task_table_updater.cpp
View File

@ -143,9 +143,7 @@ int ObDRTaskTableUpdater::init(
? MINI_MODE_UPDATE_THREAD_CNT
: UPDATE_THREAD_CNT;
const int64_t queue_size =
lib::is_mini_mode()
? MINI_MODE_TASK_QUEUE_SIZE
: TASK_QUEUE_SIZE;
TASK_QUEUE_SIZE;
if (OB_UNLIKELY(inited_)) {
ret = OB_INIT_TWICE;
LOG_WARN("init twice", KR(ret));

3
src/rootserver/ob_disaster_recovery_task_table_updater.h
View File

@ -154,8 +154,7 @@ private:
private:
const int64_t MINI_MODE_UPDATE_THREAD_CNT = 1;
const int64_t UPDATE_THREAD_CNT = 1;
const int64_t MINI_MODE_TASK_QUEUE_SIZE = 20 * 10000;
const int64_t TASK_QUEUE_SIZE = 100 * 10000;
const int64_t TASK_QUEUE_SIZE = 10000;
bool inited_;
bool stopped_;
ObDRTaskTableUpdateTaskQueue update_queue_;

48
src/rootserver/ob_partition_balance.cpp
View File

@ -91,6 +91,7 @@ void ObPartitionBalance::destroy()
iter->second.at(i)->~ObLSPartGroupDesc();
}
}
iter->second.destroy();
}
//reset
bg_builder_.destroy();
@ -243,17 +244,16 @@ int ObPartitionBalance::on_new_partition(
} else {
// create partition group if in new partition group
if (in_new_partition_group) {
ObTransferPartGroup *part_group = nullptr;
if (OB_ISNULL(part_group = reinterpret_cast<ObTransferPartGroup*>(allocator_.alloc(sizeof(ObTransferPartGroup))))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("alloc mem fail", KR(ret));
} else if (FALSE_IT(cur_part_group_ = new(part_group) ObTransferPartGroup(allocator_))) {
} else if (OB_FAIL(add_part_to_bg_map_(src_ls_id, *cur_part_group_, bg))) {
LOG_WARN("add partition to balance group fail", KR(ret), K(src_ls_id), K(bg),
K(cur_part_group_));
ObTransferPartGroup *new_pg = NULL;
if (OB_FAIL(add_new_pg_to_bg_map_(src_ls_id, bg, new_pg))) {
LOG_WARN("add new partition group to balance group failed", KR(ret), K(src_ls_id), K(bg), K(new_pg));
} else if (OB_ISNULL(new_pg)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("new_pg is null", KR(ret), K(src_ls_id), K(bg), K(new_pg));
} else {
// add new partition group
src_ls_desc->add_partgroup(1, 0);
cur_part_group_ = new_pg;
}
}
// if not in new partition group, current part group should be valid
@ -274,9 +274,10 @@ int ObPartitionBalance::on_new_partition(
return ret;
}
int ObPartitionBalance::add_part_to_bg_map_(const ObLSID &ls_id,
ObTransferPartGroup &part_group,
ObBalanceGroup &bg)
int ObPartitionBalance::add_new_pg_to_bg_map_(
const ObLSID &ls_id,
ObBalanceGroup &bg,
ObTransferPartGroup *&part_group)
{
int ret = OB_SUCCESS;
ObArray<ObLSPartGroupDesc *> *ls_part_desc_arr = NULL;
@ -291,12 +292,13 @@ int ObPartitionBalance::add_part_to_bg_map_(const ObLSID &ls_id,
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get part from balance group fail", KR(ret), K(ls_id), K(bg));
} else {
ls_part_desc_arr->set_block_allocator(ModulePageAllocator(allocator_, "LSPartDescArr"));
for (int64_t i = 0; OB_SUCC(ret) && i < ls_desc_array_.count(); i++) {
ObLSPartGroupDesc *ls_part_desc = nullptr;
if (OB_ISNULL(ls_part_desc = reinterpret_cast<ObLSPartGroupDesc*>(allocator_.alloc(sizeof(ObLSPartGroupDesc))))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("alloc mem fail", KR(ret), K(ls_id), K(bg));
} else if (FALSE_IT(new(ls_part_desc) ObLSPartGroupDesc(ls_desc_array_.at(i)->get_ls_id()))) {
} else if (FALSE_IT(new(ls_part_desc) ObLSPartGroupDesc(ls_desc_array_.at(i)->get_ls_id(), allocator_))) {
} else if (OB_FAIL(ls_part_desc_arr->push_back(ls_part_desc))) {
LOG_WARN("push_back fail", KR(ret), K(ls_id), K(bg));
}
@ -308,8 +310,8 @@ int ObPartitionBalance::add_part_to_bg_map_(const ObLSID &ls_id,
bool find_ls = false;
for (int64_t idx = 0; OB_SUCC(ret) && idx < ls_part_desc_arr->count(); idx++) {
if (ls_part_desc_arr->at(idx)->get_ls_id() == ls_id) {
if (OB_FAIL(ls_part_desc_arr->at(idx)->get_part_groups().push_back(&part_group))) {
LOG_WARN("push_back fail", KR(ret));
if (OB_FAIL(ls_part_desc_arr->at(idx)->add_new_part_group(part_group))) {
LOG_WARN("add_new_part_group failed", KR(ret), K(ls_id), K(idx), KPC(ls_part_desc_arr), K(part_group));
}
find_ls = true;
break;
@ -887,5 +889,23 @@ int ObPartitionHelper::get_sub_part_info(const schema::ObSimpleTableSchemaV2 &ta
return ret;
}
int ObPartitionBalance::ObLSPartGroupDesc::add_new_part_group(ObTransferPartGroup *&part_group)
{
int ret = OB_SUCCESS;
part_group = NULL;
const int64_t part_group_size = sizeof(ObTransferPartGroup);
void *buf = alloc_.alloc(part_group_size);
if (OB_ISNULL(buf)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("allocate memory for partition group fail", KR(ret), K(buf), K(part_group_size));
} else if (OB_ISNULL(part_group = new(buf) ObTransferPartGroup(alloc_))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("construct ObTransferPartGroup fail", KR(ret), K(buf), K(part_group_size));
} else if (OB_FAIL(part_groups_.push_back(part_group))) {
LOG_WARN("push back new partition group fail", KR(ret), K(part_group), K(part_groups_));
}
return ret;
}
} // end rootserver
} // end oceanbase

11
src/rootserver/ob_partition_balance.h
View File

@ -83,21 +83,28 @@ public:
class ObLSPartGroupDesc
{
public:
ObLSPartGroupDesc(ObLSID ls_id) : ls_id_(ls_id) {}
ObLSPartGroupDesc(ObLSID ls_id, ObIAllocator &alloc) :
ls_id_(ls_id),
alloc_(alloc),
part_groups_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator(alloc, "LSPartGroupDesc")) {}
~ObLSPartGroupDesc() {
ls_id_.reset();
for (int64_t i = 0; i < part_groups_.count(); i++) {
if (OB_NOT_NULL(part_groups_.at(i))) {
part_groups_.at(i)->~ObTransferPartGroup();
alloc_.free(part_groups_.at(i));
part_groups_.at(i) = NULL;
}
}
part_groups_.reset();
}
ObLSID get_ls_id() const { return ls_id_; }
ObArray<ObTransferPartGroup *> &get_part_groups() { return part_groups_; }
int add_new_part_group(ObTransferPartGroup *&part_gourp);
TO_STRING_KV(K_(ls_id), K_(part_groups));
private:
ObLSID ls_id_;
ObIAllocator &alloc_;
ObArray<ObTransferPartGroup *> part_groups_;
};
@ -175,7 +182,7 @@ private:
int generate_balance_job_from_logical_task_();
int prepare_ls_();
int add_part_to_bg_map_(const ObLSID &ls_id, ObTransferPartGroup &part_group, ObBalanceGroup &bg);
int add_new_pg_to_bg_map_(const ObLSID &ls_id, ObBalanceGroup &bg, ObTransferPartGroup *&part_group);
int add_transfer_task_(const ObLSID &src_ls_id, const ObLSID &dest_ls_id, ObTransferPartGroup *part_group, bool modify_ls_desc = true);
int update_ls_desc_(const ObLSID &ls_id, int64_t cnt, int64_t size);
int try_swap_part_group_(ObLSDesc &src_ls, ObLSDesc &dest_ls, int64_t part_group_min_size ,int64_t &swap_cnt);

70
src/rootserver/ob_tenant_balance_service.cpp
View File

@ -28,6 +28,7 @@
#include "storage/tablelock/ob_lock_utils.h" // ObInnerTableLockUtil
#include "share/ob_cluster_version.h"
#include "share/ob_share_util.h" // ObShareUtil
#include "share/transfer/ob_transfer_task_operator.h"
#define ISTAT(fmt, args...) FLOG_INFO("[TENANT_BALANCE] " fmt, ##args)
#define WSTAT(fmt, args...) FLOG_WARN("[TENANT_BALANCE] " fmt, ##args)
@ -83,8 +84,10 @@ void ObTenantBalanceService::do_work()
int64_t idle_time_us = 10 * 1000 * 1000L;
int tmp_ret = OB_SUCCESS;
int64_t job_cnt = 0;
int64_t last_statistic_bg_stat_time = ObTimeUtility::current_time();
int64_t last_partition_balance_time = last_statistic_bg_stat_time;
int64_t last_partition_balance_time = ObTimeUtility::current_time();
int64_t last_statistic_bg_stat_time = OB_INVALID_TIMESTAMP; // statistic once when thread starts
int64_t last_statistic_schema_version = OB_INVALID_VERSION;
ObTransferTaskID last_statistic_max_transfer_task_id;
while (!has_set_stop()) {
ObCurTraceId::init(GCONF.self_addr_);
reset();
@ -120,9 +123,14 @@ void ObTenantBalanceService::do_work()
}
// separate statistic to avoid affecting balance jobs
// statistics balance group status periodically when tenant schema version changes or transfer occurs
if (OB_FAIL(ret)) {
} else if (OB_TMP_FAIL(try_statistic_balance_group_status_(last_statistic_bg_stat_time))) {
LOG_WARN("try statistic balance group status failed", KR(tmp_ret), K(last_statistic_bg_stat_time));
} else if (OB_TMP_FAIL(try_statistic_balance_group_status_(
last_statistic_bg_stat_time,
last_statistic_schema_version,
last_statistic_max_transfer_task_id))) {
LOG_WARN("try statistic balance group status failed", KR(tmp_ret), K(last_statistic_bg_stat_time),
K(last_statistic_schema_version), K(last_statistic_max_transfer_task_id));
}
if (OB_FAIL(ret) && OB_NEED_WAIT != ret) {
@ -134,6 +142,7 @@ void ObTenantBalanceService::do_work()
ISTAT("finish one round", KR(ret), KR(tmp_ret), K_(tenant_id), K(job_cnt),
K(primary_zone_num_), K(unit_group_array_),
K(ls_array_), K(idle_time_us), K(last_partition_balance_time), K(last_statistic_bg_stat_time),
K(last_statistic_schema_version), K(last_statistic_max_transfer_task_id),
"enable_rebalance", ObShareUtil::is_tenant_enable_rebalance(tenant_id_),
"enable_transfer", ObShareUtil::is_tenant_enable_transfer(tenant_id_));
reset();
@ -663,26 +672,53 @@ int ObTenantBalanceService::try_do_partition_balance_(int64_t &last_partition_ba
return ret;
}
int ObTenantBalanceService::try_statistic_balance_group_status_(int64_t &last_statistic_bg_stat_time)
// when running normally, it will statistic balance group status every 10min when tenant schema version changes or transfer occurs
// when thread starts, it will try to statistic bg stat every 10s until it is successful
int ObTenantBalanceService::try_statistic_balance_group_status_(
int64_t &last_statistic_bg_stat_time,
int64_t &last_statistic_schema_version,
ObTransferTaskID &last_statistic_max_transfer_task_id)
{
int ret = OB_SUCCESS;
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id_));
const int64_t curr_time = ObTimeUtility::current_time();
const int64_t STATISTIC_BG_STAT_INTERVAL = 600 * 1000 * 1000L; // 10min
ObRefreshSchemaStatus schema_status;
schema_status.tenant_id_ = tenant_id_;
int64_t latest_tenant_schema_version = OB_INVALID_VERSION;
ObTransferTaskID latest_max_transfer_task_id; // default -1
if (OB_UNLIKELY(!inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("not init", KR(ret));
} else if (OB_UNLIKELY(!tenant_config.is_valid())) {
} else if (OB_ISNULL(GCTX.sql_proxy_) || OB_ISNULL(GCTX.schema_service_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tenant config is invalid", KR(ret), K_(tenant_id));
LOG_WARN("GCTX has null ptr", KR(ret), KP(GCTX.sql_proxy_), KP(GCTX.schema_service_));
} else if (last_statistic_schema_version > OB_INVALID_VERSION
&& curr_time - last_statistic_bg_stat_time < STATISTIC_BG_STAT_INTERVAL) {
// no need to statistic because interval is not reached
} else if (OB_FAIL(GCTX.schema_service_->get_schema_version_in_inner_table(
*GCTX.sql_proxy_,
schema_status,
latest_tenant_schema_version))) {
LOG_WARN("failed to get schema version in inner table", KR(ret), K(schema_status));
} else if (OB_FAIL(ObTransferTaskOperator::get_max_task_id_from_history(
*GCTX.sql_proxy_,
tenant_id_,
latest_max_transfer_task_id))) { // -1 when transfer history is empty
LOG_WARN("get max transfer task if from history failed",
KR(ret), K_(tenant_id), K(latest_max_transfer_task_id));
} else if (latest_tenant_schema_version <= last_statistic_schema_version
&& latest_max_transfer_task_id <= last_statistic_max_transfer_task_id) {
// no need to statistics because distribution of tablets is not changed
} else if (OB_FAIL(partition_balance_(false/*need_balance*/))) { // just statistic balance group status
LOG_WARN("failed to save balance group status",
KR(ret), K(curr_time), K(last_statistic_bg_stat_time));
} else {
const int64_t curr_time = ObTimeUtility::current_time();
if (curr_time - last_statistic_bg_stat_time > tenant_config->balancer_idle_time) {
if (OB_FAIL(partition_balance_(false/*need_balance*/))) { // just statistic balance group status
LOG_WARN("failed to save balance group status",
KR(ret), K(curr_time), K(last_statistic_bg_stat_time));
} else {
last_statistic_bg_stat_time = curr_time;
}
}
ISTAT("statistic balance group status successfully", K(curr_time), K(last_statistic_bg_stat_time),
K(latest_tenant_schema_version), K(last_statistic_schema_version),
K(latest_max_transfer_task_id), K(last_statistic_max_transfer_task_id));
last_statistic_bg_stat_time = curr_time;
last_statistic_schema_version = latest_tenant_schema_version;
last_statistic_max_transfer_task_id = latest_max_transfer_task_id;
}
return ret;
}

6
src/rootserver/ob_tenant_balance_service.h
View File

@ -19,6 +19,7 @@
#include "lib/lock/ob_thread_cond.h"//ObThreadCond
#include "rootserver/ob_tenant_thread_helper.h"//ObTenantTheadHelper
#include "share/ls/ob_ls_status_operator.h"//ObLSStatusInfoArray
#include "share/ob_balance_define.h"
namespace oceanbase
{
@ -100,7 +101,10 @@ private:
int lock_and_check_balance_job_(common::ObMySQLTransaction &trans, const uint64_t tenant_id);
int try_update_job_comment_(const share::ObBalanceJob &job, const common::ObSqlString &comment);
int try_do_partition_balance_(int64_t &last_partition_balance_time);
int try_statistic_balance_group_status_(int64_t &last_statistic_bg_stat_time);
int try_statistic_balance_group_status_(
int64_t &last_statistic_bg_stat_time,
int64_t &last_statistic_schema_version,
share::ObTransferTaskID &last_statistic_max_transfer_task_id);
int get_active_unit_num_(int64_t &active_unit_num) const;
private:
bool inited_;

4
src/rootserver/ob_unit_stat_manager.cpp
View File

@ -37,7 +37,7 @@ int ObUnitStatManager::init(common::ObMySQLProxy &sql_proxy)
if (OB_UNLIKELY(inited_)) {
ret = OB_INIT_TWICE;
LOG_WARN("init twice", KR(ret), K(inited_));
} else if (OB_FAIL(unit_stat_map_.init(500000))) { /// FIXME: use more accurate CONSTANT
} else if (OB_FAIL(unit_stat_map_.init(1000))) { /// FIXME: use more accurate CONSTANT
LOG_WARN("fail init unit stat map", KR(ret));
} else if (OB_FAIL(ut_operator_.init(sql_proxy))) {
LOG_WARN("fail to init ut_operator_", KR(ret));
@ -112,5 +112,3 @@ int ObUnitStatManager::get_unit_stat(
}
return ret;
}

30
src/share/cache/ob_kvcache_inst_map.cpp vendored
View File

@ -176,12 +176,11 @@ ObKVCacheInstHandle& ObKVCacheInstHandle::operator = (const ObKVCacheInstHandle&
ObKVCacheInstMap::ObKVCacheInstMap()
: lock_(common::ObLatchIds::KV_CACHE_INST_LOCK),
inst_map_(),
inst_pool_(),
list_lock_(common::ObLatchIds::KV_CACHE_LIST_LOCK),
list_map_(),
list_pool_(),
configs_(NULL),
allocator_("CACHE_INST"),
allocator_("TenantMBList"),
inst_keys_(),
mem_limit_getter_(NULL),
is_inited_(false)
@ -209,19 +208,6 @@ int ObKVCacheInstMap::init(const int64_t max_entry_cnt, const ObKVCacheConfig *c
if (OB_SUCC(ret)) {
if (OB_FAIL(inst_map_.create(max_entry_cnt, "CACHE_INST_MAP", "CACHE_INST_MAP"))) {
COMMON_LOG(WARN, "Fail to create inst map, ", K(ret));
} else if (NULL == (buf = (char*) allocator_.alloc((sizeof(ObKVCacheInst) + sizeof(ObKVCacheInst*))* max_entry_cnt))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
COMMON_LOG(ERROR, "Fail to allocate memory, ", K(max_entry_cnt), K(ret));
} else if (OB_FAIL(inst_pool_.init(max_entry_cnt, buf + sizeof(ObKVCacheInst) * max_entry_cnt))) {
COMMON_LOG(WARN, "Fail to init inst pool, ", K(ret));
} else {
ObKVCacheInst *inst = NULL;
for (int64_t i = 0; OB_SUCC(ret) && i < max_entry_cnt; ++i) {
inst = new (buf + sizeof(ObKVCacheInst) * i) ObKVCacheInst();
if (OB_FAIL(inst_pool_.push(inst))) {
COMMON_LOG(WARN, "Fail to push inst to pool, ", K(ret));
}
}
}
}
@ -270,7 +256,6 @@ void ObKVCacheInstMap::destroy()
{
inst_map_.destroy();
tenant_set_.destroy();
inst_pool_.destroy();
list_map_.destroy();
list_pool_.destroy();
allocator_.reset();
@ -310,7 +295,9 @@ int ObKVCacheInstMap::get_cache_inst(
} else if (OB_HASH_NOT_EXIST == ret) {
lib::ObMemAttr attr(inst_key.tenant_id_, "CACHE_MAP_NODE");
SET_USE_500(attr);
if (OB_FAIL(inst_pool_.pop(inst))) {
inst = OB_NEW(ObKVCacheInst, ObMemAttr(inst_key.tenant_id_, "CACHE_INST"));
if (OB_ISNULL(inst)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
COMMON_LOG(WARN, "Fail to alloc cache inst, ", K(ret));
} else if (OB_FAIL(get_mb_list(inst_key.tenant_id_, inst->mb_list_handle_))) {
COMMON_LOG(WARN, "get mb list failed", K(ret), "tenant_id", inst_key.tenant_id_);
@ -331,9 +318,7 @@ int ObKVCacheInstMap::get_cache_inst(
if (OB_FAIL(ret) && NULL != inst) {
inst->reset();
int tmp_ret = OB_SUCCESS;
if (OB_SUCCESS != (tmp_ret = inst_pool_.push(inst))) {
COMMON_LOG(ERROR, "Fail to push inst to pool, ", K(tmp_ret));
}
ob_delete(inst);
if (OB_SUCCESS != (tmp_ret = inst_map_.erase_refactored(inst_key))) {
if (OB_HASH_NOT_EXIST != tmp_ret) {
COMMON_LOG(ERROR, "Fail to erase inst key, ", K(ret));
@ -417,9 +402,8 @@ int ObKVCacheInstMap::erase_tenant(const uint64_t tenant_id)
if (OB_FAIL(inst_map_.erase_refactored(tmp_key))) {
COMMON_LOG(WARN, "Fail to erase cache inst from inst map", K(ret));
} else if (FALSE_IT(inst->reset())) {
} else if (OB_FAIL(inst_pool_.push(inst))) {
COMMON_LOG(ERROR, "Fail to push inst back to inst pool", K(ret));
ob_abort();
} else {
ob_delete(inst);
}
}
}

1
src/share/cache/ob_kvcache_inst_map.h vendored
View File

@ -166,7 +166,6 @@ private:
private:
DRWLock lock_;
KVCacheInstMap inst_map_;
ObFixedQueue<ObKVCacheInst> inst_pool_;
DRWLock list_lock_;
TenantMBListMap list_map_;
ObFixedQueue<ObTenantMBList> list_pool_;

154
src/share/config/ob_config_manager.cpp
View File

@ -214,82 +214,96 @@ int ObConfigManager::dump2file(const char* path) const
path = dump_path_;
}
PageArena<> pa;
if (OB_ISNULL(path) || STRLEN(path) <= 0) {
ret = OB_INVALID_ERROR;
LOG_WARN("NO dump path specified!", K(ret));
} else {
// write server config
char *buf = nullptr;
char *tmp_path = nullptr;
char *hist_path = nullptr;
int64_t pos = 0;
if (OB_ISNULL(buf = pa.alloc(OB_MAX_PACKET_LENGTH))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("ob tc malloc memory for buf failed", K(ret));
}
if (OB_ISNULL(tmp_path = pa.alloc(MAX_PATH_SIZE))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("ob tc malloc memory for tmp configure path failed", K(ret));
} else {
snprintf(tmp_path, MAX_PATH_SIZE, "%s.tmp", path);
}
if (OB_ISNULL(hist_path = pa.alloc(MAX_PATH_SIZE))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("ob tc malloc memory for history configure path fail", K(ret));
} else {
snprintf(hist_path, MAX_PATH_SIZE, "%s.history", path);
const int64_t min_buffer_size = 2LL << 20;
const int64_t max_buffer_size = 64LL << 20;
int64_t buf_size = min_buffer_size;
bool need_retry = true;
while (OB_SUCC(ret) && need_retry) {
PageArena<> pa;
char *buf = nullptr;
char *tmp_path = nullptr;
char *hist_path = nullptr;
int64_t pos = 0;
need_retry = false;
if (OB_ISNULL(buf = pa.alloc(buf_size))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("ob tc malloc memory for buf failed", K(ret));
}
if (OB_ISNULL(tmp_path = pa.alloc(MAX_PATH_SIZE))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("ob tc malloc memory for tmp configure path failed", K(ret));
} else {
snprintf(tmp_path, MAX_PATH_SIZE, "%s.tmp", path);
}
if (OB_ISNULL(hist_path = pa.alloc(MAX_PATH_SIZE))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("ob tc malloc memory for history configure path fail", K(ret));
} else {
snprintf(hist_path, MAX_PATH_SIZE, "%s.history", path);
}
#ifdef ERRSIM
ret = OB_E(EventTable::EN_WRITE_CONFIG_FILE_FAILED) OB_SUCCESS;;
if (OB_FAIL(ret)) {
ret = OB_IO_ERROR;
LOG_WARN("ERRSIM, write config file failed", K(ret));
}
#endif
if (OB_SUCC(ret)) {
if (OB_FAIL(server_config_.serialize(buf, buf_size, pos))) {
LOG_WARN("Serialize server config fail!", K(pos), K(buf_size), K(ret));
if (OB_SIZE_OVERFLOW == ret && (buf_size << 1) <= max_buffer_size) {
buf_size = buf_size << 1;
need_retry = true;
}
} else if (OB_FAIL(check_header_change(path, buf)) && OB_EAGAIN == ret) {
LOG_INFO("Header not change, no need to write server config!");
} else if ((fd = ::open(tmp_path, O_WRONLY | O_CREAT | O_TRUNC,
S_IRUSR | S_IWUSR | S_IRGRP)) < 0) {
ret = OB_IO_ERROR;
LOG_WARN("fail to create config file", K(tmp_path), KERRMSG, K(ret));
} else if (pos != (size = unintr_write(fd, buf, pos))) {
ret = OB_IO_ERROR;
LOG_WARN("Write server config fail!", K(errno), KERRMSG, K(pos), K(size), K(ret));
if (0 != close(fd)) {
LOG_WARN("fail to close file fd", K(fd), K(errno), KERRMSG, K(ret));
}
} else if (::fsync(fd) != 0) {
ret = OB_IO_ERROR;
LOG_WARN("Sync server config fail!", K(errno), KERRMSG, K(pos), K(size), K(ret));
if (0 != close(fd)) {
LOG_WARN("fail to close file fd", K(fd), K(errno), KERRMSG, K(ret));
}
} else if (0 != close(fd)) {
ret = OB_IO_ERROR;
LOG_WARN("fail to close file fd", K(fd), KERRMSG, K(ret));
} else {
LOG_INFO("Write server config successfully!", K(pos), K(buf_size));
}
}
if (OB_SUCC(ret)) {
if (0 != ::rename(path, hist_path) && errno != ENOENT) {
ret = OB_ERR_SYS;
LOG_WARN("fail to backup history config file", KERRMSG, K(ret));
}
// 运行到这里的时候可能掉电，导致没有 conf 文件，需要 DBA 手工拷贝 tmp 文件到这里
if (0 != ::rename(tmp_path, path) && errno != ENOENT) {
ret = OB_ERR_SYS;
LOG_WARN("fail to move tmp config file", KERRMSG, K(ret));
}
} else if (OB_EAGAIN == ret) {
ret = OB_SUCCESS;
} else if (need_retry) {
ret = OB_SUCCESS;
}
}
#ifdef ERRSIM
ret = OB_E(EventTable::EN_WRITE_CONFIG_FILE_FAILED) OB_SUCCESS;;
if (OB_FAIL(ret)) {
ret = OB_IO_ERROR;
LOG_WARN("ERRSIM, write config file failed", K(ret));
}
#endif
if (OB_SUCC(ret)) {
if (OB_FAIL(server_config_.serialize(buf, OB_MAX_PACKET_LENGTH, pos))) {
LOG_WARN("Serialize server config fail!", K(ret));
} else if (OB_FAIL(check_header_change(path, buf)) && OB_EAGAIN == ret) {
LOG_INFO("Header not change, no need to write server config!");
} else if ((fd = ::open(tmp_path, O_WRONLY | O_CREAT | O_TRUNC,
S_IRUSR | S_IWUSR | S_IRGRP)) < 0) {
ret = OB_IO_ERROR;
LOG_WARN("fail to create config file", K(tmp_path), KERRMSG, K(ret));
} else if (pos != (size = unintr_write(fd, buf, pos))) {
ret = OB_IO_ERROR;
LOG_WARN("Write server config fail!", K(errno), KERRMSG, K(pos), K(size), K(ret));
if (0 != close(fd)) {
LOG_WARN("fail to close file fd", K(fd), K(errno), KERRMSG, K(ret));
}
} else if (::fsync(fd) != 0) {
ret = OB_IO_ERROR;
LOG_WARN("Sync server config fail!", K(errno), KERRMSG, K(pos), K(size), K(ret));
if (0 != close(fd)) {
LOG_WARN("fail to close file fd", K(fd), K(errno), KERRMSG, K(ret));
}
} else if (0 != close(fd)) {
ret = OB_IO_ERROR;
LOG_WARN("fail to close file fd", K(fd), KERRMSG, K(ret));
} else {
LOG_INFO("Write server config successfully!");
}
}
if (OB_SUCC(ret)) {
if (0 != ::rename(path, hist_path) && errno != ENOENT) {
ret = OB_ERR_SYS;
LOG_WARN("fail to backup history config file", KERRMSG, K(ret));
}
// 运行到这里的时候可能掉电，导致没有 conf 文件，需要 DBA 手工拷贝 tmp 文件到这里
if (0 != ::rename(tmp_path, path) && errno != ENOENT) {
ret = OB_ERR_SYS;
LOG_WARN("fail to move tmp config file", KERRMSG, K(ret));
}
} else if (OB_EAGAIN == ret) {
ret = OB_SUCCESS;
}
// write server config
}
return ret;
}

164
src/share/config/ob_server_config.cpp
View File

@ -211,7 +211,7 @@ int ObServerConfig::deserialize_with_compat(const char *buf, const int64_t data_
}
ObServerMemoryConfig::ObServerMemoryConfig()
: memory_limit_(0), system_memory_(0)
: memory_limit_(0), system_memory_(0), hidden_sys_memory_(0)
{}
ObServerMemoryConfig &ObServerMemoryConfig::get_instance()
@ -220,75 +220,126 @@ ObServerMemoryConfig &ObServerMemoryConfig::get_instance()
return memory_config;
}
int64_t ObServerMemoryConfig::get_capacity_default_memory(CapacityType type, int64_t memory_limit)
int64_t ObServerMemoryConfig::get_adaptive_memory_config(const int64_t memory_size,
DependentMemConfig dep_mem_config,
AdaptiveMemConfig adap_mem_config)
{
// According to different memory_limit, the kernel can provide adaptive memory_size for default capacity.
// For example, memory_limit = 16G, adaptive system_memory and hidden_sys_memory are 6G and 2G.
static const int64_t memory_limit_array[] = {4LL<<30, 8LL<<30, 14LL<<30, 28LL<<30, 48LL<<30, 56LL<<30, 65LL<<30, 96LL<<30, 128LL<<30};
static const int64_t system_memory_array[] = {2LL<<30, 3LL<<30, 6LL<<30, 10LL<<30, 12LL<<30, 13LL<<30, 15LL<<30, 18LL<<30, 20LL<<30};
static const int64_t hidden_sys_memory_array[] = {1LL<<30, 2LL<<30, 2LL<<30, 2LL<<30, 4LL<<30, 4LL<<30, 6LL<<30, 7LL<<30, 8LL<<30};
static const int64_t memory_limit_array[] = {4LL<<30, 12LL<<30, 20LL<<30, 36LL<<30, 60LL<<30, 80LL<<30, 100LL<<30, 120LL<<30, 130LL<<30};
static const int64_t system_memory_array[] = {1LL<<30, 1LL<<30, 4LL<<30, 5LL<<30, 6LL<<30, 7LL<<30, 8LL<<30, 9LL<<30, 9LL<<30};
static const int64_t hidden_sys_memory_array[] = {1LL<<30, 1LL<<30, 1LL<<30, 1LL<<30, 2LL<<30, 2LL<<30, 2LL<<30, 3LL<<30, 3LL<<30};
static const int64_t array_size = ARRAYSIZEOF(memory_limit_array);
int64_t memory_size = 0;
for (int i = ARRAYSIZEOF(memory_limit_array) - 1; i >= 0; --i) {
if (memory_limit_array[i] <= memory_limit) {
switch (type) {
case SYSTEM_MEMORY:
memory_size = system_memory_array[i];
break;
case HIDDEN_SYS_MEMORY:
memory_size = hidden_sys_memory_array[i];
break;
}
int64_t adap_memory_size = 0;
int64_t dep_memory_limit = 0;
const int64_t *dep_array = NULL;
switch (dep_mem_config) {
case MEMORY_LIMIT:
dep_array = memory_limit_array;
dep_memory_limit = memory_size;
break;
case SYSTEM_MEMORY:
dep_array = system_memory_array;
dep_memory_limit = memory_size / 0.08;
break;
}
if (memory_size < dep_array[array_size - 1]) {
// When memory_limit < 130G, adaptive memory is calculated by array.
// For example, memory_limit = 16G, adaptive system_memory and hidden_sys_memory are 1G and 1G.
for (int i = 0; i < array_size; ++i) {
if (memory_size < dep_array[i]) {
switch (adap_mem_config) {
case ADAPTIVE_SYSTEM_MEMORY:
adap_memory_size = system_memory_array[i];
break;
case ADAPTIVE_HIDDEN_SYS_MEMORY:
adap_memory_size = hidden_sys_memory_array[i];
break;
}
break;
}
}
} else {
// When memory_limit >= 130G or system_memory >= 9G, system_memory = memory_limit*0.08, hidden_sys_memory = memory_limit*0.03.
// For example, memory_limit = 200G, adaptive system_memory and hidden_sys_memory are 16G and 6G.
switch (adap_mem_config) {
case ADAPTIVE_SYSTEM_MEMORY:
adap_memory_size = dep_memory_limit * 0.08;
break;
case ADAPTIVE_HIDDEN_SYS_MEMORY:
adap_memory_size = dep_memory_limit * 0.03;
break;
}
}
return memory_size;
return adap_memory_size;
}
int64_t ObServerMemoryConfig::get_extra_memory()
{
return memory_limit_ < lib::ObRunningModeConfig::MINI_MEM_UPPER ? 0 : hidden_sys_memory_;
}
int ObServerMemoryConfig::reload_config(const ObServerConfig& server_config)
{
int ret = OB_SUCCESS;
const bool is_arbitration_mode = OBSERVER.is_arbitration_mode();
int64_t memory_limit = server_config.memory_limit;
if (0 == memory_limit) {
memory_limit = get_phy_mem_size() * server_config.memory_limit_percentage / 100;
}
int64_t system_memory = server_config.system_memory;
if (memory_limit < (1 << 30) ) {
// The memory_limit should not be less than 1G for arbitration mode
ret = OB_INVALID_CONFIG;
LOG_ERROR("memory_limit with unexpected value", K(ret), K(memory_limit), "phy mem", get_phy_mem_size());
} else if (is_arbitration_mode) {
// do nothing
} else if (0 == system_memory) {
system_memory = get_capacity_default_memory(SYSTEM_MEMORY, memory_limit);
if (0 == system_memory) {
if (is_arbitration_mode) {
if (memory_limit < (1LL << 30) ) {
// The memory_limit should not be less than 1G for arbitration mode
ret = OB_INVALID_CONFIG;
LOG_ERROR("memory_limit with unexpected value", K(ret), K(memory_limit), "phy mem", get_phy_mem_size());
}
}
if (OB_FAIL(ret)) {
// do nothing
} else if (memory_limit > system_memory) {
memory_limit_ = memory_limit;
system_memory_ = system_memory;
LOG_INFO("update memory_limit or system_memory success",
K(memory_limit_), K(system_memory_));
} else {
ret = OB_INVALID_CONFIG;
LOG_ERROR("update memory_limit or system_memory failed",
K(memory_limit), K(system_memory));
}
int64_t observer_tenant_hold = lib::get_tenant_memory_hold(OB_SERVER_TENANT_ID);
if (observer_tenant_hold > system_memory_ && false == is_arbitration_mode) {
if (server_config._ignore_system_memory_over_limit_error) {
LOG_WARN("the hold of tenant_500 is over the system_memory",
K(observer_tenant_hold), K_(system_memory));
} else {
LOG_ERROR("the hold of tenant_500 is over the system_memory",
K(observer_tenant_hold), K_(system_memory));
memory_limit_ = memory_limit;
LOG_INFO("update observer memory config success", K_(memory_limit));
}
} else if (memory_limit < (4LL << 30)) {
// The memory_limit should not be less than 4G for observer
ret = OB_INVALID_CONFIG;
LOG_ERROR("memory_limit with unexpected value", K(ret), K(memory_limit), "phy mem", get_phy_mem_size());
} else {
// update observer memory config
int64_t system_memory = server_config.system_memory;
int64_t hidden_sys_memory = server_config._hidden_sys_tenant_memory;
int64_t min_server_avail_memory = max(ObUnitResource::UNIT_MIN_MEMORY, server_config.__min_full_resource_pool_memory);
if (0 != system_memory && 0 != hidden_sys_memory) {
// do-nothing
} else if (0 == system_memory && 0 == hidden_sys_memory) {
system_memory = get_adaptive_memory_config(memory_limit, MEMORY_LIMIT, ADAPTIVE_SYSTEM_MEMORY);
hidden_sys_memory = get_adaptive_memory_config(memory_limit, MEMORY_LIMIT, ADAPTIVE_HIDDEN_SYS_MEMORY);
} else if (0 == hidden_sys_memory) {
hidden_sys_memory = get_adaptive_memory_config(system_memory, SYSTEM_MEMORY, ADAPTIVE_HIDDEN_SYS_MEMORY);
} else {
system_memory = get_adaptive_memory_config(memory_limit, MEMORY_LIMIT, ADAPTIVE_SYSTEM_MEMORY);
}
if (memory_limit - system_memory >= min_server_avail_memory &&
system_memory >= hidden_sys_memory) {
memory_limit_ = memory_limit;
system_memory_ = system_memory;
hidden_sys_memory_ = hidden_sys_memory;
LOG_INFO("update observer memory config success",
K_(memory_limit), K_(system_memory), K_(hidden_sys_memory));
} else {
ret = OB_INVALID_CONFIG;
LOG_ERROR("update observer memory config failed",
K(memory_limit), K(system_memory), K(hidden_sys_memory), K(min_server_avail_memory));
}
//check the hold memory of tenant 500
int64_t tenant_500_hold = lib::get_tenant_memory_hold(OB_SERVER_TENANT_ID);
int64_t tenant_500_reserved = system_memory_ - get_extra_memory();
if (tenant_500_hold > tenant_500_reserved) {
if (server_config._ignore_system_memory_over_limit_error) {
LOG_ERROR("the hold memory of tenant_500 is over the reserved memory",
K(tenant_500_hold), K(tenant_500_reserved));
} else {
LOG_WARN("the hold memory of tenant_500 is over the reserved memory",
K(tenant_500_hold), K(tenant_500_reserved));
}
}
}
@ -305,15 +356,6 @@ int ObServerMemoryConfig::reload_config(const ObServerConfig& server_config)
return ret;
}
void ObServerMemoryConfig::set_server_memory_limit(int64_t memory_limit)
{
if (memory_limit > system_memory_) {
LOG_INFO("update memory_limit success", K(memory_limit), K(system_memory_));
} else {
LOG_ERROR_RET(OB_ERR_UNEXPECTED, "update memory_limit failed", K(memory_limit), K(system_memory_));
}
}
int ObServerConfig::serialize_(char *buf, const int64_t buf_len, int64_t &pos) const
{
int ret = OB_SUCCESS;

19
src/share/config/ob_server_config.h
View File

@ -149,25 +149,34 @@ private:
class ObServerMemoryConfig
{
public:
enum CapacityType {
enum DependentMemConfig {
MEMORY_LIMIT,
SYSTEM_MEMORY,
HIDDEN_SYS_MEMORY,
};
enum AdaptiveMemConfig {
ADAPTIVE_SYSTEM_MEMORY,
ADAPTIVE_HIDDEN_SYS_MEMORY,
};
friend class unittest::ObSimpleClusterTestBase;
friend class unittest::ObMultiReplicaTestBase;
ObServerMemoryConfig();
static ObServerMemoryConfig &get_instance();
int reload_config(const ObServerConfig& server_config);
int64_t get_capacity_default_memory(CapacityType type, int64_t memory_limit);
int64_t get_server_memory_limit() { return memory_limit_; }
int64_t get_reserved_server_memory() { return system_memory_; }
int64_t get_server_memory_avail() { return memory_limit_ - system_memory_; }
int64_t get_hidden_sys_memory() { return hidden_sys_memory_; }
//the extra_memory just used by real sys when non_mini_mode
int64_t get_extra_memory();
private:
// set_server_memory_limit just for mittest
void set_server_memory_limit(int64_t memory_limit);
int64_t get_adaptive_memory_config(const int64_t memory_size,
DependentMemConfig dep_mem_config,
AdaptiveMemConfig adap_mem_config);
private:
int64_t memory_limit_;
int64_t system_memory_;
int64_t hidden_sys_memory_;
private:
DISALLOW_COPY_AND_ASSIGN(ObServerMemoryConfig);
};

Some files were not shown because too many files have changed in this diff Show More