From 81b8923b1425dd16f9f0145022b4fb1cf0d21fb7 Mon Sep 17 00:00:00 2001 From: obdev Date: Tue, 29 Aug 2023 10:41:01 +0000 Subject: [PATCH] [FEAT MERGE] system resource optimization Co-authored-by: tushicheng <18829573815@163.com> Co-authored-by: HaHaJeff Co-authored-by: dimstars --- deps/easy/src/io/easy_baseth_pool.c | 14 +- deps/easy/src/io/easy_io.c | 27 +- deps/easy/src/io/easy_io_struct.h | 4 +- deps/easy/src/io/easy_maccept.c | 11 +- deps/easy/src/io/easy_request.c | 7 +- deps/oblib/src/lib/alloc/alloc_struct.cpp | 51 +- deps/oblib/src/lib/alloc/alloc_struct.h | 10 +- deps/oblib/src/lib/alloc/malloc_hook.cpp | 47 +- .../src/lib/alloc/ob_malloc_sample_struct.h | 4 + .../allocator/ob_fixed_size_block_allocator.h | 3 +- deps/oblib/src/lib/geo/ob_srs_info.cpp | 6 +- deps/oblib/src/lib/geo/ob_srs_info.h | 93 +-- deps/oblib/src/lib/hash/ob_hashset.h | 9 + deps/oblib/src/lib/hash/ob_hashutils.h | 42 +- deps/oblib/src/lib/lock/ob_latch.h | 52 +- deps/oblib/src/lib/ob_define.h | 2 +- .../lib/objectpool/ob_server_object_pool.h | 3 +- deps/oblib/src/lib/queue/ob_dedup_queue.h | 4 +- deps/oblib/src/lib/queue/ob_priority_queue.h | 16 +- deps/oblib/src/lib/resource/achunk_mgr.cpp | 8 +- deps/oblib/src/lib/stat/ob_di_cache.cpp | 16 +- deps/oblib/src/lib/stat/ob_di_cache.h | 10 +- deps/oblib/src/lib/stat/ob_diagnose_info.cpp | 72 +- deps/oblib/src/lib/stat/ob_diagnose_info.h | 24 +- .../src/lib/statistic_event/ob_stat_event.h | 3 + deps/oblib/src/lib/task/ob_timer.cpp | 131 ++- deps/oblib/src/lib/task/ob_timer.h | 10 +- .../src/lib/thread/ob_async_task_queue.cpp | 4 +- .../src/lib/thread/ob_async_task_queue.h | 4 +- deps/oblib/src/lib/thread/ob_tenant_hook.cpp | 34 - deps/oblib/src/lib/thread/thread.h | 1 + deps/oblib/src/lib/thread/thread_mgr.h | 33 + deps/oblib/src/lib/thread/threads.cpp | 50 +- deps/oblib/src/lib/thread/threads.h | 23 + .../src/lib/timezone/ob_timezone_info.cpp | 68 +- .../oblib/src/lib/timezone/ob_timezone_info.h | 18 +- .../src/rpc/frame/ob_req_queue_thread.cpp | 18 +- .../oblib/src/rpc/frame/ob_req_queue_thread.h | 7 +- deps/oblib/src/rpc/pnio/interface/group.c | 5 +- deps/oblib/src/rpc/pnio/interface/group.h | 2 +- deps/oblib/src/rpc/pnio/r0/log.h | 9 +- deps/oblib/unittest/lib/task/test_timer.cpp | 56 +- .../unittest/lib/thread/test_threads.cpp | 20 + deps/ussl-hook/ussl-loop.c | 11 +- .../test_ob_simple_log_disk_mgr.cpp | 41 + mittest/mtlenv/CMakeLists.txt | 2 +- mittest/mtlenv/mock_tenant_module_env.h | 8 +- .../env/ob_multi_replica_test_base.cpp | 1 - .../env/ob_simple_cluster_test_base.cpp | 1 - .../test_observer_expand_shrink.cpp | 99 --- .../test_transfer_task_operator.cpp | 27 +- .../src/ob_log_timezone_info_getter.cpp | 6 +- src/logservice/ob_server_log_block_mgr.cpp | 9 +- src/logservice/palf/palf_env_impl.cpp | 8 +- src/logservice/palf/palf_options.cpp | 2 +- src/observer/CMakeLists.txt | 1 - src/observer/dbms_job/ob_dbms_job_master.cpp | 16 +- src/observer/dbms_job/ob_dbms_job_master.h | 11 +- src/observer/dbms_job/ob_dbms_job_utils.h | 2 + .../ob_dbms_sched_job_master.cpp | 15 +- .../dbms_scheduler/ob_dbms_sched_job_master.h | 12 +- src/observer/mysql/ob_eliminate_task.cpp | 4 +- src/observer/ob_rpc_processor_simple.cpp | 13 +- src/observer/ob_server.cpp | 41 +- src/observer/ob_server.h | 4 - src/observer/ob_server_reload_config.cpp | 5 +- src/observer/ob_uniq_task_queue.h | 44 +- src/observer/omt/ob_multi_tenant.cpp | 76 +- src/observer/omt/ob_multi_tenant.h | 17 +- src/observer/omt/ob_tenant_config.h | 1 - src/observer/omt/ob_tenant_node_balancer.cpp | 17 +- src/observer/omt/ob_tenant_srs.cpp | 190 +++-- src/observer/omt/ob_tenant_srs.h | 54 +- src/observer/omt/ob_tenant_srs_mgr.cpp | 369 --------- src/observer/omt/ob_tenant_srs_mgr.h | 143 ---- src/observer/omt/ob_tenant_timezone.cpp | 40 +- src/observer/omt/ob_tenant_timezone.h | 18 +- src/observer/omt/ob_tenant_timezone_mgr.cpp | 16 +- src/observer/report/ob_tablet_table_updater.h | 4 +- .../table_load/ob_table_load_store.cpp | 2 +- src/observer/virtual_table/ob_all_latch.cpp | 4 +- .../ob_all_virtual_dml_stats.cpp | 77 +- .../virtual_table/ob_all_virtual_dml_stats.h | 33 +- .../virtual_table/ob_all_virtual_sys_stat.cpp | 6 + src/pl/ob_pl_package_state.h | 2 +- src/pl/ob_pl_router.h | 2 +- src/pl/sys_package/ob_dbms_stats.cpp | 2 +- .../ob_disaster_recovery_task_mgr.h | 8 +- ...b_disaster_recovery_task_table_updater.cpp | 4 +- .../ob_disaster_recovery_task_table_updater.h | 3 +- src/rootserver/ob_partition_balance.cpp | 48 +- src/rootserver/ob_partition_balance.h | 11 +- src/rootserver/ob_tenant_balance_service.cpp | 70 +- src/rootserver/ob_tenant_balance_service.h | 6 +- src/rootserver/ob_unit_stat_manager.cpp | 4 +- src/share/cache/ob_kvcache_inst_map.cpp | 30 +- src/share/cache/ob_kvcache_inst_map.h | 1 - src/share/config/ob_config_manager.cpp | 154 ++-- src/share/config/ob_server_config.cpp | 164 ++-- src/share/config/ob_server_config.h | 19 +- .../ob_inner_table_schema.21451_21500.cpp | 130 +++ src/share/inner_table/ob_inner_table_schema.h | 10 +- .../ob_inner_table_schema_constants.h | 4 + .../inner_table/ob_inner_table_schema_def.py | 68 +- .../interrupt/ob_global_interrupt_call.cpp | 4 +- .../interrupt/ob_global_interrupt_call.h | 4 +- .../location_cache/ob_location_struct.cpp | 21 +- src/share/location_cache/ob_location_struct.h | 12 +- src/share/location_cache/ob_tablet_ls_map.cpp | 20 +- src/share/location_cache/ob_tablet_ls_map.h | 2 +- .../location_cache/ob_tablet_ls_service.h | 2 +- src/share/ob_common_id.h | 2 + src/share/ob_debug_sync.cpp | 28 +- src/share/ob_debug_sync.h | 3 +- src/share/ob_scanner.h | 6 +- src/share/ob_tenant_mgr.cpp | 55 +- src/share/ob_tenant_mgr.h | 6 +- src/share/ob_thread_define.h | 1 + src/share/ob_time_zone_info_manager.cpp | 204 +++-- src/share/ob_time_zone_info_manager.h | 30 +- src/share/object/ob_obj_cast.cpp | 4 +- src/share/rc/ob_tenant_base.h | 8 +- src/share/scheduler/ob_dag_scheduler.cpp | 2 +- src/share/schema/ob_dependency_info.h | 4 +- .../stat/ob_opt_stat_monitor_manager.cpp | 751 ++++++------------ src/share/stat/ob_opt_stat_monitor_manager.h | 157 ++-- .../transfer/ob_transfer_task_operator.cpp | 42 + .../transfer/ob_transfer_task_operator.h | 16 + src/share/unit/ob_unit_config.cpp | 2 +- src/share/unit/ob_unit_resource.cpp | 110 +-- src/share/unit/ob_unit_resource.h | 8 +- src/sql/das/ob_das_utils.cpp | 6 +- .../engine/expr/ob_expr_priv_st_asewkb.cpp | 2 +- .../expr/ob_expr_priv_st_geogfromtext.cpp | 4 +- src/sql/engine/expr/ob_expr_priv_st_point.cpp | 4 +- .../engine/expr/ob_expr_priv_st_setsrid.cpp | 2 +- .../engine/expr/ob_expr_priv_st_transform.cpp | 4 +- .../engine/expr/ob_expr_spatial_cellid.cpp | 2 +- src/sql/engine/expr/ob_expr_spatial_mbr.cpp | 2 +- src/sql/engine/expr/ob_expr_st_area.cpp | 2 +- src/sql/engine/expr/ob_expr_st_asewkt.cpp | 2 +- src/sql/engine/expr/ob_expr_st_astext.cpp | 2 +- src/sql/engine/expr/ob_expr_st_bestsrid.cpp | 2 +- src/sql/engine/expr/ob_expr_st_buffer.h | 2 +- src/sql/engine/expr/ob_expr_st_contains.cpp | 2 +- src/sql/engine/expr/ob_expr_st_covers.cpp | 2 +- src/sql/engine/expr/ob_expr_st_distance.cpp | 2 +- src/sql/engine/expr/ob_expr_st_dwithin.cpp | 2 +- .../engine/expr/ob_expr_st_geomfromewkb.cpp | 2 +- .../engine/expr/ob_expr_st_geomfromewkt.cpp | 4 +- .../engine/expr/ob_expr_st_geomfromtext.cpp | 4 +- .../engine/expr/ob_expr_st_geomfromwkb.cpp | 4 +- src/sql/engine/expr/ob_expr_st_intersects.cpp | 2 +- src/sql/engine/expr/ob_expr_st_isvalid.cpp | 2 +- src/sql/engine/expr/ob_expr_st_srid.cpp | 4 +- src/sql/engine/expr/ob_expr_st_transform.cpp | 2 +- src/sql/engine/expr/ob_expr_st_within.cpp | 2 +- src/sql/engine/expr/ob_expr_st_x.cpp | 2 +- src/sql/engine/expr/ob_geo_expr_utils.cpp | 4 +- src/sql/engine/expr/ob_geo_expr_utils.h | 2 +- src/sql/engine/table/ob_table_scan_op.cpp | 6 +- .../ob_maintain_dependency_info_task.cpp | 2 +- .../executor/ob_remote_executor_processor.cpp | 39 +- src/sql/ob_sql_utils.cpp | 4 +- src/sql/optimizer/ob_optimizer.cpp | 17 +- src/sql/rewrite/ob_query_range.cpp | 14 +- src/storage/blocksstable/ob_tmp_file.cpp | 5 +- src/storage/blocksstable/ob_tmp_file.h | 3 - src/storage/ddl/ob_ddl_redo_log_writer.cpp | 7 +- src/storage/ddl/ob_ddl_redo_log_writer.h | 3 +- src/storage/ddl/ob_ddl_replay_executor.cpp | 3 +- src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp | 8 +- src/storage/ddl/ob_tablet_ddl_kv_mgr.h | 2 +- src/storage/ls/ob_ls_ddl_log_handler.cpp | 168 +++- src/storage/ls/ob_ls_ddl_log_handler.h | 38 + .../meta_mem/ob_storage_meta_cache.cpp | 15 +- src/storage/meta_mem/ob_storage_meta_cache.h | 1 - .../meta_mem/ob_tenant_meta_mem_mgr.cpp | 9 +- src/storage/meta_mem/ob_tenant_meta_mem_mgr.h | 5 +- src/storage/ob_resource_map.h | 49 +- src/storage/tablet/ob_tablet.cpp | 14 +- src/storage/tx_storage/ob_access_service.cpp | 10 +- .../tx_storage/ob_tenant_memory_printer.cpp | 6 +- .../r/mysql/inner_table_overall.result | 2 + unittest/share/CMakeLists.txt | 2 +- .../share/cache/test_cache_working_set.cpp | 1 - .../location_cache/test_tablet_ls_map.cpp | 2 +- unittest/share/test_debug_sync.cpp | 21 + unittest/share/test_geo_bin.cpp | 18 +- unittest/share/test_geo_srs.cpp | 2 +- unittest/sql/test_sql_utils.cpp | 2 +- 191 files changed, 2817 insertions(+), 2464 deletions(-) delete mode 100644 src/observer/omt/ob_tenant_srs_mgr.cpp delete mode 100644 src/observer/omt/ob_tenant_srs_mgr.h create mode 100644 src/share/inner_table/ob_inner_table_schema.21451_21500.cpp diff --git a/deps/easy/src/io/easy_baseth_pool.c b/deps/easy/src/io/easy_baseth_pool.c index 52739d58f..eab7cd19b 100644 --- a/deps/easy/src/io/easy_baseth_pool.c +++ b/deps/easy/src/io/easy_baseth_pool.c @@ -25,8 +25,8 @@ static void easy_baseth_pool_invoke_debug(struct ev_loop *loop); static int easy_monitor_interval = 100; static const int64_t easy_monitor_signal = 34; -int ob_pthread_create(pthread_t *thread, const pthread_attr_t *attr, - void *(*start_routine) (void *), void *arg); +int ob_pthread_create(void **ptr, void *(*start_routine) (void *), void *arg); +pthread_t ob_pthread_get_pth(void *ptr); void ob_set_thread_name(const char* type); int64_t ob_update_loop_ts(); void ob_usleep(const useconds_t v); @@ -274,7 +274,7 @@ static void *easy_baseth_pool_monitor_func(void *args) if (EASY_REACH_TIME_INTERVAL(1 * 1000 * 1000)) { easy_warn_log("EASY SLOW: thread: %lx, lastrun: %f cost: %f loop:%d, slowcnt: %ld", - th->tid, last, now - last, id, slowcnts[th->idx]); + ob_pthread_get_pth(th->tid), last, now - last, id, slowcnts[th->idx]); } } } @@ -325,12 +325,12 @@ void easy_baseth_pool_monitor(easy_thread_pool_t *tp) sigemptyset(&sa.sa_mask); rc = sigaction(easy_monitor_signal, &sa, NULL); - err = ob_pthread_create(&tp->monitor_tid, NULL, easy_baseth_pool_monitor_func, tp); + err = ob_pthread_create(&tp->monitor_tid, easy_baseth_pool_monitor_func, tp); if (err != 0) { - tp->monitor_tid = 0; - easy_error_log("sigaction: %d, monitor_thread: 0x%lx, err:%d, errno:%d", rc, tp->monitor_tid, err, errno); + tp->monitor_tid = NULL; + easy_error_log("sigaction: %d, monitor_tid: %p, err:%d, errno:%d", rc, tp->monitor_tid, err, errno); } else { - easy_info_log("monitor thread created, tp=0x%lx tid=%lx\n", tp, tp->monitor_tid); + easy_info_log("monitor thread created, tp=0x%lx tid=%p\n", tp, tp->monitor_tid); } } } diff --git a/deps/easy/src/io/easy_io.c b/deps/easy/src/io/easy_io.c index 0687f9f45..c21e3d313 100644 --- a/deps/easy/src/io/easy_io.c +++ b/deps/easy/src/io/easy_io.c @@ -47,8 +47,8 @@ static void easy_io_print_status(easy_io_t *eio); static void easy_signal_handler(int sig); static void easy_listen_close(easy_listen_t *l); -int ob_pthread_create(pthread_t *thread, const pthread_attr_t *attr, - void *(*start_routine) (void *), void *arg); +int ob_pthread_create(void **ptr, void *(*start_routine) (void *), void *arg); +void ob_pthread_join(void *ptr); /** * 初始化easy_io */ @@ -275,9 +275,9 @@ int easy_eio_start(easy_io_t *eio) easy_list_for_each_entry(tp, &eio->thread_pool_list, list_node) { easy_thread_pool_for_each(th, tp, 0) { int err = 0; - if ((err = ob_pthread_create(&(th->tid), NULL, th->on_start, (void *)th))) { + if ((err = ob_pthread_create(&(th->tid), th->on_start, (void *)th))) { ret = EASY_ERROR; - th->tid = 0; + th->tid = NULL; easy_error_log("easy_io_start, pthread_create error: %d(%d), idx: %d", err, errno, th->idx); } } @@ -302,27 +302,18 @@ int easy_eio_wait(easy_io_t *eio) { easy_baseth_t *th; easy_thread_pool_t *tp; - // 等待thread easy_spin_lock(&eio->lock); easy_list_for_each_entry(tp, &eio->thread_pool_list, list_node) { easy_spin_unlock(&eio->lock); easy_thread_pool_for_each(th, tp, 0) { - if (th->tid && pthread_join(th->tid, NULL) == EDEADLK) { - easy_fatal_log("easy_io_wait fatal, eio=%p, tid=%lx\n", eio, th->tid); - abort(); - } - - th->tid = 0; + ob_pthread_join(th->tid); + th->tid = NULL; } easy_spin_lock(&eio->lock); - easy_info_log("easy_io_wait join monitor, tp=0x%lx tid=%lx\n", tp, tp->monitor_tid); - if (tp->monitor_tid && pthread_join(tp->monitor_tid, NULL) == EDEADLK) { - easy_fatal_log("easy_io_wait fatal, eio=%p, tid=%lx\n", eio, tp->monitor_tid); - abort(); - } - - tp->monitor_tid = 0; + easy_info_log("easy_io_wait join monitor, tp=0x%lx tid=%p\n", tp, tp->monitor_tid); + ob_pthread_join(tp->monitor_tid); + tp->monitor_tid = NULL; } easy_spin_unlock(&eio->lock); diff --git a/deps/easy/src/io/easy_io_struct.h b/deps/easy/src/io/easy_io_struct.h index a416d5ede..8e738f23b 100644 --- a/deps/easy/src/io/easy_io_struct.h +++ b/deps/easy/src/io/easy_io_struct.h @@ -148,7 +148,7 @@ enum { // async + spinlock #define EASY_BASETH_DEFINE \ easy_baseth_on_start_pt *on_start; \ - pthread_t tid; \ + void *tid; \ int idx, iot; \ struct ev_loop *loop; \ ev_tstamp lastrun; \ @@ -690,7 +690,7 @@ struct easy_thread_pool_t { easy_list_t list_node; easy_thread_pool_t *next; char *last; - pthread_t monitor_tid; + void *monitor_tid; easy_io_thread_t *ratelimit_thread; char data[0]; }; diff --git a/deps/easy/src/io/easy_maccept.c b/deps/easy/src/io/easy_maccept.c index 11c8da20c..6107f702b 100644 --- a/deps/easy/src/io/easy_maccept.c +++ b/deps/easy/src/io/easy_maccept.c @@ -30,7 +30,7 @@ typedef struct easy_ma_t { int flags; int efd; int lfd; - pthread_t th; + void *th; volatile int stop; int g_count; int g_start[MAX_GROUP_COUNT]; @@ -39,8 +39,8 @@ typedef struct easy_ma_t { } easy_ma_t; easy_ma_t g_ma; -int ob_pthread_create(pthread_t *thread, const pthread_attr_t *attr, - void *(*start_routine) (void *), void *arg); +int ob_pthread_create(void **ptr, void *(*start_routine) (void *), void *arg); +void ob_pthread_join(void *ptr); int ob_epoll_wait(int __epfd, struct epoll_event *__events, int __maxevents, int __timeout); void easy_ma_init(int port) @@ -219,7 +219,7 @@ int easy_ma_start() g_ma.efd = efd; g_ma.lfd = lfd; g_ma.stop = 0; - if (0 != ob_pthread_create(&g_ma.th, NULL, (void*)easy_ma_thread_func, (void*)&g_ma)) { + if (0 != ob_pthread_create(&g_ma.th, (void*)easy_ma_thread_func, (void*)&g_ma)) { goto error_exit; } @@ -236,7 +236,8 @@ void easy_ma_stop() { if (!g_ma.stop) { g_ma.stop = 1; - pthread_join(g_ma.th, NULL); + ob_pthread_join(g_ma.th); + g_ma.th = NULL; } } diff --git a/deps/easy/src/io/easy_request.c b/deps/easy/src/io/easy_request.c index c79eada68..9187bf4f2 100644 --- a/deps/easy/src/io/easy_request.c +++ b/deps/easy/src/io/easy_request.c @@ -24,6 +24,7 @@ static void easy_request_dosess(easy_request_thread_t *th, easy_list_t *session_ static void easy_request_cleanup(easy_buf_t *b, void *args); static const int64_t REQUEST_ITEM_COST_RT = 100 * 1000; +pthread_t ob_pthread_get_pth(void *ptr); /** * 对request回复响应 * @@ -37,9 +38,9 @@ int easy_request_do_reply(easy_request_t *r) // encode m = (easy_message_t *)r->ms; c = m->c; - - if (c->ioth->tid != pthread_self()) { - easy_fatal_log("not run at other thread: %lx <> %lx\n", r, pthread_self(), c->ioth->tid); + pthread_t pth = ob_pthread_get_pth(c->ioth->tid); + if (pth != pthread_self()) { + easy_fatal_log("not run at other thread: %lx <> %lx\n", r, pthread_self(), pth); return EASY_ERROR; } diff --git a/deps/oblib/src/lib/alloc/alloc_struct.cpp b/deps/oblib/src/lib/alloc/alloc_struct.cpp index 7497ae7ce..d3a8aa70c 100644 --- a/deps/oblib/src/lib/alloc/alloc_struct.cpp +++ b/deps/oblib/src/lib/alloc/alloc_struct.cpp @@ -17,13 +17,19 @@ #include "lib/oblog/ob_log.h" #include "lib/utility/ob_fast_convert.h" -using namespace oceanbase; -using namespace lib; +namespace oceanbase +{ + using namespace common; +namespace lib +{ + thread_local ObMemAttr ObMallocHookAttrGuard::tl_mem_attr(OB_SERVER_TENANT_ID, "glibc_malloc", ObCtxIds::GLIBC); +static int64_t g_divisive_mem_size[OB_MAX_CPU_NUM]; +static thread_local bool g_is_ob_mem_mgr_path = false; ObMallocHookAttrGuard::ObMallocHookAttrGuard(const ObMemAttr& attr) : old_attr_(tl_mem_attr) @@ -86,4 +92,43 @@ void Label::fmt(char *buf, int64_t buf_len, int64_t &pos, const char *str) pos = buf_len; } } -} \ No newline at end of file +} + +int64_t get_divisive_mem_size() +{ + int64_t total_size = 0; + for (int64_t i = 0; i < OB_MAX_CPU_NUM; i++) { + total_size += g_divisive_mem_size[i]; + } + return total_size; +} + +void inc_divisive_mem_size(const int64_t size) +{ + const int64_t idx = ob_gettid() % OB_MAX_CPU_NUM; + __sync_fetch_and_add(&(g_divisive_mem_size[idx]), size); +} + +void dec_divisive_mem_size(const int64_t size) +{ + const int64_t idx = ob_gettid() % OB_MAX_CPU_NUM; + __sync_fetch_and_add(&(g_divisive_mem_size[idx]), 0 - size); +} + +void set_ob_mem_mgr_path() +{ + g_is_ob_mem_mgr_path = true; +} + +void unset_ob_mem_mgr_path() +{ + g_is_ob_mem_mgr_path = false; +} + +bool is_ob_mem_mgr_path() +{ + return g_is_ob_mem_mgr_path; +} + +} // end of namespace lib +} // end of namespace oceanbase diff --git a/deps/oblib/src/lib/alloc/alloc_struct.h b/deps/oblib/src/lib/alloc/alloc_struct.h index 0cf450d6f..e6f9b4b9f 100644 --- a/deps/oblib/src/lib/alloc/alloc_struct.h +++ b/deps/oblib/src/lib/alloc/alloc_struct.h @@ -52,7 +52,7 @@ static const int64_t ALLOC_ABLOCK_CONCURRENCY = 4; static ssize_t get_page_size() { - static ssize_t ps = sysconf(_SC_PAGESIZE); + static ssize_t ps = getpagesize(); return ps; } @@ -604,6 +604,14 @@ private: ObMemAttr old_attr_; }; +extern void inc_divisive_mem_size(const int64_t size); +extern void dec_divisive_mem_size(const int64_t size); +extern int64_t get_divisive_mem_size(); + +extern void set_ob_mem_mgr_path(); +extern void unset_ob_mem_mgr_path(); +extern bool is_ob_mem_mgr_path(); + #define FORCE_EXPLICT_500_MALLOC() \ OB_UNLIKELY(oceanbase::lib::ObMallocAllocator::get_instance()->force_explict_500_malloc_) diff --git a/deps/oblib/src/lib/alloc/malloc_hook.cpp b/deps/oblib/src/lib/alloc/malloc_hook.cpp index 6d295f667..39b9ed7dd 100644 --- a/deps/oblib/src/lib/alloc/malloc_hook.cpp +++ b/deps/oblib/src/lib/alloc/malloc_hook.cpp @@ -19,7 +19,7 @@ using namespace oceanbase; using namespace oceanbase::common; using namespace oceanbase::lib; -bool g_malloc_hook_inited = false; +static bool g_malloc_hook_inited = false; void init_malloc_hook() { @@ -74,6 +74,25 @@ void *ob_malloc_retry(size_t size) return ptr; } +static inline void *ob_mmap(void *addr, size_t length, int prot, int flags, int fd, loff_t offset) +{ + void *ptr = (void*)syscall(SYS_mmap, addr, length, prot, flags, fd, offset); + if (OB_UNLIKELY(!is_ob_mem_mgr_path()) && OB_LIKELY(MAP_FAILED != ptr)) { + const int64_t page_size = get_page_size(); + inc_divisive_mem_size(upper_align(length, page_size)); + } + return ptr; +} + +static inline int ob_munmap(void *addr, size_t length) +{ + if (OB_UNLIKELY(!is_ob_mem_mgr_path())) { + const int64_t page_size = get_page_size(); + dec_divisive_mem_size(upper_align(length, page_size)); + } + return syscall(SYS_munmap, addr, length); +} + void *ob_malloc_hook(size_t size, const void *) { void *ptr = nullptr; @@ -81,7 +100,7 @@ void *ob_malloc_hook(size_t size, const void *) void *tmp_ptr = nullptr; bool from_mmap = false; if (OB_UNLIKELY(!g_malloc_hook_inited || in_hook())) { - if (MAP_FAILED == (tmp_ptr = ::mmap(nullptr, real_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0))) { + if (MAP_FAILED == (tmp_ptr = ob_mmap(nullptr, real_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0))) { tmp_ptr = nullptr; } from_mmap = true; @@ -107,7 +126,7 @@ void ob_free_hook(void *ptr, const void *) header->mark_unused(); void *orig_ptr = (char*)header - header->offset_; if (OB_UNLIKELY(header->from_mmap_)) { - ::munmap(orig_ptr, header->data_size_ + Header::SIZE + header->offset_); + ob_munmap(orig_ptr, header->data_size_ + Header::SIZE + header->offset_); } else { bool in_hook_bak = in_hook(); in_hook()= true; @@ -124,7 +143,7 @@ void *ob_realloc_hook(void *ptr, size_t size, const void *caller) void *tmp_ptr = nullptr; bool from_mmap = false; if (OB_UNLIKELY(!g_malloc_hook_inited || in_hook())) { - if (MAP_FAILED == (tmp_ptr = ::mmap(nullptr, real_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0))) { + if (MAP_FAILED == (tmp_ptr = ob_mmap(nullptr, real_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0))) { tmp_ptr = nullptr; } from_mmap = true; @@ -163,7 +182,7 @@ void *ob_memalign_hook(size_t alignment, size_t size, const void *) void *tmp_ptr = nullptr; bool from_mmap = false; if (OB_UNLIKELY(!g_malloc_hook_inited || in_hook())) { - if (MAP_FAILED == (tmp_ptr = ::mmap(nullptr, real_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0))) { + if (MAP_FAILED == (tmp_ptr = ob_mmap(nullptr, real_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0))) { tmp_ptr = nullptr; } from_mmap = true; @@ -187,6 +206,20 @@ void *ob_memalign_hook(size_t alignment, size_t size, const void *) return ptr; } +EXTERN_C_BEGIN + +void *ob_mmap_hook(void *addr, size_t length, int prot, int flags, int fd, loff_t offset) +{ + return ob_mmap(addr, length, prot, flags, fd, offset); +} + +int ob_munmap_hook(void *addr, size_t length) +{ + return ob_munmap(addr, length); +} + +EXTERN_C_END + #if !defined(__MALLOC_HOOK_VOLATILE) #define MALLOC_HOOK_MAYBE_VOLATILE /**/ #else @@ -200,6 +233,10 @@ __attribute__((visibility("default"))) void (*MALLOC_HOOK_MAYBE_VOLATILE __free_ __attribute__((visibility("default"))) void *(*MALLOC_HOOK_MAYBE_VOLATILE __realloc_hook)(void *, size_t, const void *) = ob_realloc_hook; __attribute__((visibility("default"))) void *(*MALLOC_HOOK_MAYBE_VOLATILE __memalign_hook)(size_t, size_t, const void *) = ob_memalign_hook; +__attribute__((visibility("default"))) void *mmap(void *addr, size_t, int, int, int, loff_t) __attribute__((weak,alias("ob_mmap_hook"))); +__attribute__((visibility("default"))) void *mmap64(void *addr, size_t, int, int, int, loff_t) __attribute__((weak,alias("ob_mmap_hook"))); +__attribute__((visibility("default"))) int munmap(void *addr, size_t length) __attribute__((weak,alias("ob_munmap_hook"))); + size_t malloc_usable_size(void *ptr) { size_t ret = 0; diff --git a/deps/oblib/src/lib/alloc/ob_malloc_sample_struct.h b/deps/oblib/src/lib/alloc/ob_malloc_sample_struct.h index 6e79c6865..4e86042f5 100644 --- a/deps/oblib/src/lib/alloc/ob_malloc_sample_struct.h +++ b/deps/oblib/src/lib/alloc/ob_malloc_sample_struct.h @@ -116,6 +116,7 @@ inline bool ObMallocSampleLimiter::malloc_sample_allowed(const int64_t size, con inline void ObMallocSampleLimiter::set_interval(int32_t max_interval, int32_t min_interval) { +#if defined(__x86_64__) if (min_interval < 1 || max_interval > INTERVAL_UPPER_LIMIT || max_interval < min_interval) { _OB_LOG_RET(WARN, common::OB_INVALID_ARGUMENT, "set the min or max malloc times between two samples unexpected," @@ -126,6 +127,9 @@ inline void ObMallocSampleLimiter::set_interval(int32_t max_interval, int32_t mi _OB_LOG_RET(INFO, common::OB_SUCCESS, "set the min or max malloc times between two samples succeed," "max_interval=%d, min_interval=%d", max_interval, min_interval); } +#else + UNUSEDx(max_interval, min_interval); +#endif } inline int64_t ObMallocSampleKey::hash() const diff --git a/deps/oblib/src/lib/allocator/ob_fixed_size_block_allocator.h b/deps/oblib/src/lib/allocator/ob_fixed_size_block_allocator.h index 599e6af5e..6e5e60701 100644 --- a/deps/oblib/src/lib/allocator/ob_fixed_size_block_allocator.h +++ b/deps/oblib/src/lib/allocator/ob_fixed_size_block_allocator.h @@ -61,8 +61,7 @@ private: bool contains_internal(void* ptr) const; public: - static const int64_t MAX_MEMORY_ALLOCATION = OB_MAX_SYS_BKGD_THREAD_NUM * 6 - * OB_DEFAULT_MACRO_BLOCK_SIZE; + static const int64_t MAX_MEMORY_ALLOCATION = OB_MAX_SYS_BKGD_THREAD_NUM * 2 * OB_DEFAULT_MACRO_BLOCK_SIZE; //256MB private: // TODO(zhuixin.gsy) reduce 128*2M size to 64*2M, which is expanded temporarily for generating index blocks diff --git a/deps/oblib/src/lib/geo/ob_srs_info.cpp b/deps/oblib/src/lib/geo/ob_srs_info.cpp index 79271aecf..0fa870956 100644 --- a/deps/oblib/src/lib/geo/ob_srs_info.cpp +++ b/deps/oblib/src/lib/geo/ob_srs_info.cpp @@ -84,7 +84,7 @@ int ObSrsUtils::check_is_wgs84(const ObGeographicRs *rs, bool &is_wgs84) // todo@dazhi: compare the param_name and param_alias when epsg isnot comparable ? int ObSrsUtils::get_simple_proj_params(const ObProjectionPrams &parsed_params, - ObVector ¶ms) + ObVector ¶ms) { int ret = OB_SUCCESS; FOREACH_X(parsed_param, parsed_params.vals, OB_SUCC(ret)) { @@ -307,7 +307,7 @@ int ObSpatialReferenceSystemBase::create_srs_internal(ObIAllocator* allocator, u ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc projected srs failed", K(ret), K(srs_id)); } else { - tmp_srs_info = new(buf)SRS_T(static_cast(allocator)); + tmp_srs_info = new(buf)SRS_T(static_cast(allocator)); if (OB_FAIL(tmp_srs_info->init(srs_id, rs))) { LOG_WARN("srs info init failed", K(ret), KP(rs), K(srs_id)); } else { @@ -327,7 +327,7 @@ int ObSpatialReferenceSystemBase::create_geographic_srs(ObIAllocator* allocator, return create_srs_internal(allocator, srs_id, rs, srs_info); } -ObGeographicSrs::ObGeographicSrs(common::ObArenaAllocator* alloc) +ObGeographicSrs::ObGeographicSrs(common::ObIAllocator* alloc) : semi_major_axis_(NAN), inverse_flattening_(NAN), is_wgs84_(false), prime_meridian_(NAN), angular_factor_(NAN), bounds_info_(), proj4text_() diff --git a/deps/oblib/src/lib/geo/ob_srs_info.h b/deps/oblib/src/lib/geo/ob_srs_info.h index d7ce778ea..021a9f548 100644 --- a/deps/oblib/src/lib/geo/ob_srs_info.h +++ b/deps/oblib/src/lib/geo/ob_srs_info.h @@ -18,6 +18,7 @@ #include "lib/container/ob_vector.h" #include "lib/allocator/page_arena.h" #include "lib/allocator/ob_allocator.h" +#include "lib/allocator/ob_fifo_allocator.h" #include "lib/ob_errno.h" #include "common/data_buffer.h" @@ -276,7 +277,7 @@ private: class ObGeographicSrs : public ObSpatialReferenceSystemBase { public: - ObGeographicSrs(common::ObArenaAllocator* alloc); + ObGeographicSrs(common::ObIAllocator* alloc); virtual ~ObGeographicSrs() {} ObSrsType srs_type() const override { return ObSrsType::GEOGRAPHIC_SRS; } virtual int init(uint32_t srid, const ObGeographicRs *rs); @@ -320,9 +321,9 @@ struct ObSimpleProjPram class ObProjectedSrs : public ObSpatialReferenceSystemBase { public: - ObProjectedSrs(common::ObArenaAllocator *allocator) : geographic_srs_(allocator), - linear_unit_(NAN), - simple_proj_prams_(allocator) + ObProjectedSrs(common::ObIAllocator *allocator) : geographic_srs_(allocator), + linear_unit_(NAN), + simple_proj_prams_(static_cast(allocator)) { for (int i = 0; i < AXIS_DIRECTION_NUM; i++) { axis_dir_[i] = ObAxisDirection::INIT; @@ -357,7 +358,7 @@ private: ObAxisDirection axis_dir_[AXIS_DIRECTION_NUM]; // direction of x and y axis; protected: - ObVector simple_proj_prams_; // should be filled by subclass + ObVector simple_proj_prams_; // should be filled by subclass }; #define OB_GEO_REG_PROJ_PARAMS(...) \ @@ -374,7 +375,7 @@ for (int32_t i = 0; i < ARRAYSIZEOF(arr); ++i) { \ class ObUnknownProjectedSrs : public ObProjectedSrs { public: - ObUnknownProjectedSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObUnknownProjectedSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObUnknownProjectedSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::UNKNOWN; }; @@ -387,7 +388,7 @@ public: class ObPopularVisualPseudoMercatorSrs : public ObProjectedSrs { public: - ObPopularVisualPseudoMercatorSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObPopularVisualPseudoMercatorSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObPopularVisualPseudoMercatorSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::POPULAR_VISUAL_PSEUDO_MERCATOR; }; @@ -400,7 +401,7 @@ public: class ObLambertAzimuthalEqualAreaSphericalSrs: public ObProjectedSrs { public: - ObLambertAzimuthalEqualAreaSphericalSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObLambertAzimuthalEqualAreaSphericalSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObLambertAzimuthalEqualAreaSphericalSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::LAMBERT_AZIMUTHAL_EQUAL_AREA_SPHERICAL; }; @@ -413,7 +414,7 @@ public: class ObEquidistantCylindricalSrs : public ObProjectedSrs { public: - ObEquidistantCylindricalSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObEquidistantCylindricalSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObEquidistantCylindricalSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::EQUIDISTANT_CYLINDRICAL; }; @@ -426,7 +427,7 @@ public: class ObEquidistantCylindricalSphericalSrs : public ObProjectedSrs { public: - ObEquidistantCylindricalSphericalSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObEquidistantCylindricalSphericalSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObEquidistantCylindricalSphericalSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::EQUIDISTANT_CYLINDRICAL_SPHERICAL; }; @@ -439,7 +440,7 @@ public: class ObKrovakNorthOrientatedSrs : public ObProjectedSrs { public: - ObKrovakNorthOrientatedSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObKrovakNorthOrientatedSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObKrovakNorthOrientatedSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::KROVAK_NORTH_ORIENTATED; }; @@ -452,7 +453,7 @@ public: class ObKrovakModifiedSrs : public ObProjectedSrs { public: - ObKrovakModifiedSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObKrovakModifiedSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObKrovakModifiedSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::KROVAK_MODIFIED; }; @@ -465,7 +466,7 @@ public: class ObKrovakModifiedNorthOrientatedSrs : public ObProjectedSrs { public: - ObKrovakModifiedNorthOrientatedSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObKrovakModifiedNorthOrientatedSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObKrovakModifiedNorthOrientatedSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::KROVAK_MODIFIED_NORTH_ORIENTATED; }; @@ -478,7 +479,7 @@ public: class ObLambertConicConformal2SPMichiganSrs : public ObProjectedSrs { public: - ObLambertConicConformal2SPMichiganSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObLambertConicConformal2SPMichiganSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObLambertConicConformal2SPMichiganSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::LAMBERT_CONIC_CONFORMAL_2SP_MICHIGAN; }; @@ -491,7 +492,7 @@ public: class ObColombiaUrbanSrs : public ObProjectedSrs { public: - ObColombiaUrbanSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObColombiaUrbanSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObColombiaUrbanSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::COLOMBIA_URBAN; }; @@ -504,7 +505,7 @@ public: class ObLambertConicConformal1SPSrs : public ObProjectedSrs { public: - ObLambertConicConformal1SPSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObLambertConicConformal1SPSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObLambertConicConformal1SPSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::LAMBERT_CONIC_CONFORMAL_1SP; }; @@ -517,7 +518,7 @@ public: class ObLambertConicConformal2SPSrs : public ObProjectedSrs { public: - ObLambertConicConformal2SPSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObLambertConicConformal2SPSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObLambertConicConformal2SPSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::LAMBERT_CONIC_CONFORMAL_2SP; }; @@ -530,7 +531,7 @@ public: class ObLambertConicConformal2SPBelgiumSrs : public ObProjectedSrs { public: - ObLambertConicConformal2SPBelgiumSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObLambertConicConformal2SPBelgiumSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObLambertConicConformal2SPBelgiumSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::LAMBERT_CONIC_CONFORMAL_2SP_BELGIUM; }; @@ -543,7 +544,7 @@ public: class ObMercatorvariantASrs : public ObProjectedSrs { public: - ObMercatorvariantASrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObMercatorvariantASrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObMercatorvariantASrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::MERCATOR_VARIANT_A; }; @@ -556,7 +557,7 @@ public: class ObMercatorvariantBSrs : public ObProjectedSrs { public: - ObMercatorvariantBSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObMercatorvariantBSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObMercatorvariantBSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::MERCATOR_VARIANT_B; }; @@ -569,7 +570,7 @@ public: class ObCassiniSoldnerSrs : public ObProjectedSrs { public: - ObCassiniSoldnerSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObCassiniSoldnerSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObCassiniSoldnerSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::MERCATOR_VARIANT_B; }; @@ -582,7 +583,7 @@ public: class ObTransverseMercatorSrs : public ObProjectedSrs { public: - ObTransverseMercatorSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObTransverseMercatorSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObTransverseMercatorSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::TRANSVERSE_MERCATOR; }; @@ -595,7 +596,7 @@ public: class ObTransverseMercatorSouthOrientatedSrs : public ObProjectedSrs { public: - ObTransverseMercatorSouthOrientatedSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObTransverseMercatorSouthOrientatedSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObTransverseMercatorSouthOrientatedSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::TRANSVERSE_MERCATOR_SOUTH_ORIENTATED; }; @@ -608,7 +609,7 @@ public: class ObObliqueStereographicSrs : public ObProjectedSrs { public: - ObObliqueStereographicSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObObliqueStereographicSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObObliqueStereographicSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::OBLIQUE_STEREOGRAPHIC; }; @@ -621,7 +622,7 @@ public: class ObPolarStereographicVariantASrs : public ObProjectedSrs { public: - ObPolarStereographicVariantASrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObPolarStereographicVariantASrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObPolarStereographicVariantASrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::POLAR_STEREOGRAPHIC_VARIANT_A; }; @@ -634,7 +635,7 @@ public: class ObNewZealandMapGridSrs : public ObProjectedSrs { public: - ObNewZealandMapGridSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObNewZealandMapGridSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObNewZealandMapGridSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::NEW_ZEALAND_MAP_GRID; }; @@ -647,7 +648,7 @@ public: class ObHotineObliqueMercatorvariantASrs : public ObProjectedSrs { public: - ObHotineObliqueMercatorvariantASrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObHotineObliqueMercatorvariantASrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObHotineObliqueMercatorvariantASrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::HOTINE_OBLIQUE_MERCATOR_VARIANT_A; }; @@ -660,7 +661,7 @@ public: class ObLabordeObliqueMercatorSrs : public ObProjectedSrs { public: - ObLabordeObliqueMercatorSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObLabordeObliqueMercatorSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObLabordeObliqueMercatorSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::LABORDE_OBLIQUE_MERCATOR; }; @@ -673,7 +674,7 @@ public: class ObHotineObliqueMercatorVariantBSrs : public ObProjectedSrs { public: - ObHotineObliqueMercatorVariantBSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObHotineObliqueMercatorVariantBSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObHotineObliqueMercatorVariantBSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::HOTINE_OBLIQUE_MERCATOR_VARIANT_B; }; @@ -686,7 +687,7 @@ public: class ObTunisiaMiningGridSrs : public ObProjectedSrs { public: - ObTunisiaMiningGridSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObTunisiaMiningGridSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObTunisiaMiningGridSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::TUNISIA_MINING_GRID; }; @@ -699,7 +700,7 @@ public: class ObLambertConicNearConformalSrs : public ObProjectedSrs { public: - ObLambertConicNearConformalSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObLambertConicNearConformalSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObLambertConicNearConformalSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::LAMBERT_CONIC_NEAR_CONFORMAL; }; @@ -712,7 +713,7 @@ public: class ObAmericanPolyconicSrs : public ObProjectedSrs { public: - ObAmericanPolyconicSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObAmericanPolyconicSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObAmericanPolyconicSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::AMERICAN_POLYCONIC; }; @@ -725,7 +726,7 @@ public: class ObKrovakSrs : public ObProjectedSrs { public: - ObKrovakSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObKrovakSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObKrovakSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::KROVAK; }; @@ -738,7 +739,7 @@ public: class ObLambertAzimuthalEqualAreaSrs : public ObProjectedSrs { public: - ObLambertAzimuthalEqualAreaSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObLambertAzimuthalEqualAreaSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObLambertAzimuthalEqualAreaSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::LAMBERT_AZIMUTHAL_EQUAL_AREA; }; @@ -751,7 +752,7 @@ public: class ObAlbersEqualAreaSrs : public ObProjectedSrs { public: - ObAlbersEqualAreaSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObAlbersEqualAreaSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObAlbersEqualAreaSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::ALBERS_EQUAL_AREA; }; @@ -764,7 +765,7 @@ public: class ObTransverseMercatorZonedGridSystemSrs : public ObProjectedSrs { public: - ObTransverseMercatorZonedGridSystemSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObTransverseMercatorZonedGridSystemSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObTransverseMercatorZonedGridSystemSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::TRANSVERSE_MERCATOR_ZONED_GRID_SYSTEM; }; @@ -777,7 +778,7 @@ public: class ObLambertConicConformalWestOrientatedSrs : public ObProjectedSrs { public: - ObLambertConicConformalWestOrientatedSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObLambertConicConformalWestOrientatedSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObLambertConicConformalWestOrientatedSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::LAMBERT_CONIC_CONFORMAL_WEST_ORIENTATED; }; @@ -790,7 +791,7 @@ public: class ObBonneSouthOrientatedSrs : public ObProjectedSrs { public: - ObBonneSouthOrientatedSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObBonneSouthOrientatedSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObBonneSouthOrientatedSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::BONNE_SOUTH_ORIENTATED; }; @@ -803,7 +804,7 @@ public: class ObPolarStereographicVariantBSrs : public ObProjectedSrs { public: - ObPolarStereographicVariantBSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObPolarStereographicVariantBSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObPolarStereographicVariantBSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::POLAR_STEREOGRAPHIC_VARIANT_B; }; @@ -816,7 +817,7 @@ public: class ObPolarStereographicVariantCSrs : public ObProjectedSrs { public: - ObPolarStereographicVariantCSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObPolarStereographicVariantCSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObPolarStereographicVariantCSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::POLAR_STEREOGRAPHIC_VARIANT_C; }; @@ -829,7 +830,7 @@ public: class ObGuamProjectionSrs : public ObProjectedSrs { public: - ObGuamProjectionSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObGuamProjectionSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObGuamProjectionSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::GUAM_PROJECTION; }; @@ -842,7 +843,7 @@ public: class ObModifiedAzimuthalEquidistantSrs : public ObProjectedSrs { public: - ObModifiedAzimuthalEquidistantSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObModifiedAzimuthalEquidistantSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObModifiedAzimuthalEquidistantSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::MODIFIED_AZIMUTHAL_EQUIDISTANT; }; @@ -855,7 +856,7 @@ public: class ObHyperbolicCassiniSoldnerSrs : public ObProjectedSrs { public: - ObHyperbolicCassiniSoldnerSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObHyperbolicCassiniSoldnerSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObHyperbolicCassiniSoldnerSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::HYPERBOLIC_CASSINI_SOLDNER; }; @@ -868,7 +869,7 @@ public: class ObLambertCylindricalEqualAreaSphericalSrs : public ObProjectedSrs { public: - ObLambertCylindricalEqualAreaSphericalSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObLambertCylindricalEqualAreaSphericalSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObLambertCylindricalEqualAreaSphericalSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::LAMBERT_CYLINDRICAL_EQUAL_AREA_SPHERICAL; }; @@ -881,7 +882,7 @@ public: class ObLambertCylindricalEqualAreaSrs : public ObProjectedSrs { public: - ObLambertCylindricalEqualAreaSrs(common::ObArenaAllocator *allocator) : ObProjectedSrs(allocator) {} + ObLambertCylindricalEqualAreaSrs(common::ObIAllocator *allocator) : ObProjectedSrs(allocator) {} virtual ~ObLambertCylindricalEqualAreaSrs() {} ObProjectionType get_projection_type() const override { return ObProjectionType::LAMBERT_CYLINDRICAL_EQUAL_AREA; }; @@ -932,7 +933,7 @@ public: static int check_is_wgs84(const ObGeographicRs *rs, bool &is_wgs84); static int get_simple_proj_params(const ObProjectionPrams &parsed_params, - ObVector ¶ms); + ObVector ¶ms); static int check_authority(const ObRsAuthority& auth, const char *target_auth_name, int target_auth_code, bool allow_invalid, bool &res); constexpr static double WGS_SEMI_MAJOR_AXIS = 6378137.0; diff --git a/deps/oblib/src/lib/hash/ob_hashset.h b/deps/oblib/src/lib/hash/ob_hashset.h index d48709f0e..fbdcb45e7 100644 --- a/deps/oblib/src/lib/hash/ob_hashset.h +++ b/deps/oblib/src/lib/hash/ob_hashset.h @@ -137,6 +137,15 @@ public: { clear(); } + _key_type *get(const _key_type &key) const + { + const _key_type *ret = NULL; + const pair_type *pair = NULL; + if (OB_SUCCESS == const_cast(ht_).get_refactored(key, pair)) { + ret = &(pair->first); + } + return const_cast<_key_type*>(ret); + } // return: // OB_HASH_EXIST node exists // OB_HASH_NOT_EXIST node not exists diff --git a/deps/oblib/src/lib/hash/ob_hashutils.h b/deps/oblib/src/lib/hash/ob_hashutils.h index d31cab0c7..17eb9f2b6 100644 --- a/deps/oblib/src/lib/hash/ob_hashutils.h +++ b/deps/oblib/src/lib/hash/ob_hashutils.h @@ -1239,28 +1239,18 @@ struct SimpleAllocerBlock Block *next; }; -template +template struct NodeNumTraits { + /* + 24 : sizeof(SimpleAllocerBlock 's members except nodes) + 32 : sizeof(SimpleAllocerNode's members except data) + 128: for robust + */ + static const int32_t NODE_NUM = ((NODE_PAGE_SIZE - 24 - 128) < (32 + sizeof(T))) ? 1 : ((NODE_PAGE_SIZE - + 24 - 128) / + (32 + sizeof(T))); }; - -template -struct NodeNumTraits -{ - static const int32_t NODE_NUM = (common::OB_MALLOC_NORMAL_BLOCK_SIZE - - 24/*=sizeof(SimpleAllocerBlock 's members except nodes)*/ - 128/*for robust*/) / - (32/*sizeof(SimpleAllocerNode's members except data)*/ + sizeof(T)); -}; - -template -struct NodeNumTraits -{ - static const int32_t NODE_NUM = 1; -}; - -#define IS_BIG_OBJ(T) \ - ((common::OB_MALLOC_NORMAL_BLOCK_SIZE - 24 - 128) < (32 + sizeof(T))) - /* block_free_list_: Block C: node1->node2->node3... @@ -1284,7 +1274,7 @@ free: 4. neither */ template ::NODE_NUM, + int32_t NODE_NUM = NodeNumTraits::NODE_NUM, class DefendMode = SpinMutexDefendMode, class Allocer = DefaultSimpleAllocerAllocator> class SimpleAllocer @@ -1297,15 +1287,20 @@ class SimpleAllocer typedef typename DefendMode::lock_type lock_type; typedef typename DefendMode::lock_initer lock_initer; public: - SimpleAllocer() : block_remainder_(NULL), block_free_list_(NULL) + SimpleAllocer() : leak_check_(true), block_remainder_(NULL), block_free_list_(NULL) { lock_initer initer(lock_); } ~SimpleAllocer() { - OB_ASSERT(NULL == block_remainder_ && - NULL == block_free_list_); + if (leak_check_) { + if (NULL != block_remainder_ || + NULL != block_free_list_) { + HASH_WRITE_LOG_RET(HASH_FATAL, OB_ERR_UNEXPECTED, "SimpleAllocer memory leak"); + } + } } + void set_leak_check(const bool check) { leak_check_ = check; } void set_attr(const ObMemAttr &attr) { allocer_.set_attr(attr); } void set_label(const lib::ObLabel &label) { allocer_.set_label(label); } template @@ -1413,6 +1408,7 @@ public: } } private: + bool leak_check_; Block *block_remainder_; Block *block_free_list_; lock_type lock_; diff --git a/deps/oblib/src/lib/lock/ob_latch.h b/deps/oblib/src/lib/lock/ob_latch.h index c8afcd0b4..70970eb53 100644 --- a/deps/oblib/src/lib/lock/ob_latch.h +++ b/deps/oblib/src/lib/lock/ob_latch.h @@ -42,13 +42,17 @@ extern bool USE_CO_LATCH; if (lib::is_diagnose_info_enabled()) { \ ObDiagnoseTenantInfo *di = ObDiagnoseTenantInfo::get_local_diagnose_info(); \ if (NULL != di) { \ - ObLatchStat &latch_stat = di->get_latch_stats().items_[latch_id]; \ - if (OB_SUCC(ret)) { \ - ++latch_stat.immediate_gets_; \ - } else { \ - ++latch_stat.immediate_misses_; \ - } \ - latch_stat.spin_gets_ += spin_cnt; \ + ObLatchStat *p_latch_stat = di->get_latch_stats().get_or_create_item(latch_id); \ + if (OB_ISNULL(p_latch_stat)) { \ + } else { \ + ObLatchStat &latch_stat = *p_latch_stat; \ + if (OB_SUCC(ret)) { \ + ++latch_stat.immediate_gets_; \ + } else { \ + ++latch_stat.immediate_misses_; \ + } \ + latch_stat.spin_gets_ += spin_cnt; \ + } \ } \ } \ } while(0) @@ -58,21 +62,25 @@ extern bool USE_CO_LATCH; if (lib::is_diagnose_info_enabled()) { \ ObDiagnoseTenantInfo *di = ObDiagnoseTenantInfo::get_local_diagnose_info(); \ if (NULL != di) { \ - ObLatchStat &latch_stat = di->get_latch_stats().items_[latch_id]; \ - ++latch_stat.gets_; \ - latch_stat.spin_gets_ += spin_cnt; \ - latch_stat.sleeps_ += yield_cnt; \ - if (OB_UNLIKELY(waited)) { \ - ++latch_stat.misses_; \ - ObDiagnoseSessionInfo *dsi = ObDiagnoseSessionInfo::get_local_diagnose_info(); \ - if (NULL != dsi) { \ - latch_stat.wait_time_ += dsi->get_curr_wait().wait_time_; \ - if (dsi->get_curr_wait().wait_time_ > 1000 * 1000) { \ - COMMON_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "The Latch wait too much time, ", \ - K(dsi->get_curr_wait()), KCSTRING(lbt())); \ - } \ - } \ - } \ + ObLatchStat *p_latch_stat = di->get_latch_stats().get_or_create_item(latch_id); \ + if (OB_ISNULL(p_latch_stat)) { \ + } else { \ + ObLatchStat &latch_stat = *p_latch_stat; \ + ++latch_stat.gets_; \ + latch_stat.spin_gets_ += spin_cnt; \ + latch_stat.sleeps_ += yield_cnt; \ + if (OB_UNLIKELY(waited)) { \ + ++latch_stat.misses_; \ + ObDiagnoseSessionInfo *dsi = ObDiagnoseSessionInfo::get_local_diagnose_info(); \ + if (NULL != dsi) { \ + latch_stat.wait_time_ += dsi->get_curr_wait().wait_time_; \ + if (dsi->get_curr_wait().wait_time_ > 1000 * 1000) { \ + COMMON_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "The Latch wait too much time, ", \ + K(dsi->get_curr_wait()), KCSTRING(lbt())); \ + } \ + } \ + } \ + } \ } \ } \ } while(0) diff --git a/deps/oblib/src/lib/ob_define.h b/deps/oblib/src/lib/ob_define.h index c16235598..d8e4c25d1 100644 --- a/deps/oblib/src/lib/ob_define.h +++ b/deps/oblib/src/lib/ob_define.h @@ -1595,7 +1595,7 @@ OB_INLINE bool is_bootstrap_resource_pool(const uint64_t resource_pool_id) } // ob_malloc & ob_tc_malloc -const int64_t OB_MALLOC_NORMAL_BLOCK_SIZE = (1LL << 13) - 128; // 8KB +const int64_t OB_MALLOC_NORMAL_BLOCK_SIZE = (1LL << 13) - 256; // 8KB const int64_t OB_MALLOC_MIDDLE_BLOCK_SIZE = (1LL << 16) - 128; // 64KB const int64_t OB_MALLOC_BIG_BLOCK_SIZE = (1LL << 21) - ACHUNK_PRESERVE_SIZE;// 2MB (-17KB) diff --git a/deps/oblib/src/lib/objectpool/ob_server_object_pool.h b/deps/oblib/src/lib/objectpool/ob_server_object_pool.h index ac0d9172a..c8d509f99 100644 --- a/deps/oblib/src/lib/objectpool/ob_server_object_pool.h +++ b/deps/oblib/src/lib/objectpool/ob_server_object_pool.h @@ -177,7 +177,8 @@ private: ObPoolArenaHead *array; int32_t array_size; }; - static const int64_t MAX_POOL_NUM = 1L << 21; + // This structure is used to index all object pools. It is currently invalid and needs to be redesigned later + static const int64_t MAX_POOL_NUM = 1024; static PoolPair pool_list_[MAX_POOL_NUM]; static int64_t pool_num_; }; diff --git a/deps/oblib/src/lib/queue/ob_dedup_queue.h b/deps/oblib/src/lib/queue/ob_dedup_queue.h index 33187398a..c01917937 100644 --- a/deps/oblib/src/lib/queue/ob_dedup_queue.h +++ b/deps/oblib/src/lib/queue/ob_dedup_queue.h @@ -155,7 +155,7 @@ class ObDedupQueue : public lib::ThreadPool public: static const int64_t TOTAL_LIMIT = 1024L * 1024L * 1024L; static const int64_t HOLD_LIMIT = 512L * 1024L * 1024L; - static const int64_t PAGE_SIZE = common::OB_MALLOC_BIG_BLOCK_SIZE; + static const int64_t PAGE_SIZE = common::OB_MALLOC_MIDDLE_BLOCK_SIZE; static const int64_t TASK_MAP_SIZE = 20L * 1000; static const int64_t TASK_QUEUE_SIZE = 20L * 1000; public: @@ -186,7 +186,7 @@ private: HashAllocator; typedef hash::ObHashMap, hash::equal_to, HashAllocator, diff --git a/deps/oblib/src/lib/queue/ob_priority_queue.h b/deps/oblib/src/lib/queue/ob_priority_queue.h index 9320fd27b..83fd6624d 100644 --- a/deps/oblib/src/lib/queue/ob_priority_queue.h +++ b/deps/oblib/src/lib/queue/ob_priority_queue.h @@ -77,14 +77,14 @@ public: ret = OB_INVALID_ARGUMENT; COMMON_LOG(ERROR, "timeout is invalid", K(ret), K(timeout_us)); } else { - auto key = sem_.get_key(); - sem_.wait(key, timeout_us); for(int i = 0; OB_ENTRY_NOT_EXIST == ret && i < PRIO_CNT; i++) { if (OB_SUCCESS == queue_[i].pop(data)) { ret = OB_SUCCESS; } } if (OB_FAIL(ret)) { + auto key = sem_.get_key(); + sem_.wait(key, timeout_us); data = NULL; } else { (void)ATOMIC_FAA(&size_, -1); @@ -93,6 +93,18 @@ public: return ret; } + void destroy() + { + clear(); + } + + void clear() + { + ObLink* p = NULL; + while(OB_SUCCESS == pop(p, 0)) + ; + } + private: SimpleCond sem_; ObLinkQueue queue_[PRIO_CNT]; diff --git a/deps/oblib/src/lib/resource/achunk_mgr.cpp b/deps/oblib/src/lib/resource/achunk_mgr.cpp index e77936fd0..a27705df5 100644 --- a/deps/oblib/src/lib/resource/achunk_mgr.cpp +++ b/deps/oblib/src/lib/resource/achunk_mgr.cpp @@ -23,6 +23,7 @@ #include "lib/alloc/alloc_struct.h" #include "lib/alloc/alloc_failed_reason.h" #include "lib/alloc/memory_sanity.h" +#include "deps/oblib/src/lib/alloc/malloc_hook.h" using namespace oceanbase::lib; @@ -143,9 +144,11 @@ void *AChunkMgr::low_alloc(const uint64_t size, const bool can_use_huge_page, bo huge_flags = flags | MAP_HUGETLB; } #endif - const int fd = -1; + // for debug more efficiently + const int fd = -1234; const int offset = 0; const int large_page_type = ObLargePageHelper::get_type(); + set_ob_mem_mgr_path(); if (SANITY_BOOL_EXPR(alloc_shadow)) { int64_t new_addr = ATOMIC_FAA(&global_canonical_addr, size); if (!SANITY_ADDR_IN_RANGE((void*)new_addr)) { @@ -188,11 +191,13 @@ void *AChunkMgr::low_alloc(const uint64_t size, const bool can_use_huge_page, bo //SANITY_UNPOISON(ptr, size); // maybe no need? } } + unset_ob_mem_mgr_path(); return ptr; } void AChunkMgr::low_free(const void *ptr, const uint64_t size) { + set_ob_mem_mgr_path(); if (SANITY_ADDR_IN_RANGE(ptr)) { void *shad_ptr = SANITY_TO_SHADOW((void*)ptr); ssize_t shad_size = SANITY_TO_SHADOW_SIZE(size); @@ -200,6 +205,7 @@ void AChunkMgr::low_free(const void *ptr, const uint64_t size) ::munmap(shad_ptr, shad_size); } ::munmap((void*)ptr, size); + unset_ob_mem_mgr_path(); } AChunk *AChunkMgr::alloc_chunk(const uint64_t size, bool high_prio) diff --git a/deps/oblib/src/lib/stat/ob_di_cache.cpp b/deps/oblib/src/lib/stat/ob_di_cache.cpp index 2bf8a782b..eb24f6bb5 100644 --- a/deps/oblib/src/lib/stat/ob_di_cache.cpp +++ b/deps/oblib/src/lib/stat/ob_di_cache.cpp @@ -37,10 +37,10 @@ void ObDISessionCollect::clean() base_value_.reset(); } -ObDITenantCollect::ObDITenantCollect() +ObDITenantCollect::ObDITenantCollect(ObIAllocator *allocator) : tenant_id_(0), last_access_time_(0), - base_value_() + base_value_(allocator) { } @@ -78,7 +78,7 @@ int ObDISessionCache::get_node(uint64_t session_id, ObDISessionCollect *&session { int ret = OB_SUCCESS; thread_local ObRandom random; - ObSessionBucket &bucket = di_map_[session_id % OB_MAX_SERVER_SESSION_CNT]; + ObSessionBucket &bucket = di_map_[session_id % MAX_SESSION_COLLECT_NUM]; while (1) { bucket.lock_.rdlock(); if (OB_SUCCESS == (ret = bucket.get_the_node(session_id, session_collect))) { @@ -91,14 +91,14 @@ int ObDISessionCache::get_node(uint64_t session_id, ObDISessionCollect *&session bucket.lock_.unlock(); int64_t pos = 0; while (1) { - pos = random.get(0, OB_MAX_SERVER_SESSION_CNT-1); + pos = random.get(0, MAX_SESSION_COLLECT_NUM - 1); if (OB_SUCCESS == (ret = collects_[pos].lock_.try_wrlock())) { break; } } if (OB_SUCCESS == ret) { if (0 != collects_[pos].session_id_) { - ObSessionBucket &des_bucket = di_map_[collects_[pos].session_id_ % OB_MAX_SERVER_SESSION_CNT]; + ObSessionBucket &des_bucket = di_map_[collects_[pos].session_id_ % MAX_SESSION_COLLECT_NUM]; des_bucket.lock_.wrlock(); des_bucket.list_.remove(&collects_[pos]); collects_[pos].clean(); @@ -135,7 +135,7 @@ int ObDISessionCache::get_all_diag_info(ObIArray pair; ObDISessionCollect *head = NULL; ObDISessionCollect *node = NULL; - for (int64_t i = 0; OB_SUCC(ret) && i < OB_MAX_SERVER_SESSION_CNT; ++i) { + for (int64_t i = 0; OB_SUCC(ret) && i < MAX_SESSION_COLLECT_NUM; ++i) { ObSessionBucket &bucket = di_map_[i]; bucket.lock_.rdlock(); head = bucket.list_.get_header(); @@ -158,7 +158,7 @@ int ObDISessionCache::get_the_diag_info( ObDISessionCollect *&diag_infos) { int ret = OB_SUCCESS; - ObSessionBucket &bucket = di_map_[session_id % OB_MAX_SERVER_SESSION_CNT]; + ObSessionBucket &bucket = di_map_[session_id % MAX_SESSION_COLLECT_NUM]; bucket.lock_.rdlock(); ObDISessionCollect *collect = NULL; if (OB_SUCCESS == (ret = bucket.get_the_node(session_id, collect))) { @@ -501,7 +501,7 @@ int ObDIGlobalTenantCache::get_tenant_stat(ObIAllocator &allocator, if (NULL == (buf = allocator.alloc(sizeof(ObDITenantCollect)))) { ret = OB_ALLOCATE_MEMORY_FAILED; } else { - got_collect = new (buf) ObDITenantCollect(); + got_collect = new (buf) ObDITenantCollect(&allocator); got_collect->tenant_id_ = collect->tenant_id_; bucket.list_.add_last(got_collect); } diff --git a/deps/oblib/src/lib/stat/ob_di_cache.h b/deps/oblib/src/lib/stat/ob_di_cache.h index 70f1b851d..288be13ef 100644 --- a/deps/oblib/src/lib/stat/ob_di_cache.h +++ b/deps/oblib/src/lib/stat/ob_di_cache.h @@ -42,7 +42,7 @@ public: class ObDITenantCollect : public ObDINode { public: - ObDITenantCollect(); + ObDITenantCollect(ObIAllocator *allocator = NULL); virtual ~ObDITenantCollect(); void clean(); uint64_t tenant_id_; @@ -62,6 +62,10 @@ public: uint64_t session_id, ObDISessionCollect *&diag_infos); int get_node(uint64_t session_id, ObDISessionCollect *&session_collect); +private: + static const int64_t MAX_SESSION_COLLECT_NUM = OB_MAX_SERVER_SESSION_CNT / 4; + STATIC_ASSERT((MAX_SESSION_COLLECT_NUM > 4 * 1024 && MAX_SESSION_COLLECT_NUM < 10 * 1024), + "unexpected MAX_SESSION_COLLECT_NUM"); private: struct ObSessionBucket { @@ -89,8 +93,8 @@ private: }; ObDISessionCache(); virtual ~ObDISessionCache(); - ObSessionBucket di_map_[OB_MAX_SERVER_SESSION_CNT]; - ObDISessionCollect collects_[OB_MAX_SERVER_SESSION_CNT]; + ObSessionBucket di_map_[MAX_SESSION_COLLECT_NUM]; + ObDISessionCollect collects_[MAX_SESSION_COLLECT_NUM]; }; struct ObTenantBucket diff --git a/deps/oblib/src/lib/stat/ob_diagnose_info.cpp b/deps/oblib/src/lib/stat/ob_diagnose_info.cpp index 05dd9fc17..f2a740c86 100644 --- a/deps/oblib/src/lib/stat/ob_diagnose_info.cpp +++ b/deps/oblib/src/lib/stat/ob_diagnose_info.cpp @@ -75,16 +75,31 @@ void ObLatchStat::reset() /** * ----------------------------------------------------------ObLatchStatArray----------------------------------------------------- */ -ObLatchStatArray::ObLatchStatArray() - : items_() +ObLatchStatArray::ObLatchStatArray(ObIAllocator *allocator) + : allocator_(allocator), items_() { } +ObLatchStatArray::~ObLatchStatArray() +{ + for (int64_t i = 0; i < ObLatchIds::LATCH_END; ++i) { + if (OB_ISNULL(items_[i])) { + } else { + free_item(items_[i]); + items_[i] = NULL; + } + } +} + int ObLatchStatArray::add(const ObLatchStatArray &other) { int ret = OB_SUCCESS; for (int64_t i = 0; i < ObLatchIds::LATCH_END && OB_SUCCESS == ret; ++i) { - ret = items_[i].add(other.items_[i]); + if (OB_ISNULL(other.get_item(i))) continue; + auto *item = get_or_create_item(i); + if (OB_NOT_NULL(item)) { + ret = item->add(*other.get_item(i)); + } } return ret; } @@ -92,7 +107,52 @@ int ObLatchStatArray::add(const ObLatchStatArray &other) void ObLatchStatArray::reset() { for (int64_t i = 0; i < ObLatchIds::LATCH_END; ++i) { - items_[i].reset(); + if (OB_ISNULL(items_[i])) { + } else { + items_[i]->reset(); + } + } +} + +static constexpr int NODE_NUM = + common::hash::NodeNumTraits::NODE_NUM; +using LatchStatAlloc = hash::SimpleAllocer; + +LatchStatAlloc &get_latch_stat_alloc() +{ + struct Wrapper + { + Wrapper() + { + instance_.set_attr(SET_USE_500("LatchStat")); + instance_.set_leak_check(false); + } + LatchStatAlloc instance_; + }; + static Wrapper w; + return w.instance_; +} + +ObLatchStat *ObLatchStatArray::create_item() +{ + ObLatchStat *stat = NULL; + lib::ObDisableDiagnoseGuard disable_diagnose_guard; + if (OB_ISNULL(allocator_)) { + stat = get_latch_stat_alloc().alloc(); + } else { + stat = OB_NEWx(ObLatchStat, allocator_); + } + return stat; +} + +void ObLatchStatArray::free_item(ObLatchStat *stat) +{ + lib::ObDisableDiagnoseGuard disable_diagnose_guard; + if (OB_ISNULL(allocator_)) { + get_latch_stat_alloc().free(stat); + } else { + stat->~ObLatchStat(); + allocator_->free(stat); } } @@ -567,11 +627,11 @@ ObDiagnoseSessionInfo *ObDiagnoseSessionInfo::get_local_diagnose_info() return di; } -ObDiagnoseTenantInfo::ObDiagnoseTenantInfo() +ObDiagnoseTenantInfo::ObDiagnoseTenantInfo(ObIAllocator *allocator) : event_stats_(), stat_add_stats_(), stat_set_stats_(), - latch_stats_() + latch_stats_(allocator) { } diff --git a/deps/oblib/src/lib/stat/ob_diagnose_info.h b/deps/oblib/src/lib/stat/ob_diagnose_info.h index d0a2b4642..e4c5dfc9f 100644 --- a/deps/oblib/src/lib/stat/ob_diagnose_info.h +++ b/deps/oblib/src/lib/stat/ob_diagnose_info.h @@ -50,10 +50,28 @@ struct ObLatchStat struct ObLatchStatArray { - ObLatchStatArray(); +public: + ObLatchStatArray(ObIAllocator *allocator = NULL); + ~ObLatchStatArray(); int add(const ObLatchStatArray &other); void reset(); - ObLatchStat items_[ObLatchIds::LATCH_END]; + ObLatchStat *get_item(int32_t idx) const + { + return items_[idx]; + } + ObLatchStat *get_or_create_item(int32_t idx) + { + if (OB_ISNULL(items_[idx])) { + items_[idx] = create_item(); + } + return items_[idx]; + } +private: + ObLatchStat *create_item(); + void free_item(ObLatchStat *stat); +private: + ObIAllocator *allocator_; + ObLatchStat *items_[ObLatchIds::LATCH_END] = {NULL}; }; class ObWaitEventHistoryIter @@ -140,7 +158,7 @@ private: class ObDiagnoseTenantInfo final { public: - ObDiagnoseTenantInfo(); + ObDiagnoseTenantInfo(ObIAllocator *allocator = NULL); ~ObDiagnoseTenantInfo(); void add(const ObDiagnoseTenantInfo &other); void add_wait_event(const ObDiagnoseTenantInfo &other); diff --git a/deps/oblib/src/lib/statistic_event/ob_stat_event.h b/deps/oblib/src/lib/statistic_event/ob_stat_event.h index ca86c9b38..a3c0a170b 100644 --- a/deps/oblib/src/lib/statistic_event/ob_stat_event.h +++ b/deps/oblib/src/lib/statistic_event/ob_stat_event.h @@ -531,6 +531,9 @@ STAT_EVENT_SET_DEF(IS_MINI_MODE, "is mini mode", ObStatClassIds::RESOURCE, 14001 STAT_EVENT_SET_DEF(MEMORY_HOLD_SIZE, "observer memory hold size", ObStatClassIds::RESOURCE, 140011, false, true) STAT_EVENT_SET_DEF(WORKER_TIME, "worker time", ObStatClassIds::RESOURCE, 140012, false, true) STAT_EVENT_SET_DEF(CPU_TIME, "cpu time", ObStatClassIds::RESOURCE, 140013, false, true) +STAT_EVENT_SET_DEF(MEMORY_LIMIT, "effective observer memory limit", ObStatClassIds::RESOURCE, 140014, false, true) +STAT_EVENT_SET_DEF(SYSTEM_MEMORY, "effective system memory", ObStatClassIds::RESOURCE, 140015, false, true) +STAT_EVENT_SET_DEF(HIDDEN_SYS_MEMORY, "effective hidden sys memory", ObStatClassIds::RESOURCE, 140016, false, true) //CLOG diff --git a/deps/oblib/src/lib/task/ob_timer.cpp b/deps/oblib/src/lib/task/ob_timer.cpp index 6e247bab3..51b46edb5 100644 --- a/deps/oblib/src/lib/task/ob_timer.cpp +++ b/deps/oblib/src/lib/task/ob_timer.cpp @@ -104,7 +104,7 @@ void ObTimer::stop() void ObTimer::wait() { ObMonitor::Lock guard(monitor_); - while (has_running_task_) { + while (running_task_ != NULL) { static const int64_t WAIT_INTERVAL_US = 2000000; // 2s (void)monitor_.timed_wait(ObSysTime(WAIT_INTERVAL_US)); } @@ -125,7 +125,6 @@ void ObTimer::destroy() { ObMonitor::Lock guard(monitor_); for (int64_t i = 0; i < tasks_num_; ++i) { - tokens_[i].task->cancelCallBack(); ATOMIC_STORE(&(tokens_[i].task->timer_), nullptr); } tasks_num_ = 0; @@ -245,41 +244,112 @@ int ObTimer::insert_token(const Token &token) return ret; } -int ObTimer::cancel(const ObTimerTask &task) +int ObTimer::cancel_task(const ObTimerTask &task) { int ret = OB_SUCCESS; ObMonitor::Lock guard(monitor_); if (!is_inited_) { ret = OB_NOT_INIT; } else { - int64_t pos = -1; - for (int64_t i = 0; i < tasks_num_; ++i) { - if (&task == tokens_[i].task) { - pos = i; - break; + if (&task == uncanceled_task_) { + // repeat cancel, do-nothing + } else if (&task == running_task_) { + if (uncanceled_task_ != NULL) { + ret = OB_ERR_UNEXPECTED; + } else { + ATOMIC_STORE(&const_cast(task).timer_, nullptr); + uncanceled_task_ = &const_cast(task); + OB_LOG(INFO, "cancel task", KP(this), K_(thread_id), K(wakeup_time_), K(tasks_num_), K(task)); + } + } else { + int64_t pos = -1; + for (int64_t i = 0; i < tasks_num_; ++i) { + if (&task == tokens_[i].task) { + pos = i; + break; + } + } + if (-1 == pos) { + // not found, do-nothing + } else { + ATOMIC_STORE(&const_cast(task).timer_, nullptr); + memmove(&tokens_[pos], &tokens_[pos + 1], + sizeof(tokens_[0]) * (tasks_num_ - pos - 1)); + --tasks_num_; + OB_LOG(INFO, "cancel task", KP(this), K_(thread_id), K(wakeup_time_), K(tasks_num_), K(task)); } } - if (pos != -1) { - tokens_[pos].task->cancelCallBack(); - ATOMIC_STORE(&(tokens_[pos].task->timer_), nullptr); - memmove(&tokens_[pos], &tokens_[pos + 1], - sizeof(tokens_[0]) * (tasks_num_ - pos - 1)); - --tasks_num_; - OB_LOG(INFO, "cancel task", KP(this), K_(thread_id), K(pos), K(wakeup_time_), K(tasks_num_), K(task)); - } + } + return ret; +} + +int ObTimer::wait_task(const ObTimerTask &task) +{ + int ret = OB_SUCCESS; + ObMonitor::Lock guard(monitor_); + if (!is_inited_) { + ret = OB_NOT_INIT; + } else { + do { + bool exist = &task == running_task_; + if (!exist) exist = &task == uncanceled_task_; + for (int64_t i = 0; !exist && i < tasks_num_; ++i) { + if (&task == tokens_[i].task) { + exist = true; + break; + } + } + if (!exist) { + break; + } else { + (void)monitor_.timed_wait(ObSysTime(10 * 1000/*10ms*/)); + } + } while (true); + } + return ret; +} + +int ObTimer::cancel(const ObTimerTask &task) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(cancel_task(task))) { + OB_LOG(WARN, "failed to cancel_task", K(ret), K(task)); + // @TODO: nijia.nj + //} else if (OB_FAIL(wait_task(task))) { + // OB_LOG(WARN, "failed to wait_task", K(ret), K(task)); } return ret; } void ObTimer::cancel_all() { - ObMonitor::Lock guard(monitor_); - for (int64_t i = 0; i < tasks_num_; ++i) { - tokens_[i].task->cancelCallBack(); - ATOMIC_STORE(&(tokens_[i].task->timer_), nullptr); - } - tasks_num_ = 0; - OB_LOG(INFO, "cancel all", KP(this), K_(thread_id), K(wakeup_time_), K(tasks_num_)); + int ret = OB_SUCCESS; + do { + ObMonitor::Lock guard(monitor_); + for (int64_t i = 0; i < tasks_num_; ++i) { + ATOMIC_STORE(&(tokens_[i].task->timer_), nullptr); + } + tasks_num_ = 0; + if (running_task_ != NULL) { + if (uncanceled_task_ == running_task_) { + // do-nothing + } else if (uncanceled_task_ != NULL) { + ret = OB_ERR_UNEXPECTED; + } else { + ATOMIC_STORE(&running_task_->timer_, nullptr); + uncanceled_task_ = running_task_; + } + if (OB_SUCC(ret)) { + (void)monitor_.timed_wait(ObSysTime(10 * 1000/*10ms*/)); + } + } else { + if (uncanceled_task_ != NULL) { + ret = OB_ERR_UNEXPECTED; + } + break; + } + } while (OB_SUCC(ret)); + OB_LOG(INFO, "cancel all", K(ret), KP(this), K_(thread_id), K(wakeup_time_), K(tasks_num_)); } void ObTimer::run1() @@ -308,8 +378,9 @@ void ObTimer::run1() if (is_destroyed_) { break; } + //add repeated task to tasks_ again - if (token.delay != 0 && token.task->need_retry() && !is_stopped_) { + if (token.delay != 0 && !is_stopped_) { has_running_repeat_task_ = false; token.scheduled_time = ObSysTime::now(ObSysTime::Monotonic).toMicroSeconds() + token.delay; if (OB_SUCCESS != (tmp_ret = insert_token( @@ -317,7 +388,7 @@ void ObTimer::run1() OB_LOG_RET(WARN, tmp_ret, "insert token error", K(tmp_ret), K(token)); } } - has_running_task_ = false; + running_task_ = NULL; if (is_stopped_) { monitor_.notify_all(); } @@ -340,8 +411,8 @@ void ObTimer::run1() while (tasks_num_ > 0 && !is_destroyed_ && !is_stopped_) { const int64_t now = ObSysTime::now(ObSysTime::Monotonic).toMicroSeconds(); if (tokens_[0].scheduled_time <= now) { - has_running_task_ = true; token = tokens_[0]; + running_task_ = token.task; memmove(tokens_, tokens_ + 1, (tasks_num_ - 1) * sizeof(tokens_[0])); --tasks_num_; ATOMIC_STORE(&(token.task->timer_), nullptr); @@ -374,13 +445,19 @@ void ObTimer::run1() } } - if (token.task != NULL && has_running_task_ && !is_destroyed_ && !is_stopped_) { + if (token.task != NULL && running_task_ != NULL && !is_destroyed_ && !is_stopped_) { bool timeout_check = token.task->timeout_check(); const int64_t start_time = ::oceanbase::common::ObTimeUtility::current_time(); if (timeout_check) { ObTimerMonitor::get_instance().start_task(thread_id_, start_time, token.delay, token.task); } token.task->runTimerTask(); + ObMonitor::Lock guard(monitor_); + if (token.task == uncanceled_task_) { + uncanceled_task_ = NULL; + token.delay = 0; + monitor_.notify_all(); + } const int64_t end_time = ::oceanbase::common::ObTimeUtility::current_time(); const int64_t elapsed_time = end_time - start_time; diff --git a/deps/oblib/src/lib/task/ob_timer.h b/deps/oblib/src/lib/task/ob_timer.h index 9881c4eb3..58835410d 100644 --- a/deps/oblib/src/lib/task/ob_timer.h +++ b/deps/oblib/src/lib/task/ob_timer.h @@ -41,7 +41,6 @@ class ObTimerTask public: ObTimerTask() : timeout_check_(true), timer_(nullptr) {} virtual ~ObTimerTask() { abort_unless(OB_ISNULL(ATOMIC_LOAD(&timer_))); } - virtual void cancelCallBack() {} virtual void runTimerTask() = 0; virtual int64_t to_string(char *buf, const int64_t buf_len) const { @@ -55,7 +54,6 @@ public: inline void enable_timeout_check() { timeout_check_ = true; } inline void disable_timeout_check() { timeout_check_ = false; } inline bool timeout_check() { return timeout_check_; } - virtual bool need_retry() { return true; } private: bool timeout_check_; ObTimer* timer_; @@ -67,7 +65,7 @@ class ObTimer public: friend class oceanbase::tests::blocksstable::FakeTabletManager; ObTimer(int64_t max_task_num = 32): tasks_num_(0), max_task_num_(max_task_num), wakeup_time_(0), is_inited_(false), is_stopped_(false), - is_destroyed_(false), tokens_(nullptr), has_running_task_(false), has_running_repeat_task_(false), + is_destroyed_(false), tokens_(nullptr), running_task_(nullptr), uncanceled_task_(nullptr), has_running_repeat_task_(false), thread_id_(-1), thread_name_(nullptr) {} ~ObTimer(); int init(const char* thread_name = nullptr, @@ -88,6 +86,8 @@ public: return OB_SUCCESS; } int cancel(const ObTimerTask &task); + int cancel_task(const ObTimerTask &task); + int wait_task(const ObTimerTask &task); void cancel_all(); int32_t get_tasks_num() const { return tasks_num_; } void dump() const; @@ -100,6 +100,7 @@ private: TO_STRING_KV(K(scheduled_time), K(delay), KP(task), KPC(task)); int64_t scheduled_time; int64_t delay; + bool canceled_; ObTimerTask *task; }; int insert_token(const Token &token); @@ -116,7 +117,8 @@ private: bool is_destroyed_; obutil::ObMonitor monitor_; Token* tokens_; - bool has_running_task_; + ObTimerTask *running_task_; + ObTimerTask *uncanceled_task_; bool has_running_repeat_task_; int64_t thread_id_; const char* thread_name_; diff --git a/deps/oblib/src/lib/thread/ob_async_task_queue.cpp b/deps/oblib/src/lib/thread/ob_async_task_queue.cpp index 33e290541..b808d8382 100644 --- a/deps/oblib/src/lib/thread/ob_async_task_queue.cpp +++ b/deps/oblib/src/lib/thread/ob_async_task_queue.cpp @@ -35,7 +35,7 @@ ObAsyncTaskQueue::~ObAsyncTaskQueue() } } -int ObAsyncTaskQueue::init(const int64_t thread_cnt, const int64_t queue_size, const char *thread_name) +int ObAsyncTaskQueue::init(const int64_t thread_cnt, const int64_t queue_size, const char *thread_name, const int64_t page_size) { int ret = OB_SUCCESS; if (is_inited_) { @@ -44,7 +44,7 @@ int ObAsyncTaskQueue::init(const int64_t thread_cnt, const int64_t queue_size, c } else if (thread_cnt <= 0|| queue_size <= 0 || 0 != (queue_size & (queue_size - 1))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(thread_cnt), K(queue_size), K(ret)); - } else if (OB_FAIL(allocator_.init(TOTAL_LIMIT, HOLD_LIMIT, PAGE_SIZE))) { + } else if (OB_FAIL(allocator_.init(TOTAL_LIMIT, HOLD_LIMIT, page_size))) { LOG_WARN("allocator init failed", "total limit", static_cast(TOTAL_LIMIT), "hold limit", static_cast(HOLD_LIMIT), "page size", static_cast(PAGE_SIZE), K(ret)); diff --git a/deps/oblib/src/lib/thread/ob_async_task_queue.h b/deps/oblib/src/lib/thread/ob_async_task_queue.h index ab3e0167f..dabf83fd5 100644 --- a/deps/oblib/src/lib/thread/ob_async_task_queue.h +++ b/deps/oblib/src/lib/thread/ob_async_task_queue.h @@ -97,7 +97,7 @@ public: virtual ~ObAsyncTaskQueue(); //attention queue_size should be 2^n int init(const int64_t thread_cnt, const int64_t queue_size, - const char *thread_name = nullptr); + const char *thread_name = nullptr, const int64_t page_size = PAGE_SIZE); int start(); void stop(); void wait(); @@ -107,7 +107,7 @@ public: protected: static const int64_t TOTAL_LIMIT = 1024L * 1024L * 1024L; static const int64_t HOLD_LIMIT = 512L * 1024L * 1024L; - static const int64_t PAGE_SIZE = common::OB_MALLOC_BIG_BLOCK_SIZE; + static const int64_t PAGE_SIZE = common::OB_MALLOC_MIDDLE_BLOCK_SIZE; static const int64_t SLEEP_INTERVAL = 10000; //10ms virtual void run2(); virtual int blocking_run() { BLOCKING_RUN_IMPLEMENT(); } diff --git a/deps/oblib/src/lib/thread/ob_tenant_hook.cpp b/deps/oblib/src/lib/thread/ob_tenant_hook.cpp index 20484b300..e217e46f4 100644 --- a/deps/oblib/src/lib/thread/ob_tenant_hook.cpp +++ b/deps/oblib/src/lib/thread/ob_tenant_hook.cpp @@ -179,40 +179,6 @@ int futex_hook(uint32_t *uaddr, int futex_op, uint32_t val, const struct timespe return ret; } -struct PthreadCreateArgument -{ - PthreadCreateArgument(void *(*start_routine)(void *), void *arg) - { - start_routine_ = start_routine; - arg_ = arg; - in_use_ = 1; - } - void *(*start_routine_)(void *); - void *arg_; - int in_use_; // TO avoid memory alloc, there is a sync wait for pthread_create. -}; - -void* run_func(void* arg) -{ - struct PthreadCreateArgument* parg = (struct PthreadCreateArgument*)arg; - void *(*start_routine)(void *) = parg->start_routine_; - void *real_arg = parg->arg_; - ATOMIC_STORE(&(parg->in_use_), 0); - ::oceanbase::lib::ObStackHeaderGuard stack_header_guard; - return start_routine(real_arg); -} - -int ob_pthread_create(pthread_t *thread, const pthread_attr_t *attr, - void *(*start_routine) (void *), void *arg) -{ - struct PthreadCreateArgument parg(start_routine, arg); - int ret = pthread_create(thread, attr, run_func, &parg); - while (ATOMIC_LOAD(&(parg.in_use_)) != 0) { - sched_yield(); - } - return ret; -} - void ob_set_thread_name(const char* type) { ::oceanbase::lib::set_thread_name(type); diff --git a/deps/oblib/src/lib/thread/thread.h b/deps/oblib/src/lib/thread/thread.h index 312853427..67838d53a 100644 --- a/deps/oblib/src/lib/thread/thread.h +++ b/deps/oblib/src/lib/thread/thread.h @@ -55,6 +55,7 @@ public: void wait(); void destroy(); void dump_pth(); + pthread_t get_pthread() { return pth_; } /// \brief Get current thread object. /// diff --git a/deps/oblib/src/lib/thread/thread_mgr.h b/deps/oblib/src/lib/thread/thread_mgr.h index f81701487..bdf502f90 100644 --- a/deps/oblib/src/lib/thread/thread_mgr.h +++ b/deps/oblib/src/lib/thread/thread_mgr.h @@ -190,6 +190,16 @@ public: { return common::OB_NOT_SUPPORTED; } + virtual int cancel_task(const common::ObTimerTask &task) + { + UNUSED(task); + return common::OB_NOT_SUPPORTED; + } + virtual int wait_task(const common::ObTimerTask &task) + { + UNUSED(task); + return common::OB_NOT_SUPPORTED; + } virtual void set_queue_size(const int64_t qsize) { UNUSED(qsize); @@ -829,6 +839,26 @@ public: } return ret; } + int cancel_task(const common::ObTimerTask &task) override + { + int ret = common::OB_SUCCESS; + if (OB_ISNULL(timer_)) { + ret = common::OB_ERR_UNEXPECTED; + } else { + ret = timer_->cancel_task(task); + } + return ret; + } + int wait_task(const common::ObTimerTask &task) override + { + int ret = common::OB_SUCCESS; + if (OB_ISNULL(timer_)) { + ret = common::OB_ERR_UNEXPECTED; + } else { + ret = timer_->wait_task(task); + } + return ret; + } void destroy() { if (timer_ != nullptr) { @@ -1048,6 +1078,7 @@ public: TG_MGR.tgs_[tg_id]; \ if (OB_ISNULL(tg)) { \ ret = common::OB_ERR_UNEXPECTED; \ + OB_LOG(WARN, "null tg", K(tg_id)); \ } else { \ auto f = [&]() { return tg->func(args); }; \ ret = lib::FWrap::value>()(f); \ @@ -1084,6 +1115,8 @@ public: #define TG_STOP(tg_id) do { int r = TG_INVOKE(tg_id, stop); UNUSED(r); } while (0) #define TG_CANCEL_R(tg_id, args...) TG_INVOKE(tg_id, cancel, args) #define TG_CANCEL(tg_id, args...) do { int r = TG_INVOKE(tg_id, cancel, args); UNUSED(r); } while (0) +#define TG_CANCEL_TASK(tg_id, args...) do { int r = TG_INVOKE(tg_id, cancel_task, args); UNUSED(r); } while (0) +#define TG_WAIT_TASK(tg_id, args...) do { int r = TG_INVOKE(tg_id, wait_task, args); UNUSED(r); } while (0) #define TG_CANCEL_ALL(tg_id) TG_INVOKE(tg_id, cancel_all) #define TG_TASK_EXIST(tg_id, args...) TG_INVOKE(tg_id, task_exist, args) #define TG_SCHEDULE(tg_id, args...) TG_INVOKE(tg_id, schedule, args) diff --git a/deps/oblib/src/lib/thread/threads.cpp b/deps/oblib/src/lib/thread/threads.cpp index 35bf3f735..aa2484af3 100644 --- a/deps/oblib/src/lib/thread/threads.cpp +++ b/deps/oblib/src/lib/thread/threads.cpp @@ -19,7 +19,7 @@ #include "lib/oblog/ob_log.h" #include "lib/signal/ob_signal_struct.h" #include "lib/worker.h" - +using namespace oceanbase; using namespace oceanbase::lib; using namespace oceanbase::common; @@ -271,3 +271,51 @@ void Threads::destroy() threads_ = nullptr; } } + + +extern "C" { +int ob_pthread_create(void **ptr, void *(*start_routine) (void *), void *arg) +{ + int ret = OB_SUCCESS; + ObPThread *thread = NULL; + // Temporarily set expect_run_wrapper to NULL for creating normal thread + IRunWrapper *expect_run_wrapper = Threads::get_expect_run_wrapper(); + Threads::get_expect_run_wrapper() = NULL; + DEFER(Threads::get_expect_run_wrapper() = expect_run_wrapper); + OB_LOG(INFO, "ob_pthread_create start"); + if (OB_ISNULL(thread = OB_NEW(ObPThread, SET_USE_500("PThread"), start_routine, arg))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "alloc memory failed", K(ret)); + } else if (OB_FAIL(thread->start())) { + OB_LOG(WARN, "failed to start thread", K(ret)); + } + if (OB_FAIL(ret)) { + if (OB_NOT_NULL(thread)) { + OB_DELETE(ObPThread, SET_USE_500("PThread"), thread); + } + } else { + *ptr = thread; + OB_LOG(INFO, "ob_pthread_create succeed", KP(thread)); + } + return ret; +} +void ob_pthread_join(void *ptr) +{ + if (OB_NOT_NULL(ptr)) { + ObPThread *thread = (ObPThread*) ptr; + thread->wait(); + OB_LOG(INFO, "ob_pthread_join succeed", KP(thread)); + OB_DELETE(ObPThread, SET_USE_500("PThread"), thread); + } +} + +pthread_t ob_pthread_get_pth(void *ptr) +{ + pthread_t pth = 0; + if (OB_NOT_NULL(ptr)) { + ObPThread *thread = (ObPThread*) ptr; + pth = thread->get_pthread(0); + } + return pth; +} +} /* extern "C" */ \ No newline at end of file diff --git a/deps/oblib/src/lib/thread/threads.h b/deps/oblib/src/lib/thread/threads.h index b8f075c49..2356a8d1f 100644 --- a/deps/oblib/src/lib/thread/threads.h +++ b/deps/oblib/src/lib/thread/threads.h @@ -93,6 +93,14 @@ public: IGNORE_RETURN lib::Thread::update_loop_ts(); return stop_; } + pthread_t get_pthread(int64_t idx) + { + pthread_t pth = 0; + if (idx < n_threads_) { + pth = threads_[idx]->get_pthread(); + } + return pth; + } protected: int64_t get_thread_count() const { return n_threads_; } uint64_t get_thread_idx() const { return thread_idx_; } @@ -121,6 +129,21 @@ private: IRunWrapper *run_wrapper_; }; +class ObPThread : public Threads +{ +public: + ObPThread(void *(*start_routine) (void *), void *arg) + : start_routine_(start_routine), arg_(arg) + {} + void run1() override + { + start_routine_(arg_); + } +private: + void *(*start_routine_)(void *); + void *arg_; +}; + using ThreadPool = Threads; } // lib diff --git a/deps/oblib/src/lib/timezone/ob_timezone_info.cpp b/deps/oblib/src/lib/timezone/ob_timezone_info.cpp index 75bdd2a0b..23440e440 100644 --- a/deps/oblib/src/lib/timezone/ob_timezone_info.cpp +++ b/deps/oblib/src/lib/timezone/ob_timezone_info.cpp @@ -39680,6 +39680,38 @@ int ObTimeZoneInfoPos::timezone_to_str(char *buf, const int64_t buf_len, int64_t return ret; } +void ObTimeZoneInfoPos::set_tz_type_attr(const lib::ObMemAttr &attr) +{ + tz_tran_types_[0].set_attr(attr); + tz_tran_types_[1].set_attr(attr); + tz_revt_types_[0].set_attr(attr); + tz_revt_types_[1].set_attr(attr); +} + +bool ObTimeZoneInfoPos::operator==(const ObTimeZoneInfoPos &other) const +{ + bool same = (tz_id_ == other.tz_id_ + && default_type_ == other.default_type_) + && curr_idx_ == other.curr_idx_ + && (0 == curr_idx_ || 1 == curr_idx_) + && 0 == STRCASECMP(tz_name_, other.tz_name_); + if (same) { + same = tz_tran_types_[curr_idx_].count() == other.tz_tran_types_[curr_idx_].count() + && tz_revt_types_[curr_idx_].count() == other.tz_revt_types_[curr_idx_].count(); + } + if (same) { + for (int64_t i = 0; i < tz_tran_types_[curr_idx_].count() && same; i++) { + same = tz_tran_types_[curr_idx_].at(i) == other.tz_tran_types_[curr_idx_].at(i); + } + } + if (same) { + for (int64_t i = 0; i < tz_revt_types_[curr_idx_].count() && same; i++) { + same = tz_revt_types_[curr_idx_].at(i) == other.tz_revt_types_[curr_idx_].at(i); + } + } + return same; +} + OB_DEF_SERIALIZE(ObTimeZoneInfoPos) { int ret = OB_SUCCESS; @@ -39752,8 +39784,6 @@ void ObTZNameIDAlloc::free_value(ObTZNameIDInfo *info) { op_free(info); info = NULL; - - } ObTZNameHashNode* ObTZNameIDAlloc::alloc_node(ObTZNameIDInfo *value) @@ -39776,9 +39806,9 @@ int ObTZInfoMap::init(const lib::ObMemAttr &attr) if (OB_UNLIKELY(inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret)); - } else if (OB_FAIL(id_map_.init(attr))) { + } else if (OB_FAIL(id_map_buf_.init(attr))) { LOG_WARN("fail to init id map", K(ret)); - } else if (OB_FAIL(name_map_.init(attr))) { + } else if (OB_FAIL(name_map_buf_.init(attr))) { LOG_WARN("fail to init name map", K(ret)); } else { inited_ = true; @@ -39786,18 +39816,12 @@ int ObTZInfoMap::init(const lib::ObMemAttr &attr) return ret; } -int ObTZInfoMap::reset() -{ - int ret = OB_SUCCESS; - id_map_.reset(); - name_map_.reset(); - return ret; -} - void ObTZInfoMap::destroy() { - id_map_.destroy(); - name_map_.destroy(); + id_map_ = NULL; + name_map_ = NULL; + id_map_buf_.destroy(); + name_map_buf_.destroy(); } static bool print_tz_info(ObTZIDKey &key, ObTimeZoneInfoPos *tz_info) @@ -39816,7 +39840,7 @@ static bool print_tz_info(ObTZIDKey &key, ObTimeZoneInfoPos *tz_info) int ObTZInfoMap::print_tz_info_map() { int ret = OB_SUCCESS; - if (OB_FAIL(id_map_.for_each(print_tz_info))) { + if (OB_FAIL(id_map_->for_each(print_tz_info))) { LOG_WARN("fail to call for_each", K(ret)); } return ret; @@ -39826,7 +39850,7 @@ int ObTZInfoMap::get_tz_info_by_id(const int64_t tz_id, ObTimeZoneInfoPos &tz_in { int ret = OB_SUCCESS; ObTimeZoneInfoPos *tmp_tz_info = NULL; - if (OB_FAIL(id_map_.get(tz_id, tmp_tz_info))) { + if (OB_FAIL(id_map_->get(tz_id, tmp_tz_info))) { LOG_WARN("fail to get tz_info_by_id, should not happened", K(tz_id), K(ret)); } else if (OB_FAIL(tz_info_by_id.assign(*tmp_tz_info))) { LOG_WARN("assign time zone info pos failed", K(ret)); @@ -39834,7 +39858,7 @@ int ObTZInfoMap::get_tz_info_by_id(const int64_t tz_id, ObTimeZoneInfoPos &tz_in LOG_DEBUG("succ to get tz_info_by_id", K(tz_id), KPC(tmp_tz_info), K(ret)); } if (NULL != tmp_tz_info) { - id_map_.revert(tmp_tz_info); + id_map_->revert(tmp_tz_info); } return ret; } @@ -39843,7 +39867,7 @@ int ObTZInfoMap::get_tz_info_by_name(const ObString &tz_name, ObTimeZoneInfoPos { int ret = OB_SUCCESS; ObTZNameIDInfo *name_id_info = NULL; - if (OB_FAIL(name_map_.get(ObTZNameKey(tz_name), name_id_info))) { + if (OB_FAIL(name_map_->get(ObTZNameKey(tz_name), name_id_info))) { LOG_WARN("fail to get get_tz_info_by_name", K(tz_name), K(ret)); } else if (OB_FAIL(get_tz_info_by_id(name_id_info->tz_id_, tz_info_by_name))) { LOG_WARN("fail to get get_tz_info_by_name", KPC(name_id_info), K(ret)); @@ -39856,7 +39880,7 @@ int ObTZInfoMap::get_tz_info_by_name(const ObString &tz_name, ObTimeZoneInfoPos } if (NULL != name_id_info) { - name_map_.revert(name_id_info); + name_map_->revert(name_id_info); name_id_info = NULL; } @@ -39870,7 +39894,7 @@ int ObTZInfoMap::get_tz_info_by_id(const int64_t tz_id, ObTimeZoneInfoPos *&tz_i if (OB_NOT_NULL(tz_info_by_id)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("tz_info_by_id should be null here", K(ret)); - } else if (OB_FAIL(id_map_.get(tz_id, tz_info_by_id))) { + } else if (OB_FAIL(id_map_->get(tz_id, tz_info_by_id))) { LOG_WARN("fail to get tz_info_by_id, should not happened", K(tz_id), K(ret)); } return ret; @@ -39883,7 +39907,7 @@ int ObTZInfoMap::get_tz_info_by_name(const ObString &tz_name, ObTimeZoneInfoPos if (OB_NOT_NULL(tz_info_by_name)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("v should be null here", K(ret)); - } else if (OB_FAIL(name_map_.get(ObTZNameKey(tz_name), name_id_info))) { + } else if (OB_FAIL(name_map_->get(ObTZNameKey(tz_name), name_id_info))) { LOG_WARN("fail to get get_tz_info_by_name", K(tz_name), K(ret)); } else if (OB_FAIL(get_tz_info_by_id(name_id_info->tz_id_, tz_info_by_name))) { LOG_WARN("fail to get get_tz_info_by_name", KPC(name_id_info), K(ret)); @@ -39892,7 +39916,7 @@ int ObTZInfoMap::get_tz_info_by_name(const ObString &tz_name, ObTimeZoneInfoPos ret = OB_ERR_UNKNOWN_TIME_ZONE; } if (NULL != name_id_info) { - name_map_.revert(name_id_info); + name_map_->revert(name_id_info); name_id_info = NULL; } diff --git a/deps/oblib/src/lib/timezone/ob_timezone_info.h b/deps/oblib/src/lib/timezone/ob_timezone_info.h index 75eddb476..72ea2425a 100644 --- a/deps/oblib/src/lib/timezone/ob_timezone_info.h +++ b/deps/oblib/src/lib/timezone/ob_timezone_info.h @@ -236,6 +236,7 @@ public: { } const ObTZInfoMap *get_tz_map() const { return tz_info_map_; } void set_tz_map(const common::ObTZInfoMap *tz_info_map); + TO_STRING_KV(KP_(tz_info_map)); private: ObTZInfoMap *tz_info_map_; }; @@ -541,6 +542,8 @@ public: common::ObSArray &get_next_tz_tran_types() { return tz_tran_types_[get_next_idx() % 2]; } common::ObSArray &get_next_tz_revt_types() { return tz_revt_types_[get_next_idx() % 2]; } int calc_revt_types(); + bool operator==(const ObTimeZoneInfoPos &other) const; + void set_tz_type_attr(const lib::ObMemAttr &attr); virtual int timezone_to_str(char *buf, const int64_t len, int64_t &pos) const; VIRTUAL_TO_STRING_KV("tz_name", common::ObString(common::OB_MAX_TZ_NAME_LEN, tz_name_), "tz_id", tz_id_, @@ -594,7 +597,12 @@ public: } ~ObTZNameIDInfo() {} TO_STRING_KV(K_(tz_id), KCSTRING_(tz_name)); + bool operator==(const ObTZNameIDInfo &other) const + { + return (tz_id_ == other.tz_id_ + && 0 == STRCASECMP(tz_name_, other.tz_name_)); + } public: int64_t tz_id_; char tz_name_[common::OB_MAX_TZ_NAME_LEN]; @@ -630,10 +638,9 @@ typedef common::ObLinkHashMap ObTZ class ObTZInfoMap { public: - ObTZInfoMap() : inited_(false), id_map_(), name_map_() {} + ObTZInfoMap() : inited_(false), id_map_(&id_map_buf_), name_map_(&name_map_buf_) {} ~ObTZInfoMap() {} int init(const lib::ObMemAttr &attr); - int reset(); void destroy(); int print_tz_info_map(); bool is_inited() { return inited_; } @@ -641,11 +648,12 @@ public: int get_tz_info_by_name(const common::ObString &tz_name, ObTimeZoneInfoPos &tz_info_by_name); int get_tz_info_by_id(const int64_t tz_id, ObTimeZoneInfoPos *&tz_info_by_id); int get_tz_info_by_name(const common::ObString &tz_name, ObTimeZoneInfoPos *&tz_info_by_name); - void free_tz_info_pos(ObTimeZoneInfoPos *&tz_info) { id_map_.revert(tz_info); tz_info = NULL; } public: bool inited_; - ObTZInfoIDPosMap id_map_; // tz_id => ObTimeZoneInfoPos - ObTZInfoNameIDMap name_map_; // tz_name => tz_id + ObTZInfoIDPosMap *id_map_; + ObTZInfoNameIDMap *name_map_; + ObTZInfoIDPosMap id_map_buf_; // tz_id => ObTimeZoneInfoPos + ObTZInfoNameIDMap name_map_buf_; // tz_name => tz_id private: DISALLOW_COPY_AND_ASSIGN(ObTZInfoMap); diff --git a/deps/oblib/src/rpc/frame/ob_req_queue_thread.cpp b/deps/oblib/src/rpc/frame/ob_req_queue_thread.cpp index aeb65b78c..973d8cdd7 100644 --- a/deps/oblib/src/rpc/frame/ob_req_queue_thread.cpp +++ b/deps/oblib/src/rpc/frame/ob_req_queue_thread.cpp @@ -34,22 +34,22 @@ using namespace oceanbase::lib; ObReqQueue::ObReqQueue(int capacity) : wait_finish_(true), push_worker_count_(0), - capacity_(capacity), queue_(), qhandler_(NULL), host_() { + queue_.set_limit(capacity); } ObReqQueue::~ObReqQueue() { LOG_INFO("begin to destroy queue", K(queue_.size())); - queue_.destroy(); } int ObReqQueue::init(const int64_t tenant_id) { - return queue_.init(capacity_, "ReqQueue", tenant_id); + UNUSED(tenant_id); + return OB_SUCCESS; } void ObReqQueue::set_qhandler(ObiReqQHandler *qhandler) @@ -74,18 +74,18 @@ bool ObReqQueue::push(ObRequest *req, int max_queue_len, bool block) } if (bret) { - bret = OB_LIKELY(OB_SUCCESS == queue_.push(req)); + bret = OB_LIKELY(OB_SUCCESS == queue_.push(req, 0)); } return bret; } oceanbase::rpc::ObRequest *ObReqQueue::pop() { - void *task = NULL; + ObLink *task = NULL; int64_t timeout = 0; ObRequest *req = NULL; if (queue_.size() > 0 && OB_LIKELY(OB_SUCCESS == queue_.pop(task, timeout)) && OB_NOT_NULL(task)) { - req = reinterpret_cast(task); + req = static_cast(task); } return req; } @@ -95,7 +95,7 @@ void ObReqQueue::set_host(const ObAddr &host) host_ = host; } -int ObReqQueue::process_task(void *task) +int ObReqQueue::process_task(ObLink *task) { int ret = OB_SUCCESS; if (OB_ISNULL(task) || OB_ISNULL(qhandler_)) { @@ -106,7 +106,7 @@ int ObReqQueue::process_task(void *task) param.set_mem_attr(common::OB_SERVER_TENANT_ID, ObModIds::OB_ROOT_CONTEXT, ObCtxIds::WORK_AREA) .set_properties(USE_TL_PAGE_OPTIONAL); CREATE_WITH_TEMP_CONTEXT(param) { - ObRequest *req = reinterpret_cast(task); + ObRequest *req = static_cast(task); // init trace id if (ObRequest::OB_RPC == req->get_type()) { @@ -164,7 +164,7 @@ void ObReqQueue::loop() { int ret = OB_SUCCESS; int64_t timeout = 3000 * 1000; - void *task = NULL; + ObLink *task = NULL; if (OB_ISNULL(qhandler_)) { ret = OB_INVALID_ARGUMENT; LOG_ERROR("invalid argument", K(qhandler_)); diff --git a/deps/oblib/src/rpc/frame/ob_req_queue_thread.h b/deps/oblib/src/rpc/frame/ob_req_queue_thread.h index eca7639bd..6e9b6a285 100644 --- a/deps/oblib/src/rpc/frame/ob_req_queue_thread.h +++ b/deps/oblib/src/rpc/frame/ob_req_queue_thread.h @@ -15,7 +15,7 @@ #include "lib/ob_define.h" -#include "lib/queue/ob_lighty_queue.h" +#include "lib/queue/ob_priority_queue.h" #include "lib/profile/ob_trace_id.h" #include "lib/net/ob_addr.h" #include "rpc/frame/obi_req_qhandler.h" @@ -66,7 +66,7 @@ public: } private: - int process_task(void *task); + int process_task(ObLink *task); DISALLOW_COPY_AND_ASSIGN(ObReqQueue); @@ -74,8 +74,7 @@ protected: bool wait_finish_; int push_worker_count_; - int64_t capacity_; - common::ObLightyQueue queue_; + common::ObPriorityQueue<1> queue_; ObiReqQHandler *qhandler_; static const int64_t MAX_PACKET_SIZE = 2 * 1024 * 1024L; // 2M diff --git a/deps/oblib/src/rpc/pnio/interface/group.c b/deps/oblib/src/rpc/pnio/interface/group.c index 6b760a262..e58527f7b 100644 --- a/deps/oblib/src/rpc/pnio/interface/group.c +++ b/deps/oblib/src/rpc/pnio/interface/group.c @@ -240,7 +240,7 @@ PN_API int pn_provision(int listen_id, int gid, int thread_count) pn_t* pn = pn_create(listen_id, gid, count); if (NULL == pn) { err = ENOMEM; - } else if (0 != (err = ob_pthread_create(&pn->pd, NULL, pn_thread_func, pn))) { + } else if (0 != (err = ob_pthread_create(&pn->pd, pn_thread_func, pn))) { pn_destroy(pn); } else { pn->has_stopped_ = false; @@ -384,7 +384,8 @@ PN_API void pn_wait(uint64_t gid) for (int tid = 0; tid < pgrp->count; tid++) { pn_t *pn = get_pn_for_send(pgrp, tid); if (!pn->has_stopped_) { - pthread_join(pn->pd, NULL); + ob_pthread_join(pn->pd); + pn->pd = NULL; pn->has_stopped_ = true; } } diff --git a/deps/oblib/src/rpc/pnio/interface/group.h b/deps/oblib/src/rpc/pnio/interface/group.h index 6d31c6e2d..a3051956f 100644 --- a/deps/oblib/src/rpc/pnio/interface/group.h +++ b/deps/oblib/src/rpc/pnio/interface/group.h @@ -40,7 +40,7 @@ typedef struct pn_grp_comm_t #define PN_COMM \ bool is_stop_; \ bool has_stopped_; \ - pthread_t pd; \ + void *pd; \ int accept_qfd; \ int gid; \ int tid; \ diff --git a/deps/oblib/src/rpc/pnio/r0/log.h b/deps/oblib/src/rpc/pnio/r0/log.h index 6c8cc22bb..f341151be 100644 --- a/deps/oblib/src/rpc/pnio/r0/log.h +++ b/deps/oblib/src/rpc/pnio/r0/log.h @@ -7,8 +7,8 @@ typedef void (*log_func_t)(int level, const char *file, int line, const char *function, const char *fmt, va_list ap); extern void do_log(int level, const char* file, int line, const char* func, const char* format, ...) __attribute__((format(printf, 5, 6))); extern void ob_set_thread_name(const char* type); -extern int ob_pthread_create(pthread_t *thread, const pthread_attr_t *attr, - void *(*start_routine) (void *), void *arg); +extern int ob_pthread_create(void **ptr, void *(*start_routine) (void *), void *arg); +extern void ob_pthread_join(void *ptr); extern int64_t ob_update_loop_ts(); extern log_func_t g_log_func; extern int g_log_level; @@ -22,11 +22,6 @@ void ob_set_thread_name(const char* type) { prctl(PR_SET_NAME, type); } -int ob_pthread_create(pthread_t *thread, const pthread_attr_t *attr, - void *(*start_routine) (void *), void *arg) -{ - return pthread_create(thread, attr, start_routine, arg); -} int64_t ob_update_loop_ts() { return 0; diff --git a/deps/oblib/unittest/lib/task/test_timer.cpp b/deps/oblib/unittest/lib/task/test_timer.cpp index eff594470..dd0eff48f 100644 --- a/deps/oblib/unittest/lib/task/test_timer.cpp +++ b/deps/oblib/unittest/lib/task/test_timer.cpp @@ -28,14 +28,17 @@ public: void runTimerTask() { + has_run_ = true; running_ = true; ++task_run_count_; - ::usleep(50000);//50ms + ::usleep(exec_time_); running_ = false; } volatile bool running_; int64_t task_run_count_; + int64_t exec_time_ = 50000; // 50ms + bool has_run_ = false; }; TEST(TestTimer, timer_task) @@ -120,6 +123,57 @@ TEST(TestTimer, start_stop) timer.destroy(); } +TEST(TestTimer, task_cancel_wait) +{ + TestTimerTask task; + ObTimer timer; + ASSERT_EQ(OB_SUCCESS, timer.init()); + ASSERT_EQ(OB_SUCCESS, timer.start()); + // cancel from non-running + { + ASSERT_EQ(OB_SUCCESS, timer.schedule(task, 1000000, true)); + ASSERT_EQ(1, timer.get_tasks_num()); + ASSERT_EQ(OB_SUCCESS, timer.cancel_task(task)); + ASSERT_FALSE(timer.task_exist(task)); + // repeat cancel + ASSERT_EQ(OB_SUCCESS, timer.cancel_task(task)); + ASSERT_FALSE(timer.task_exist(task)); + } + // cancel from running + { + task.exec_time_ = 1000000; + int64_t cur_time = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, timer.schedule(task, 0, true)); + usleep(10000); + ASSERT_EQ(OB_SUCCESS, timer.cancel_task(task)); + // repeat cancel + ASSERT_EQ(OB_SUCCESS, timer.cancel_task(task)); + ASSERT_LT(ObTimeUtility::current_time() - cur_time, 1000000); + ASSERT_EQ(OB_SUCCESS, timer.wait_task(task)); + ASSERT_GT(ObTimeUtility::current_time() - cur_time, 1000000); + ASSERT_FALSE(timer.task_exist(task)); + // wait non-exist task + ASSERT_EQ(OB_SUCCESS, timer.wait_task(task)); + ASSERT_TRUE(NULL == timer.running_task_); + ASSERT_TRUE(NULL == timer.uncanceled_task_); + } + // cancel all + { + TestTimerTask task2; + TestTimerTask task3; + task2.exec_time_ = 1000000; + ASSERT_EQ(OB_SUCCESS, timer.schedule(task2, 0, true)); + ASSERT_EQ(OB_SUCCESS, timer.schedule(task3, 1000, true)); + usleep(10000); + timer.cancel_all(); + ASSERT_TRUE(task2.has_run_); + ASSERT_FALSE(task3.has_run_); + } + timer.stop(); + timer.wait(); + timer.destroy(); +} + } // end namespace common } // end namespace oceanbase diff --git a/deps/oblib/unittest/lib/thread/test_threads.cpp b/deps/oblib/unittest/lib/thread/test_threads.cpp index 3e2c97bf5..54c8ec103 100644 --- a/deps/oblib/unittest/lib/thread/test_threads.cpp +++ b/deps/oblib/unittest/lib/thread/test_threads.cpp @@ -96,7 +96,27 @@ TEST(TestThreads, DynamicThread) ASSERT_EQ(4, starts); ASSERT_EQ(4, exits); } +extern "C" { +int ob_pthread_create(void **ptr, void *(*start_routine) (void *), void *arg); +void ob_pthread_join(void *ptr); +pthread_t ob_pthread_get_pth(void *ptr); +} +void *my_func(void *arg) +{ + pthread_t *pth = (pthread_t*)arg; + *pth = pthread_self(); + return NULL; +} +TEST(TestThreads, ObPthread) +{ + pthread_t pth = 0; + void *tid = NULL; + ASSERT_EQ(0, ob_pthread_create(&tid, my_func, &pth)); + sleep(1); + ASSERT_EQ(ob_pthread_get_pth(tid), pth); + ob_pthread_join(tid); +} int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); diff --git a/deps/ussl-hook/ussl-loop.c b/deps/ussl-hook/ussl-loop.c index a1ea2eee0..181fe97a4 100644 --- a/deps/ussl-hook/ussl-loop.c +++ b/deps/ussl-hook/ussl-loop.c @@ -30,10 +30,10 @@ uloop_t global_ussl_loop_struct; static int ussl_has_listened = 0; static ussl_sf_t acceptfd_fty; static ussl_sf_t clientfd_fty; -static pthread_t ussl_bg_thread_id; +static void *ussl_bg_thread_id; -int ob_pthread_create(pthread_t *thread, const pthread_attr_t *attr, - void *(*start_routine) (void *), void *arg); +int ob_pthread_create(void **ptr, void *(*start_routine) (void *), void *arg); +void ob_pthread_join(void *ptr); void ob_set_thread_name(const char* type); static int uloop_init(uloop_t *l) @@ -102,7 +102,7 @@ int ussl_init_bg_thread() if (0 == ret) { if (0 != uloop_init(&global_ussl_loop_struct)) { ussl_log_error("initialize uloop failed.") - } else if (0 != ob_pthread_create(&ussl_bg_thread_id, NULL, bg_thread_func, NULL)) { + } else if (0 != ob_pthread_create(&ussl_bg_thread_id, bg_thread_func, NULL)) { ret = EIO; ussl_log_error("create background thread failed, errno:%d", errno); } else { @@ -116,7 +116,8 @@ extern int is_ussl_bg_thread_started; void ussl_wait_bg_thread() { if (ATOMIC_LOAD(&is_ussl_bg_thread_started)) { - pthread_join(ussl_bg_thread_id, NULL); + ob_pthread_join(ussl_bg_thread_id); + ussl_bg_thread_id = NULL; ATOMIC_STORE(&is_ussl_bg_thread_started, 0); } } diff --git a/mittest/logservice/test_ob_simple_log_disk_mgr.cpp b/mittest/logservice/test_ob_simple_log_disk_mgr.cpp index ae238c147..ffdbe4d7c 100644 --- a/mittest/logservice/test_ob_simple_log_disk_mgr.cpp +++ b/mittest/logservice/test_ob_simple_log_disk_mgr.cpp @@ -389,6 +389,47 @@ TEST_F(TestObSimpleLogDiskMgr, overshelling) } +TEST_F(TestObSimpleLogDiskMgr, hidden_sys) +{ + SET_CASE_LOG_FILE(TEST_NAME, "hidden_sys"); + int server_idx = 0; + PalfEnv *palf_env = NULL; + int64_t leader_idx = 0; + int64_t id = ATOMIC_AAF(&palf_id_, 1); + int64_t total_used_size = 0, total_size = 0; + share::SCN create_scn = share::SCN::base_scn(); + EXPECT_EQ(OB_SUCCESS, get_palf_env(0, palf_env)); + { + PalfHandleImplGuard leader; + EXPECT_EQ(OB_SUCCESS, palf_env->get_stable_disk_usage(total_used_size, total_size)); + EXPECT_EQ(0, total_used_size); + EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, create_scn, leader_idx, leader)); + EXPECT_EQ(OB_SUCCESS, palf_env->get_stable_disk_usage(total_used_size, total_size)); + EXPECT_NE(0, total_used_size); + EXPECT_EQ(OB_NOT_SUPPORTED, update_disk_options(0)); + } + EXPECT_EQ(OB_SUCCESS, delete_paxos_group(id)); + EXPECT_EQ(OB_SUCCESS, update_disk_options(0)); + // tenant unit中记录的disk_opts直接生效 + PalfDiskOptions disk_opts; + EXPECT_EQ(OB_SUCCESS, get_disk_options(0, disk_opts)); + EXPECT_EQ(0, disk_opts.log_disk_usage_limit_size_); + EXPECT_EQ(PalfDiskOptionsWrapper::Status::SHRINKING_STATUS, + palf_env->palf_env_impl_.disk_options_wrapper_.status_); + usleep(palf::BlockGCTimerTask::BLOCK_GC_TIMER_INTERVAL_MS + 5*10000); + usleep(ObLooper::INTERVAL_US * 2); + EXPECT_EQ(PalfDiskOptionsWrapper::Status::NORMAL_STATUS, + palf_env->palf_env_impl_.disk_options_wrapper_.status_); + EXPECT_EQ(OB_SUCCESS, palf_env->get_stable_disk_usage(total_used_size, total_size)); + EXPECT_EQ(0, total_used_size); + EXPECT_EQ(0, total_size); + EXPECT_EQ(OB_SUCCESS, update_disk_options(8)); + PalfHandleImplGuard leader; + EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, create_scn, leader_idx, leader)); + EXPECT_EQ(OB_SUCCESS, palf_env->get_stable_disk_usage(total_used_size, total_size)); + EXPECT_NE(0, total_used_size); +} + } // namespace unittest } // namespace oceanbase diff --git a/mittest/mtlenv/CMakeLists.txt b/mittest/mtlenv/CMakeLists.txt index 843ca1e61..f13e94963 100644 --- a/mittest/mtlenv/CMakeLists.txt +++ b/mittest/mtlenv/CMakeLists.txt @@ -1,7 +1,7 @@ sql_unittest(test_session_serde) #ob_unittest(test_ob_election_priority) storage_unittest(test_tx_data_table) -storage_unittest(test_multi_tenant test_multi_tenant.cpp) +#storage_unittest(test_multi_tenant test_multi_tenant.cpp) storage_unittest(test_buffer_ctx_node test_buffer_ctx_node.cpp) add_subdirectory(storage) diff --git a/mittest/mtlenv/mock_tenant_module_env.h b/mittest/mtlenv/mock_tenant_module_env.h index a437045d0..eba7d5f63 100644 --- a/mittest/mtlenv/mock_tenant_module_env.h +++ b/mittest/mtlenv/mock_tenant_module_env.h @@ -32,6 +32,8 @@ #include "observer/omt/ob_tenant.h" #include "observer/omt/ob_worker_processor.h" #include "observer/omt/ob_tenant_meta.h" +#include "observer/omt/ob_multi_tenant.h" +#include "observer/omt/ob_tenant_srs.h" #include "share/allocator/ob_tenant_mutil_allocator_mgr.h" #include "share/ob_alive_server_tracer.h" #include "share/ob_device_manager.h" @@ -600,7 +602,7 @@ int MockTenantModuleEnv::init_before_start_mtl() STORAGE_LOG(WARN, "fail to init env", K(ret)); } else if (OB_FAIL(session_mgr_.init())) { STORAGE_LOG(WARN, "fail to init env", K(ret)); - } else if (OB_FAIL(ObVirtualTenantManager::get_instance().init(10))) { + } else if (OB_FAIL(ObVirtualTenantManager::get_instance().init())) { STORAGE_LOG(WARN, "fail to init env", K(ret)); } else if (OB_FAIL(OB_SERVER_BLOCK_MGR.init(THE_IO_DEVICE, 2 * 1024 * 1024))) { STORAGE_LOG(WARN, "fail to init env", K(ret)); @@ -638,8 +640,6 @@ int MockTenantModuleEnv::init_before_start_mtl() STORAGE_LOG(ERROR, "init timer fail", KR(ret)); } else if (OB_FAIL(TG_START(lib::TGDefIDs::MemDumpTimer))) { STORAGE_LOG(ERROR, "init memory dump timer fail", KR(ret)); - } else if (OB_FAIL(ObOptStatMonitorManager::get_instance().init(&sql_proxy_))) { - STORAGE_LOG(ERROR, "failed to init opt stat monitor manager", KR(ret)); } else { obrpc::ObRpcNetHandler::CLUSTER_ID = 1; oceanbase::palf::election::INIT_TS = 1; @@ -665,6 +665,7 @@ int MockTenantModuleEnv::init() } else { oceanbase::ObClusterVersion::get_instance().update_data_version(DATA_CURRENT_VERSION); MTL_BIND(ObTenantIOManager::mtl_init, ObTenantIOManager::mtl_destroy); + MTL_BIND2(mtl_new_default, omt::ObSharedTimer::mtl_init, omt::ObSharedTimer::mtl_start, omt::ObSharedTimer::mtl_stop, omt::ObSharedTimer::mtl_wait, mtl_destroy_default); MTL_BIND2(mtl_new_default, ObTenantSchemaService::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); MTL_BIND2(mtl_new_default, ObStorageLogger::mtl_init, ObStorageLogger::mtl_start, ObStorageLogger::mtl_stop, ObStorageLogger::mtl_wait, mtl_destroy_default); MTL_BIND2(ObTenantMetaMemMgr::mtl_new, mtl_init_default, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); @@ -699,6 +700,7 @@ int MockTenantModuleEnv::init() MTL_BIND2(server_obj_pool_mtl_new, nullptr, nullptr, nullptr, nullptr, server_obj_pool_mtl_destroy); MTL_BIND(ObTenantSQLSessionMgr::mtl_init, ObTenantSQLSessionMgr::mtl_destroy); MTL_BIND2(mtl_new_default, ObRebuildService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); + MTL_BIND2(mtl_new_default, omt::ObTenantSrs::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); } if (OB_FAIL(ret)) { diff --git a/mittest/multi_replica/env/ob_multi_replica_test_base.cpp b/mittest/multi_replica/env/ob_multi_replica_test_base.cpp index c9acc4a25..938673ccc 100644 --- a/mittest/multi_replica/env/ob_multi_replica_test_base.cpp +++ b/mittest/multi_replica/env/ob_multi_replica_test_base.cpp @@ -507,7 +507,6 @@ int ObMultiReplicaTestBase::start() GCONF.enable_perf_event = false; GCONF.enable_sql_audit = true; GCONF.enable_record_trace_log = false; - GMEMCONF.set_server_memory_limit(10 * 1024 * 1024 * 1024ul); int32_t log_level; bool change_log_level = false; diff --git a/mittest/simple_server/env/ob_simple_cluster_test_base.cpp b/mittest/simple_server/env/ob_simple_cluster_test_base.cpp index 5722b74c3..99dee8f12 100644 --- a/mittest/simple_server/env/ob_simple_cluster_test_base.cpp +++ b/mittest/simple_server/env/ob_simple_cluster_test_base.cpp @@ -161,7 +161,6 @@ int ObSimpleClusterTestBase::start() GCONF.enable_perf_event = false; GCONF.enable_sql_audit = true; GCONF.enable_record_trace_log = false; - GMEMCONF.set_server_memory_limit(10 * 1024 * 1024 * 1024ul); int32_t log_level; bool change_log_level = false; diff --git a/mittest/simple_server/test_observer_expand_shrink.cpp b/mittest/simple_server/test_observer_expand_shrink.cpp index 79b3b2be5..a15c60bea 100644 --- a/mittest/simple_server/test_observer_expand_shrink.cpp +++ b/mittest/simple_server/test_observer_expand_shrink.cpp @@ -210,105 +210,6 @@ TEST_F(ObserverExpandShink, resize_tenant_log_disk) // EXPECT_NE(0, GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_); // SERVER_LOG(INFO, "create_tenant_after_restart trace", KPC(GCTX.log_block_mgr_)); //} - -TEST_F(ObserverExpandShink, test_hidden_sys_tenant) -{ - omt::ObMultiTenant *omt = GCTX.omt_; - bool remove_tenant_succ = false; - int64_t log_disk_size_in_use = GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_; - - share::TenantUnits units; - EXPECT_EQ(OB_SUCCESS, omt->get_tenant_units(units, false)); - EXPECT_EQ(false, units.empty()); - int64_t origin_sys_log_disk_size = 0; - int64_t hidden_sys_log_disk_size = 0; - for (int i = 0; i < units.count(); i++) { - if (OB_SYS_TENANT_ID == units[i].tenant_id_) { - origin_sys_log_disk_size = units[i].config_.log_disk_size(); - EXPECT_EQ(OB_SUCCESS, omt->convert_real_to_hidden_sys_tenant()); - } - } - - EXPECT_EQ(OB_SUCCESS, omt->get_tenant_units(units, false)); - EXPECT_EQ(false, units.empty()); - ObUnitInfoGetter::ObTenantConfig sys_unit_config; - for (int i = 0; i < units.count(); i++) { - if (OB_SYS_TENANT_ID == units[i].tenant_id_) { - hidden_sys_log_disk_size = units[i].config_.log_disk_size(); - sys_unit_config = units[i]; - sys_unit_config.config_.resource_.log_disk_size_ = origin_sys_log_disk_size + 512*1024*1024; - } - } - CLOG_LOG(INFO, "runlin trace convert_hidden_to_real_sys_tenant", K(log_disk_size_in_use), KPC(GCTX.log_block_mgr_), K(origin_sys_log_disk_size), K(hidden_sys_log_disk_size)); - - // 类型转换后，sys租户的unit规格可能会发生变化（隐藏sys租户的规格会被重新生成，具体逻辑参见gen_sys_tenant_unit_config） - EXPECT_EQ(log_disk_size_in_use-origin_sys_log_disk_size+hidden_sys_log_disk_size, - GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_); - log_disk_size_in_use = GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_; - EXPECT_EQ(OB_SUCCESS, omt->convert_hidden_to_real_sys_tenant(sys_unit_config)); - EXPECT_EQ(log_disk_size_in_use-hidden_sys_log_disk_size+sys_unit_config.config_.log_disk_size(), - GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_); - log_disk_size_in_use = GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_; - CLOG_LOG(INFO, "runlin trace after convert_hidden_to_real_sys_tenant", K(log_disk_size_in_use), KPC(GCTX.log_block_mgr_), - K(origin_sys_log_disk_size)); - int64_t new_sys_log_disk_size = sys_unit_config.config_.log_disk_size()+512*1024*1024; - omt::ObTenant *tenant = nullptr; - EXPECT_EQ(OB_SUCCESS, omt->get_tenant(OB_SYS_TENANT_ID, tenant)); - share::ObUnitInfoGetter::ObTenantConfig unit_config = tenant->get_unit(); - unit_config.config_.resource_.log_disk_size_ = new_sys_log_disk_size; - // 扩容直接成功 - EXPECT_EQ(OB_SUCCESS, omt->update_tenant_unit(unit_config)); - unit_config = tenant->get_unit(); - EXPECT_EQ(unit_config.config_.log_disk_size(), new_sys_log_disk_size); - EXPECT_EQ(log_disk_size_in_use-sys_unit_config.config_.log_disk_size()+new_sys_log_disk_size, - GCTX.log_block_mgr_->min_log_disk_size_for_all_tenants_); - CLOG_LOG(INFO, "runlin trace after convert_real_to_hidden_sys_tenant", K(log_disk_size_in_use), KPC(GCTX.log_block_mgr_), - K(origin_sys_log_disk_size)); -} - - -// don't has any case after this. -TEST_F(ObserverExpandShink, paralle_set) -{ - omt::ObTenantNodeBalancer::get_instance().refresh_interval_ = 1000 * 1000 * 1000; - sleep(3); - LOG_INFO("start to test parallel_set"); - share::ObTenantSwitchGuard tguard; - ASSERT_EQ(OB_SUCCESS, tguard.switch_to(1)); - ObLogService *log_service = MTL(ObLogService*); - palf::PalfOptions opts; - ASSERT_NE(nullptr, log_service); - EXPECT_EQ(OB_SUCCESS, log_service->get_palf_options(opts)); - EXPECT_EQ(OB_INVALID_ARGUMENT, log_service->update_log_disk_usage_limit_size(1000)); - opts.disk_options_.log_disk_utilization_limit_threshold_ = 10; - opts.disk_options_.log_disk_utilization_threshold_ = 11; - EXPECT_EQ(OB_INVALID_ARGUMENT, log_service->palf_env_->update_options(opts)); - { - const int64_t new_log_disk_size = opts.disk_options_.log_disk_usage_limit_size_*50/100; - EXPECT_EQ(OB_SUCCESS, log_service->update_log_disk_usage_limit_size(new_log_disk_size)); - sleep(1); - EXPECT_EQ(log_service->palf_env_->palf_env_impl_.disk_options_wrapper_.disk_opts_for_stopping_writing_.log_disk_usage_limit_size_, - new_log_disk_size); - } - { - const int64_t count1 = 10000; - const int64_t count2 = 9000; - std::atomic new_log_disk_size(1*1024*1024*1024); - auto update_size = [&log_service, &new_log_disk_size](int64_t count) - { - for (int64_t i = 0; i < count; i++) { - log_service->update_log_disk_usage_limit_size(++new_log_disk_size); - } - }; - std::thread t1(update_size, count1); - std::thread t2(update_size, count2); - t1.join(); - t2.join(); - EXPECT_EQ(OB_SUCCESS, log_service->get_palf_options(opts)); - EXPECT_EQ(opts.disk_options_.log_disk_usage_limit_size_, 1*1024*1024*1024+count1+count2); - } -} - } // end unittest } // end oceanbase diff --git a/mittest/simple_server/test_transfer_task_operator.cpp b/mittest/simple_server/test_transfer_task_operator.cpp index 9302b447f..45164bb03 100644 --- a/mittest/simple_server/test_transfer_task_operator.cpp +++ b/mittest/simple_server/test_transfer_task_operator.cpp @@ -191,16 +191,16 @@ TEST_F(TestTransferTaskOperator, test_basic_func) // ObTrasnferTaskComment ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::EMPTY_COMMENT), "")); - ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::WAIT_FOR_MEMBER_LIST), "WAIT FOR MEMBER LIST TO BE SAME")); - ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::TASK_COMPLETED_AS_NO_VALID_PARTITION), "TASK COMPLETED AS NO VALID PARTITION")); - ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::TASK_CANCELED), "TASK CANCELED")); - ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::MAX_COMMENT), "UNKNOW")); + ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::WAIT_FOR_MEMBER_LIST), "Wait for member list to be same")); + ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::TASK_COMPLETED_AS_NO_VALID_PARTITION), "Task completed as no valid partition")); + ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::TASK_CANCELED), "Task canceled")); + ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(ObTransferTaskComment::MAX_COMMENT), "Unknow")); - ASSERT_TRUE(ObTransferTaskComment::WAIT_FOR_MEMBER_LIST == str_to_transfer_task_comment("WAIT FOR MEMBER LIST TO BE SAME")); - ASSERT_TRUE(ObTransferTaskComment::TASK_COMPLETED_AS_NO_VALID_PARTITION == str_to_transfer_task_comment("TASK COMPLETED AS NO VALID PARTITION")); - ASSERT_TRUE(ObTransferTaskComment::TASK_CANCELED == str_to_transfer_task_comment("TASK CANCELED")); + ASSERT_TRUE(ObTransferTaskComment::WAIT_FOR_MEMBER_LIST == str_to_transfer_task_comment("Wait for member list to be same")); + ASSERT_TRUE(ObTransferTaskComment::TASK_COMPLETED_AS_NO_VALID_PARTITION == str_to_transfer_task_comment("Task completed as no valid partition")); + ASSERT_TRUE(ObTransferTaskComment::TASK_CANCELED == str_to_transfer_task_comment("Task canceled")); ASSERT_TRUE(ObTransferTaskComment::EMPTY_COMMENT == str_to_transfer_task_comment("")); - ASSERT_TRUE(ObTransferTaskComment::MAX_COMMENT == str_to_transfer_task_comment("UNKNOW")); + ASSERT_TRUE(ObTransferTaskComment::MAX_COMMENT == str_to_transfer_task_comment("Unknow")); ASSERT_TRUE(ObTransferTaskComment::MAX_COMMENT == str_to_transfer_task_comment("XXXXX")); } @@ -218,6 +218,11 @@ TEST_F(TestTransferTaskOperator, test_operator) ASSERT_EQ(OB_ENTRY_NOT_EXIST, ObTransferTaskOperator::get_all_task_status(sql_proxy, tenant_id_, task_status)); ASSERT_TRUE(task_status.empty()); + // get_max_task_id_from_history when history is empty + ObTransferTaskID max_task_id; + ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get_max_task_id_from_history(sql_proxy, tenant_id_, max_task_id)); + ASSERT_TRUE(!max_task_id.is_valid()); + // insert ObTransferTask other_task; ObTransferTaskID other_task_id(222); @@ -235,7 +240,7 @@ TEST_F(TestTransferTaskOperator, test_operator) ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(sql_proxy, tenant_id_, task_id_, false, task)); ASSERT_TRUE(task.get_task_id() == task_id_); ASSERT_TRUE(task.get_tablet_list().empty()); - ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(task.get_comment()), "TASK CANCELED")); + ASSERT_TRUE(0 == strcmp(transfer_task_comment_to_str(task.get_comment()), "Task canceled")); LOG_INFO("get from table", K(task)); task.reset(); ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get(sql_proxy, tenant_id_, task_id_, true, task)); @@ -375,6 +380,10 @@ TEST_F(TestTransferTaskOperator, test_operator) ASSERT_TRUE(0 == compare(lock_conflict_part_list_str, history_lock_conflict_part_list_str)); ASSERT_EQ(OB_ENTRY_NOT_EXIST, ObTransferTaskOperator::get_history_task(sql_proxy, tenant_id_, ObTransferTaskID(555), history_task, create_time, finish_time)); + + max_task_id.reset(); + ASSERT_EQ(OB_SUCCESS, ObTransferTaskOperator::get_max_task_id_from_history(sql_proxy, tenant_id_, max_task_id)); + ASSERT_TRUE(max_task_id == task_id_); } } // namespace share diff --git a/src/logservice/libobcdc/src/ob_log_timezone_info_getter.cpp b/src/logservice/libobcdc/src/ob_log_timezone_info_getter.cpp index b1e1647fd..5ace3bc43 100644 --- a/src/logservice/libobcdc/src/ob_log_timezone_info_getter.cpp +++ b/src/logservice/libobcdc/src/ob_log_timezone_info_getter.cpp @@ -547,7 +547,7 @@ int ObLogTimeZoneInfoGetter::export_timezone_info_(common::ObTZInfoMap &tz_info_ return OB_SUCCESS == tmp_ret; }; - if (OB_FAIL(tz_info_map.id_map_.for_each(tz_info_op))) { + if (OB_FAIL(tz_info_map.id_map_->for_each(tz_info_op))) { LOG_ERROR("generate ObRequestTZInfoResult failed", KR(ret)); } else if (0 >= tz_info_res.tz_array_.count()) { // skip empty tz_info_result. @@ -599,7 +599,7 @@ int ObLogTimeZoneInfoGetter::import_timezone_info_(common::ObTZInfoMap &tz_info_ for (int idx= 0; OB_SUCC(ret) && idx < tz_info_cnt; idx++) { ObTimeZoneInfoPos &new_tz_info = tz_info_res.tz_array_.at(idx); - if (OB_FAIL(tz_info_map.id_map_.get(new_tz_info.get_tz_id(), stored_tz_info))) { + if (OB_FAIL(tz_info_map.id_map_->get(new_tz_info.get_tz_id(), stored_tz_info))) { if (OB_ENTRY_NOT_EXIST == ret) { ret = OB_SUCCESS; } else { @@ -619,7 +619,7 @@ int ObLogTimeZoneInfoGetter::import_timezone_info_(common::ObTZInfoMap &tz_info_ } if (OB_NOT_NULL(stored_tz_info)) { - tz_info_map.id_map_.revert(stored_tz_info); + tz_info_map.id_map_->revert(stored_tz_info); stored_tz_info = nullptr; } } // end for diff --git a/src/logservice/ob_server_log_block_mgr.cpp b/src/logservice/ob_server_log_block_mgr.cpp index bbd1077a2..354ffcfdd 100644 --- a/src/logservice/ob_server_log_block_mgr.cpp +++ b/src/logservice/ob_server_log_block_mgr.cpp @@ -362,7 +362,7 @@ int ObServerLogBlockMgr::update_tenant(const int64_t old_log_disk_size, if (IS_NOT_INIT) { ret = OB_NOT_INIT; CLOG_LOG(WARN, "ObServerLogBlockMGR is not inited", K(old_log_disk_size), K(new_log_disk_size), KPC(this)); - } else if (old_log_disk_size <= 0 || new_log_disk_size <= 0 || OB_ISNULL(log_service)) { + } else if (old_log_disk_size < 0 || new_log_disk_size < 0 || OB_ISNULL(log_service)) { ret = OB_INVALID_ARGUMENT; CLOG_LOG(WARN, "invalid argument", K(old_log_disk_size), K(new_log_disk_size), KP(log_service), KPC(this)); } else if (OB_FAIL(log_service->get_palf_stable_disk_usage(used_log_disk_size, palf_log_disk_size))) { @@ -387,7 +387,12 @@ int ObServerLogBlockMgr::update_tenant(const int64_t old_log_disk_size, // 2. if 'palf_log_disk_size' which get from palf is 50G, we think palf has been in normal status. and we will // construct 'new_unit' with 80G because 'palf_log_disk_size' is smaller than new log disk size(80G), but udpate // palf with 80G. - } else if (FALSE_IT(can_update_log_disk_size_with_expected_log_disk = (new_log_disk_size >= palf_log_disk_size))) { + // + // NB: when new_log_disk_size is zero(means convert real sys tenant to hidden sys tenant), need make allowed_new_log_disk_size + // to zero directlly, otherwise, there is no chance to update log disk size to zero because hidden sys tenant is invisible + // for ObTenantNodeBalancer, and min_log_disk_size_for_all_tenants_ can not be reduce. + } else if (FALSE_IT(can_update_log_disk_size_with_expected_log_disk = + (new_log_disk_size >= palf_log_disk_size || 0 == new_log_disk_size))) { // For expanding log disk, we can update 'allowed_new_log_disk_size' to 'new_log_disk_size' directlly. } else if (can_update_log_disk_size_with_expected_log_disk && FALSE_IT(allowed_new_log_disk_size = new_log_disk_size)) { // For shrinking log disk, we still update log disk size of 'new_unit' to 'old_log_disk_size'. diff --git a/src/logservice/palf/palf_env_impl.cpp b/src/logservice/palf/palf_env_impl.cpp index 83cdf9cdd..2f38e334d 100644 --- a/src/logservice/palf/palf_env_impl.cpp +++ b/src/logservice/palf/palf_env_impl.cpp @@ -700,19 +700,19 @@ int PalfEnvImpl::try_recycle_blocks() total_size_to_stop_write * disk_opts_for_stopping_writing.log_disk_utilization_limit_threshold_ / 100LL; const bool need_recycle = - usable_disk_size_to_recycle_blocks > total_used_size_byte ? false : true; + usable_disk_size_to_recycle_blocks >= total_used_size_byte ? false : true; const bool is_shrinking = disk_options_wrapper_.is_shrinking(); // Assume that, recycle speed is higher than write speed, therefor, the abnormal case // is that, after each 'recycle_blocks_', the 'total_used_size_byte' is one PALF_BLOCK_SIZE // more than 'usable_disk_size'. const bool curr_diskspace_enough = - usable_disk_limit_size_to_stop_writing > total_used_size_byte ? true : false; + usable_disk_limit_size_to_stop_writing >= total_used_size_byte ? true : false; constexpr int64_t MB = 1024 * 1024LL; const int64_t print_error_log_disk_size = disk_opts_for_stopping_writing.log_disk_usage_limit_size_ * disk_opts_for_stopping_writing.log_disk_utilization_threshold_ / 100LL; const bool need_print_error_log = - print_error_log_disk_size > total_used_size_byte ? false : true; + print_error_log_disk_size >= total_used_size_byte ? false : true; // step1. change SHRINKING_STATUS to normal // 1. when there is no possibility to stop writing, @@ -720,7 +720,7 @@ int PalfEnvImpl::try_recycle_blocks() bool has_recycled = false; int64_t oldest_palf_id = INVALID_PALF_ID; if (OB_SUCC(ret) && PalfDiskOptionsWrapper::Status::SHRINKING_STATUS == status) { - if (total_used_size_byte < usable_disk_size_to_recycle_blocks) { + if (total_used_size_byte <= usable_disk_size_to_recycle_blocks) { disk_options_wrapper_.change_to_normal(sequence); PALF_LOG(INFO, "change_to_normal success", K(disk_options_wrapper_), K(total_used_size_byte), K(usable_disk_size_to_recycle_blocks)); diff --git a/src/logservice/palf/palf_options.cpp b/src/logservice/palf/palf_options.cpp index 564f8dd61..61e9838f5 100644 --- a/src/logservice/palf/palf_options.cpp +++ b/src/logservice/palf/palf_options.cpp @@ -46,7 +46,7 @@ bool PalfDiskOptions::is_valid() const { const int64_t MIN_DURATION = 1 * 1000 * 1000L; const int64_t MAX_DURATION = 3 * 24 * 60 * 60 * 1000 * 1000L; - return -1 != log_disk_usage_limit_size_ && log_disk_usage_limit_size_ >= 4 * PALF_PHY_BLOCK_SIZE + return 0 <= log_disk_usage_limit_size_ && 1 <=log_disk_utilization_threshold_ && 100 >= log_disk_utilization_threshold_ && 1 <=log_disk_utilization_limit_threshold_ && 100 >= log_disk_utilization_limit_threshold_ && log_disk_utilization_limit_threshold_ > log_disk_utilization_threshold_ diff --git a/src/observer/CMakeLists.txt b/src/observer/CMakeLists.txt index 483ab4252..30ffe7522 100644 --- a/src/observer/CMakeLists.txt +++ b/src/observer/CMakeLists.txt @@ -118,7 +118,6 @@ ob_set_subtarget(ob_server omt omt/ob_th_worker.cpp omt/ob_worker_processor.cpp omt/ob_multi_tenant_operator.cpp - omt/ob_tenant_srs_mgr.cpp omt/ob_tenant_srs.cpp ) diff --git a/src/observer/dbms_job/ob_dbms_job_master.cpp b/src/observer/dbms_job/ob_dbms_job_master.cpp index fba4ec46e..36e894979 100644 --- a/src/observer/dbms_job/ob_dbms_job_master.cpp +++ b/src/observer/dbms_job/ob_dbms_job_master.cpp @@ -41,7 +41,7 @@ using namespace obrpc; namespace dbms_job { -int ObDBMSJobTask::init(ObLightyQueue *ready_queue) +int ObDBMSJobTask::init(ObDBMSJobQueue *ready_queue) { int ret = OB_SUCCESS; if (inited_) { @@ -110,7 +110,7 @@ void ObDBMSJobTask::runTimerTask() || OB_ISNULL(ready_queue_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("null ptr", K(ret), K(job_key_), K(ready_queue_)); - } else if (OB_FAIL(ready_queue_->push(static_cast(job_key_)))) { + } else if (OB_FAIL(ready_queue_->push(job_key_, 0))) { LOG_WARN("fail to push ready job to queue", K(ret), K(*job_key_)); } else { job_key_ = NULL; @@ -198,7 +198,7 @@ int ObDBMSJobTask::immediately(ObDBMSJobKey *job_key) LOG_WARN("NULL ptr", K(ret), K(job_key), K(ready_queue_)); } else { ObSpinLockGuard guard(lock_); - if (OB_FAIL(ready_queue_->push(static_cast(job_key)))) { + if (OB_FAIL(ready_queue_->push(job_key, 0))) { LOG_WARN("fail to push ready job to queue", K(ret), K(*job_key)); } } @@ -240,10 +240,6 @@ int ObDBMSJobMaster::init(ObISQLClient *sql_client, ObMultiVersionSchemaService *schema_service) { int ret = OB_SUCCESS; - uint64_t ready_queue_size = MAX_READY_JOBS_CAPACITY; - if (is_mini_mode()) { - ready_queue_size *= lib::mini_mode_resource_ratio(); - } if (inited_) { ret = OB_INIT_TWICE; LOG_WARN("dbms job master already inited", K(ret), K(inited_)); @@ -253,8 +249,8 @@ int ObDBMSJobMaster::init(ObISQLClient *sql_client, ) { ret = OB_ERR_UNEXPECTED; LOG_WARN("null ptr", K(ret), K(sql_client), K(schema_service)); - } else if (OB_FAIL(ready_queue_.init(ready_queue_size))) { - LOG_WARN("fail to init ready job queue for all jobs", K(ret)); + } else if (FALSE_IT(ready_queue_.set_limit(MAX_READY_JOBS_CAPACITY))) { + // do-nothing } else if (OB_FAIL(scheduler_task_.init(&ready_queue_))) { LOG_WARN("fail to init ready queue", K(ret)); } else if (OB_FAIL(scheduler_thread_.init(1, 1))) { @@ -324,7 +320,7 @@ int ObDBMSJobMaster::scheduler() LOG_INFO("NOTICE: DBMS Job master start running!", K(ret), K(running_)); lib::set_thread_name("DBMS_JOB_MASTER"); while (OB_SUCC(ret) && !stoped_) { - void* ptr = NULL; + ObLink* ptr = NULL; int64_t timeout = MIN_SCHEDULER_INTERVAL; ObDBMSJobKey *job_key = NULL; if (OB_FAIL(ready_queue_.pop(ptr, timeout))) { diff --git a/src/observer/dbms_job/ob_dbms_job_master.h b/src/observer/dbms_job/ob_dbms_job_master.h index 8cf8372a7..3f54d3e02 100644 --- a/src/observer/dbms_job/ob_dbms_job_master.h +++ b/src/observer/dbms_job/ob_dbms_job_master.h @@ -23,7 +23,6 @@ #include "lib/lock/ob_spin_lock.h" #include "lib/thread/ob_simple_thread_pool.h" #include "lib/task/ob_timer.h" -#include "lib/queue/ob_lighty_queue.h" #include "lib/container/ob_iarray.h" #include "share/schema/ob_schema_service.h" @@ -43,7 +42,7 @@ class ObDBMSJobThread : public ObSimpleThreadPool }; -class ObDBMSJobKey +class ObDBMSJobKey : public common::ObLink { public: ObDBMSJobKey( @@ -52,7 +51,7 @@ public: bool check_job, bool check_new, bool check_new_tenant) : tenant_id_(tenant_id), job_id_(job_id), - execute_at_(execute_at), + execute_at_(execute_at), delay_(delay), check_job_(check_job), check_new_(check_new), @@ -120,7 +119,7 @@ public: virtual ~ObDBMSJobTask() {} - int init(common::ObLightyQueue *ready_queue); + int init(ObDBMSJobQueue *ready_queue); int start(); int stop(); int destroy(); @@ -139,7 +138,7 @@ public: private: bool inited_; ObDBMSJobKey *job_key_; - common::ObLightyQueue *ready_queue_; + ObDBMSJobQueue *ready_queue_; WaitVector wait_vector_; ObSpinLock lock_; @@ -217,7 +216,7 @@ private: obrpc::ObDBMSJobRpcProxy *job_rpc_proxy_; common::ObAddr self_addr_; - common::ObLightyQueue ready_queue_; + ObDBMSJobQueue ready_queue_; ObDBMSJobTask scheduler_task_; ObDBMSJobThread scheduler_thread_; ObDBMSJobUtils job_utils_; diff --git a/src/observer/dbms_job/ob_dbms_job_utils.h b/src/observer/dbms_job/ob_dbms_job_utils.h index 264c080db..4ab893c86 100644 --- a/src/observer/dbms_job/ob_dbms_job_utils.h +++ b/src/observer/dbms_job/ob_dbms_job_utils.h @@ -18,6 +18,7 @@ #include "lib/utility/ob_print_utils.h" #include "lib/mysqlclient/ob_isql_client.h" #include "lib/container/ob_iarray.h" +#include "lib/queue/ob_priority_queue.h" namespace oceanbase { @@ -174,6 +175,7 @@ private: common::ObISQLClient *sql_proxy_; }; +using ObDBMSJobQueue = common::ObPriorityQueue<1>; } } diff --git a/src/observer/dbms_scheduler/ob_dbms_sched_job_master.cpp b/src/observer/dbms_scheduler/ob_dbms_sched_job_master.cpp index 1263f8a8f..af68cb602 100644 --- a/src/observer/dbms_scheduler/ob_dbms_sched_job_master.cpp +++ b/src/observer/dbms_scheduler/ob_dbms_sched_job_master.cpp @@ -56,7 +56,7 @@ int ObDBMSSchedJobTask::init() return ret; } -int ObDBMSSchedJobTask::start(ObLightyQueue *ready_queue) +int ObDBMSSchedJobTask::start(dbms_job::ObDBMSJobQueue *ready_queue) { int ret = OB_SUCCESS; if (!inited_) { @@ -111,7 +111,7 @@ void ObDBMSSchedJobTask::runTimerTask() || OB_ISNULL(ready_queue_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("null ptr", K(ret), K(job_key_), K(ready_queue_)); - } else if (OB_FAIL(ready_queue_->push(static_cast(job_key_)))) { + } else if (OB_FAIL(ready_queue_->push(job_key_, 0))) { LOG_WARN("fail to push ready job to queue", K(ret), K(*job_key_)); } else { job_key_ = NULL; @@ -199,7 +199,7 @@ int ObDBMSSchedJobTask::immediately(ObDBMSSchedJobKey *job_key) LOG_WARN("NULL ptr", K(ret), K(job_key), K(ready_queue_)); } else { ObSpinLockGuard guard(lock_); - if (OB_FAIL(ready_queue_->push(static_cast(job_key)))) { + if (OB_FAIL(ready_queue_->push(job_key, 0))) { LOG_WARN("fail to push ready job to queue", K(ret), K(*job_key)); } } @@ -258,8 +258,8 @@ int ObDBMSSchedJobMaster::init(ObUnitManager *unit_mgr, ) { ret = OB_ERR_UNEXPECTED; LOG_WARN("null ptr", K(ret), K(unit_mgr), K(sql_client), K(schema_service)); - } else if (OB_FAIL(ready_queue_.init(MAX_READY_JOBS_CAPACITY))) { - LOG_WARN("fail to init ready job queue for all jobs", K(ret)); + } else if (FALSE_IT(ready_queue_.set_limit(MAX_READY_JOBS_CAPACITY))) { + // do-nothing } else if (OB_FAIL(scheduler_task_.init())) { LOG_WARN("fail to init ready queue", K(ret)); } else if (OB_FAIL(scheduler_thread_.init(1, 1))) { @@ -332,7 +332,7 @@ int ObDBMSSchedJobMaster::scheduler() LOG_INFO("NOTICE: DBMS Sched Job master start running!", K(ret), K(running_)); lib::set_thread_name("DBMS_SCHEDULER"); while (OB_SUCC(ret) && !stoped_) { - void* ptr = NULL; + ObLink* ptr = NULL; int64_t timeout = MIN_SCHEDULER_INTERVAL; ObDBMSSchedJobKey *job_key = NULL; if (OB_FAIL(ready_queue_.pop(ptr, timeout))) { @@ -507,7 +507,7 @@ int ObDBMSSchedJobMaster::server_random_pick(int64_t tenant_id, ObString &pick_z } } } - } + } } } if (OB_SUCC(ret) && 0 == total_server.count()) { @@ -730,4 +730,3 @@ int ObDBMSSchedJobMaster::register_job( } // end for namespace dbms_scheduler } // end for namespace oceanbase - diff --git a/src/observer/dbms_scheduler/ob_dbms_sched_job_master.h b/src/observer/dbms_scheduler/ob_dbms_sched_job_master.h index 58252edcf..f1a4ecbd1 100644 --- a/src/observer/dbms_scheduler/ob_dbms_sched_job_master.h +++ b/src/observer/dbms_scheduler/ob_dbms_sched_job_master.h @@ -24,12 +24,12 @@ #include "lib/lock/ob_spin_lock.h" #include "lib/thread/ob_simple_thread_pool.h" #include "lib/task/ob_timer.h" -#include "lib/queue/ob_lighty_queue.h" #include "lib/container/ob_iarray.h" #include "share/schema/ob_schema_service.h" #include "share/schema/ob_multi_version_schema_service.h" +#include "observer/dbms_job/ob_dbms_job_utils.h" #include "rootserver/ob_ddl_service.h" @@ -47,7 +47,7 @@ private: virtual void handle(void *task); }; -class ObDBMSSchedJobKey +class ObDBMSSchedJobKey : public common::ObLink { public: ObDBMSSchedJobKey( @@ -57,7 +57,7 @@ public: : tenant_id_(tenant_id), is_oracle_tenant_(is_oracle_tenant), job_id_(job_id), - execute_at_(execute_at), + execute_at_(execute_at), delay_(delay), check_job_(check_job), check_new_(check_new), @@ -130,7 +130,7 @@ public: virtual ~ObDBMSSchedJobTask() {} int init(); - int start(common::ObLightyQueue *ready_queue); + int start(dbms_job::ObDBMSJobQueue *ready_queue); int stop(); int destroy(); @@ -148,7 +148,7 @@ public: private: bool inited_; ObDBMSSchedJobKey *job_key_; - common::ObLightyQueue *ready_queue_; + dbms_job::ObDBMSJobQueue *ready_queue_; WaitVector wait_vector_; ObSpinLock lock_; @@ -226,7 +226,7 @@ private: obrpc::ObDBMSSchedJobRpcProxy *job_rpc_proxy_; common::ObAddr self_addr_; - common::ObLightyQueue ready_queue_; + dbms_job::ObDBMSJobQueue ready_queue_; ObDBMSSchedJobTask scheduler_task_; ObDBMSSchedJobThread scheduler_thread_; ObDBMSSchedTableOperator table_operator_; diff --git a/src/observer/mysql/ob_eliminate_task.cpp b/src/observer/mysql/ob_eliminate_task.cpp index c8f436023..046be5431 100644 --- a/src/observer/mysql/ob_eliminate_task.cpp +++ b/src/observer/mysql/ob_eliminate_task.cpp @@ -38,8 +38,8 @@ int ObEliminateTask::init(const ObMySQLRequestManager *request_manager) } else { request_manager_ = const_cast(request_manager); // can't call ObMySQLRequestManager::get_mem_limit for now, tenant not inited - // set config_mem_limit_ to 64M - config_mem_limit_ = 64 * 1024 * 1024; // 64M + // set config_mem_limit_ to 16M + config_mem_limit_ = 16 * 1024 * 1024; // 16M common::ObConcurrentFIFOAllocator *allocator = request_manager_->get_allocator(); if (OB_ISNULL(allocator)) { ret = OB_NOT_INIT; diff --git a/src/observer/ob_rpc_processor_simple.cpp b/src/observer/ob_rpc_processor_simple.cpp index c4eaa8e32..0d02e6815 100644 --- a/src/observer/ob_rpc_processor_simple.cpp +++ b/src/observer/ob_rpc_processor_simple.cpp @@ -2152,10 +2152,15 @@ int ObHandlePartTransCtxP::process() int ObFlushLocalOptStatMonitoringInfoP::process() { int ret = OB_SUCCESS; - ObOptStatMonitorManager &opt_stat_monitor_mgr = ObOptStatMonitorManager::get_instance(); - if (OB_FAIL(opt_stat_monitor_mgr.update_opt_stat_monitoring_info(arg_))) { - LOG_WARN("failed to flush opt stat monitoring info", K(ret)); - } else {/*do nothing*/} + MTL_SWITCH(arg_.tenant_id_) { + ObOptStatMonitorManager *optstat_monitor_mgr = NULL; + if (OB_ISNULL(optstat_monitor_mgr = MTL(ObOptStatMonitorManager*))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(optstat_monitor_mgr)); + } else if (OB_FAIL(optstat_monitor_mgr->update_opt_stat_monitoring_info(arg_))) { + LOG_WARN("failed to flush opt stat monitoring info", K(ret)); + } else {/*do nothing*/} + } return ret; } diff --git a/src/observer/ob_server.cpp b/src/observer/ob_server.cpp index 266a81fa2..58c4384c4 100644 --- a/src/observer/ob_server.cpp +++ b/src/observer/ob_server.cpp @@ -36,7 +36,7 @@ #include "observer/ob_server_utils.h" #include "observer/ob_rpc_extra_payload.h" #include "observer/omt/ob_tenant_timezone_mgr.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "observer/table/ob_table_rpc_processor.h" #include "observer/mysql/ob_query_retry_ctrl.h" #include "rpc/obrpc/ob_rpc_handler.h" @@ -166,7 +166,6 @@ ObServer::ObServer() reload_config_(config_, gctx_), config_mgr_(config_, reload_config_), tenant_config_mgr_(omt::ObTenantConfigMgr::get_instance()), tenant_timezone_mgr_(omt::ObTenantTimezoneMgr::get_instance()), - tenant_srs_mgr_(omt::ObTenantSrsMgr::get_instance()), schema_service_(share::schema::ObMultiVersionSchemaService::get_instance()), lst_operator_(), tablet_operator_(), server_tracer_(), @@ -291,8 +290,6 @@ int ObServer::init(const ObServerOptions &opts, const ObPLogWriterCfg &log_cfg) ret = OB_ELECTION_ASYNC_LOG_WARN_INIT; } else if (OB_FAIL(init_tz_info_mgr())) { LOG_ERROR("init tz_info_mgr failed", KR(ret)); - } else if (OB_FAIL(init_srs_mgr())) { - LOG_ERROR("init srs_mgr fail", K(ret)); } else if (OB_FAIL(ObSqlTaskFactory::get_instance().init())) { LOG_ERROR("init sql task factory failed", KR(ret)); } @@ -434,8 +431,6 @@ int ObServer::init(const ObServerOptions &opts, const ObPLogWriterCfg &log_cfg) } else if (OB_FAIL(ObOptStatManager::get_instance().init( &sql_proxy_, &config_))) { LOG_ERROR("init opt stat manager failed", KR(ret)); - } else if (OB_FAIL(ObOptStatMonitorManager::get_instance().init(&sql_proxy_))) { - LOG_ERROR("init opt stat monitor manager failed", KR(ret)); } else if (OB_FAIL(lst_operator_.set_callback_for_obs( rs_rpc_proxy_, srv_rpc_proxy_, rs_mgr_, sql_proxy_))) { LOG_ERROR("set_use_rpc_table failed", KR(ret)); @@ -720,10 +715,6 @@ void ObServer::destroy() tenant_timezone_mgr_.destroy(); FLOG_INFO("tenant timezone manager destroyed"); - FLOG_INFO("begin to destroy tenant srs manager"); - tenant_srs_mgr_.destroy(); - FLOG_INFO("tenant srs manager destroyed"); - FLOG_INFO("begin to destroy ObMdsEventBuffer"); ObMdsEventBuffer::destroy(); FLOG_INFO("ObMdsEventBuffer destroyed"); @@ -1045,11 +1036,6 @@ int ObServer::start() } FLOG_INFO("check if timezone usable", KR(ret), K(stop_), K(timezone_usable)); - while (OB_SUCC(ret) && !stop_ && !tenant_srs_mgr_.is_sys_load_completed()) { - SLEEP(1); - } - LOG_INFO("[NOTICE] check if sys srs usable", K(ret), K(stop_)); - // check log replay and user tenant schema refresh status if (OB_SUCC(ret)) { if (stop_) { @@ -1354,10 +1340,6 @@ int ObServer::stop() //ObPartitionScheduler::get_instance().stop_merge(); //FLOG_INFO("partition scheduler stopped", KR(ret)); - FLOG_INFO("begin to stop tenant srs manager"); - tenant_srs_mgr_.stop(); - FLOG_INFO("tenant srs manager stopped"); - FLOG_INFO("begin to stop opt stat manager "); ObOptStatManager::get_instance().stop(); FLOG_INFO("opt stat manager stopped"); @@ -1647,10 +1629,6 @@ int ObServer::wait() TG_WAIT(lib::TGDefIDs::DiskUseReport); FLOG_INFO("wait disk usage report task success"); - FLOG_INFO("begin to wait tenant srs manager"); - tenant_srs_mgr_.wait(); - FLOG_INFO("wait tenant srs manager success"); - FLOG_INFO("begin to wait ob_server_block_mgr"); OB_SERVER_BLOCK_MGR.wait(); FLOG_INFO("wait ob_server_block_mgr success"); @@ -1744,16 +1722,6 @@ int ObServer::init_tz_info_mgr() return ret; } -int ObServer::init_srs_mgr() -{ - int ret = OB_SUCCESS; - - if (OB_FAIL(tenant_srs_mgr_.init(&sql_proxy_, self_addr_, &schema_service_))) { - LOG_WARN("tenant_srs_mgr_ init failed", K_(self_addr), K(ret)); - } - return ret; -} - int ObServer::init_config() { int ret = OB_SUCCESS; @@ -2005,6 +1973,8 @@ int ObServer::init_pre_setting() int ret = OB_SUCCESS; reset_mem_leak_checker_label(GCONF.leak_mod_to_check.str()); + ObMallocSampleLimiter::set_interval(GCONF._max_malloc_sample_interval, + GCONF._min_malloc_sample_interval); // oblog configuration if (OB_SUCC(ret)) { @@ -2372,10 +2342,7 @@ int ObServer::init_global_kvcache() { int ret = OB_SUCCESS; int64_t bucket_num = ObKVGlobalCache::get_instance().get_suitable_bucket_num(); - int64_t max_cache_size = ObKVGlobalCache::DEFAULT_MAX_CACHE_SIZE; - if (is_mini_mode()) { - max_cache_size *= lib::mini_mode_resource_ratio(); - } + const int64_t max_cache_size = MIN(sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE), ObKVGlobalCache::DEFAULT_MAX_CACHE_SIZE); if (OB_FAIL(ObKVGlobalCache::get_instance().init(&ObTenantMemLimitGetter::get_instance(), bucket_num, max_cache_size))) { diff --git a/src/observer/ob_server.h b/src/observer/ob_server.h index 08c434187..164d6ed13 100644 --- a/src/observer/ob_server.h +++ b/src/observer/ob_server.h @@ -85,7 +85,6 @@ namespace oceanbase namespace omt { class ObTenantTimezoneMgr; -class ObTenantSrsMgr; } namespace share { @@ -281,7 +280,6 @@ private: int init_table_lock_rpc_client(); int start_log_mgr(); int stop_log_mgr(); - int init_srs_mgr(); int reload_bandwidth_throttle_limit(int64_t network_speed); int get_network_speed_from_sysfs(int64_t &network_speed); int get_network_speed_from_config_file(int64_t &network_speed); @@ -361,8 +359,6 @@ private: common::ObConfigManager config_mgr_; omt::ObTenantConfigMgr &tenant_config_mgr_; omt::ObTenantTimezoneMgr &tenant_timezone_mgr_; - // gis: srs mgr - omt::ObTenantSrsMgr &tenant_srs_mgr_; // The Oceanbase schema relating to. share::schema::ObMultiVersionSchemaService &schema_service_; diff --git a/src/observer/ob_server_reload_config.cpp b/src/observer/ob_server_reload_config.cpp index d7f414c41..36698d42f 100644 --- a/src/observer/ob_server_reload_config.cpp +++ b/src/observer/ob_server_reload_config.cpp @@ -157,10 +157,8 @@ int ObServerReloadConfig::operator()() #ifdef OB_USE_ASAN __MemoryContext__::set_enable_asan_allocator(GCONF.enable_asan_for_memory_context); #endif -#if defined(__x86_64__) ObMallocSampleLimiter::set_interval(GCONF._max_malloc_sample_interval, GCONF._min_malloc_sample_interval); -#endif if (!is_arbitration_mode) { ObIOConfig io_config; int64_t cpu_cnt = GCONF.cpu_count; @@ -199,6 +197,9 @@ int ObServerReloadConfig::operator()() int64_t cache_size = GCONF.memory_chunk_cache_size; if (0 == cache_size) { cache_size = GMEMCONF.get_server_memory_limit(); + if (cache_size >= (32L<<30)) { + cache_size -= (4L<<30); + } } int64_t large_cache_size = GCONF._memory_large_chunk_cache_size; if (0 == large_cache_size) { diff --git a/src/observer/ob_uniq_task_queue.h b/src/observer/ob_uniq_task_queue.h index 9dbaee611..23603a3a5 100644 --- a/src/observer/ob_uniq_task_queue.h +++ b/src/observer/ob_uniq_task_queue.h @@ -15,6 +15,7 @@ #include "lib/container/ob_se_array.h" #include "lib/hash/ob_hashmap.h" +#include "lib/hash/ob_hashset.h" #include "lib/list/ob_dlink_node.h" #include "lib/thread/ob_thread_name.h" #include "lib/list/ob_dlist.h" @@ -94,8 +95,8 @@ public: friend class TestBatchProcessQueue_test_update_process_Test; friend class TestBatchProcessQueue_test_single_update_Test; static const int64_t QUEUE_WAIT_INTERVAL_MS = 200; // 200ms - ObUniqTaskQueue() : inited_(false), queue_size_(0), thread_name_(nullptr), task_map_(), - task_count_(0), group_map_(), processing_task_map_(), cur_group_(NULL), + ObUniqTaskQueue() : inited_(false), queue_size_(0), thread_name_(nullptr), task_set_(), + task_count_(0), group_map_(), processing_task_set_(), cur_group_(NULL), processing_thread_count_(0), barrier_task_count_(0), updater_(NULL) {} virtual ~ObUniqTaskQueue() { } @@ -146,14 +147,13 @@ private: const char *thread_name_; common::ObThreadCond cond_; // FIXME baihua: half memory wasted here. - common::hash::ObHashMap, common::hash::equal_to, - common::hash::SimpleAllocer::AllocType, - common::hash::NodeNumTraits::NODE_NUM, - common::hash::NoPthreadDefendMode, ObHighPrioMemAllocator> > task_map_; + common::hash::SimpleAllocer::AllocType, + common::hash::NodeNumTraits::NODE_NUM, + common::hash::NoPthreadDefendMode, ObHighPrioMemAllocator> > task_set_; int64_t task_count_; common::hash::ObHashMap, common::hash::SimpleAllocer< typename common::hash::HashMapTypes::AllocType, - common::hash::NodeNumTraits::NODE_NUM, + common::hash::NodeNumTraits::NODE_NUM, common::hash::NoPthreadDefendMode, ObHighPrioMemAllocator> > group_map_; - common::hash::ObHashMap, CompareT, - common::hash::SimpleAllocer::AllocType, - common::hash::NodeNumTraits::NODE_NUM, - common::hash::NoPthreadDefendMode, ObHighPrioMemAllocator> > processing_task_map_; + common::hash::SimpleAllocer::AllocType, + common::hash::NodeNumTraits::NODE_NUM, + common::hash::NoPthreadDefendMode, ObHighPrioMemAllocator> > processing_task_set_; common::ObDList groups_; Group *cur_group_; int64_t processing_thread_count_; @@ -211,12 +210,13 @@ int ObUniqTaskQueue::init_only(Process *updater, const int64_t th SERVER_LOG(WARN, "invalid argument", K(thread_num), K(queue_size), K(updater)); } else if (OB_FAIL(cond_.init(common::ObWaitEventIds::PARTITION_TABLE_UPDATER_COND_WAIT))) { SERVER_LOG(WARN, "fai to init condition, ", K(ret)); - } else if (OB_FAIL(task_map_.create(queue_size, attr, attr))) { + } else if (OB_FAIL(task_set_.create(queue_size, attr, attr))) { SERVER_LOG(WARN, "create hash map failed", K(ret), K(queue_size)); } else if (OB_FAIL(group_map_.create(group_count, attr, attr))) { SERVER_LOG(WARN, "create hash map failed", K(ret), K(group_count)); - } else if (OB_FAIL(processing_task_map_.create(queue_size, attr, attr))) { + } else if (OB_FAIL(processing_task_set_.create(common::UNIQ_TASK_QUEUE_BATCH_EXECUTE_NUM * thread_num, + attr, attr))) { SERVER_LOG(WARN, "create hash map failed", K(ret)); } else { this->set_thread_count(static_cast(thread_num)); @@ -259,7 +259,7 @@ int ObUniqTaskQueue::check_exist(const Task &task, bool &exist) exist = false; common::ObThreadCondGuard guard(cond_); const Task *stored_task = nullptr; - if (nullptr == (stored_task = task_map_.get(task))) { + if (nullptr == (stored_task = task_set_.get(task))) { exist = false; } else { exist = true; @@ -288,10 +288,10 @@ int ObUniqTaskQueue::add(const Task &task) } } else { const Task *stored_task = NULL; - if (OB_FAIL(task_map_.set_refactored(task, task))) { + if (OB_FAIL(task_set_.set_refactored(task, 0))) { if (common::OB_HASH_EXIST == ret) { if (task.need_assign_when_equal()) { - if (NULL == (stored_task = task_map_.get(task))) { + if (NULL == (stored_task = task_set_.get(task))) { ret = common::OB_ERR_SYS; SERVER_LOG(WARN, "get inserted task failed", K(ret), K(task)); } else if (OB_FAIL(const_cast(stored_task)->assign_when_equal(task))) { @@ -304,7 +304,7 @@ int ObUniqTaskQueue::add(const Task &task) } else { SERVER_LOG(WARN, "insert into hash failed", K(ret), K(task)); } - } else if (NULL == (stored_task = task_map_.get(task))) { + } else if (NULL == (stored_task = task_set_.get(task))) { ret = common::OB_ERR_SYS; SERVER_LOG(WARN, "get inserted task failed", K(ret), K(task)); } else { @@ -381,7 +381,7 @@ void ObUniqTaskQueue::run1() } else if (OB_FAIL(tasks.push_back(*t))) { SERVER_LOG(WARN, "push_back failed", K(ret)); } else { - if (OB_FAIL(task_map_.erase_refactored(*t))) { + if (OB_FAIL(task_set_.erase_refactored(*t))) { SERVER_LOG(WARN, "erase task from task map failed", K(ret), "task", *t); } else { --task_count_; @@ -550,7 +550,7 @@ int ObUniqTaskQueue::try_lock(const Task &task) if (!task.is_valid()) { ret = common::OB_INVALID_ARGUMENT; SERVER_LOG(WARN, "get invalid task", K(ret), K(task)); - } else if (OB_FAIL(processing_task_map_.set_refactored(task, 1))) { + } else if (OB_FAIL(processing_task_set_.set_refactored(task, 0))) { if (common::OB_HASH_EXIST == ret) { ret = common::OB_EAGAIN; SERVER_LOG(TRACE, "same task exist", K(task)); @@ -570,7 +570,7 @@ int ObUniqTaskQueue::batch_unlock(const common::ObIArray &t if (OB_ISNULL(task)) { tmp_ret = common::OB_ERR_UNEXPECTED; SERVER_LOG(WARN, "get invalid task", K(tmp_ret), K(task)); - } else if (common::OB_SUCCESS != (tmp_ret = processing_task_map_.erase_refactored(*task))) { + } else if (common::OB_SUCCESS != (tmp_ret = processing_task_set_.erase_refactored(*task))) { SERVER_LOG(ERROR, "fail to erase task", K(tmp_ret), K(*task)); } if (common::OB_SUCCESS != tmp_ret && OB_SUCC(ret)) { diff --git a/src/observer/omt/ob_multi_tenant.cpp b/src/observer/omt/ob_multi_tenant.cpp index dd6ea034c..5d53d2bc0 100644 --- a/src/observer/omt/ob_multi_tenant.cpp +++ b/src/observer/omt/ob_multi_tenant.cpp @@ -88,6 +88,7 @@ #include "share/io/ob_io_manager.h" #include "rootserver/freeze/ob_major_freeze_service.h" #include "observer/omt/ob_tenant_config_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "observer/report/ob_tenant_meta_checker.h" #include "storage/high_availability/ob_storage_ha_service.h" #include "rootserver/ob_tenant_info_loader.h"//ObTenantInfoLoader @@ -523,6 +524,9 @@ int ObMultiTenant::init(ObAddr myaddr, MTL_BIND2(mtl_new_default, ObTenantErrsimEventMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); #endif + MTL_BIND2(mtl_new_default, ObSharedTimer::mtl_init, ObSharedTimer::mtl_start, ObSharedTimer::mtl_stop, ObSharedTimer::mtl_wait, mtl_destroy_default); + MTL_BIND2(mtl_new_default, ObOptStatMonitorManager::mtl_init, ObOptStatMonitorManager::mtl_start, ObOptStatMonitorManager::mtl_stop, ObOptStatMonitorManager::mtl_wait, mtl_destroy_default); + MTL_BIND2(mtl_new_default, ObTenantSrs::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); } if (OB_SUCC(ret)) { @@ -768,7 +772,6 @@ int ObMultiTenant::convert_hidden_to_real_sys_tenant(const ObUnitInfoGetter::ObT int64_t bucket_lock_idx = -1; int64_t lock_timeout_ts = abs_timeout_us - 3000000; // reserve 3s for converting tenant - if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); @@ -1071,12 +1074,14 @@ int ObMultiTenant::update_tenant_unit_no_lock(const ObUnitInfoGetter::ObTenantCo return ret; } -int ObMultiTenant::update_tenant_memory(const ObUnitInfoGetter::ObTenantConfig &unit) +int ObMultiTenant::update_tenant_memory(const ObUnitInfoGetter::ObTenantConfig &unit, + const int64_t extra_memory /* = 0 */) { int ret = OB_SUCCESS; ObTenant *tenant = nullptr; const uint64_t tenant_id = unit.tenant_id_; int64_t allowed_mem_limit = 0; + int64_t memory_size = unit.config_.memory_size() + extra_memory; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); @@ -1085,9 +1090,9 @@ int ObMultiTenant::update_tenant_memory(const ObUnitInfoGetter::ObTenantConfig & } else if (OB_ISNULL(tenant)) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("tenant is nullptr", K(tenant_id)); - } else if (OB_FAIL(update_tenant_memory(tenant_id, unit.config_.memory_size(), allowed_mem_limit))) { + } else if (OB_FAIL(update_tenant_memory(tenant_id, memory_size, allowed_mem_limit))) { LOG_WARN("fail to update tenant memory", K(ret), K(tenant_id)); - } else if (OB_FAIL(update_tenant_freezer_mem_limit(tenant_id, unit.config_.memory_size(), allowed_mem_limit))) { + } else if (OB_FAIL(update_tenant_freezer_mem_limit(tenant_id, memory_size, allowed_mem_limit))) { LOG_WARN("fail to update_tenant_freezer_mem_limit", K(ret), K(tenant_id)); } else if (FALSE_IT(tenant->set_unit_memory_size(allowed_mem_limit))) { // unreachable @@ -1100,7 +1105,7 @@ int ObMultiTenant::construct_allowed_unit_config(const int64_t allowed_new_log_d ObUnitInfoGetter::ObTenantConfig &allowed_new_unit) { int ret = OB_SUCCESS; - if (0 >= allowed_new_log_disk_size + if (0 > allowed_new_log_disk_size || !expected_unit_config.is_valid()) { ret= OB_INVALID_ARGUMENT; } else if (OB_FAIL(allowed_new_unit.assign(expected_unit_config))) { @@ -1639,15 +1644,6 @@ int ObMultiTenant::remove_tenant(const uint64_t tenant_id, bool &remove_tenant_s allocator->try_purge(); } } - if (OB_SUCC(ret)) { - if (OB_FAIL(ObOptStatMonitorManager::get_instance().erase_opt_stat_monitoring_info_map(tenant_id))) { - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_SUCCESS; - } else { - LOG_WARN("failed to erase column usage map", K(ret)); - } - } - } if (OB_SUCC(ret)) { if (is_virtual_tenant_id(tenant_id) && OB_FAIL(ObVirtualTenantManager::get_instance().del_tenant(tenant_id))) { @@ -2392,3 +2388,55 @@ int ObSrvNetworkFrame::reload_sql_thread_config() } return ret; } + +int ObSharedTimer::mtl_init(ObSharedTimer *&st) +{ + int ret = common::OB_SUCCESS; + if (st != NULL) { + int &tg_id = st->tg_id_; + if (OB_FAIL(TG_CREATE_TENANT(lib::TGDefIDs::TntSharedTimer, tg_id))) { + LOG_WARN("init shared timer failed", K(ret)); + } + } + return ret; +} + +int ObSharedTimer::mtl_start(ObSharedTimer *&st) +{ + int ret = common::OB_SUCCESS; + if (st != NULL) { + int &tg_id = st->tg_id_; + if (OB_FAIL(TG_START(tg_id))) { + LOG_WARN("init shared timer failed", K(ret), K(tg_id)); + } + } + return ret; +} + +void ObSharedTimer::mtl_stop(ObSharedTimer *&st) +{ + if (st != NULL) { + int &tg_id = st->tg_id_; + if (tg_id > 0) { + TG_STOP(tg_id); + } + } +} + +void ObSharedTimer::mtl_wait(ObSharedTimer *&st) +{ + if (st != NULL) { + int &tg_id = st->tg_id_; + if (tg_id > 0) { + TG_WAIT(tg_id); + } + } +} + +void ObSharedTimer::destroy() +{ + if (tg_id_ > 0) { + TG_DESTROY(tg_id_); + tg_id_ = -1; + } +} diff --git a/src/observer/omt/ob_multi_tenant.h b/src/observer/omt/ob_multi_tenant.h index 7bac65ae9..b1fb5395a 100644 --- a/src/observer/omt/ob_multi_tenant.h +++ b/src/observer/omt/ob_multi_tenant.h @@ -106,7 +106,8 @@ public: int convert_real_to_hidden_sys_tenant(); int update_tenant_cpu(const uint64_t tenant_id, const double min_cpu, const double max_cpu); int update_tenant_memory(const uint64_t tenant_id, const int64_t mem_limit, int64_t &allowed_mem_limit); - int update_tenant_memory(const share::ObUnitInfoGetter::ObTenantConfig &unit); + int update_tenant_memory(const share::ObUnitInfoGetter::ObTenantConfig &unit, + const int64_t extra_memory = 0); int update_tenant_log_disk_size(const uint64_t tenant_id, const int64_t old_log_disk_size, const int64_t new_log_disk_size, @@ -256,6 +257,20 @@ bool ObMultiTenant::has_synced() const return has_synced_; } +class ObSharedTimer +{ +public: + ObSharedTimer() : tg_id_(-1) {} + static int mtl_init(ObSharedTimer *&st); + static int mtl_start(ObSharedTimer *&st); + static void mtl_stop(ObSharedTimer *&st); + static void mtl_wait(ObSharedTimer *&st); + void destroy(); + int get_tg_id() const { return tg_id_; } +private: + int tg_id_; +}; + } // end of namespace omt } // end of namespace oceanbase diff --git a/src/observer/omt/ob_tenant_config.h b/src/observer/omt/ob_tenant_config.h index 730879dd5..5d836b0cb 100644 --- a/src/observer/omt/ob_tenant_config.h +++ b/src/observer/omt/ob_tenant_config.h @@ -52,7 +52,6 @@ public: virtual ~TenantConfigUpdateTask() {} TenantConfigUpdateTask(const TenantConfigUpdateTask &) = delete; TenantConfigUpdateTask &operator=(const TenantConfigUpdateTask &) = delete; - void cancelCallBack() override {} void set_tenant_config(ObTenantConfig *config) { tenant_config_ = config; } void runTimerTask(void) override; ObTenantConfigMgr *config_mgr_; diff --git a/src/observer/omt/ob_tenant_node_balancer.cpp b/src/observer/omt/ob_tenant_node_balancer.cpp index b7ee7661f..4d477141e 100644 --- a/src/observer/omt/ob_tenant_node_balancer.cpp +++ b/src/observer/omt/ob_tenant_node_balancer.cpp @@ -259,8 +259,8 @@ int ObTenantNodeBalancer::get_server_allocated_resource(ServerResource &server_r server_resource.max_cpu_ += tenant_units.at(i).config_.max_cpu(); server_resource.min_cpu_ += tenant_units.at(i).config_.min_cpu(); } - - server_resource.memory_size_ += max(ObMallocAllocator::get_instance()->get_tenant_limit(tenant_units.at(i).tenant_id_), + int64_t extra_memory = is_sys_tenant(tenant_units.at(i).tenant_id_) ? GMEMCONF.get_extra_memory() : 0; + server_resource.memory_size_ += max(ObMallocAllocator::get_instance()->get_tenant_limit(tenant_units.at(i).tenant_id_) - extra_memory, tenant_units.at(i).config_.memory_size()); server_resource.log_disk_size_ += tenant_units.at(i).config_.log_disk_size(); } @@ -346,17 +346,19 @@ int ObTenantNodeBalancer::check_new_tenant(const ObUnitInfoGetter::ObTenantConfi } } } else { - if (is_sys_tenant(tenant_id) && tenant->is_hidden()) { - if (OB_FAIL(omt_->convert_hidden_to_real_sys_tenant(unit, abs_timeout_us))) { + int64_t extra_memory = 0; + if (is_sys_tenant(tenant_id)) { + if (tenant->is_hidden() && OB_FAIL(omt_->convert_hidden_to_real_sys_tenant(unit, abs_timeout_us))) { LOG_WARN("fail to create real sys tenant", K(unit)); } + extra_memory = GMEMCONF.get_extra_memory(); } if (OB_SUCC(ret) && !(unit == tenant->get_unit())) { if (OB_FAIL(omt_->update_tenant_unit(unit))) { LOG_WARN("fail to update tenant unit", K(ret), K(tenant_id)); } } - if (OB_SUCC(ret) && OB_FAIL(omt_->update_tenant_memory(unit))) { + if (OB_SUCC(ret) && OB_FAIL(omt_->update_tenant_memory(unit, extra_memory))) { LOG_ERROR("fail to update tenant memory", K(ret), K(tenant_id)); } } @@ -371,16 +373,13 @@ int ObTenantNodeBalancer::check_new_tenant(const ObUnitInfoGetter::ObTenantConfi int ObTenantNodeBalancer::refresh_hidden_sys_memory() { int ret = OB_SUCCESS; - int64_t sys_tenant_memory = 0; int64_t allowed_mem_limit = 0; ObTenant *tenant = nullptr; if (OB_FAIL(omt_->get_tenant(OB_SYS_TENANT_ID, tenant))) { LOG_WARN("get sys tenant failed", K(ret)); } else if (OB_ISNULL(tenant) || !tenant->is_hidden()) { // do nothing - } else if (OB_FAIL(ObUnitResource::get_sys_tenant_default_memory(sys_tenant_memory))) { - LOG_WARN("get hidden sys tenant default memory failed", K(ret)); - } else if (OB_FAIL(omt_->update_tenant_memory(OB_SYS_TENANT_ID, sys_tenant_memory, allowed_mem_limit))) { + } else if (OB_FAIL(omt_->update_tenant_memory(OB_SYS_TENANT_ID, GMEMCONF.get_hidden_sys_memory(), allowed_mem_limit))) { LOG_WARN("update hidden sys tenant memory failed", K(ret)); } else { LOG_INFO("update hidden sys tenant memory succeed ", K(allowed_mem_limit)); diff --git a/src/observer/omt/ob_tenant_srs.cpp b/src/observer/omt/ob_tenant_srs.cpp index 12c7691f4..0d6a34a31 100644 --- a/src/observer/omt/ob_tenant_srs.cpp +++ b/src/observer/omt/ob_tenant_srs.cpp @@ -12,15 +12,16 @@ #define USING_LOG_PREFIX SERVER_OMT #include "ob_tenant_srs.h" -#include "ob_tenant_srs_mgr.h" +#include "ob_tenant_srs.h" #include "lib/string/ob_sql_string.h" #include "common/ob_smart_var.h" #include "observer/ob_sql_client_decorator.h" -#include "lib/mysqlclient/ob_mysql_proxy.h" #include "share/ob_thread_mgr.h" #include "share/inner_table/ob_inner_table_schema.h" #include "share/schema/ob_multi_version_schema_service.h" #include "share/rc/ob_tenant_base.h" +#include "observer/omt/ob_multi_tenant.h" +#include "observer/ob_server_struct.h" #include "lib/geo/ob_geo_utils.h" using namespace oceanbase::share; @@ -32,21 +33,74 @@ namespace oceanbase namespace omt { -int ObTenantSrs::init(ObTenantSrsMgr *srs_mgr) +int ObTenantSrs::mtl_init(ObTenantSrs* &tenant_srs) { int ret = OB_SUCCESS; - if (OB_ISNULL(srs_mgr)) { - ret = OB_ERR_NULL_VALUE; - LOG_WARN("unexpected null srs mgr", K(ret)); - } else if (OB_ISNULL(srs_mgr->sql_proxy_)) { - ret = OB_ERR_NULL_VALUE; - LOG_WARN("unexpected null sql proxy_ in srs mgr", K(ret)); - } else if (OB_FAIL(srs_update_periodic_task_.init(srs_mgr, this))) { - LOG_WARN("failed to init srs update task", K(ret)); + if (OB_FAIL(tenant_srs->init())) { + LOG_WARN("fail to init tenant srs", K(ret)); + } + return ret; +} + +int ObTenantSrs::start() +{ + int ret = OB_SUCCESS; + if (!inited_) { + ret = OB_NOT_INIT; + LOG_WARN("tenant srs isn't inited", K(ret)); + } else if (OB_FAIL(TG_SCHEDULE(MTL(omt::ObSharedTimer*)->get_tg_id(), srs_update_periodic_task_, 0, false))) { + LOG_WARN("failed to schedule tenant srs update task", K(ret)); + } + return ret; +} + +void ObTenantSrs::stop() +{ + if (OB_LIKELY(inited_)) { + TG_CANCEL_TASK(MTL(omt::ObSharedTimer*)->get_tg_id(), srs_update_periodic_task_); + } +} + +void ObTenantSrs::wait() +{ + if (OB_LIKELY(inited_)) { + TG_WAIT_TASK(MTL(omt::ObSharedTimer*)->get_tg_id(), srs_update_periodic_task_); + } +} + +void ObTenantSrs::destroy() +{ + if (OB_LIKELY(inited_)) { + cancle_update_task(); + recycle_old_snapshots(); + recycle_last_snapshots(); + allocator_.~ObFIFOAllocator(); + } +} + +int ObTenantSrs::init() +{ + int ret = OB_SUCCESS; + sql_proxy_ = GCTX.sql_proxy_; + lib::ObMemAttr mem_attr(MTL_ID(), "TenantSrs"); + if (inited_) { + ret = OB_INIT_TWICE; + LOG_WARN("ObTenantSrs init twice.", K(ret)); + } else if (FALSE_IT(alloc_.set_tenant_id(MTL_ID()))) { + } else if (OB_FAIL(allocator_.init(&alloc_, OB_MALLOC_MIDDLE_BLOCK_SIZE, mem_attr))) { + LOG_WARN("ObTenantSrs allocator init failed.", K(ret)); } else { - srs_mgr_ = srs_mgr; - if (OB_FAIL(TG_SCHEDULE(lib::TGDefIDs::SRS_MGR, srs_update_periodic_task_, 0, false))) { - LOG_WARN("failed to schedule tenant srs update task", K(ret)); + page_allocator_.set_allocator(&allocator_); + page_allocator_.set_attr(mem_attr); + mode_arena_.init(DEFAULT_PAGE_SIZE, page_allocator_); + if (OB_FAIL(srs_update_periodic_task_.init(this))) { + LOG_WARN("failed to init srs update task", K(ret)); + } else { + inited_ = true; + infinite_plane_.minX_ = INT32_MIN; + infinite_plane_.minY_ = INT32_MIN; + infinite_plane_.maxX_ = INT32_MAX; + infinite_plane_.maxY_ = INT32_MAX; } } return ret; @@ -81,6 +135,42 @@ int ObSrsCacheGuard::get_srs_item(uint64_t srs_id, const ObSrsItem *&srs_item) return ret; } +int ObTenantSrs::get_tenant_srs_guard(ObSrsCacheGuard &srs_guard) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(try_get_last_snapshot(srs_guard))) { + if (ret == OB_ERR_EMPTY_QUERY) { + ret = OB_ERR_SRS_EMPTY; + LOG_WARN("srs table might be empty", K(ret), K(MTL_ID())); + LOG_USER_ERROR(OB_ERR_SRS_EMPTY); + } else { + LOG_WARN("failed to get tenant srs", K(ret), K(MTL_ID())); + } + } + return ret; +} + +int ObTenantSrs::get_srs_bounds(uint64_t srid, const ObSrsItem *srs_item, const ObSrsBoundsItem *&bounds_item) +{ + int ret = OB_SUCCESS; + if (srid == 0) { + bounds_item = &infinite_plane_; + } else if (OB_ISNULL(srs_item)) { + ret = OB_ERR_NULL_VALUE; + LOG_ERROR("srs item is null", K(ret)); + } else { + const ObSrsBoundsItem *tmp_bounds = srs_item->get_bounds(); + if (isnan(tmp_bounds->minX_) || isnan(tmp_bounds->minY_) + || isnan(tmp_bounds->maxX_) || isnan(tmp_bounds->maxY_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid bounds info", K(ret), K(srid), K(srs_item->get_srid()), K(*tmp_bounds)); + } else { + bounds_item = tmp_bounds; + } + } + return ret; +} + int ObTenantSrs::get_last_sys_snapshot(ObSrsCacheSnapShot *&sys_cache) { int ret = OB_SUCCESS; @@ -151,7 +241,8 @@ int ObTenantSrs::refresh_srs(bool is_sys) int ret = OB_SUCCESS; ObSrsCacheSnapShot *srs = NULL; uint64_t tenant_data_version = 0; - if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id_, tenant_data_version))) { + const uint64_t tenant_id = MTL_ID(); + if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, tenant_data_version))) { LOG_WARN("get tenant data version failed", K(ret)); } else if (tenant_data_version < DATA_VERSION_4_1_0_0) { ret = OB_ERR_EMPTY_QUERY; @@ -170,15 +261,15 @@ int ObTenantSrs::refresh_srs(bool is_sys) if (last_snapshot != NULL) { if (last_snapshot->get_ref_count() > 0 && OB_FAIL(srs_old_snapshots_.push_back(last_snapshot))) { - LOG_WARN("failed to push last_snapshot to recycle queue", K(ret), K(tenant_id_), K(is_sys)); + LOG_WARN("failed to push last_snapshot to recycle queue", K(ret), K(tenant_id), K(is_sys)); } else { - OB_DELETE(ObSrsCacheSnapShot, ObModIds::OMT, last_snapshot); + allocator_.free(last_snapshot); } } last_snapshot = srs; local_version = srs->get_srs_version(); LOG_INFO("fetch srs cache snapshot success", K(local_version), K(remote_version), - K(srs->get_srs_count()), K(srs_old_snapshots_.size()), K(tenant_id_), K(is_sys)); + K(srs->get_srs_count()), K(srs_old_snapshots_.size()), K(tenant_id), K(is_sys)); } return ret; } @@ -193,16 +284,14 @@ int ObTenantSrs::refresh_usr_srs() return refresh_srs(false); } -int ObTenantSrs::TenantSrsUpdatePeriodicTask::init(ObTenantSrsMgr *srs_mgr, ObTenantSrs *srs) +int ObTenantSrs::TenantSrsUpdatePeriodicTask::init(ObTenantSrs *srs) { - tenant_srs_mgr_ = srs_mgr; tenant_srs_ = srs; return OB_SUCCESS; } -int ObTenantSrs::TenantSrsUpdateTask::init(ObTenantSrsMgr *srs_mgr, ObTenantSrs *srs) +int ObTenantSrs::TenantSrsUpdateTask::init(ObTenantSrs *srs) { - tenant_srs_mgr_ = srs_mgr; tenant_srs_ = srs; return OB_SUCCESS; } @@ -212,12 +301,14 @@ void ObTenantSrs::recycle_last_snapshots() TCWLockGuard guard(lock_); if (OB_NOT_NULL(last_sys_snapshot_) && last_sys_snapshot_->get_ref_count() <= 0) { - OB_DELETE(ObSrsCacheSnapShot, ObModIds::OMT, last_sys_snapshot_); + last_sys_snapshot_->~ObSrsCacheSnapShot(); + allocator_.free(last_sys_snapshot_); last_sys_snapshot_ = NULL; } if (OB_NOT_NULL(last_user_snapshot_) && last_user_snapshot_->get_ref_count() <= 0) { - OB_DELETE(ObSrsCacheSnapShot, ObModIds::OMT, last_user_snapshot_); + last_user_snapshot_->~ObSrsCacheSnapShot(); + allocator_.free(last_user_snapshot_); last_user_snapshot_ = NULL; } } @@ -246,7 +337,8 @@ void ObTenantSrs::recycle_old_snapshots() if (OB_FAIL(srs_old_snapshots_.remove(i))) { LOG_WARN("failed to remove old snapshot", K(ret)); } else { - OB_DELETE(ObSrsCacheSnapShot, ObModIds::OMT, snap); + snap->~ObSrsCacheSnapShot(); + allocator_.free(snap); } } } @@ -269,12 +361,15 @@ void ObTenantSrs::TenantSrsUpdateTask::runTimerTask() void ObTenantSrs::TenantSrsUpdatePeriodicTask::runTimerTask() { int ret = OB_SUCCESS; + ObMultiVersionSchemaService *schema_service = nullptr; + const uint64_t tenant_id = MTL_ID(); bool is_sys_overdue = false; bool is_user_overdue = false; uint32_t delay = SLEEP_USECONDS; - // check tenant schema whether is ready - if ((tenant_srs_->tenant_id_ == OB_SYS_TENANT_ID && !tenant_srs_mgr_->is_sys_schema_ready()) || - (tenant_srs_->tenant_id_ != OB_SYS_TENANT_ID && !tenant_srs_mgr_->schema_service_->is_tenant_full_schema(tenant_srs_->tenant_id_))) { + + + if ((tenant_id == OB_SYS_TENANT_ID && !GSCHEMASERVICE.is_sys_full_schema()) || + (tenant_id != OB_SYS_TENANT_ID && !GSCHEMASERVICE.is_tenant_full_schema(tenant_id))) { delay = BOOTSTRAP_PERIOD; } else { uint32_t old_snapshot_size = 0; @@ -297,16 +392,12 @@ void ObTenantSrs::TenantSrsUpdatePeriodicTask::runTimerTask() if (is_user_overdue) { // to do:user srs refresh } - if (OB_UNLIKELY(tenant_srs_->tenant_id_ == OB_SYS_TENANT_ID && !tenant_srs_mgr_->is_sys_load_completed())) { - LOG_INFO("sys_tenant init load completed"); - tenant_srs_mgr_->set_sys_load_completed(); - } if (old_snapshot_size > 0) { tenant_srs_->recycle_old_snapshots(); } } // timer task, ignore error code - if (OB_FAIL(TG_SCHEDULE(lib::TGDefIDs::SRS_MGR, *this, delay, false))) { + if (OB_FAIL(TG_SCHEDULE(MTL(omt::ObSharedTimer*)->get_tg_id(), *this, delay, false))) { LOG_WARN("schedule srs update task failed", K(ret)); } } @@ -315,11 +406,11 @@ int ObTenantSrs::cancle_update_task() { int ret = OB_SUCCESS; bool is_exist = true; - if (OB_FAIL(TG_TASK_EXIST(lib::TGDefIDs::SRS_MGR, srs_update_periodic_task_, is_exist))) { - LOG_WARN("failed to check tenant srs update task", K(ret), K(tenant_id_)); + if (OB_FAIL(TG_TASK_EXIST(MTL(omt::ObSharedTimer*)->get_tg_id(), srs_update_periodic_task_, is_exist))) { + LOG_WARN("failed to check tenant srs update task", K(ret)); } else if (is_exist) { - if (OB_FAIL(TG_CANCEL_R(lib::TGDefIDs::SRS_MGR, srs_update_periodic_task_))) { - LOG_WARN("failed to cancel tenant srs update task", K(ret), K(tenant_id_)); + if (OB_FAIL(TG_CANCEL_R(MTL(omt::ObSharedTimer*)->get_tg_id(), srs_update_periodic_task_))) { + LOG_WARN("failed to cancel tenant srs update task", K(ret)); } } return ret; @@ -343,9 +434,10 @@ int ObTenantSrs::fetch_all_srs(ObSrsCacheSnapShot *&srs_snapshot, bool is_sys_sr ObSrsCacheType snapshot_type; ObSrsCacheSnapShot *snapshot = NULL; uint32_t res_count = 0; + const uint64_t tenant_id = MTL_ID(); ObSqlString sql; - ObSQLClientRetryWeak sql_client_retry_weak(srs_mgr_->sql_proxy_, tenant_id_, OB_ALL_SPATIAL_REFERENCE_SYSTEMS_TID); + ObSQLClientRetryWeak sql_client_retry_weak(sql_proxy_, tenant_id, OB_ALL_SPATIAL_REFERENCE_SYSTEMS_TID); SMART_VAR(ObMySQLProxy::MySQLResult, res) { ObMySQLResult *result = NULL; if (is_sys_srs) { @@ -359,8 +451,8 @@ int ObTenantSrs::fetch_all_srs(ObSrsCacheSnapShot *&srs_snapshot, bool is_sys_sr } if (OB_FAIL(ret)) { LOG_WARN("append sql failed", K(ret)); - } else if (OB_FAIL(sql_client_retry_weak.read(res, tenant_id_, sql.ptr()))) { - LOG_WARN("execute sql failed", K(sql), K(ret), K(tenant_id_)); + } else if (OB_FAIL(sql_client_retry_weak.read(res, tenant_id, sql.ptr()))) { + LOG_WARN("execute sql failed", K(sql), K(ret), K(tenant_id)); } else if (OB_UNLIKELY(NULL == (result = res.get_result()))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("fail to get result. ", K(ret)); @@ -370,7 +462,7 @@ int ObTenantSrs::fetch_all_srs(ObSrsCacheSnapShot *&srs_snapshot, bool is_sys_sr const ObSrsItem *tmp = NULL; res_count++; if (OB_ISNULL(snapshot)) { - snapshot = OB_NEW(ObSrsCacheSnapShot, ObModIds::OMT, snapshot_type); + snapshot = OB_NEWx(ObSrsCacheSnapShot, &allocator_, &allocator_, snapshot_type); if (OB_ISNULL(snapshot)) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to create ObSrsCacheSnapShot", K(ret)); @@ -403,14 +495,14 @@ int ObTenantSrs::fetch_all_srs(ObSrsCacheSnapShot *&srs_snapshot, bool is_sys_sr } else { if (OB_FAIL(generate_pg_reserved_srs(snapshot))) { LOG_WARN("failed to geneate pg reserved srs", K(ret)); - OB_DELETE(ObSrsCacheSnapShot, ObModIds::OMT, snapshot); + allocator_.free(snapshot); } else { snapshot->set_srs_version(srs_version); srs_snapshot = snapshot; } } } else if (snapshot != NULL) { - OB_DELETE(ObSrsCacheSnapShot, ObModIds::OMT, snapshot); + allocator_.free(snapshot); LOG_WARN("failed to get all srs item, iter quit", K(ret)); } } @@ -478,16 +570,16 @@ int ObSrsCacheSnapShot::parse_srs_item(ObMySQLResult *result, const ObSrsItem *& LOG_WARN("failed to extract maxx value", K(ret)); } else if (OB_FAIL(extract_bounds_numberic(result, "maxY", max_y))) { LOG_WARN("failed to extract maxy value", K(ret)); - } else if (OB_FAIL(ObSrsWktParser::parse_srs_wkt(allocator_, srs_id, definition, srs_info))) { + } else if (OB_FAIL(ObSrsWktParser::parse_srs_wkt(*allocator_, srs_id, definition, srs_info))) { LOG_WARN("failed to parse srs wkt from definition", K(ret), K(definition)); } else { - ObSrsItem *new_srs_item = OB_NEWx(ObSrsItem, (&allocator_), srs_info); + ObSrsItem *new_srs_item = OB_NEWx(ObSrsItem, allocator_, srs_info); if (OB_ISNULL(new_srs_item)) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to alloc memory for srs item", K(ret)); } else if (!proj4text.empty()) { srs_info->set_bounds(min_x, min_y, max_x, max_y); - if (OB_FAIL(srs_info->set_proj4text(allocator_, proj4text))) { + if (OB_FAIL(srs_info->set_proj4text(*allocator_, proj4text))) { LOG_WARN("fail to set proj4text for srs item", K(ret), K(srs_id)); } } @@ -505,14 +597,14 @@ int ObSrsCacheSnapShot::add_pg_reserved_srs_item(const ObString &pg_wkt, const u ObSpatialReferenceSystemBase *srs_info = NULL; lib::ObMallocHookAttrGuard malloc_guard(lib::ObMemAttr(common::OB_SERVER_TENANT_ID, "SRSWKTParser")); - if (OB_FAIL(ObSrsWktParser::parse_srs_wkt(allocator_, srs_id, pg_wkt, srs_info))) { + if (OB_FAIL(ObSrsWktParser::parse_srs_wkt(*allocator_, srs_id, pg_wkt, srs_info))) { LOG_WARN("failed to parse pg reserved srs wkt", K(ret), K(srs_id), K(pg_wkt)); } else { - ObSrsItem *new_srs_item = OB_NEWx(ObSrsItem, (&allocator_), srs_info); + ObSrsItem *new_srs_item = OB_NEWx(ObSrsItem, allocator_, srs_info); if (OB_ISNULL(new_srs_item)) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to alloc memory for srs item", K(ret)); - } else if (OB_FAIL(ObGeoTypeUtil::get_pg_reserved_prj4text(&allocator_, srs_id, proj4text))) { + } else if (OB_FAIL(ObGeoTypeUtil::get_pg_reserved_prj4text(allocator_, srs_id, proj4text))) { LOG_WARN("fail to generate proj4text for pg srs item", K(ret)); } else if (OB_FAIL(add_srs_item(new_srs_item->get_srid(), new_srs_item))) { LOG_WARN("failed to add pg srs item to snapshot", K(ret), K(new_srs_item->get_srid())); diff --git a/src/observer/omt/ob_tenant_srs.h b/src/observer/omt/ob_tenant_srs.h index 50ebe3893..192b6d3ec 100644 --- a/src/observer/omt/ob_tenant_srs.h +++ b/src/observer/omt/ob_tenant_srs.h @@ -14,6 +14,8 @@ #define OCEANBASE_TENANT_SRS_H_ #include "share/ob_define.h" +#include "share/rc/ob_tenant_base.h" +#include "lib/mysqlclient/ob_mysql_proxy.h" #include "lib/hash/ob_pointer_hashmap.h" #include "lib/container/ob_vector.h" #include "lib/lock/ob_tc_rwlock.h" @@ -47,10 +49,10 @@ class ObSrsCacheSnapShot { public: static const uint32_t SRS_ITEM_BUCKET_NUM = 6144; - explicit ObSrsCacheSnapShot(ObSrsCacheType srs_type) - : allocator_("SrsSnapShot"), srs_type_(srs_type), srs_version_(0), ref_count_(0) {} + explicit ObSrsCacheSnapShot(common::ObIAllocator *allocator, ObSrsCacheType srs_type) + : allocator_(allocator), srs_type_(srs_type), srs_version_(0), ref_count_(0) {} virtual ~ObSrsCacheSnapShot() { srs_item_map_.destroy(); } - int init() { return srs_item_map_.create(SRS_ITEM_BUCKET_NUM, "SrsSnapShot", "SrsSnapShot"); } + int init() { return srs_item_map_.create(SRS_ITEM_BUCKET_NUM, "SrsSnapShot", "SrsSnapShot", MTL_ID()); } int add_srs_item(uint64_t srid, const common::ObSrsItem* srs_item) { return srs_item_map_.set_refactored(srid, srs_item); } int get_srs_item(uint64_t srid, const common::ObSrsItem *&srs_item); void set_srs_version(uint64_t version) { srs_version_ = version; } @@ -64,7 +66,7 @@ public: int add_pg_reserved_srs_item(const common::ObString &pg_wkt, const uint32_t srs_id); private: - common::ObArenaAllocator allocator_; + common::ObIAllocator *allocator_; ObSrsCacheType srs_type_; uint64_t srs_version_; volatile int64_t ref_count_; @@ -86,7 +88,6 @@ private: ObSrsCacheSnapShot *srs_cache_; }; -class ObTenantSrsMgr; class ObTenantSrs { @@ -94,32 +95,28 @@ public: class TenantSrsUpdatePeriodicTask : public common::ObTimerTask { public: - TenantSrsUpdatePeriodicTask() : tenant_srs_mgr_(nullptr), - tenant_srs_(nullptr) {} + TenantSrsUpdatePeriodicTask() : tenant_srs_(nullptr) {} virtual ~TenantSrsUpdatePeriodicTask() {} - int init(ObTenantSrsMgr *srs_mgr, ObTenantSrs *srs); + int init(ObTenantSrs *srs); TenantSrsUpdatePeriodicTask(const TenantSrsUpdatePeriodicTask &) = delete; TenantSrsUpdatePeriodicTask &operator=(const TenantSrsUpdatePeriodicTask &) = delete; void runTimerTask(void) override; private: static const uint64_t SLEEP_USECONDS = 5000000; static const uint64_t BOOTSTRAP_PERIOD = 1000000; - ObTenantSrsMgr *tenant_srs_mgr_; ObTenantSrs *tenant_srs_; }; class TenantSrsUpdateTask : public common::ObTimerTask { public: - TenantSrsUpdateTask() : tenant_srs_mgr_(nullptr), - tenant_srs_(nullptr) {} + TenantSrsUpdateTask() : tenant_srs_(nullptr) {} virtual ~TenantSrsUpdateTask() {} - int init(ObTenantSrsMgr *srs_mgr, ObTenantSrs *srs); + int init(ObTenantSrs *srs); TenantSrsUpdateTask(const TenantSrsUpdateTask &) = delete; TenantSrsUpdateTask &operator=(const TenantSrsUpdateTask &) = delete; void runTimerTask(void) override; private: - ObTenantSrsMgr *tenant_srs_mgr_; ObTenantSrs *tenant_srs_; }; @@ -131,19 +128,17 @@ public: static const uint32_t RETRY_TIMES = 45; static const uint32_t RETRY_INTERVAL_US = 100000; - explicit ObTenantSrs(common::ObArenaAllocator *allocator, uint64_t tenant_id) - : allocator_(allocator),tenant_id_(tenant_id), - page_allocator_(*allocator, common::ObModIds::OB_MODULE_PAGE_ALLOCATOR), - mode_arena_(DEFAULT_PAGE_SIZE, page_allocator_), + explicit ObTenantSrs() + : alloc_("TenantSrs"), sql_proxy_(nullptr), inited_(false), last_sys_snapshot_(nullptr), last_user_snapshot_(nullptr), srs_old_snapshots_(&mode_arena_, common::ObModIds::OB_MODULE_PAGE_ALLOCATOR), remote_sys_srs_version_(0), remote_user_srs_version_(0), - local_sys_srs_version_(0), local_user_srs_version_(0), - srs_mgr_(nullptr) {} + local_sys_srs_version_(0), local_user_srs_version_(0), infinite_plane_() {} virtual ~ObTenantSrs() {}; - int init(ObTenantSrsMgr *srs_mgr); - inline uint64_t tenant_id() { return tenant_id_; } - + int init(); + inline uint64_t tenant_id() { return MTL_ID(); } + int get_tenant_srs_guard(ObSrsCacheGuard &srs_guard); + int get_srs_bounds(uint64_t srid, const ObSrsItem *srs_item, const ObSrsBoundsItem *&bounds_item); int get_last_snapshot(ObSrsCacheGuard &srs_guard); TenantSrsUpdatePeriodicTask &get_update_srs_task() { return srs_update_periodic_task_; } int try_get_last_snapshot(ObSrsCacheGuard &srs_guard); @@ -151,6 +146,11 @@ public: void recycle_last_snapshots(); uint32_t get_snapshots_size(); int cancle_update_task(); + static int mtl_init(ObTenantSrs* &tenant_srs); + int start(); + void stop(); + void wait(); + void destroy(); private: typedef common::PageArena ObCGeoModuleArena; @@ -165,9 +165,10 @@ private: int get_last_user_snapshot(ObSrsCacheSnapShot *&user_cache); int generate_pg_reserved_srs(ObSrsCacheSnapShot *&srs_snapshot); - - common::ObArenaAllocator *allocator_; - uint64_t tenant_id_; + common::ObFIFOAllocator allocator_; + common::ObArenaAllocator alloc_; + common::ObMySQLProxy *sql_proxy_; + bool inited_; common::ModulePageAllocator page_allocator_; ObCGeoModuleArena mode_arena_; common::TCRWLock lock_; @@ -187,10 +188,11 @@ private: uint64_t local_user_srs_version_; TenantSrsUpdatePeriodicTask srs_update_periodic_task_; TenantSrsUpdateTask srs_update_task_; - ObTenantSrsMgr *srs_mgr_; + common::ObSrsBoundsItem infinite_plane_; DISALLOW_COPY_AND_ASSIGN(ObTenantSrs); }; +#define OTSRS_MGR (MTL(omt::ObTenantSrs*)) } // namespace omt } // namespace oceanbase diff --git a/src/observer/omt/ob_tenant_srs_mgr.cpp b/src/observer/omt/ob_tenant_srs_mgr.cpp deleted file mode 100644 index 507e64ff6..000000000 --- a/src/observer/omt/ob_tenant_srs_mgr.cpp +++ /dev/null @@ -1,369 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#define USING_LOG_PREFIX SERVER_OMT -#include "ob_tenant_srs_mgr.h" -#include "share/ob_thread_mgr.h" -#include "lib/thread/thread_mgr.h" -#include "share/schema/ob_multi_version_schema_service.h" -#include "share/schema/ob_schema_getter_guard.h" - -using namespace oceanbase::common; - -namespace oceanbase -{ - -namespace omt -{ - -ObTenantSrsMgr::ObTenantSrsMgr() - : allocator_("TenantSrs"), sql_proxy_(nullptr), self_(), - rwlock_(ObLatchIds::SRS_LOCK), tenant_srs_map_(), - add_tenant_task_(this), del_tenant_task_(this), - is_inited_(false), is_sys_load_completed_(false), - schema_service_(nullptr), nonexist_tenant_srs_(allocator_), infinite_plane_() -{ -} - -ObTenantSrsMgr::~ObTenantSrsMgr() -{ -} - -ObTenantSrsMgr &ObTenantSrsMgr::get_instance() -{ - static ObTenantSrsMgr ob_tenant_srs_mgr; - return ob_tenant_srs_mgr; -} - -void ObTenantSrsMgr::AddTenantTask::runTimerTask() -{ - int ret = OB_SUCCESS; - uint32_t delay = DEFAULT_PERIOD; - if (OB_ISNULL(tenant_srs_mgr_)) { - ret = OB_ERR_NULL_VALUE; - LOG_ERROR("failed to do add tenant srs task, tenant srs mgr is null", K(ret)); - } else if (!tenant_srs_mgr_->is_sys_schema_ready()) { - delay = BOOTSTRAP_PERIOD; - } else if (OB_FAIL(tenant_srs_mgr_->try_to_add_new_tenants())) { - LOG_WARN("failed to update tenants srs", K(ret)); - } - if (OB_FAIL(TG_SCHEDULE(lib::TGDefIDs::SRS_MGR, *this, delay, false))) { - LOG_ERROR("failed to schedule add tenant srs task", K(ret)); - } -} - -void ObTenantSrsMgr::DeleteTenantTask::runTimerTask() -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(tenant_srs_mgr_)) { - ret = OB_ERR_NULL_VALUE; - LOG_ERROR("failed to do add tenant srs task, tenant srs mgr is null", K(ret)); - } else if (!tenant_srs_mgr_->is_sys_schema_ready()) { - // do nothing - } else if (OB_FAIL(tenant_srs_mgr_->remove_nonexist_tenants())) { - LOG_WARN("failed to add nonexist tenants to del list", K(ret)); - } else if (OB_FAIL(tenant_srs_mgr_->delete_nonexist_tenants_srs())) { - LOG_WARN("failed to delete nonexist tenants", K(ret)); - } - - if (OB_FAIL(TG_SCHEDULE(lib::TGDefIDs::SRS_MGR, *this, DEFAULT_PERIOD, false))) { - LOG_ERROR("failed to schedule add tenant srs task", K(ret)); - } -} - -int ObTenantSrsMgr::init(ObMySQLProxy *sql_proxy, const ObAddr &server, - share::schema::ObMultiVersionSchemaService *schema_service) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(TG_START(lib::TGDefIDs::SRS_MGR))) { - LOG_WARN("fail to init timer", K(ret)); - } else { - sql_proxy_ = sql_proxy; - self_ = server; - schema_service_ = schema_service; - is_inited_ = true; - infinite_plane_.minX_ = INT32_MIN; - infinite_plane_.minY_ = INT32_MIN; - infinite_plane_.maxX_ = INT32_MAX; - infinite_plane_.maxY_ = INT32_MAX; - if (OB_FAIL(add_tenant_srs(OB_SYS_TENANT_ID))) { - LOG_WARN("add sys tenant srs info failed", K(ret)); - } else if (OB_FAIL(TG_SCHEDULE(lib::TGDefIDs::SRS_MGR, add_tenant_task_, 0, false))) { - LOG_WARN("schedule add tenant srs task failed", K(ret)); - } else if (OB_FAIL(TG_SCHEDULE(lib::TGDefIDs::SRS_MGR, del_tenant_task_, 0, false))) { - LOG_WARN("schedule del tenant srs task failed", K(ret)); - } - } - return ret; -} - -int ObTenantSrsMgr::get_srs_bounds(uint64_t srid, const ObSrsItem *srs_item, - const ObSrsBoundsItem *&bounds_item) -{ - int ret = OB_SUCCESS; - if (srid == 0) { - bounds_item = &infinite_plane_; - } else if (OB_ISNULL(srs_item)) { - ret = OB_ERR_NULL_VALUE; - LOG_ERROR("srs item is null", K(ret)); - } else { - const ObSrsBoundsItem *tmp_bounds = srs_item->get_bounds(); - if (isnan(tmp_bounds->minX_) || isnan(tmp_bounds->minY_) - || isnan(tmp_bounds->maxX_) || isnan(tmp_bounds->maxY_)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid bounds info", K(ret), K(srid), K(srs_item->get_srid()), K(*tmp_bounds)); - } else { - bounds_item = tmp_bounds; - } - } - return ret; -} - -int ObTenantSrsMgr::remove_nonexist_tenants() -{ - int ret = OB_SUCCESS; - ObArray nonexist_tenants_id; - { - share::schema::ObSchemaGetterGuard sys_schema_guard; - if (OB_ISNULL(schema_service_)) { - ret = OB_ERR_NULL_VALUE; - LOG_WARN("not init", K(ret)); - } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, - sys_schema_guard))) { - LOG_WARN("get sys tenant schema guard failed", K(ret)); - } - DRWLock::RDLockGuard guard(rwlock_); - TenantSrsMap::const_iterator it = tenant_srs_map_.begin(); - for (; OB_SUCC(ret) && it != tenant_srs_map_.end(); it++) { - bool is_dropped = false; - if (OB_FAIL(sys_schema_guard.check_if_tenant_has_been_dropped(it->first, is_dropped))) { - LOG_WARN("check if tenant has been dropped failed", K(ret)); - } else if (is_dropped && OB_FAIL(nonexist_tenants_id.push_back(it->first))) { - LOG_WARN("push back failed", K(ret), K(it->first)); - } - } - } - for (uint64_t i = 0; i < nonexist_tenants_id.count() && OB_SUCC(ret); i++) { - if (OB_FAIL(move_tenant_srs_to_nonexist_list(nonexist_tenants_id.at(i)))) { - LOG_WARN("failed to move tenant srs to nonexist list", K(ret), K(nonexist_tenants_id.at(i))); - } - } - return ret; -} - -int ObTenantSrsMgr::delete_nonexist_tenants_srs() -{ - int ret = OB_SUCCESS; - FOREACH_X(iter, nonexist_tenant_srs_, OB_SUCC(ret)) { - ObTenantSrs *tenant_srs = *iter; - if (OB_ISNULL(tenant_srs)) { - ret = OB_ERR_NULL_VALUE; - LOG_WARN("tenant srs in recycle list is null", K(ret)); - } else { - if (OB_FAIL(tenant_srs->cancle_update_task())) { - LOG_WARN("failed to cancle tenant srs update task", K(ret), K(tenant_srs->tenant_id())); - } else { - tenant_srs->recycle_old_snapshots(); - tenant_srs->recycle_last_snapshots(); - if (tenant_srs->get_snapshots_size() == 0) { - if (OB_FAIL(nonexist_tenant_srs_.erase(iter))) { - LOG_WARN("failed to erase tenant srs from nonexist list", K(ret)); - } else { - tenant_srs->~ObTenantSrs(); - allocator_.free(tenant_srs); - } - } - } - } - } - return ret; -} - -int ObTenantSrsMgr::move_tenant_srs_to_nonexist_list(uint64_t tenant_id) -{ - int ret = OB_SUCCESS; - ObTenantSrs *tenant_srs = nullptr; - DRWLock::WRLockGuard guard(rwlock_); - if (is_virtual_tenant_id(tenant_id)) { - // do nothing - } else if (OB_FAIL(tenant_srs_map_.get_refactored(tenant_id, tenant_srs))) { - if (OB_HASH_NOT_EXIST == ret) { - ret = OB_SUCCESS; - } else { - LOG_WARN("get tenant tenant_srs failed", K(tenant_id), K(ret)); - } - } else if (OB_ISNULL(tenant_srs)) { - ret = OB_ERR_NULL_VALUE; - LOG_WARN("tenant srs is null", K(ret), K(tenant_id)); - } else if (OB_FAIL(tenant_srs_map_.erase_refactored(tenant_id))) { - LOG_WARN("erase tenant tenant_srs failed", K(ret), K(tenant_id)); - } else if (OB_FAIL(nonexist_tenant_srs_.push_back(tenant_srs))) { - LOG_WARN("push back tenant_srs failed", K(ret)); - } else { - LOG_INFO("drop tenant srs push back succeed", K(tenant_srs->tenant_id())); - } - return ret; -} - -int ObTenantSrsMgr::try_to_add_new_tenants() -{ - int ret = OB_SUCCESS; - ObArray new_tenants_id; - { - share::schema::ObSchemaGetterGuard sys_schema_guard; - if (OB_ISNULL(schema_service_)) { - ret = OB_ERR_NULL_VALUE; - LOG_WARN("schema service is null", K(ret)); - } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, - sys_schema_guard))) { - LOG_WARN("get sys tenant schema guard failed", K(ret)); - } else if (OB_FAIL(sys_schema_guard.get_tenant_ids(new_tenants_id))) { - LOG_WARN("get tenant ids failed", K(ret)); - } - } - if (OB_SUCC(ret) && OB_FAIL(add_tenants_srs(new_tenants_id))) { - LOG_WARN("failed to add tenants srs", K(ret), K(new_tenants_id)); - } - return ret; -} - -int ObTenantSrsMgr::add_tenants_srs(const common::ObIArray &new_tenants_id) -{ - int ret = OB_SUCCESS; - for (uint32_t i = 0; i < new_tenants_id.count() && OB_SUCC(ret); i++) { - uint64_t tenant_id = new_tenants_id.at(i); - if (OB_FAIL(add_tenant_srs(tenant_id))) { - LOG_WARN("failed to add tenant srs", K(ret), K(tenant_id)); - } - } - return ret; -} - -int ObTenantSrsMgr::add_tenant_srs(const uint64_t tenant_id) -{ - int ret = OB_SUCCESS; - ObTenantSrs *const *old_tenant_srs = NULL; - DRWLock::WRLockGuard guard(rwlock_); - if (!is_inited_) { - ret = OB_NOT_INIT; - LOG_WARN("tenant srs mgr not inited", K(ret)); - } else if (is_virtual_tenant_id(tenant_id) - || OB_NOT_NULL(old_tenant_srs = tenant_srs_map_.get(tenant_id))) { - LOG_INFO("try to add exist tenant or virtual tenant", K(tenant_id)); - } else { - void *buf = allocator_.alloc(sizeof(ObTenantSrs)); - if (OB_ISNULL(buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memory for tenant srs", K(ret)); - } else { - ObTenantSrs *new_tenant_srs = new (buf) ObTenantSrs(&allocator_, tenant_id); - if (OB_FAIL(new_tenant_srs->init(this))) { - LOG_WARN("failed to init new tenant srs", K(ret)); - } else if (OB_FAIL(tenant_srs_map_.set_refactored(tenant_id, new_tenant_srs, 1))) { - LOG_WARN("failed to set new tenant srs", K(ret), K(tenant_id)); - } - if (OB_FAIL(ret)) { - if (OB_FAIL(new_tenant_srs->cancle_update_task())) { - LOG_WARN("failed to cancle update srs task", K(ret), K(tenant_id)); - } else { - new_tenant_srs->~ObTenantSrs(); - allocator_.free(new_tenant_srs); - } - } - } - } - - return OB_SUCCESS; -} - -int ObTenantSrsMgr::refresh_tenant_srs(uint64_t tenant_id) -{ - int ret = OB_SUCCESS; - bool is_exist = false; - if (OB_LIKELY(OB_INVALID_ID != tenant_id)) { - share::schema::ObSchemaGetterGuard sys_schema_guard; - if (OB_ISNULL(schema_service_)) { - ret = OB_NOT_INIT; - LOG_WARN("not init", K(ret)); - } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, - sys_schema_guard))) { - LOG_WARN("get sys tenant schema guard failed", K(ret)); - } else if (OB_FAIL(sys_schema_guard.check_tenant_exist(tenant_id, is_exist))) { - LOG_WARN("get tenant ids failed", K(ret)); - } - } - if (OB_SUCC(ret) && is_exist && OB_FAIL(add_tenant_srs(tenant_id))) { - LOG_WARN("add tenant timezone failed", K(ret)); - } - return ret; -} - -int ObTenantSrsMgr::get_tenant_guard_inner(uint64_t tenant_id, ObSrsCacheGuard &srs_guard) -{ - int ret = OB_SUCCESS; - ObTenantSrs *tenant_srs = NULL; - DRWLock::RDLockGuard guard(rwlock_); - if (OB_FAIL(tenant_srs_map_.get_refactored(tenant_id, tenant_srs))) { - if (ret != OB_HASH_NOT_EXIST) { - LOG_WARN("failed to get tenant srs", K(ret), K(tenant_id)); - } - } else if (OB_FAIL(tenant_srs->try_get_last_snapshot(srs_guard))) { - LOG_WARN("failed to get last srs snapshot", K(ret), K(tenant_id)); - } - return ret; -} - -int ObTenantSrsMgr::get_tenant_srs_guard(uint64_t tenant_id, ObSrsCacheGuard &srs_guard) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(get_tenant_guard_inner(tenant_id, srs_guard))) { - if (ret == OB_HASH_NOT_EXIST) { - if (OB_FAIL(refresh_tenant_srs(tenant_id))) { - LOG_WARN("update srs tenant map failed", K(ret), K(tenant_id)); - } else if (OB_FAIL(get_tenant_guard_inner(tenant_id, srs_guard))) { - LOG_WARN("failed to get tenant srs", K(ret), K(tenant_id)); - } - } else if (ret == OB_ERR_EMPTY_QUERY) { - ret = OB_ERR_SRS_EMPTY; - LOG_WARN("srs table might be empty", K(ret), K(tenant_id)); - LOG_USER_ERROR(OB_ERR_SRS_EMPTY); - } else { - LOG_WARN("failed to get tenant srs", K(ret), K(tenant_id)); - } - } - return ret; -} - -bool ObTenantSrsMgr::is_sys_schema_ready() -{ - return schema_service_->is_sys_full_schema(); -} - -void ObTenantSrsMgr::stop() -{ - TG_STOP(lib::TGDefIDs::SRS_MGR); -} - -void ObTenantSrsMgr::wait() -{ - TG_WAIT(lib::TGDefIDs::SRS_MGR); -} - -void ObTenantSrsMgr::destroy() -{ - TG_DESTROY(lib::TGDefIDs::SRS_MGR); - tenant_srs_map_.destroy(); -} - - -} // namespace omt -} // namespace oceanbase \ No newline at end of file diff --git a/src/observer/omt/ob_tenant_srs_mgr.h b/src/observer/omt/ob_tenant_srs_mgr.h deleted file mode 100644 index bae3aacbf..000000000 --- a/src/observer/omt/ob_tenant_srs_mgr.h +++ /dev/null @@ -1,143 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#ifndef OCEANBASE_OBSERVER_OMT_OB_TENANT_SRS_MGR_H_ -#define OCEANBASE_OBSERVER_OMT_OB_TENANT_SRS_MGR_H_ - -#include "ob_tenant_srs.h" -#include "share/ob_define.h" -#include "lib/hash/ob_pointer_hashmap.h" -#include "lib/container/ob_vector.h" -#include "lib/list/ob_list.h" -#include "lib/allocator/page_arena.h" -#include "lib/lock/ob_drw_lock.h" -#include "lib/task/ob_timer.h" -#include "lib/geo/ob_srs_wkt_parser.h" -#include "share/schema/ob_schema_struct.h" - -namespace oceanbase -{ - -namespace share -{ -namespace schema -{ -class ObMultiVersionSchemaService; -} -} - -namespace omt -{ - -class ObTenantSrsMgr; - -class ObTenantSrsMgr -{ - template - class ObSrsContainer - : public common::hash::ObHashMap - { - public: - ObSrsContainer() - { - this->create(num, "TenantSrsMgr","TenantSrs"); - } - virtual ~ObSrsContainer() {} - - private: - DISALLOW_COPY_AND_ASSIGN(ObSrsContainer); - }; - -class AddTenantTask : public common::ObTimerTask -{ -public: - AddTenantTask(ObTenantSrsMgr *tenant_srs_mgr) - : tenant_srs_mgr_(tenant_srs_mgr) {} - virtual ~AddTenantTask() {} - AddTenantTask(const AddTenantTask &) = delete; - AddTenantTask &operator=(const AddTenantTask &) = delete; - void runTimerTask(void) override; - int add_tenants_to_map(common::ObIArray &latest_tenant_ids); - -private: - static const uint64_t DEFAULT_PERIOD = 5000000; - static const uint64_t BOOTSTRAP_PERIOD = 500000; - ObTenantSrsMgr *tenant_srs_mgr_; -}; - -class DeleteTenantTask : public common::ObTimerTask -{ -public: - DeleteTenantTask(ObTenantSrsMgr *tenant_srs_mgr) - : tenant_srs_mgr_(tenant_srs_mgr) {} - virtual ~DeleteTenantTask() {} - DeleteTenantTask(const DeleteTenantTask &) = delete; - DeleteTenantTask &operator=(const DeleteTenantTask &) = delete; - void runTimerTask(void) override; - -private: - static const uint64_t DEFAULT_PERIOD = 5000000; - ObTenantSrsMgr *tenant_srs_mgr_; -}; - - friend ObTenantSrs; -public: - using TenantSrsMap = ObSrsContainer; - - virtual ~ObTenantSrsMgr(); - static ObTenantSrsMgr &get_instance(); - int init(common::ObMySQLProxy *sql_proxy, const common::ObAddr &server, - share::schema::ObMultiVersionSchemaService *schema_service); - int add_tenant_srs(const uint64_t tenant_id); - int get_tenant_srs_guard(uint64_t tenant_id, ObSrsCacheGuard &srs_guard); - void stop(); - void wait(); - void destroy(); - bool is_sys_load_completed() { return is_sys_load_completed_; } - void set_sys_load_completed() { is_sys_load_completed_ = true; } - bool is_sys_schema_ready(); - int get_srs_bounds(uint64_t srid, const common::ObSrsItem *srs_item, - const common::ObSrsBoundsItem *&bounds_item); - -private: - ObTenantSrsMgr(); - int refresh_tenant_srs(uint64_t tenant_id); - int get_tenant_guard_inner(uint64_t tenant_id, ObSrsCacheGuard &srs_guard); - int add_tenants_srs(const common::ObIArray &new_tenants_id); - int try_to_add_new_tenants(); - int remove_nonexist_tenants(); - int move_tenant_srs_to_nonexist_list(uint64_t tenant_id); - int delete_nonexist_tenants_srs(); - - common::ObArenaAllocator allocator_; - common::ObMySQLProxy *sql_proxy_; - common::ObAddr self_; - // protect tenant_srs_map_ - common::DRWLock rwlock_; - TenantSrsMap tenant_srs_map_; - AddTenantTask add_tenant_task_; - DeleteTenantTask del_tenant_task_; - bool is_inited_; - bool is_sys_load_completed_; - share::schema::ObMultiVersionSchemaService *schema_service_; - common::ObList nonexist_tenant_srs_; - common::ObSrsBoundsItem infinite_plane_; - - DISALLOW_COPY_AND_ASSIGN(ObTenantSrsMgr); -}; - -} // namespace omt -} // namespace oceanbase - -#define OTSRS_MGR (::oceanbase::omt::ObTenantSrsMgr::get_instance()) - -#endif diff --git a/src/observer/omt/ob_tenant_timezone.cpp b/src/observer/omt/ob_tenant_timezone.cpp index f65485927..42596a1b6 100644 --- a/src/observer/omt/ob_tenant_timezone.cpp +++ b/src/observer/omt/ob_tenant_timezone.cpp @@ -21,54 +21,34 @@ #include "observer/omt/ob_tenant_timezone_mgr.h" #include "share/ob_thread_mgr.h" #include "share/ob_time_zone_info_manager.h" -#include "observer/ob_server.h" using namespace oceanbase::common; namespace oceanbase { namespace omt { - -ObTenantTimezone::ObTenantTimezone() : tenant_id_(OB_INVALID_TENANT_ID) -{ -} - -ObTenantTimezone::ObTenantTimezone(uint64_t tenant_id) - : is_inited_(false), tenant_id_(tenant_id), tenant_tz_mgr_(nullptr), - tz_info_mgr_(nullptr), tz_info_map_(nullptr), update_task_not_exist_(false) +ObTenantTimezone::ObTenantTimezone(common::ObMySQLProxy &sql_proxy, uint64_t tenant_id) + : is_inited_(false), tenant_id_(tenant_id), + tz_info_mgr_(sql_proxy, tenant_id), update_task_not_exist_(false) { } ObTenantTimezone::~ObTenantTimezone() { } -int ObTenantTimezone::init(ObTenantTimezoneMgr *tz_mgr) +int ObTenantTimezone::init() { int ret = OB_SUCCESS; - tenant_tz_mgr_ = tz_mgr; - is_inited_ = true; - tz_info_map_ = OB_NEW(ObTZInfoMap, SET_USE_500("TZInfoMap")); - tz_info_mgr_ = OB_NEW(ObTimeZoneInfoManager, SET_USE_500("TZInfoMgr"), OBSERVER.get_common_rpc_proxy(), - OBSERVER.get_mysql_proxy(), OBSERVER.get_root_service(), - *tz_info_map_, tenant_id_); - if (OB_ISNULL(tz_info_map_) || OB_ISNULL(tz_info_mgr_)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to allocate mem for tz_info", K(ret), K(tz_info_map_), K(tz_info_mgr_)); - } else if (OB_FAIL(tz_info_map_->init(SET_USE_500("TZInfoMap")))) { - LOG_WARN("fail to init tz_info_map_", K(ret)); - } else if (OB_FAIL(tz_info_mgr_->init())) { + if (OB_UNLIKELY(is_inited_)) { + ret = OB_INIT_TWICE; + LOG_WARN("init twice", K(ret), K(tenant_id_)); + } else if (OB_FAIL(tz_info_mgr_.init())) { LOG_WARN("fail to init tz_info_mgr_", K(ret)); } else { - LOG_INFO("tenant timezone init", K(tz_info_map_), K(tenant_id_)); + is_inited_ = true; } + LOG_INFO("tenant timezone init", K(ret), K(tenant_id_), K(sizeof(ObTimeZoneInfoManager))); return ret; } -void ObTenantTimezone::destroy() -{ - if (NULL != tz_info_map_) { - tz_info_map_->destroy(); - } -} - } // omt } // oceanbase diff --git a/src/observer/omt/ob_tenant_timezone.h b/src/observer/omt/ob_tenant_timezone.h index c59a7abb9..8e0dc8dfa 100644 --- a/src/observer/omt/ob_tenant_timezone.h +++ b/src/observer/omt/ob_tenant_timezone.h @@ -13,7 +13,7 @@ #ifndef OCEANBASE_TENANT_TIMEZONE_H_ #define OCEANBASE_TENANT_TIMEZONE_H_ -#include "lib/timezone/ob_timezone_info.h" +#include "src/share/ob_time_zone_info_manager.h" namespace oceanbase { @@ -25,34 +25,28 @@ class ObTenantTimezone { friend class ObTenantTimezoneMgr; public: - ObTenantTimezone(); - ObTenantTimezone(uint64_t tenant_id); + ObTenantTimezone(common::ObMySQLProxy &sql_proxy, uint64_t tenant_id); virtual ~ObTenantTimezone(); ObTenantTimezone(const ObTenantTimezone &)=delete; ObTenantTimezone &operator=(const ObTenantTimezone &)=delete; - int init(ObTenantTimezoneMgr *tz_mgr); + int init(); int update_timezone(int64_t tz_version); bool is_inited() { return is_inited_; } bool get_update_task_not_exist() { return update_task_not_exist_; } int get_ref_count(int64_t &ref_count); uint64_t get_tenant_id() const { return tenant_id_; } - common::ObTZInfoMap *get_tz_map() { return tz_info_map_; } - common::ObTimeZoneInfoManager *get_tz_mgr() { return tz_info_mgr_; } + common::ObTZInfoMap *get_tz_map() { return tz_info_mgr_.get_tz_info_map(); } + common::ObTimeZoneInfoManager &get_tz_mgr() { return tz_info_mgr_; } void set_update_task_not_exist() { update_task_not_exist_ = true; } - void set_tz_mgr(common::ObTimeZoneInfoManager *tz_mgr) { tz_info_mgr_ = tz_mgr; } - void set_tenant_tz_mgr(ObTenantTimezoneMgr *tz_mgr) { tenant_tz_mgr_ = tz_mgr; }; - void destroy(); VIRTUAL_TO_STRING_KV(K_(is_inited), K_(tenant_id)); private: bool is_inited_; uint64_t tenant_id_; - ObTenantTimezoneMgr *tenant_tz_mgr_; - common::ObTimeZoneInfoManager *tz_info_mgr_; - common::ObTZInfoMap *tz_info_map_; + common::ObTimeZoneInfoManager tz_info_mgr_; bool update_task_not_exist_; }; diff --git a/src/observer/omt/ob_tenant_timezone_mgr.cpp b/src/observer/omt/ob_tenant_timezone_mgr.cpp index 5bbd7620e..c0e639cc3 100644 --- a/src/observer/omt/ob_tenant_timezone_mgr.cpp +++ b/src/observer/omt/ob_tenant_timezone_mgr.cpp @@ -18,6 +18,7 @@ #include "share/ob_time_zone_info_manager.h" #include "share/schema/ob_multi_version_schema_service.h" #include "lib/hash/ob_hashset.h" +#include "observer/ob_server.h" using namespace oceanbase::common; @@ -51,7 +52,7 @@ int ObTenantTimezoneMgr::UpdateTenantTZOp::operator() (common::hash::HashMapPair { int ret = OB_SUCCESS; ObTenantTimezone &tenant_tz = *entry.second; - if (OB_FAIL(tenant_tz.get_tz_mgr()->fetch_time_zone_info())) { + if (OB_FAIL(tenant_tz.get_tz_mgr().fetch_time_zone_info())) { LOG_WARN("fail to update time zone info", K(ret)); } return ret; @@ -157,16 +158,16 @@ int ObTenantTimezoneMgr::add_tenant_timezone(uint64_t tenant_id) || OB_NOT_NULL(timezone = timezone_map_.get(tenant_id))) { } else { ObTenantTimezone *new_timezone = nullptr; - new_timezone = OB_NEW(ObTenantTimezone, "TenantTZ", tenant_id); + new_timezone = OB_NEW(ObTenantTimezone, "TenantTZ", OBSERVER.get_mysql_proxy(), tenant_id); if (OB_ISNULL(new_timezone)) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc new tenant timezone failed", K(ret)); - } else if(OB_FAIL(new_timezone->init(this))) { + } else if(OB_FAIL(new_timezone->init())) { LOG_WARN("new tenant timezone init failed", K(ret)); } else if (OB_FAIL(timezone_map_.set_refactored(tenant_id, new_timezone, 1))) { LOG_WARN("add new tenant timezone failed", K(ret)); } else { - LOG_INFO("add tenant timezone success!", K(tenant_id)); + LOG_INFO("add tenant timezone success!", K(tenant_id), K(sizeof(ObTenantTimezone))); } if (OB_FAIL(ret)) { ob_delete(new_timezone); @@ -193,10 +194,7 @@ int ObTenantTimezoneMgr::del_tenant_timezone(uint64_t tenant_id) } else if (OB_FAIL(timezone_map_.erase_refactored(tenant_id))) { LOG_WARN("erase tenant timezone failed", K(ret), K(tenant_id)); } else { - ObTZInfoMap *tz_map = timezone->get_tz_map(); - LOG_INFO("drop tenant tz push back succeed", K(timezone->get_tz_map()), - K(timezone->get_tenant_id())); - timezone->destroy(); + LOG_INFO("drop tenant tz push back succeed", K(timezone->get_tenant_id())); ob_delete(timezone); } return ret; @@ -218,7 +216,7 @@ int ObTenantTimezoneMgr::get_tenant_timezone_inner(const uint64_t tenant_id, LOG_WARN("tenant tz is null", K(ret)); } else { timezone_wrap.set_tz_map(timezone->get_tz_map()); - tz_info_mgr = timezone->get_tz_mgr(); + tz_info_mgr = &(timezone->get_tz_mgr()); } return ret; } diff --git a/src/observer/report/ob_tablet_table_updater.h b/src/observer/report/ob_tablet_table_updater.h index 2e374fc5c..defff4777 100644 --- a/src/observer/report/ob_tablet_table_updater.h +++ b/src/observer/report/ob_tablet_table_updater.h @@ -220,8 +220,8 @@ private: private: const int64_t MINI_MODE_UPDATE_TASK_THREAD_CNT = 1; const int64_t UPDATE_TASK_THREAD_CNT = 7; - const int64_t MINI_MODE_UPDATE_QUEUE_SIZE = 20 * 10000; - const int64_t UPDATE_QUEUE_SIZE = 100 * 10000; + const int64_t MINI_MODE_UPDATE_QUEUE_SIZE = 5 * 10000; + const int64_t UPDATE_QUEUE_SIZE = 10 * 10000; bool inited_; bool stopped_; ObService *ob_service_; diff --git a/src/observer/table_load/ob_table_load_store.cpp b/src/observer/table_load/ob_table_load_store.cpp index 0b021be4a..e9dadb3d2 100644 --- a/src/observer/table_load/ob_table_load_store.cpp +++ b/src/observer/table_load/ob_table_load_store.cpp @@ -311,7 +311,7 @@ int ObTableLoadStore::commit(ObTableLoadResultInfo &result_info) LOG_WARN("fail to commit sql stats", KR(ret)); } else if (OB_FAIL(store_ctx_->merger_->collect_dml_stat(dml_stats))) { LOG_WARN("fail to build dml stat", KR(ret)); - } else if (OB_FAIL(ObOptStatMonitorManager::get_instance().update_dml_stat_info_from_direct_load(dml_stats.dml_stat_array_))) { + } else if (OB_FAIL(ObOptStatMonitorManager::update_dml_stat_info_from_direct_load(dml_stats.dml_stat_array_))) { LOG_WARN("fail to update dml stat info", KR(ret)); } else if (OB_FAIL(store_ctx_->set_status_commit())) { LOG_WARN("fail to set store status commit", KR(ret)); diff --git a/src/observer/virtual_table/ob_all_latch.cpp b/src/observer/virtual_table/ob_all_latch.cpp index 02fad2e0f..701332024 100644 --- a/src/observer/virtual_table/ob_all_latch.cpp +++ b/src/observer/virtual_table/ob_all_latch.cpp @@ -131,7 +131,9 @@ int ObAllLatch::inner_get_next_row(ObNewRow *&row) OB_SUCC(ret) && cell_idx < output_column_ids_.count(); ++cell_idx) { const uint64_t column_id = output_column_ids_.at(cell_idx); - const ObLatchStat& latch_stat = dipair.second->get_latch_stats().items_[latch_iter_]; + ObLatchStat *p_latch_stat = dipair.second->get_latch_stats().get_item(latch_iter_); + if (OB_ISNULL(p_latch_stat)) continue; + const ObLatchStat& latch_stat = *p_latch_stat; switch(column_id) { case TENANT_ID: { cells[cell_idx].set_int(dipair.first); diff --git a/src/observer/virtual_table/ob_all_virtual_dml_stats.cpp b/src/observer/virtual_table/ob_all_virtual_dml_stats.cpp index daff499f6..c936daf53 100644 --- a/src/observer/virtual_table/ob_all_virtual_dml_stats.cpp +++ b/src/observer/virtual_table/ob_all_virtual_dml_stats.cpp @@ -22,6 +22,24 @@ using namespace sql; namespace observer { +int ObAllVirtualDMmlStats::inner_open() +{ + int ret = OB_SUCCESS; + // sys tenant show all tenant infos + if (is_sys_tenant(effective_tenant_id_)) { + if (OB_FAIL(GCTX.omt_->get_mtl_tenant_ids(tenant_ids_))) { + SERVER_LOG(WARN, "failed to add tenant id", K(ret)); + } + } else { + // user tenant show self tenant infos + if (OB_FAIL(tenant_ids_.push_back(effective_tenant_id_))) { + SERVER_LOG(WARN, "failed to push back tenant id", KR(ret), K(effective_tenant_id_), + K(tenant_ids_)); + } + } + return ret; +} + int ObOptDmlStatMapGetter::operator()(common::hash::HashMapPair &entry) { int ret = OB_SUCCESS; @@ -77,27 +95,8 @@ int ObOptDmlStatMapGetter::operator()(common::hash::HashMapPair &entry) -{ - int ret = OB_SUCCESS; - uint64_t tenant_id = entry.first; - DmlStatMap *dml_stat_map = entry.second; - if (effective_tenant_id_ != tenant_id && !is_sys_tenant(effective_tenant_id_)) { - SERVER_LOG(TRACE, "not full fill tenant id", K(effective_tenant_id_), K(tenant_id)); - } else if (OB_ISNULL(dml_stat_map)) { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "get unexpected null", K(ret), K(dml_stat_map)); - } else { - ObOptDmlStatMapGetter getter(scanner_, output_column_ids_, svr_ip_, port_, cur_row_, effective_tenant_id_); - if (OB_FAIL(dml_stat_map->foreach_refactored(getter))) { - SERVER_LOG(WARN, "fail to get dml stat", K(ret)); - } else {/*do nothing*/} - } - return ret; -} - ObAllVirtualDMmlStats::ObAllVirtualDMmlStats() - : port_(0) + : port_(0), tenant_ids_(), tenant_idx_(0) { MEMSET(svr_ip_, 0, sizeof(svr_ip_)); } @@ -122,19 +121,27 @@ void ObAllVirtualDMmlStats::reset() int ObAllVirtualDMmlStats::inner_get_next_row(ObNewRow *&row) { int ret = OB_SUCCESS; - ObObj *cells = cur_row_.cells_; - if (!start_to_read_) { - if (OB_FAIL(fill_scanner())) { + if (tenant_idx_ >= tenant_ids_.count()) { + ret = OB_ITER_END; + } else if (!start_to_read_) { + if (OB_FAIL(fill_scanner(tenant_ids_.at(tenant_idx_)))) { SERVER_LOG(WARN, "fill scanner failed", K(ret)); } else { start_to_read_ = true; } } - - if (OB_SUCCESS == ret && start_to_read_) { + if (OB_SUCC(ret) && start_to_read_) { if (OB_FAIL(scanner_it_.get_next_row(cur_row_))) { if (OB_ITER_END != ret) { SERVER_LOG(WARN, "fail to get next row", K(ret)); + } else if (++ tenant_idx_ < tenant_ids_.count()) {//load a new tenant info + ret = OB_SUCCESS; + start_to_read_ = false; + if (OB_FAIL(SMART_CALL(inner_get_next_row(row)))) { + if (OB_ITER_END != ret) { + SERVER_LOG(WARN, "failed to inner get next row", K(ret)); + } + } } } else { row = &cur_row_; @@ -143,7 +150,7 @@ int ObAllVirtualDMmlStats::inner_get_next_row(ObNewRow *&row) return ret; } -int ObAllVirtualDMmlStats::fill_scanner() +int ObAllVirtualDMmlStats::fill_scanner(uint64_t tenant_id) { int ret = OB_SUCCESS; ObObj *cells = NULL; @@ -156,12 +163,18 @@ int ObAllVirtualDMmlStats::fill_scanner() ret = OB_ERR_UNEXPECTED; } else { port_ = addr.get_port(); - ObOptDmlStatMapsGetter getter(scanner_, output_column_ids_, svr_ip_, port_, cur_row_, effective_tenant_id_); - if (OB_FAIL(ObOptStatMonitorManager::get_instance().generate_opt_stat_monitoring_info_rows(getter))) { - SERVER_LOG(WARN, "generate monitor info array failed", K(ret)); - } else { - scanner_it_ = scanner_.begin(); - start_to_read_ = true; + MTL_SWITCH(tenant_id) { + ObOptDmlStatMapGetter getter(scanner_, output_column_ids_, svr_ip_, port_, cur_row_, tenant_id); + ObOptStatMonitorManager *optstat_monitor_mgr = MTL(ObOptStatMonitorManager*); + if (OB_ISNULL(optstat_monitor_mgr)) { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "optstat monitor mgr is NULL", K(ret), K(tenant_id), K(effective_tenant_id_)); + } else if (OB_FAIL(optstat_monitor_mgr->generate_opt_stat_monitoring_info_rows(getter))) { + SERVER_LOG(WARN, "generate monitor info array failed", K(ret)); + } else { + scanner_it_ = scanner_.begin(); + start_to_read_ = true; + } } } return ret; diff --git a/src/observer/virtual_table/ob_all_virtual_dml_stats.h b/src/observer/virtual_table/ob_all_virtual_dml_stats.h index 3fa803d81..0848f9ca5 100644 --- a/src/observer/virtual_table/ob_all_virtual_dml_stats.h +++ b/src/observer/virtual_table/ob_all_virtual_dml_stats.h @@ -54,34 +54,6 @@ private: uint64_t effective_tenant_id_; }; -class ObOptDmlStatMapsGetter -{ -public: - explicit ObOptDmlStatMapsGetter(common::ObScanner &scanner, - common::ObIArray &output_column_ids, - char *svr_ip, - int32_t port, - common::ObNewRow &cur_row, - uint64_t effective_tenant_id) - : scanner_(scanner), - output_column_ids_(output_column_ids), - svr_ip_(svr_ip), - port_(port), - cur_row_(cur_row), - effective_tenant_id_(effective_tenant_id) - {} - virtual ~ObOptDmlStatMapsGetter() {}; - int operator() (common::hash::HashMapPair &entry); - DISALLOW_COPY_AND_ASSIGN(ObOptDmlStatMapsGetter); -private: - common::ObScanner &scanner_; - common::ObIArray &output_column_ids_; - char *svr_ip_; - int32_t port_; - common::ObNewRow &cur_row_; - uint64_t effective_tenant_id_; -}; - class ObAllVirtualDMmlStats : public ObVirtualTableScannerIterator { friend class ObOptDmlStatMapGetter; @@ -91,6 +63,7 @@ public: void destroy(); virtual void reset() override; virtual int inner_get_next_row(common::ObNewRow *&row) override; + virtual int inner_open() override; private: enum COLUMNS { @@ -105,7 +78,9 @@ private: }; int32_t port_; char svr_ip_[common::OB_IP_STR_BUFF]; - int fill_scanner(); + int fill_scanner(uint64_t tenant_id); + common::ObSEArray tenant_ids_; + int64_t tenant_idx_; DISALLOW_COPY_AND_ASSIGN(ObAllVirtualDMmlStats); }; diff --git a/src/observer/virtual_table/ob_all_virtual_sys_stat.cpp b/src/observer/virtual_table/ob_all_virtual_sys_stat.cpp index cd3082e10..9a92def9b 100644 --- a/src/observer/virtual_table/ob_all_virtual_sys_stat.cpp +++ b/src/observer/virtual_table/ob_all_virtual_sys_stat.cpp @@ -193,6 +193,12 @@ int ObAllVirtualSysStat::update_all_stats_(const int64_t tenant_id, ObStatEventS (OB_SYS_TENANT_ID == tenant_id) ? global_poc_server.get_ratelimit_rxbytes() : -1; stat_events.get(ObStatEventIds::STANDBY_FETCH_LOG_BANDWIDTH_LIMIT - ObStatEventIds::STAT_EVENT_ADD_END -1)->stat_value_ = (OB_SYS_TENANT_ID == tenant_id) ? global_poc_server.get_ratelimit() : -1; + stat_events.get(ObStatEventIds::MEMORY_LIMIT - ObStatEventIds::STAT_EVENT_ADD_END -1)->stat_value_ = + (OB_SYS_TENANT_ID == tenant_id) ? GMEMCONF.get_server_memory_limit() : 0; + stat_events.get(ObStatEventIds::SYSTEM_MEMORY - ObStatEventIds::STAT_EVENT_ADD_END -1)->stat_value_ = + (OB_SYS_TENANT_ID == tenant_id) ? GMEMCONF.get_reserved_server_memory() : 0; + stat_events.get(ObStatEventIds::HIDDEN_SYS_MEMORY - ObStatEventIds::STAT_EVENT_ADD_END -1)->stat_value_ = + (OB_SYS_TENANT_ID == tenant_id) ? GMEMCONF.get_hidden_sys_memory() : 0; int ret_bk = ret; if (NULL != GCTX.omt_) { diff --git a/src/pl/ob_pl_package_state.h b/src/pl/ob_pl_package_state.h index 4332b006b..3a3c8ba2b 100644 --- a/src/pl/ob_pl_package_state.h +++ b/src/pl/ob_pl_package_state.h @@ -114,7 +114,7 @@ public: const ObPackageStateVersion &state_version, bool serially_reusable) : inner_allocator_(this), - cursor_allocator_(), + cursor_allocator_("PlPkgCursor", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), package_id_(package_id), state_version_(state_version), serially_reusable_(serially_reusable), diff --git a/src/pl/ob_pl_router.h b/src/pl/ob_pl_router.h index 3bd3b78dc..e6b9f41f5 100644 --- a/src/pl/ob_pl_router.h +++ b/src/pl/ob_pl_router.h @@ -29,7 +29,7 @@ public: session_info_(session_info), schema_guard_(schema_guard), sql_proxy_(sql_proxy), - inner_allocator_(), + inner_allocator_(ObModIds::OB_PL_TEMP, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), expr_factory_(inner_allocator_) {} virtual ~ObPLRouter() {} diff --git a/src/pl/sys_package/ob_dbms_stats.cpp b/src/pl/sys_package/ob_dbms_stats.cpp index f87202166..240092a22 100644 --- a/src/pl/sys_package/ob_dbms_stats.cpp +++ b/src/pl/sys_package/ob_dbms_stats.cpp @@ -4814,7 +4814,7 @@ int ObDbmsStats::process_not_size_manual_column(sql::ObExecContext &ctx, if (OB_SUCC(ret) && !auto_columns.empty()) { if (OB_FAIL(ObOptStatMonitorManager::flush_database_monitoring_info(ctx, true, false))) { LOG_WARN("failed to do flush database monitoring info", K(ret)); - } else if (OB_FAIL(ObOptStatMonitorManager::get_instance().get_column_usage_from_table( + } else if (OB_FAIL(ObOptStatMonitorManager::get_column_usage_from_table( ctx, auto_columns, table_param.tenant_id_, table_param.table_id_))) { LOG_WARN("failed to get column usage from table", K(ret)); } else { diff --git a/src/rootserver/ob_disaster_recovery_task_mgr.h b/src/rootserver/ob_disaster_recovery_task_mgr.h index 3981ac421..92d3ce67a 100644 --- a/src/rootserver/ob_disaster_recovery_task_mgr.h +++ b/src/rootserver/ob_disaster_recovery_task_mgr.h @@ -107,7 +107,7 @@ public: // @param [in] task_mgr, to execute over a task int handle_not_in_progress_task( ObDRTaskMgr &task_mgr); - + // remove task from schedule_list and clean it // @param [in] task, the task to finish schedule int finish_schedule( @@ -195,7 +195,7 @@ private: class ObDRTaskMgr : public ObRsReentrantThread { public: - const static int64_t TASK_QUEUE_LIMIT = 1L << 20; + const static int64_t TASK_QUEUE_LIMIT = 10000; //1w const static int64_t ONCE_ADD_TASK_CNT = TASK_QUEUE_LIMIT / 2; const static int64_t SAFE_DISCARD_TASK_INTERVAL = 200L * 1000L; // 200ms const static int64_t DATA_IN_CLEAR_INTERVAL = 20L * 60L * 1000000L; // 20min @@ -298,7 +298,7 @@ public: const ObDRTaskRetComment &ret_comment); // operations of reach_concurrency_limit - void set_reach_concurrency_limit() { + void set_reach_concurrency_limit() { concurrency_limited_ts_ = common::ObTimeUtility::current_time(); } void clear_reach_concurrency_limit() { @@ -365,7 +365,7 @@ private: const ObDRTask &task); // try to log inmemory task infos according to balancer_log_interval - // @param [in] last_dump_ts, last time do logging + // @param [in] last_dump_ts, last time do logging int try_dump_statistic_( int64_t &last_dump_ts) const; int inner_dump_statistic_() const; diff --git a/src/rootserver/ob_disaster_recovery_task_table_updater.cpp b/src/rootserver/ob_disaster_recovery_task_table_updater.cpp index 97f395299..73d056589 100644 --- a/src/rootserver/ob_disaster_recovery_task_table_updater.cpp +++ b/src/rootserver/ob_disaster_recovery_task_table_updater.cpp @@ -143,9 +143,7 @@ int ObDRTaskTableUpdater::init( ? MINI_MODE_UPDATE_THREAD_CNT : UPDATE_THREAD_CNT; const int64_t queue_size = - lib::is_mini_mode() - ? MINI_MODE_TASK_QUEUE_SIZE - : TASK_QUEUE_SIZE; + TASK_QUEUE_SIZE; if (OB_UNLIKELY(inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", KR(ret)); diff --git a/src/rootserver/ob_disaster_recovery_task_table_updater.h b/src/rootserver/ob_disaster_recovery_task_table_updater.h index 09b9e97bf..df7ac7d24 100644 --- a/src/rootserver/ob_disaster_recovery_task_table_updater.h +++ b/src/rootserver/ob_disaster_recovery_task_table_updater.h @@ -154,8 +154,7 @@ private: private: const int64_t MINI_MODE_UPDATE_THREAD_CNT = 1; const int64_t UPDATE_THREAD_CNT = 1; - const int64_t MINI_MODE_TASK_QUEUE_SIZE = 20 * 10000; - const int64_t TASK_QUEUE_SIZE = 100 * 10000; + const int64_t TASK_QUEUE_SIZE = 10000; bool inited_; bool stopped_; ObDRTaskTableUpdateTaskQueue update_queue_; diff --git a/src/rootserver/ob_partition_balance.cpp b/src/rootserver/ob_partition_balance.cpp index 26b92aaa8..1c9bd867c 100644 --- a/src/rootserver/ob_partition_balance.cpp +++ b/src/rootserver/ob_partition_balance.cpp @@ -91,6 +91,7 @@ void ObPartitionBalance::destroy() iter->second.at(i)->~ObLSPartGroupDesc(); } } + iter->second.destroy(); } //reset bg_builder_.destroy(); @@ -243,17 +244,16 @@ int ObPartitionBalance::on_new_partition( } else { // create partition group if in new partition group if (in_new_partition_group) { - ObTransferPartGroup *part_group = nullptr; - if (OB_ISNULL(part_group = reinterpret_cast(allocator_.alloc(sizeof(ObTransferPartGroup))))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("alloc mem fail", KR(ret)); - } else if (FALSE_IT(cur_part_group_ = new(part_group) ObTransferPartGroup(allocator_))) { - } else if (OB_FAIL(add_part_to_bg_map_(src_ls_id, *cur_part_group_, bg))) { - LOG_WARN("add partition to balance group fail", KR(ret), K(src_ls_id), K(bg), - K(cur_part_group_)); + ObTransferPartGroup *new_pg = NULL; + if (OB_FAIL(add_new_pg_to_bg_map_(src_ls_id, bg, new_pg))) { + LOG_WARN("add new partition group to balance group failed", KR(ret), K(src_ls_id), K(bg), K(new_pg)); + } else if (OB_ISNULL(new_pg)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("new_pg is null", KR(ret), K(src_ls_id), K(bg), K(new_pg)); } else { // add new partition group src_ls_desc->add_partgroup(1, 0); + cur_part_group_ = new_pg; } } // if not in new partition group, current part group should be valid @@ -274,9 +274,10 @@ int ObPartitionBalance::on_new_partition( return ret; } -int ObPartitionBalance::add_part_to_bg_map_(const ObLSID &ls_id, - ObTransferPartGroup &part_group, - ObBalanceGroup &bg) +int ObPartitionBalance::add_new_pg_to_bg_map_( + const ObLSID &ls_id, + ObBalanceGroup &bg, + ObTransferPartGroup *&part_group) { int ret = OB_SUCCESS; ObArray *ls_part_desc_arr = NULL; @@ -291,12 +292,13 @@ int ObPartitionBalance::add_part_to_bg_map_(const ObLSID &ls_id, ret = OB_ERR_UNEXPECTED; LOG_WARN("get part from balance group fail", KR(ret), K(ls_id), K(bg)); } else { + ls_part_desc_arr->set_block_allocator(ModulePageAllocator(allocator_, "LSPartDescArr")); for (int64_t i = 0; OB_SUCC(ret) && i < ls_desc_array_.count(); i++) { ObLSPartGroupDesc *ls_part_desc = nullptr; if (OB_ISNULL(ls_part_desc = reinterpret_cast(allocator_.alloc(sizeof(ObLSPartGroupDesc))))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc mem fail", KR(ret), K(ls_id), K(bg)); - } else if (FALSE_IT(new(ls_part_desc) ObLSPartGroupDesc(ls_desc_array_.at(i)->get_ls_id()))) { + } else if (FALSE_IT(new(ls_part_desc) ObLSPartGroupDesc(ls_desc_array_.at(i)->get_ls_id(), allocator_))) { } else if (OB_FAIL(ls_part_desc_arr->push_back(ls_part_desc))) { LOG_WARN("push_back fail", KR(ret), K(ls_id), K(bg)); } @@ -308,8 +310,8 @@ int ObPartitionBalance::add_part_to_bg_map_(const ObLSID &ls_id, bool find_ls = false; for (int64_t idx = 0; OB_SUCC(ret) && idx < ls_part_desc_arr->count(); idx++) { if (ls_part_desc_arr->at(idx)->get_ls_id() == ls_id) { - if (OB_FAIL(ls_part_desc_arr->at(idx)->get_part_groups().push_back(&part_group))) { - LOG_WARN("push_back fail", KR(ret)); + if (OB_FAIL(ls_part_desc_arr->at(idx)->add_new_part_group(part_group))) { + LOG_WARN("add_new_part_group failed", KR(ret), K(ls_id), K(idx), KPC(ls_part_desc_arr), K(part_group)); } find_ls = true; break; @@ -887,5 +889,23 @@ int ObPartitionHelper::get_sub_part_info(const schema::ObSimpleTableSchemaV2 &ta return ret; } +int ObPartitionBalance::ObLSPartGroupDesc::add_new_part_group(ObTransferPartGroup *&part_group) +{ + int ret = OB_SUCCESS; + part_group = NULL; + const int64_t part_group_size = sizeof(ObTransferPartGroup); + void *buf = alloc_.alloc(part_group_size); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate memory for partition group fail", KR(ret), K(buf), K(part_group_size)); + } else if (OB_ISNULL(part_group = new(buf) ObTransferPartGroup(alloc_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("construct ObTransferPartGroup fail", KR(ret), K(buf), K(part_group_size)); + } else if (OB_FAIL(part_groups_.push_back(part_group))) { + LOG_WARN("push back new partition group fail", KR(ret), K(part_group), K(part_groups_)); + } + return ret; +} + } // end rootserver } // end oceanbase diff --git a/src/rootserver/ob_partition_balance.h b/src/rootserver/ob_partition_balance.h index 9a5e960bd..404ec45b5 100644 --- a/src/rootserver/ob_partition_balance.h +++ b/src/rootserver/ob_partition_balance.h @@ -83,21 +83,28 @@ public: class ObLSPartGroupDesc { public: - ObLSPartGroupDesc(ObLSID ls_id) : ls_id_(ls_id) {} + ObLSPartGroupDesc(ObLSID ls_id, ObIAllocator &alloc) : + ls_id_(ls_id), + alloc_(alloc), + part_groups_(OB_MALLOC_NORMAL_BLOCK_SIZE, ModulePageAllocator(alloc, "LSPartGroupDesc")) {} ~ObLSPartGroupDesc() { ls_id_.reset(); for (int64_t i = 0; i < part_groups_.count(); i++) { if (OB_NOT_NULL(part_groups_.at(i))) { part_groups_.at(i)->~ObTransferPartGroup(); + alloc_.free(part_groups_.at(i)); + part_groups_.at(i) = NULL; } } part_groups_.reset(); } ObLSID get_ls_id() const { return ls_id_; } ObArray &get_part_groups() { return part_groups_; } + int add_new_part_group(ObTransferPartGroup *&part_gourp); TO_STRING_KV(K_(ls_id), K_(part_groups)); private: ObLSID ls_id_; + ObIAllocator &alloc_; ObArray part_groups_; }; @@ -175,7 +182,7 @@ private: int generate_balance_job_from_logical_task_(); int prepare_ls_(); - int add_part_to_bg_map_(const ObLSID &ls_id, ObTransferPartGroup &part_group, ObBalanceGroup &bg); + int add_new_pg_to_bg_map_(const ObLSID &ls_id, ObBalanceGroup &bg, ObTransferPartGroup *&part_group); int add_transfer_task_(const ObLSID &src_ls_id, const ObLSID &dest_ls_id, ObTransferPartGroup *part_group, bool modify_ls_desc = true); int update_ls_desc_(const ObLSID &ls_id, int64_t cnt, int64_t size); int try_swap_part_group_(ObLSDesc &src_ls, ObLSDesc &dest_ls, int64_t part_group_min_size ,int64_t &swap_cnt); diff --git a/src/rootserver/ob_tenant_balance_service.cpp b/src/rootserver/ob_tenant_balance_service.cpp index 233004768..0ada3c52a 100755 --- a/src/rootserver/ob_tenant_balance_service.cpp +++ b/src/rootserver/ob_tenant_balance_service.cpp @@ -28,6 +28,7 @@ #include "storage/tablelock/ob_lock_utils.h" // ObInnerTableLockUtil #include "share/ob_cluster_version.h" #include "share/ob_share_util.h" // ObShareUtil +#include "share/transfer/ob_transfer_task_operator.h" #define ISTAT(fmt, args...) FLOG_INFO("[TENANT_BALANCE] " fmt, ##args) #define WSTAT(fmt, args...) FLOG_WARN("[TENANT_BALANCE] " fmt, ##args) @@ -83,8 +84,10 @@ void ObTenantBalanceService::do_work() int64_t idle_time_us = 10 * 1000 * 1000L; int tmp_ret = OB_SUCCESS; int64_t job_cnt = 0; - int64_t last_statistic_bg_stat_time = ObTimeUtility::current_time(); - int64_t last_partition_balance_time = last_statistic_bg_stat_time; + int64_t last_partition_balance_time = ObTimeUtility::current_time(); + int64_t last_statistic_bg_stat_time = OB_INVALID_TIMESTAMP; // statistic once when thread starts + int64_t last_statistic_schema_version = OB_INVALID_VERSION; + ObTransferTaskID last_statistic_max_transfer_task_id; while (!has_set_stop()) { ObCurTraceId::init(GCONF.self_addr_); reset(); @@ -120,9 +123,14 @@ void ObTenantBalanceService::do_work() } // separate statistic to avoid affecting balance jobs + // statistics balance group status periodically when tenant schema version changes or transfer occurs if (OB_FAIL(ret)) { - } else if (OB_TMP_FAIL(try_statistic_balance_group_status_(last_statistic_bg_stat_time))) { - LOG_WARN("try statistic balance group status failed", KR(tmp_ret), K(last_statistic_bg_stat_time)); + } else if (OB_TMP_FAIL(try_statistic_balance_group_status_( + last_statistic_bg_stat_time, + last_statistic_schema_version, + last_statistic_max_transfer_task_id))) { + LOG_WARN("try statistic balance group status failed", KR(tmp_ret), K(last_statistic_bg_stat_time), + K(last_statistic_schema_version), K(last_statistic_max_transfer_task_id)); } if (OB_FAIL(ret) && OB_NEED_WAIT != ret) { @@ -134,6 +142,7 @@ void ObTenantBalanceService::do_work() ISTAT("finish one round", KR(ret), KR(tmp_ret), K_(tenant_id), K(job_cnt), K(primary_zone_num_), K(unit_group_array_), K(ls_array_), K(idle_time_us), K(last_partition_balance_time), K(last_statistic_bg_stat_time), + K(last_statistic_schema_version), K(last_statistic_max_transfer_task_id), "enable_rebalance", ObShareUtil::is_tenant_enable_rebalance(tenant_id_), "enable_transfer", ObShareUtil::is_tenant_enable_transfer(tenant_id_)); reset(); @@ -663,26 +672,53 @@ int ObTenantBalanceService::try_do_partition_balance_(int64_t &last_partition_ba return ret; } -int ObTenantBalanceService::try_statistic_balance_group_status_(int64_t &last_statistic_bg_stat_time) +// when running normally, it will statistic balance group status every 10min when tenant schema version changes or transfer occurs +// when thread starts, it will try to statistic bg stat every 10s until it is successful +int ObTenantBalanceService::try_statistic_balance_group_status_( + int64_t &last_statistic_bg_stat_time, + int64_t &last_statistic_schema_version, + ObTransferTaskID &last_statistic_max_transfer_task_id) { int ret = OB_SUCCESS; - omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id_)); + const int64_t curr_time = ObTimeUtility::current_time(); + const int64_t STATISTIC_BG_STAT_INTERVAL = 600 * 1000 * 1000L; // 10min + ObRefreshSchemaStatus schema_status; + schema_status.tenant_id_ = tenant_id_; + int64_t latest_tenant_schema_version = OB_INVALID_VERSION; + ObTransferTaskID latest_max_transfer_task_id; // default -1 if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); - } else if (OB_UNLIKELY(!tenant_config.is_valid())) { + } else if (OB_ISNULL(GCTX.sql_proxy_) || OB_ISNULL(GCTX.schema_service_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("tenant config is invalid", KR(ret), K_(tenant_id)); + LOG_WARN("GCTX has null ptr", KR(ret), KP(GCTX.sql_proxy_), KP(GCTX.schema_service_)); + } else if (last_statistic_schema_version > OB_INVALID_VERSION + && curr_time - last_statistic_bg_stat_time < STATISTIC_BG_STAT_INTERVAL) { + // no need to statistic because interval is not reached + } else if (OB_FAIL(GCTX.schema_service_->get_schema_version_in_inner_table( + *GCTX.sql_proxy_, + schema_status, + latest_tenant_schema_version))) { + LOG_WARN("failed to get schema version in inner table", KR(ret), K(schema_status)); + } else if (OB_FAIL(ObTransferTaskOperator::get_max_task_id_from_history( + *GCTX.sql_proxy_, + tenant_id_, + latest_max_transfer_task_id))) { // -1 when transfer history is empty + LOG_WARN("get max transfer task if from history failed", + KR(ret), K_(tenant_id), K(latest_max_transfer_task_id)); + } else if (latest_tenant_schema_version <= last_statistic_schema_version + && latest_max_transfer_task_id <= last_statistic_max_transfer_task_id) { + // no need to statistics because distribution of tablets is not changed + } else if (OB_FAIL(partition_balance_(false/*need_balance*/))) { // just statistic balance group status + LOG_WARN("failed to save balance group status", + KR(ret), K(curr_time), K(last_statistic_bg_stat_time)); } else { - const int64_t curr_time = ObTimeUtility::current_time(); - if (curr_time - last_statistic_bg_stat_time > tenant_config->balancer_idle_time) { - if (OB_FAIL(partition_balance_(false/*need_balance*/))) { // just statistic balance group status - LOG_WARN("failed to save balance group status", - KR(ret), K(curr_time), K(last_statistic_bg_stat_time)); - } else { - last_statistic_bg_stat_time = curr_time; - } - } + ISTAT("statistic balance group status successfully", K(curr_time), K(last_statistic_bg_stat_time), + K(latest_tenant_schema_version), K(last_statistic_schema_version), + K(latest_max_transfer_task_id), K(last_statistic_max_transfer_task_id)); + last_statistic_bg_stat_time = curr_time; + last_statistic_schema_version = latest_tenant_schema_version; + last_statistic_max_transfer_task_id = latest_max_transfer_task_id; } return ret; } diff --git a/src/rootserver/ob_tenant_balance_service.h b/src/rootserver/ob_tenant_balance_service.h index eb5b5c228..a1eb7192c 100644 --- a/src/rootserver/ob_tenant_balance_service.h +++ b/src/rootserver/ob_tenant_balance_service.h @@ -19,6 +19,7 @@ #include "lib/lock/ob_thread_cond.h"//ObThreadCond #include "rootserver/ob_tenant_thread_helper.h"//ObTenantTheadHelper #include "share/ls/ob_ls_status_operator.h"//ObLSStatusInfoArray +#include "share/ob_balance_define.h" namespace oceanbase { @@ -100,7 +101,10 @@ private: int lock_and_check_balance_job_(common::ObMySQLTransaction &trans, const uint64_t tenant_id); int try_update_job_comment_(const share::ObBalanceJob &job, const common::ObSqlString &comment); int try_do_partition_balance_(int64_t &last_partition_balance_time); - int try_statistic_balance_group_status_(int64_t &last_statistic_bg_stat_time); + int try_statistic_balance_group_status_( + int64_t &last_statistic_bg_stat_time, + int64_t &last_statistic_schema_version, + share::ObTransferTaskID &last_statistic_max_transfer_task_id); int get_active_unit_num_(int64_t &active_unit_num) const; private: bool inited_; diff --git a/src/rootserver/ob_unit_stat_manager.cpp b/src/rootserver/ob_unit_stat_manager.cpp index 731d6c337..8f3379a86 100644 --- a/src/rootserver/ob_unit_stat_manager.cpp +++ b/src/rootserver/ob_unit_stat_manager.cpp @@ -37,7 +37,7 @@ int ObUnitStatManager::init(common::ObMySQLProxy &sql_proxy) if (OB_UNLIKELY(inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", KR(ret), K(inited_)); - } else if (OB_FAIL(unit_stat_map_.init(500000))) { /// FIXME: use more accurate CONSTANT + } else if (OB_FAIL(unit_stat_map_.init(1000))) { /// FIXME: use more accurate CONSTANT LOG_WARN("fail init unit stat map", KR(ret)); } else if (OB_FAIL(ut_operator_.init(sql_proxy))) { LOG_WARN("fail to init ut_operator_", KR(ret)); @@ -112,5 +112,3 @@ int ObUnitStatManager::get_unit_stat( } return ret; } - - diff --git a/src/share/cache/ob_kvcache_inst_map.cpp b/src/share/cache/ob_kvcache_inst_map.cpp index da3023d55..d89c65c1b 100644 --- a/src/share/cache/ob_kvcache_inst_map.cpp +++ b/src/share/cache/ob_kvcache_inst_map.cpp @@ -176,12 +176,11 @@ ObKVCacheInstHandle& ObKVCacheInstHandle::operator = (const ObKVCacheInstHandle& ObKVCacheInstMap::ObKVCacheInstMap() : lock_(common::ObLatchIds::KV_CACHE_INST_LOCK), inst_map_(), - inst_pool_(), list_lock_(common::ObLatchIds::KV_CACHE_LIST_LOCK), list_map_(), list_pool_(), configs_(NULL), - allocator_("CACHE_INST"), + allocator_("TenantMBList"), inst_keys_(), mem_limit_getter_(NULL), is_inited_(false) @@ -209,19 +208,6 @@ int ObKVCacheInstMap::init(const int64_t max_entry_cnt, const ObKVCacheConfig *c if (OB_SUCC(ret)) { if (OB_FAIL(inst_map_.create(max_entry_cnt, "CACHE_INST_MAP", "CACHE_INST_MAP"))) { COMMON_LOG(WARN, "Fail to create inst map, ", K(ret)); - } else if (NULL == (buf = (char*) allocator_.alloc((sizeof(ObKVCacheInst) + sizeof(ObKVCacheInst*))* max_entry_cnt))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - COMMON_LOG(ERROR, "Fail to allocate memory, ", K(max_entry_cnt), K(ret)); - } else if (OB_FAIL(inst_pool_.init(max_entry_cnt, buf + sizeof(ObKVCacheInst) * max_entry_cnt))) { - COMMON_LOG(WARN, "Fail to init inst pool, ", K(ret)); - } else { - ObKVCacheInst *inst = NULL; - for (int64_t i = 0; OB_SUCC(ret) && i < max_entry_cnt; ++i) { - inst = new (buf + sizeof(ObKVCacheInst) * i) ObKVCacheInst(); - if (OB_FAIL(inst_pool_.push(inst))) { - COMMON_LOG(WARN, "Fail to push inst to pool, ", K(ret)); - } - } } } @@ -270,7 +256,6 @@ void ObKVCacheInstMap::destroy() { inst_map_.destroy(); tenant_set_.destroy(); - inst_pool_.destroy(); list_map_.destroy(); list_pool_.destroy(); allocator_.reset(); @@ -310,7 +295,9 @@ int ObKVCacheInstMap::get_cache_inst( } else if (OB_HASH_NOT_EXIST == ret) { lib::ObMemAttr attr(inst_key.tenant_id_, "CACHE_MAP_NODE"); SET_USE_500(attr); - if (OB_FAIL(inst_pool_.pop(inst))) { + inst = OB_NEW(ObKVCacheInst, ObMemAttr(inst_key.tenant_id_, "CACHE_INST")); + if (OB_ISNULL(inst)) { + ret = OB_ALLOCATE_MEMORY_FAILED; COMMON_LOG(WARN, "Fail to alloc cache inst, ", K(ret)); } else if (OB_FAIL(get_mb_list(inst_key.tenant_id_, inst->mb_list_handle_))) { COMMON_LOG(WARN, "get mb list failed", K(ret), "tenant_id", inst_key.tenant_id_); @@ -331,9 +318,7 @@ int ObKVCacheInstMap::get_cache_inst( if (OB_FAIL(ret) && NULL != inst) { inst->reset(); int tmp_ret = OB_SUCCESS; - if (OB_SUCCESS != (tmp_ret = inst_pool_.push(inst))) { - COMMON_LOG(ERROR, "Fail to push inst to pool, ", K(tmp_ret)); - } + ob_delete(inst); if (OB_SUCCESS != (tmp_ret = inst_map_.erase_refactored(inst_key))) { if (OB_HASH_NOT_EXIST != tmp_ret) { COMMON_LOG(ERROR, "Fail to erase inst key, ", K(ret)); @@ -417,9 +402,8 @@ int ObKVCacheInstMap::erase_tenant(const uint64_t tenant_id) if (OB_FAIL(inst_map_.erase_refactored(tmp_key))) { COMMON_LOG(WARN, "Fail to erase cache inst from inst map", K(ret)); } else if (FALSE_IT(inst->reset())) { - } else if (OB_FAIL(inst_pool_.push(inst))) { - COMMON_LOG(ERROR, "Fail to push inst back to inst pool", K(ret)); - ob_abort(); + } else { + ob_delete(inst); } } } diff --git a/src/share/cache/ob_kvcache_inst_map.h b/src/share/cache/ob_kvcache_inst_map.h index f72623d63..992e0ab15 100644 --- a/src/share/cache/ob_kvcache_inst_map.h +++ b/src/share/cache/ob_kvcache_inst_map.h @@ -166,7 +166,6 @@ private: private: DRWLock lock_; KVCacheInstMap inst_map_; - ObFixedQueue inst_pool_; DRWLock list_lock_; TenantMBListMap list_map_; ObFixedQueue list_pool_; diff --git a/src/share/config/ob_config_manager.cpp b/src/share/config/ob_config_manager.cpp index 192f2965d..3071498cc 100644 --- a/src/share/config/ob_config_manager.cpp +++ b/src/share/config/ob_config_manager.cpp @@ -214,82 +214,96 @@ int ObConfigManager::dump2file(const char* path) const path = dump_path_; } - PageArena<> pa; - if (OB_ISNULL(path) || STRLEN(path) <= 0) { ret = OB_INVALID_ERROR; LOG_WARN("NO dump path specified!", K(ret)); } else { - // write server config - char *buf = nullptr; - char *tmp_path = nullptr; - char *hist_path = nullptr; - int64_t pos = 0; - if (OB_ISNULL(buf = pa.alloc(OB_MAX_PACKET_LENGTH))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_ERROR("ob tc malloc memory for buf failed", K(ret)); - } - if (OB_ISNULL(tmp_path = pa.alloc(MAX_PATH_SIZE))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_ERROR("ob tc malloc memory for tmp configure path failed", K(ret)); - } else { - snprintf(tmp_path, MAX_PATH_SIZE, "%s.tmp", path); - } - if (OB_ISNULL(hist_path = pa.alloc(MAX_PATH_SIZE))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_ERROR("ob tc malloc memory for history configure path fail", K(ret)); - } else { - snprintf(hist_path, MAX_PATH_SIZE, "%s.history", path); + const int64_t min_buffer_size = 2LL << 20; + const int64_t max_buffer_size = 64LL << 20; + int64_t buf_size = min_buffer_size; + bool need_retry = true; + while (OB_SUCC(ret) && need_retry) { + PageArena<> pa; + char *buf = nullptr; + char *tmp_path = nullptr; + char *hist_path = nullptr; + int64_t pos = 0; + need_retry = false; + if (OB_ISNULL(buf = pa.alloc(buf_size))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("ob tc malloc memory for buf failed", K(ret)); + } + if (OB_ISNULL(tmp_path = pa.alloc(MAX_PATH_SIZE))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("ob tc malloc memory for tmp configure path failed", K(ret)); + } else { + snprintf(tmp_path, MAX_PATH_SIZE, "%s.tmp", path); + } + if (OB_ISNULL(hist_path = pa.alloc(MAX_PATH_SIZE))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("ob tc malloc memory for history configure path fail", K(ret)); + } else { + snprintf(hist_path, MAX_PATH_SIZE, "%s.history", path); + } + + #ifdef ERRSIM + ret = OB_E(EventTable::EN_WRITE_CONFIG_FILE_FAILED) OB_SUCCESS;; + if (OB_FAIL(ret)) { + ret = OB_IO_ERROR; + LOG_WARN("ERRSIM, write config file failed", K(ret)); + } + #endif + if (OB_SUCC(ret)) { + if (OB_FAIL(server_config_.serialize(buf, buf_size, pos))) { + LOG_WARN("Serialize server config fail!", K(pos), K(buf_size), K(ret)); + if (OB_SIZE_OVERFLOW == ret && (buf_size << 1) <= max_buffer_size) { + buf_size = buf_size << 1; + need_retry = true; + } + } else if (OB_FAIL(check_header_change(path, buf)) && OB_EAGAIN == ret) { + LOG_INFO("Header not change, no need to write server config!"); + } else if ((fd = ::open(tmp_path, O_WRONLY | O_CREAT | O_TRUNC, + S_IRUSR | S_IWUSR | S_IRGRP)) < 0) { + ret = OB_IO_ERROR; + LOG_WARN("fail to create config file", K(tmp_path), KERRMSG, K(ret)); + } else if (pos != (size = unintr_write(fd, buf, pos))) { + ret = OB_IO_ERROR; + LOG_WARN("Write server config fail!", K(errno), KERRMSG, K(pos), K(size), K(ret)); + if (0 != close(fd)) { + LOG_WARN("fail to close file fd", K(fd), K(errno), KERRMSG, K(ret)); + } + } else if (::fsync(fd) != 0) { + ret = OB_IO_ERROR; + LOG_WARN("Sync server config fail!", K(errno), KERRMSG, K(pos), K(size), K(ret)); + if (0 != close(fd)) { + LOG_WARN("fail to close file fd", K(fd), K(errno), KERRMSG, K(ret)); + } + } else if (0 != close(fd)) { + ret = OB_IO_ERROR; + LOG_WARN("fail to close file fd", K(fd), KERRMSG, K(ret)); + } else { + LOG_INFO("Write server config successfully!", K(pos), K(buf_size)); + } + } + if (OB_SUCC(ret)) { + if (0 != ::rename(path, hist_path) && errno != ENOENT) { + ret = OB_ERR_SYS; + LOG_WARN("fail to backup history config file", KERRMSG, K(ret)); + } + // 运行到这里的时候可能掉电，导致没有 conf 文件，需要 DBA 手工拷贝 tmp 文件到这里 + if (0 != ::rename(tmp_path, path) && errno != ENOENT) { + ret = OB_ERR_SYS; + LOG_WARN("fail to move tmp config file", KERRMSG, K(ret)); + } + } else if (OB_EAGAIN == ret) { + ret = OB_SUCCESS; + } else if (need_retry) { + ret = OB_SUCCESS; + } } - #ifdef ERRSIM - ret = OB_E(EventTable::EN_WRITE_CONFIG_FILE_FAILED) OB_SUCCESS;; - if (OB_FAIL(ret)) { - ret = OB_IO_ERROR; - LOG_WARN("ERRSIM, write config file failed", K(ret)); - } - #endif - if (OB_SUCC(ret)) { - if (OB_FAIL(server_config_.serialize(buf, OB_MAX_PACKET_LENGTH, pos))) { - LOG_WARN("Serialize server config fail!", K(ret)); - } else if (OB_FAIL(check_header_change(path, buf)) && OB_EAGAIN == ret) { - LOG_INFO("Header not change, no need to write server config!"); - } else if ((fd = ::open(tmp_path, O_WRONLY | O_CREAT | O_TRUNC, - S_IRUSR | S_IWUSR | S_IRGRP)) < 0) { - ret = OB_IO_ERROR; - LOG_WARN("fail to create config file", K(tmp_path), KERRMSG, K(ret)); - } else if (pos != (size = unintr_write(fd, buf, pos))) { - ret = OB_IO_ERROR; - LOG_WARN("Write server config fail!", K(errno), KERRMSG, K(pos), K(size), K(ret)); - if (0 != close(fd)) { - LOG_WARN("fail to close file fd", K(fd), K(errno), KERRMSG, K(ret)); - } - } else if (::fsync(fd) != 0) { - ret = OB_IO_ERROR; - LOG_WARN("Sync server config fail!", K(errno), KERRMSG, K(pos), K(size), K(ret)); - if (0 != close(fd)) { - LOG_WARN("fail to close file fd", K(fd), K(errno), KERRMSG, K(ret)); - } - } else if (0 != close(fd)) { - ret = OB_IO_ERROR; - LOG_WARN("fail to close file fd", K(fd), KERRMSG, K(ret)); - } else { - LOG_INFO("Write server config successfully!"); - } - } - if (OB_SUCC(ret)) { - if (0 != ::rename(path, hist_path) && errno != ENOENT) { - ret = OB_ERR_SYS; - LOG_WARN("fail to backup history config file", KERRMSG, K(ret)); - } - // 运行到这里的时候可能掉电，导致没有 conf 文件，需要 DBA 手工拷贝 tmp 文件到这里 - if (0 != ::rename(tmp_path, path) && errno != ENOENT) { - ret = OB_ERR_SYS; - LOG_WARN("fail to move tmp config file", KERRMSG, K(ret)); - } - } else if (OB_EAGAIN == ret) { - ret = OB_SUCCESS; - } + // write server config + } return ret; } diff --git a/src/share/config/ob_server_config.cpp b/src/share/config/ob_server_config.cpp index 742763686..86e46f12d 100644 --- a/src/share/config/ob_server_config.cpp +++ b/src/share/config/ob_server_config.cpp @@ -211,7 +211,7 @@ int ObServerConfig::deserialize_with_compat(const char *buf, const int64_t data_ } ObServerMemoryConfig::ObServerMemoryConfig() - : memory_limit_(0), system_memory_(0) + : memory_limit_(0), system_memory_(0), hidden_sys_memory_(0) {} ObServerMemoryConfig &ObServerMemoryConfig::get_instance() @@ -220,75 +220,126 @@ ObServerMemoryConfig &ObServerMemoryConfig::get_instance() return memory_config; } -int64_t ObServerMemoryConfig::get_capacity_default_memory(CapacityType type, int64_t memory_limit) +int64_t ObServerMemoryConfig::get_adaptive_memory_config(const int64_t memory_size, + DependentMemConfig dep_mem_config, + AdaptiveMemConfig adap_mem_config) { // According to different memory_limit, the kernel can provide adaptive memory_size for default capacity. - // For example, memory_limit = 16G, adaptive system_memory and hidden_sys_memory are 6G and 2G. - static const int64_t memory_limit_array[] = {4LL<<30, 8LL<<30, 14LL<<30, 28LL<<30, 48LL<<30, 56LL<<30, 65LL<<30, 96LL<<30, 128LL<<30}; - static const int64_t system_memory_array[] = {2LL<<30, 3LL<<30, 6LL<<30, 10LL<<30, 12LL<<30, 13LL<<30, 15LL<<30, 18LL<<30, 20LL<<30}; - static const int64_t hidden_sys_memory_array[] = {1LL<<30, 2LL<<30, 2LL<<30, 2LL<<30, 4LL<<30, 4LL<<30, 6LL<<30, 7LL<<30, 8LL<<30}; + static const int64_t memory_limit_array[] = {4LL<<30, 12LL<<30, 20LL<<30, 36LL<<30, 60LL<<30, 80LL<<30, 100LL<<30, 120LL<<30, 130LL<<30}; + static const int64_t system_memory_array[] = {1LL<<30, 1LL<<30, 4LL<<30, 5LL<<30, 6LL<<30, 7LL<<30, 8LL<<30, 9LL<<30, 9LL<<30}; + static const int64_t hidden_sys_memory_array[] = {1LL<<30, 1LL<<30, 1LL<<30, 1LL<<30, 2LL<<30, 2LL<<30, 2LL<<30, 3LL<<30, 3LL<<30}; + static const int64_t array_size = ARRAYSIZEOF(memory_limit_array); - int64_t memory_size = 0; - for (int i = ARRAYSIZEOF(memory_limit_array) - 1; i >= 0; --i) { - if (memory_limit_array[i] <= memory_limit) { - switch (type) { - case SYSTEM_MEMORY: - memory_size = system_memory_array[i]; - break; - case HIDDEN_SYS_MEMORY: - memory_size = hidden_sys_memory_array[i]; - break; - } + int64_t adap_memory_size = 0; + int64_t dep_memory_limit = 0; + const int64_t *dep_array = NULL; + switch (dep_mem_config) { + case MEMORY_LIMIT: + dep_array = memory_limit_array; + dep_memory_limit = memory_size; break; + case SYSTEM_MEMORY: + dep_array = system_memory_array; + dep_memory_limit = memory_size / 0.08; + break; + } + if (memory_size < dep_array[array_size - 1]) { + // When memory_limit < 130G, adaptive memory is calculated by array. + // For example, memory_limit = 16G, adaptive system_memory and hidden_sys_memory are 1G and 1G. + for (int i = 0; i < array_size; ++i) { + if (memory_size < dep_array[i]) { + switch (adap_mem_config) { + case ADAPTIVE_SYSTEM_MEMORY: + adap_memory_size = system_memory_array[i]; + break; + case ADAPTIVE_HIDDEN_SYS_MEMORY: + adap_memory_size = hidden_sys_memory_array[i]; + break; + } + break; + } + } + } else { + // When memory_limit >= 130G or system_memory >= 9G, system_memory = memory_limit*0.08, hidden_sys_memory = memory_limit*0.03. + // For example, memory_limit = 200G, adaptive system_memory and hidden_sys_memory are 16G and 6G. + switch (adap_mem_config) { + case ADAPTIVE_SYSTEM_MEMORY: + adap_memory_size = dep_memory_limit * 0.08; + break; + case ADAPTIVE_HIDDEN_SYS_MEMORY: + adap_memory_size = dep_memory_limit * 0.03; + break; } } - return memory_size; + return adap_memory_size; +} +int64_t ObServerMemoryConfig::get_extra_memory() +{ + return memory_limit_ < lib::ObRunningModeConfig::MINI_MEM_UPPER ? 0 : hidden_sys_memory_; } - int ObServerMemoryConfig::reload_config(const ObServerConfig& server_config) { int ret = OB_SUCCESS; const bool is_arbitration_mode = OBSERVER.is_arbitration_mode(); int64_t memory_limit = server_config.memory_limit; + if (0 == memory_limit) { memory_limit = get_phy_mem_size() * server_config.memory_limit_percentage / 100; } - int64_t system_memory = server_config.system_memory; - if (memory_limit < (1 << 30) ) { - // The memory_limit should not be less than 1G for arbitration mode - ret = OB_INVALID_CONFIG; - LOG_ERROR("memory_limit with unexpected value", K(ret), K(memory_limit), "phy mem", get_phy_mem_size()); - } else if (is_arbitration_mode) { - // do nothing - } else if (0 == system_memory) { - system_memory = get_capacity_default_memory(SYSTEM_MEMORY, memory_limit); - if (0 == system_memory) { + + if (is_arbitration_mode) { + if (memory_limit < (1LL << 30) ) { + // The memory_limit should not be less than 1G for arbitration mode ret = OB_INVALID_CONFIG; LOG_ERROR("memory_limit with unexpected value", K(ret), K(memory_limit), "phy mem", get_phy_mem_size()); - } - } - - if (OB_FAIL(ret)) { - // do nothing - } else if (memory_limit > system_memory) { - memory_limit_ = memory_limit; - system_memory_ = system_memory; - LOG_INFO("update memory_limit or system_memory success", - K(memory_limit_), K(system_memory_)); - } else { - ret = OB_INVALID_CONFIG; - LOG_ERROR("update memory_limit or system_memory failed", - K(memory_limit), K(system_memory)); - } - - int64_t observer_tenant_hold = lib::get_tenant_memory_hold(OB_SERVER_TENANT_ID); - if (observer_tenant_hold > system_memory_ && false == is_arbitration_mode) { - if (server_config._ignore_system_memory_over_limit_error) { - LOG_WARN("the hold of tenant_500 is over the system_memory", - K(observer_tenant_hold), K_(system_memory)); } else { - LOG_ERROR("the hold of tenant_500 is over the system_memory", - K(observer_tenant_hold), K_(system_memory)); + memory_limit_ = memory_limit; + LOG_INFO("update observer memory config success", K_(memory_limit)); + } + } else if (memory_limit < (4LL << 30)) { + // The memory_limit should not be less than 4G for observer + ret = OB_INVALID_CONFIG; + LOG_ERROR("memory_limit with unexpected value", K(ret), K(memory_limit), "phy mem", get_phy_mem_size()); + } else { + // update observer memory config + int64_t system_memory = server_config.system_memory; + int64_t hidden_sys_memory = server_config._hidden_sys_tenant_memory; + int64_t min_server_avail_memory = max(ObUnitResource::UNIT_MIN_MEMORY, server_config.__min_full_resource_pool_memory); + + if (0 != system_memory && 0 != hidden_sys_memory) { + // do-nothing + } else if (0 == system_memory && 0 == hidden_sys_memory) { + system_memory = get_adaptive_memory_config(memory_limit, MEMORY_LIMIT, ADAPTIVE_SYSTEM_MEMORY); + hidden_sys_memory = get_adaptive_memory_config(memory_limit, MEMORY_LIMIT, ADAPTIVE_HIDDEN_SYS_MEMORY); + } else if (0 == hidden_sys_memory) { + hidden_sys_memory = get_adaptive_memory_config(system_memory, SYSTEM_MEMORY, ADAPTIVE_HIDDEN_SYS_MEMORY); + } else { + system_memory = get_adaptive_memory_config(memory_limit, MEMORY_LIMIT, ADAPTIVE_SYSTEM_MEMORY); + } + if (memory_limit - system_memory >= min_server_avail_memory && + system_memory >= hidden_sys_memory) { + memory_limit_ = memory_limit; + system_memory_ = system_memory; + hidden_sys_memory_ = hidden_sys_memory; + LOG_INFO("update observer memory config success", + K_(memory_limit), K_(system_memory), K_(hidden_sys_memory)); + } else { + ret = OB_INVALID_CONFIG; + LOG_ERROR("update observer memory config failed", + K(memory_limit), K(system_memory), K(hidden_sys_memory), K(min_server_avail_memory)); + } + + //check the hold memory of tenant 500 + int64_t tenant_500_hold = lib::get_tenant_memory_hold(OB_SERVER_TENANT_ID); + int64_t tenant_500_reserved = system_memory_ - get_extra_memory(); + if (tenant_500_hold > tenant_500_reserved) { + if (server_config._ignore_system_memory_over_limit_error) { + LOG_ERROR("the hold memory of tenant_500 is over the reserved memory", + K(tenant_500_hold), K(tenant_500_reserved)); + } else { + LOG_WARN("the hold memory of tenant_500 is over the reserved memory", + K(tenant_500_hold), K(tenant_500_reserved)); + } } } @@ -305,15 +356,6 @@ int ObServerMemoryConfig::reload_config(const ObServerConfig& server_config) return ret; } -void ObServerMemoryConfig::set_server_memory_limit(int64_t memory_limit) -{ - if (memory_limit > system_memory_) { - LOG_INFO("update memory_limit success", K(memory_limit), K(system_memory_)); - } else { - LOG_ERROR_RET(OB_ERR_UNEXPECTED, "update memory_limit failed", K(memory_limit), K(system_memory_)); - } -} - int ObServerConfig::serialize_(char *buf, const int64_t buf_len, int64_t &pos) const { int ret = OB_SUCCESS; diff --git a/src/share/config/ob_server_config.h b/src/share/config/ob_server_config.h index 10fd0791d..e8521b959 100644 --- a/src/share/config/ob_server_config.h +++ b/src/share/config/ob_server_config.h @@ -149,25 +149,34 @@ private: class ObServerMemoryConfig { public: - enum CapacityType { + enum DependentMemConfig { + MEMORY_LIMIT, SYSTEM_MEMORY, - HIDDEN_SYS_MEMORY, + }; + enum AdaptiveMemConfig { + ADAPTIVE_SYSTEM_MEMORY, + ADAPTIVE_HIDDEN_SYS_MEMORY, }; friend class unittest::ObSimpleClusterTestBase; friend class unittest::ObMultiReplicaTestBase; ObServerMemoryConfig(); static ObServerMemoryConfig &get_instance(); int reload_config(const ObServerConfig& server_config); - int64_t get_capacity_default_memory(CapacityType type, int64_t memory_limit); int64_t get_server_memory_limit() { return memory_limit_; } int64_t get_reserved_server_memory() { return system_memory_; } int64_t get_server_memory_avail() { return memory_limit_ - system_memory_; } + int64_t get_hidden_sys_memory() { return hidden_sys_memory_; } + //the extra_memory just used by real sys when non_mini_mode + int64_t get_extra_memory(); + private: -// set_server_memory_limit just for mittest - void set_server_memory_limit(int64_t memory_limit); + int64_t get_adaptive_memory_config(const int64_t memory_size, + DependentMemConfig dep_mem_config, + AdaptiveMemConfig adap_mem_config); private: int64_t memory_limit_; int64_t system_memory_; + int64_t hidden_sys_memory_; private: DISALLOW_COPY_AND_ASSIGN(ObServerMemoryConfig); }; diff --git a/src/share/inner_table/ob_inner_table_schema.21451_21500.cpp b/src/share/inner_table/ob_inner_table_schema.21451_21500.cpp new file mode 100644 index 000000000..641a80b4b --- /dev/null +++ b/src/share/inner_table/ob_inner_table_schema.21451_21500.cpp @@ -0,0 +1,130 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SHARE_SCHEMA +#include "ob_inner_table_schema.h" + +#include "share/schema/ob_schema_macro_define.h" +#include "share/schema/ob_schema_service_sql_impl.h" +#include "share/schema/ob_table_schema.h" +#include "share/scn.h" + +namespace oceanbase +{ +using namespace share::schema; +using namespace common; +namespace share +{ + +int ObInnerTableSchema::gv_ob_tenant_runtime_info_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_GV_OB_TENANT_RUNTIME_INFO_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_GV_OB_TENANT_RUNTIME_INFO_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT svr_ip AS SVR_IP, svr_port AS SVR_PORT, tenant_id AS TENANT_ID, compat_mode AS COMPAT_MODE, unit_min_cpu AS UNIT_MIN_CPU, unit_max_cpu AS UNIT_MAX_CPU, slice AS SLICE, remain_slice AS REMAIN_SLICE, token_cnt AS TOKEN_CNT, ass_token_cnt AS ASS_TOKEN_CNT, lq_tokens AS LQ_TOKENS, used_lq_tokens AS USED_LQ_TOKENS, stopped AS STOPPED, idle_us AS IDLE_US, recv_hp_rpc_cnt AS RECV_HP_RPC_CNT, recv_np_rpc_cnt AS RECV_NP_RPC_CNT, recv_lp_rpc_cnt AS RECV_LP_RPC_CNT, recv_mysql_cnt AS RECV_MYSQL_CNT, recv_task_cnt AS RECV_TASK_CNT, recv_large_req_cnt AS RECV_LARGE_REQ_CNT, recv_large_queries AS RECV_LARGE_QUERIES, actives AS ACTIVES, workers AS WORKERS, lq_waiting_workers AS LQ_WAITING_WORKERS, req_queue_total_size AS REQ_QUEUE_TOTAL_SIZE, queue_0 AS QUEUE_0, queue_1 AS QUEUE_1, queue_2 AS QUEUE_2, queue_3 AS QUEUE_3, queue_4 AS QUEUE_4, queue_5 AS QUEUE_5, large_queued AS LARGE_QUEUED FROM oceanbase.__all_virtual_dump_tenant_info )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::v_ob_tenant_runtime_info_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_V_OB_TENANT_RUNTIME_INFO_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_V_OB_TENANT_RUNTIME_INFO_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT * FROM oceanbase.GV$OB_TENANT_RUNTIME_INFO WHERE SVR_IP = HOST_IP() AND SVR_PORT = RPC_PORT() )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + + +} // end namespace share +} // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.h b/src/share/inner_table/ob_inner_table_schema.h index 43ec07911..a328d81af 100644 --- a/src/share/inner_table/ob_inner_table_schema.h +++ b/src/share/inner_table/ob_inner_table_schema.h @@ -1507,6 +1507,8 @@ public: static int dba_ob_tenant_event_history_schema(share::schema::ObTableSchema &table_schema); static int cdb_ob_tenant_event_history_schema(share::schema::ObTableSchema &table_schema); static int gv_ob_flt_trace_config_schema(share::schema::ObTableSchema &table_schema); + static int gv_ob_tenant_runtime_info_schema(share::schema::ObTableSchema &table_schema); + static int v_ob_tenant_runtime_info_schema(share::schema::ObTableSchema &table_schema); static int dba_synonyms_schema(share::schema::ObTableSchema &table_schema); static int dba_objects_ora_schema(share::schema::ObTableSchema &table_schema); static int all_objects_schema(share::schema::ObTableSchema &table_schema); @@ -3933,6 +3935,8 @@ const schema_create_func sys_view_schema_creators [] = { ObInnerTableSchema::dba_ob_tenant_event_history_schema, ObInnerTableSchema::cdb_ob_tenant_event_history_schema, ObInnerTableSchema::gv_ob_flt_trace_config_schema, + ObInnerTableSchema::gv_ob_tenant_runtime_info_schema, + ObInnerTableSchema::v_ob_tenant_runtime_info_schema, ObInnerTableSchema::dba_synonyms_schema, ObInnerTableSchema::dba_objects_ora_schema, ObInnerTableSchema::all_objects_schema, @@ -11121,10 +11125,10 @@ static inline int get_sys_table_lob_aux_schema(const uint64_t tid, const int64_t OB_CORE_TABLE_COUNT = 4; const int64_t OB_SYS_TABLE_COUNT = 247; const int64_t OB_VIRTUAL_TABLE_COUNT = 715; -const int64_t OB_SYS_VIEW_COUNT = 757; -const int64_t OB_SYS_TENANT_TABLE_COUNT = 1724; +const int64_t OB_SYS_VIEW_COUNT = 759; +const int64_t OB_SYS_TENANT_TABLE_COUNT = 1726; const int64_t OB_CORE_SCHEMA_VERSION = 1; -const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 1727; +const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 1729; } // end namespace share } // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema_constants.h b/src/share/inner_table/ob_inner_table_schema_constants.h index 927027e73..d366372d2 100644 --- a/src/share/inner_table/ob_inner_table_schema_constants.h +++ b/src/share/inner_table/ob_inner_table_schema_constants.h @@ -1249,6 +1249,8 @@ const uint64_t OB_CDB_OB_LS_HISTORY_TID = 21446; // "CDB_OB_LS_HISTORY" const uint64_t OB_DBA_OB_TENANT_EVENT_HISTORY_TID = 21447; // "DBA_OB_TENANT_EVENT_HISTORY" const uint64_t OB_CDB_OB_TENANT_EVENT_HISTORY_TID = 21448; // "CDB_OB_TENANT_EVENT_HISTORY" const uint64_t OB_GV_OB_FLT_TRACE_CONFIG_TID = 21449; // "GV$OB_FLT_TRACE_CONFIG" +const uint64_t OB_GV_OB_TENANT_RUNTIME_INFO_TID = 21477; // "GV$OB_TENANT_RUNTIME_INFO" +const uint64_t OB_V_OB_TENANT_RUNTIME_INFO_TID = 21478; // "V$OB_TENANT_RUNTIME_INFO" const uint64_t OB_DBA_SYNONYMS_TID = 25001; // "DBA_SYNONYMS" const uint64_t OB_DBA_OBJECTS_ORA_TID = 25002; // "DBA_OBJECTS_ORA" const uint64_t OB_ALL_OBJECTS_TID = 25003; // "ALL_OBJECTS" @@ -3567,6 +3569,8 @@ const char *const OB_CDB_OB_LS_HISTORY_TNAME = "CDB_OB_LS_HISTORY"; const char *const OB_DBA_OB_TENANT_EVENT_HISTORY_TNAME = "DBA_OB_TENANT_EVENT_HISTORY"; const char *const OB_CDB_OB_TENANT_EVENT_HISTORY_TNAME = "CDB_OB_TENANT_EVENT_HISTORY"; const char *const OB_GV_OB_FLT_TRACE_CONFIG_TNAME = "GV$OB_FLT_TRACE_CONFIG"; +const char *const OB_GV_OB_TENANT_RUNTIME_INFO_TNAME = "GV$OB_TENANT_RUNTIME_INFO"; +const char *const OB_V_OB_TENANT_RUNTIME_INFO_TNAME = "V$OB_TENANT_RUNTIME_INFO"; const char *const OB_DBA_SYNONYMS_TNAME = "DBA_SYNONYMS"; const char *const OB_DBA_OBJECTS_ORA_TNAME = "DBA_OBJECTS"; const char *const OB_ALL_OBJECTS_TNAME = "ALL_OBJECTS"; diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index 31053431d..d3662cc2c 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -28646,8 +28646,69 @@ def_table_schema( # 21475: CDB_OB_IMPORT_STMT_EXEC_HISTORY # 21476: DBA_OB_IMPORT_STMT_EXEC_HISTORY -# 21477: GV$OB_DUMP_TENANT_INFO -# 21478: V$OB_DUMP_TENANT_INFO +def_table_schema( + owner = 'tushicheng.tsc', + table_name = 'GV$OB_TENANT_RUNTIME_INFO', + table_id = '21477', + table_type = 'SYSTEM_VIEW', + gm_columns = [], + rowkey_columns = [], + normal_columns = [], + view_definition = """ + SELECT + svr_ip AS SVR_IP, + svr_port AS SVR_PORT, + tenant_id AS TENANT_ID, + compat_mode AS COMPAT_MODE, + unit_min_cpu AS UNIT_MIN_CPU, + unit_max_cpu AS UNIT_MAX_CPU, + slice AS SLICE, + remain_slice AS REMAIN_SLICE, + token_cnt AS TOKEN_CNT, + ass_token_cnt AS ASS_TOKEN_CNT, + lq_tokens AS LQ_TOKENS, + used_lq_tokens AS USED_LQ_TOKENS, + stopped AS STOPPED, + idle_us AS IDLE_US, + recv_hp_rpc_cnt AS RECV_HP_RPC_CNT, + recv_np_rpc_cnt AS RECV_NP_RPC_CNT, + recv_lp_rpc_cnt AS RECV_LP_RPC_CNT, + recv_mysql_cnt AS RECV_MYSQL_CNT, + recv_task_cnt AS RECV_TASK_CNT, + recv_large_req_cnt AS RECV_LARGE_REQ_CNT, + recv_large_queries AS RECV_LARGE_QUERIES, + actives AS ACTIVES, + workers AS WORKERS, + lq_waiting_workers AS LQ_WAITING_WORKERS, + req_queue_total_size AS REQ_QUEUE_TOTAL_SIZE, + queue_0 AS QUEUE_0, + queue_1 AS QUEUE_1, + queue_2 AS QUEUE_2, + queue_3 AS QUEUE_3, + queue_4 AS QUEUE_4, + queue_5 AS QUEUE_5, + large_queued AS LARGE_QUEUED + FROM + oceanbase.__all_virtual_dump_tenant_info +""".replace("\n", " "), +) + +def_table_schema( + owner = 'tushicheng.tsc', + table_name = 'V$OB_TENANT_RUNTIME_INFO', + table_id = '21478', + table_type = 'SYSTEM_VIEW', + gm_columns = [], + rowkey_columns = [], + normal_columns = [], + view_definition = """ + SELECT * + FROM + oceanbase.GV$OB_TENANT_RUNTIME_INFO + WHERE SVR_IP = HOST_IP() AND SVR_PORT = RPC_PORT() +""".replace("\n", " "), +) + # # 余留位置 @@ -52845,9 +52906,6 @@ def_table_schema( # 28198: GV$OB_PL_CACHE_OBJECT # 28199: V$OB_PL_CACHE_OBJECT -# 28200: GV$OB_DUMP_TENANT_INFO -# 28201: V$OB_DUMP_TENANT_INFO - ################################################################################ # Lob Table (50000, 70000) ################################################################################ diff --git a/src/share/interrupt/ob_global_interrupt_call.cpp b/src/share/interrupt/ob_global_interrupt_call.cpp index 82e693124..ab2857464 100644 --- a/src/share/interrupt/ob_global_interrupt_call.cpp +++ b/src/share/interrupt/ob_global_interrupt_call.cpp @@ -238,11 +238,11 @@ int ObGlobalInterruptManager::create_checker_node(ObInterruptChecker *checker, { int ret = OB_SUCCESS; void *ptr = NULL; + ObMemAttr attr(GET_TENANT_ID(), ObModIds::OB_INTERRUPT_CHECKER_NODE, common::ObCtxIds::DEFAULT_CTX_ID); if (OB_ISNULL(checker)) { ret = OB_INVALID_ARGUMENT; LIB_LOG(ERROR, "invaild checker pointer", K(ret)); - } else if (OB_ISNULL(ptr = ob_malloc(sizeof(ObInterruptCheckerNode), - ObModIds::OB_INTERRUPT_CHECKER_NODE))) { + } else if (OB_ISNULL(ptr = ob_malloc(sizeof(ObInterruptCheckerNode), attr))) { ret = OB_ALLOCATE_MEMORY_FAILED; LIB_LOG(WARN, "fail to alloc memory in interrupt manager", K(ret)); } else { diff --git a/src/share/interrupt/ob_global_interrupt_call.h b/src/share/interrupt/ob_global_interrupt_call.h index 8e00f645b..b7df11a8c 100644 --- a/src/share/interrupt/ob_global_interrupt_call.h +++ b/src/share/interrupt/ob_global_interrupt_call.h @@ -237,9 +237,9 @@ class ObGlobalInterruptManager { private: /// Use 100W as the bucket initialization parameter of Map - static const int64_t DEFAULT_HASH_MAP_BUCKETS_COUNT = 1000000; //100w + static const int64_t DEFAULT_HASH_MAP_BUCKETS_COUNT = 250000; //25w static const int64_t MINI_MODE_HASH_MAP_BUCKETS_COUNT = 10000; //1w - static const int64_t DEFAULT_NODE_NUM = 1000000; //100w + static const int64_t DEFAULT_NODE_NUM = 20000; //2w, SimpleAllocer holds each block with memory less than 2M public: /// Hashmap with SpinLock, because the coroutine interrupt signal lock time is short, and there is almost no resource conflict, so SpinLock is used typedef hash::ObHashMap 0 - && row_scn_ > 0; + && renew_time_ > 0; } bool ObTabletLSCache::mapping_is_same_with(const ObTabletLSCache &other) const @@ -754,9 +745,7 @@ bool ObTabletLSCache::mapping_is_same_with(const ObTabletLSCache &other) const bool ObTabletLSCache::operator==(const ObTabletLSCache &other) const { return mapping_is_same_with(other) - && renew_time_ == other.renew_time_ - && row_scn_ == other.row_scn_ - && last_access_ts_ == other.last_access_ts_; + && renew_time_ == other.renew_time_; } bool ObTabletLSCache::operator!=(const ObTabletLSCache &other) const @@ -777,8 +766,6 @@ int ObTabletLSCache::init( } else { ls_id_ = ls_id; renew_time_ = renew_time; - row_scn_ = row_scn; - last_access_ts_ = 0; ObLink::reset(); } return ret; diff --git a/src/share/location_cache/ob_location_struct.h b/src/share/location_cache/ob_location_struct.h index 187fa8628..0d423598e 100644 --- a/src/share/location_cache/ob_location_struct.h +++ b/src/share/location_cache/ob_location_struct.h @@ -294,9 +294,9 @@ public: inline ObTabletID get_tablet_id() const { return cache_key_.get_tablet_id(); } inline ObLSID get_ls_id() const { return ls_id_; } inline int64_t get_renew_time() const { return renew_time_; } - inline int64_t get_row_scn() const { return row_scn_; } - void set_last_access_ts(const int64_t ts) { last_access_ts_ = ts; } - int64_t get_last_access_ts() const { return last_access_ts_; } + //inline int64_t get_row_scn() const { return row_scn_; } + //void set_last_access_ts(const int64_t ts) { last_access_ts_ = ts; } + //int64_t get_last_access_ts() const { return last_access_ts_; } const ObTabletLSKey &get_cache_key() const { return cache_key_; } int init( const uint64_t tenant_id, @@ -304,13 +304,13 @@ public: const ObLSID &ls_id, const int64_t renew_time, const int64_t row_scn); - TO_STRING_KV(K_(cache_key), K_(ls_id), K_(renew_time), K_(row_scn), K_(last_access_ts)); + TO_STRING_KV(K_(cache_key), K_(ls_id), K_(renew_time)); private: ObTabletLSKey cache_key_; ObLSID ls_id_; int64_t renew_time_; // renew by sql - int64_t row_scn_; // used for auto refresh location - int64_t last_access_ts_; // used for ObTabletLSMap + //int64_t row_scn_; // used for auto refresh location + //int64_t last_access_ts_; // used for ObTabletLSMap }; //TODO: Reserved for tableapi. Need remove. diff --git a/src/share/location_cache/ob_tablet_ls_map.cpp b/src/share/location_cache/ob_tablet_ls_map.cpp index 6e1f15ba1..088b88153 100644 --- a/src/share/location_cache/ob_tablet_ls_map.cpp +++ b/src/share/location_cache/ob_tablet_ls_map.cpp @@ -128,7 +128,7 @@ int ObTabletLSMap::update(const ObTabletLSCache &tablet_ls_cache) } else if (OB_FAIL(tmp->assign(tablet_ls_cache))) { LOG_WARN("fail to assign tablet_ls_cache", KR(ret), K(tablet_ls_cache)); } else { - try_update_access_ts_(tmp); // always update for insert + // try_update_access_ts_(tmp); // always update for insert tmp->next_ = ls_buckets_[pos]; ls_buckets_[pos] = tmp; ATOMIC_INC(&size_); @@ -138,7 +138,7 @@ int ObTabletLSMap::update(const ObTabletLSCache &tablet_ls_cache) if (OB_FAIL(curr->assign(tablet_ls_cache))) { LOG_WARN("fail to assign tablet_ls_cache", KR(ret), K(tablet_ls_cache)); } else { - try_update_access_ts_(curr); // always update for update + // try_update_access_ts_(curr); // always update for update } } } @@ -172,7 +172,7 @@ int ObTabletLSMap::get( if (OB_ISNULL(curr)) { ret = OB_ENTRY_NOT_EXIST; } else { - try_update_access_ts_(curr); + // try_update_access_ts_(curr); if (OB_FAIL(tablet_ls_cache.assign(*curr))) { LOG_WARN("fail to assign tablet_ls_cache", KR(ret), KPC(curr)); } @@ -241,13 +241,13 @@ int ObTabletLSMap::del(const ObTabletLSKey &key) return ret; } -void ObTabletLSMap::try_update_access_ts_(ObTabletLSCache *cache_ptr) -{ - OB_ASSERT(NULL != cache_ptr); - if (common::ObClockGenerator::getClock() > cache_ptr->get_last_access_ts() + MAX_ACCESS_TIME_UPDATE_THRESHOLD) { - cache_ptr->set_last_access_ts(common::ObClockGenerator::getClock()); - } -} +// void ObTabletLSMap::try_update_access_ts_(ObTabletLSCache *cache_ptr) +// { +// OB_ASSERT(NULL != cache_ptr); +// if (common::ObClockGenerator::getClock() > cache_ptr->get_last_access_ts() + MAX_ACCESS_TIME_UPDATE_THRESHOLD) { +// cache_ptr->set_last_access_ts(common::ObClockGenerator::getClock()); +// } +// } } // end namespace share } // end namespace oceanbase diff --git a/src/share/location_cache/ob_tablet_ls_map.h b/src/share/location_cache/ob_tablet_ls_map.h index 61bdc8928..8c22a6908 100644 --- a/src/share/location_cache/ob_tablet_ls_map.h +++ b/src/share/location_cache/ob_tablet_ls_map.h @@ -50,7 +50,7 @@ public: int64_t size() const { return size_; } private: - void try_update_access_ts_(ObTabletLSCache *cache_ptr); + // void try_update_access_ts_(ObTabletLSCache *cache_ptr); private: static const int64_t MAX_ACCESS_TIME_UPDATE_THRESHOLD = 10000000; // 10s diff --git a/src/share/location_cache/ob_tablet_ls_service.h b/src/share/location_cache/ob_tablet_ls_service.h index a8c57f9df..80748798c 100644 --- a/src/share/location_cache/ob_tablet_ls_service.h +++ b/src/share/location_cache/ob_tablet_ls_service.h @@ -103,7 +103,7 @@ private: bool belong_to_sys_ls_(const uint64_t tenant_id, const ObTabletID &tablet_id) const; const int64_t MINI_MODE_UPDATE_THREAD_CNT = 1; - const int64_t USER_TASK_QUEUE_SIZE = 200 * 1000; // 20W partitions + const int64_t USER_TASK_QUEUE_SIZE = 100 * 1000; // 10W partitions const int64_t MINI_MODE_USER_TASK_QUEUE_SIZE = 10 * 1000; // 1W partitions bool inited_; diff --git a/src/share/ob_common_id.h b/src/share/ob_common_id.h index b3dd17f37..0d991dfa0 100644 --- a/src/share/ob_common_id.h +++ b/src/share/ob_common_id.h @@ -49,6 +49,8 @@ public: bool operator > (const ObCommonID &other) const { return id_ > other.id_; } bool operator != (const ObCommonID &other) const { return id_ != other.id_; } bool operator < (const ObCommonID &other) const { return id_ < other.id_; } + bool operator <= (const ObCommonID &other) const { return id_ <= other.id_; } + bool operator >= (const ObCommonID &other) const { return id_ >= other.id_; } int compare(const ObCommonID &other) const { if (id_ == other.id_) { diff --git a/src/share/ob_debug_sync.cpp b/src/share/ob_debug_sync.cpp index 2860d0da9..c8645e2b3 100644 --- a/src/share/ob_debug_sync.cpp +++ b/src/share/ob_debug_sync.cpp @@ -35,6 +35,7 @@ OB_SERIALIZE_MEMBER(ObDebugSyncAction, sync_point_, timeout_, ObDSActionArray::ObDSActionArray(const bool is_const /* = false */) : active_cnt_(0), is_const_(is_const) { + STATIC_ASSERT(0 == INVALID_DEBUG_SYNC_POINT, "INVALID_DEBUG_SYNC_POINT shouble be zero"); memset(action_ptrs_, 0, sizeof(action_ptrs_)); } @@ -42,6 +43,7 @@ void ObDSActionArray::clear(const ObDebugSyncPoint sync_point) { if (sync_point > INVALID_DEBUG_SYNC_POINT && sync_point < MAX_DEBUG_SYNC_POINT) { if (NULL != action_ptrs_[sync_point]) { + action_ptrs_[sync_point]->sync_point_ = INVALID_DEBUG_SYNC_POINT; action_ptrs_[sync_point] = NULL; active_cnt_--; LOG_INFO("clear sync point", K(sync_point)); @@ -55,6 +57,7 @@ void ObDSActionArray::clear_all() { if (!is_empty()) { memset(action_ptrs_, 0, sizeof(action_ptrs_)); + memset(actions_, 0, sizeof(actions_)); active_cnt_ = 0; LOG_INFO("clear all sync point"); } @@ -68,11 +71,29 @@ int ObDSActionArray::add_action(const ObDebugSyncAction &action) LOG_WARN("invalid action", K(ret), K(action)); } else { if (!is_const_) { - actions_[action.sync_point_] = action; if (OB_ISNULL(action_ptrs_[action.sync_point_])) { - active_cnt_++; + if (active_cnt_ < MAX_DEBUG_SYNC_CACHED_POINT) { + ObDebugSyncAction *dst = NULL; + for (int i = 0; i < ARRAYSIZEOF(actions_); i++) { + if (INVALID_DEBUG_SYNC_POINT == actions_[i].sync_point_) { + dst = &actions_[i]; + break; + } + } + if (dst != NULL) { + active_cnt_++; + action_ptrs_[action.sync_point_] = dst; + } else { + ret = OB_ERR_UNEXPECTED; + } + } else { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("sync action size overflow", K(active_cnt_)); + } + } + if (OB_SUCC(ret)) { + *action_ptrs_[action.sync_point_] = action; } - action_ptrs_[action.sync_point_] = &actions_[action.sync_point_]; } } return ret; @@ -90,6 +111,7 @@ int ObDSActionArray::fetch_action(const ObDebugSyncPoint sync_point, action = *action_ptrs_[sync_point]; action_ptrs_[sync_point]->execute_--; if (action_ptrs_[sync_point]->execute_ <= 0) { + action_ptrs_[sync_point]->sync_point_ = INVALID_DEBUG_SYNC_POINT; action_ptrs_[sync_point] = NULL; active_cnt_--; } diff --git a/src/share/ob_debug_sync.h b/src/share/ob_debug_sync.h index 3c4ed6888..a74c4c015 100644 --- a/src/share/ob_debug_sync.h +++ b/src/share/ob_debug_sync.h @@ -59,6 +59,7 @@ public: class ObDSActionArray { public: + static constexpr int MAX_DEBUG_SYNC_CACHED_POINT = 32; OB_UNIS_VERSION(1); public: @@ -80,7 +81,7 @@ public: private: ObDebugSyncAction *action_ptrs_[MAX_DEBUG_SYNC_POINT]; - ObDebugSyncAction actions_[MAX_DEBUG_SYNC_POINT]; + ObDebugSyncAction actions_[MAX_DEBUG_SYNC_CACHED_POINT]; volatile int64_t active_cnt_; const bool is_const_; diff --git a/src/share/ob_scanner.h b/src/share/ob_scanner.h index 66b5a17e3..be4b6318c 100644 --- a/src/share/ob_scanner.h +++ b/src/share/ob_scanner.h @@ -135,7 +135,11 @@ public: /// dump all data for debug purpose void dump() const; - void set_tenant_id(uint64_t tenant_id) { row_store_.set_tenant_id(tenant_id); } + void set_tenant_id(uint64_t tenant_id) + { + tenant_id_ = tenant_id; + row_store_.set_tenant_id(tenant_id); + } ObIArray &get_table_row_counts() { return table_row_counts_; } int assign_implicit_cursor(const common::ObIArray &implicit_cursors) { return implicit_cursors_.assign(implicit_cursors); } diff --git a/src/share/ob_tenant_mgr.cpp b/src/share/ob_tenant_mgr.cpp index 34a66245a..61a00fa7f 100644 --- a/src/share/ob_tenant_mgr.cpp +++ b/src/share/ob_tenant_mgr.cpp @@ -57,7 +57,6 @@ using namespace oceanbase::obrpc; ObVirtualTenantManager::ObVirtualTenantManager() : tenant_map_(NULL), - tenant_pool_(), allocator_(ObModIds::OB_TENANT_INFO), memattr_(OB_SERVER_TENANT_ID, ObModIds::OB_TENANT_INFO), is_inited_(false) @@ -75,16 +74,13 @@ ObVirtualTenantManager &ObVirtualTenantManager::get_instance() return instance_; } -int ObVirtualTenantManager::init(const int64_t tenant_cnt) +int ObVirtualTenantManager::init() { int ret = OB_SUCCESS; if (is_inited_) { ret = OB_INIT_TWICE; COMMON_LOG(WARN, "init twice", K(ret)); - } else if (OB_UNLIKELY(tenant_cnt <= 0)) { - ret = OB_INVALID_ARGUMENT; - COMMON_LOG(WARN, "invalid argument", K(tenant_cnt), K(ret)); - } else if (OB_FAIL(init_tenant_map_(tenant_cnt))) { + } else if (OB_FAIL(init_tenant_map_())) { COMMON_LOG(WARN, "Fail to init tenant map, ", K(ret)); } else { is_inited_ = true; @@ -95,36 +91,15 @@ int ObVirtualTenantManager::init(const int64_t tenant_cnt) return ret; } -int ObVirtualTenantManager::init_tenant_map_(const int64_t tenant_cnt) +int ObVirtualTenantManager::init_tenant_map_() { int ret = OB_SUCCESS; char *buf = NULL; - if (OB_UNLIKELY(tenant_cnt <= 0)) { - ret = OB_INVALID_ARGUMENT; - COMMON_LOG(WARN, "invalid argument", K(tenant_cnt), K(ret)); - } else if (NULL == (buf = (char*) allocator_.alloc(( - sizeof(ObTenantInfo*) * tenant_cnt) + sizeof(ObTenantInfo) * tenant_cnt))) { + if (NULL == (buf = (char*) allocator_.alloc(sizeof(ObTenantBucket) * BUCKET_NUM))) { ret = OB_ALLOCATE_MEMORY_FAILED; COMMON_LOG(ERROR, "Fail to allocate memory, ", K(ret)); - } else if (OB_FAIL(tenant_pool_.init(tenant_cnt, buf))) { - COMMON_LOG(WARN, "Fail to init tenant pool, ", K(ret)); } else { - buf += (sizeof(ObTenantInfo*) * tenant_cnt); - ObTenantInfo *info = new (buf) ObTenantInfo[tenant_cnt]; - for (int64_t idx = 0; idx < tenant_cnt && OB_SUCC(ret); ++idx) { - info[idx].reset(); - if (OB_FAIL(tenant_pool_.push(&(info[idx])))) { - COMMON_LOG(WARN, "Fail to push info to pool, ", K(ret)); - } - } - if (OB_SUCC(ret)) { - if (NULL == (buf = (char*) allocator_.alloc(sizeof(ObTenantBucket) * BUCKET_NUM))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - COMMON_LOG(ERROR, "Fail to allocate memory, ", K(ret)); - } else { - tenant_map_ = new (buf) ObTenantBucket[BUCKET_NUM]; - } - } + tenant_map_ = new (buf) ObTenantBucket[BUCKET_NUM]; } return ret; } @@ -132,7 +107,6 @@ int ObVirtualTenantManager::init_tenant_map_(const int64_t tenant_cnt) void ObVirtualTenantManager::destroy() { tenant_map_ = NULL; - tenant_pool_.destroy(); allocator_.reset(); is_inited_ = false; } @@ -178,18 +152,18 @@ int ObVirtualTenantManager::add_tenant(const uint64_t tenant_id) if (OB_SUCC(bucket.get_the_node(tenant_id, node))) { //tenant is exist do nothing } else { - ObTenantInfo *info = NULL; - if (OB_FAIL(tenant_pool_.pop(info))) { - COMMON_LOG(WARN, "Fail to pop info from pool, ", K(ret)); + ret = OB_SUCCESS; + ObTenantInfo *info = OB_NEW(ObTenantInfo, memattr_); + if (OB_ISNULL(info)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + COMMON_LOG(WARN, "Fail to alloc ObTenantInfo", K(ret)); } else { info->reset(); info->tenant_id_ = tenant_id; if (OB_UNLIKELY(!bucket.info_list_.add_last(info))) { ret = OB_ERR_UNEXPECTED; COMMON_LOG(ERROR, "Fail to add proc to wait list, ", K(ret)); - if (OB_SUCCESS != (tmp_ret = tenant_pool_.push(info))) { - COMMON_LOG(WARN, "Fail to push collects to pool, ", K(tmp_ret)); - } + ob_delete(info); } } } @@ -221,11 +195,8 @@ int ObVirtualTenantManager::del_tenant(const uint64_t tenant_id) ret = OB_ERR_UNEXPECTED; COMMON_LOG(WARN, "The info is null, ", K(ret)); } else { - if (OB_FAIL(tenant_pool_.push(info))) { - COMMON_LOG(WARN, "Fail to push collect to pool, ", K(ret)); - } else { - COMMON_LOG(INFO, "del_tenant succeed", K(tenant_id)); - } + ob_delete(info); + COMMON_LOG(INFO, "del_tenant succeed", K(tenant_id)); } } } diff --git a/src/share/ob_tenant_mgr.h b/src/share/ob_tenant_mgr.h index ca58b00a5..7ee44c831 100644 --- a/src/share/ob_tenant_mgr.h +++ b/src/share/ob_tenant_mgr.h @@ -29,7 +29,7 @@ class ObVirtualTenantManager static const int64_t BUCKET_NUM = 16; public: static ObVirtualTenantManager &get_instance(); - int init(const int64_t tenant_cnt = OB_DEFAULT_TENANT_COUNT); + int init(); bool is_inited() const { return is_inited_; } void destroy(); // get all the virtual tenant id. @@ -110,12 +110,10 @@ private: int64_t &kv_cache_mem); ObVirtualTenantManager(); virtual ~ObVirtualTenantManager(); - int init_tenant_map_(const int64_t tenant_cnt); + int init_tenant_map_(); private: // a bucket map, every bucket content a ObTenantInfo list and a lock. ObTenantBucket *tenant_map_; - // the object pool of ObTenantInfo - ObFixedQueue tenant_pool_; // allocator and memattr for ObTenantInfo memory alloc ObArenaAllocator allocator_; ObMemAttr memattr_; diff --git a/src/share/ob_thread_define.h b/src/share/ob_thread_define.h index 37846ba8f..0e6296704 100755 --- a/src/share/ob_thread_define.h +++ b/src/share/ob_thread_define.h @@ -164,4 +164,5 @@ TG_DEF(WR_TIMER_THREAD, WrTimer, TIMER) TG_DEF(SvrStartupHandler, SvrStartupHandler, QUEUE_THREAD, ThreadCountPair(observer::ObServerStartupTaskHandler::get_thread_num(), observer::ObServerStartupTaskHandler::get_thread_num()), observer::ObServerStartupTaskHandler::MAX_QUEUED_TASK_NUM) +TG_DEF(TntSharedTimer, TntSharedTimer, TIMER) #endif diff --git a/src/share/ob_time_zone_info_manager.cpp b/src/share/ob_time_zone_info_manager.cpp index 853644c8e..07b7e68f3 100644 --- a/src/share/ob_time_zone_info_manager.cpp +++ b/src/share/ob_time_zone_info_manager.cpp @@ -21,6 +21,7 @@ #include "observer/ob_server.h" #include "observer/omt/ob_tenant_timezone_mgr.h" #include "observer/ob_sql_client_decorator.h" +#include "lib/lock/ob_spin_rwlock.h" using namespace oceanbase::share; using namespace oceanbase::observer; @@ -28,6 +29,10 @@ namespace oceanbase { namespace common { + +ObTZInfoMap ObTimeZoneInfoManager::shared_tz_info_map_; +int64_t ObTimeZoneInfoManager::loaded_tz_info_count_ = 0; +SpinRWLock ObTimeZoneInfoManager::sys_rwlock_; const char *ObTimeZoneInfoManager::FETCH_TZ_INFO_SQL = "SELECT * " "FROM (" @@ -74,7 +79,14 @@ int ObTimeZoneInfoManager::init() ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret)); } else { - inited_ = true; + ObMemAttr attr(OB_SERVER_TENANT_ID, "TZInfoMgrMap"); + if (OB_FAIL(tz_info_map_buf_.init(attr))) { + LOG_WARN("init tz info map failed", K(ret), K(tenant_id_)); + } else if (OB_FAIL(tz_info_map_.init(attr))) { + LOG_WARN("init tz info map failed", K(ret), K(tenant_id_)); + } else { + inited_ = true; + } } return ret; } @@ -118,7 +130,7 @@ int ObTimeZoneInfoManager::response_time_zone_info(ObRequestTZInfoResult &tz_res tz_result.tz_array_.reset(); tz_result.last_version_ = last_version_; FillRequestTZInfoResult fr_func(tz_result); - if (OB_FAIL(tz_info_map_.id_map_.for_each(fr_func))) { + if (OB_FAIL(tz_info_map_.id_map_->for_each(fr_func))) { LOG_WARN("fail to call for_each", K(ret)); } } @@ -201,12 +213,50 @@ int ObTimeZoneInfoManager::fetch_time_zone_info_from_tenant_table(const int64_t } else if (OB_ISNULL(result = res.get_result())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("fail to get result", K(result), K(ret)); - } else if (OB_FAIL(fill_tz_info_map(*result, tz_info_map_))) { - LOG_ERROR("fail to fill tz_info_map", K(ret)); } else { - last_version_ = current_tz_version; - LOG_INFO("success to fetch tz_info map", K(ret), K(last_version_), K(tenant_id_), - "new_last_version", current_tz_version, K(tz_info_map_.id_map_.size())); + bool same_with_sys = false; + bool first_map = 1 == ATOMIC_AAF(&loaded_tz_info_count_, 1); + if (first_map) { + SpinWLockGuard wlock_guard(sys_rwlock_); + // the first tenant load timezone map into shared_tz_info_map_. + // Other tenants load timezone map into tz_info_map_buf_ and then cmp with shared_tz_info_map_. + ObMemAttr attr(OB_SERVER_TENANT_ID, "TZInfoMgrMap"); + if (OB_FAIL(shared_tz_info_map_.init(attr))) { + LOG_WARN("init share tz info map failed", K(ret)); + } else if (OB_FAIL(fill_tz_info_map(*result, shared_tz_info_map_, tenant_id_))) { + LOG_ERROR("fail to fill tz_info_map for shared map", K(ret), K(tenant_id_)); + } else { + tz_info_map_.id_map_ = shared_tz_info_map_.id_map_; + tz_info_map_.name_map_ = shared_tz_info_map_.name_map_; + } + } else { + if (OB_FAIL(fill_tz_info_map(*result, tz_info_map_buf_, tenant_id_))) { + LOG_ERROR("fail to fill tz_info_map", K(ret)); + } else { + same_with_sys = cmp_tz_info_map(shared_tz_info_map_, tz_info_map_buf_); + if (same_with_sys) { + // same with sys tenant, reset tz_info_map_buf_. + // if reset link hash map, memory of hash node will not be released, so use destroy. + tz_info_map_buf_.~ObTZInfoMap(); + // Use destroyed ObTZInfoMap will lead to a crash. + // According to current design, upgrade timezone is not supported, + // so ObTZInfoMap will not be used again. This is a defense code. + new (&tz_info_map_buf_) ObTZInfoMap(); + LOG_INFO("reset tz info map buf", K(tenant_id_)); + tz_info_map_.id_map_ = shared_tz_info_map_.id_map_; + tz_info_map_.name_map_ = shared_tz_info_map_.name_map_; + } else { + tz_info_map_.id_map_ = tz_info_map_buf_.id_map_; + tz_info_map_.name_map_ = tz_info_map_buf_.name_map_; + } + } + } + if (OB_SUCC(ret)) { + last_version_ = current_tz_version; + LOG_INFO("success to fetch tz_info map", K(ret), K(last_version_), K(tenant_id_), + "new_last_version", current_tz_version, K(same_with_sys), + K(tz_info_map_.id_map_->size())); + } } } } @@ -226,6 +276,7 @@ int ObTimeZoneInfoManager::set_tz_info_map(ObTimeZoneInfoPos *&stored_tz_info, ObTimeZoneInfoPos *tz_pos_value = NULL; ObTZNameIDInfo *name_id_value = NULL; bool is_equal = false; + lib::ObMemAttr attr1(OB_SERVER_TENANT_ID, "TZInfoArray"); if (NULL == stored_tz_info) { if (OB_ISNULL(tz_pos_value = op_alloc(ObTimeZoneInfoPos))) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -233,19 +284,24 @@ int ObTimeZoneInfoManager::set_tz_info_map(ObTimeZoneInfoPos *&stored_tz_info, } else if (OB_ISNULL(name_id_value = op_alloc(ObTZNameIDInfo))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_ERROR("fail to alloc memory for ObTZNameIDInfo ", K(ret)); + } else if (FALSE_IT(tz_pos_value->set_tz_type_attr(attr1))) { } else if (OB_FAIL(tz_pos_value->assign(new_tz_info))) { LOG_WARN("fail to assign ObTimeZoneInfoPos", K(new_tz_info), K(ret)); } else if (FALSE_IT(name_id_value->set(new_tz_info.get_tz_id(), new_tz_info.get_tz_name()))) { - } else if (OB_FAIL(tz_info_map.id_map_.insert_and_get(new_tz_info.get_tz_id(), tz_pos_value))) { + } else if (OB_FAIL(tz_info_map.id_map_->insert_and_get(new_tz_info.get_tz_id(), tz_pos_value))) { LOG_WARN("fail to insert new_tz_info to tz_info_id_map", KPC(tz_pos_value), K(ret)); - } else if (OB_FAIL(tz_info_map.name_map_.insert_and_get(new_tz_info.get_tz_name(), name_id_value))) { - tz_info_map.id_map_.revert(tz_pos_value); - tz_info_map.id_map_.del(new_tz_info.get_tz_id()); + } else if (OB_FAIL(tz_info_map.name_map_->insert_and_get(new_tz_info.get_tz_name(), name_id_value))) { + tz_info_map.id_map_->revert(tz_pos_value); + tz_info_map.id_map_->del(new_tz_info.get_tz_id()); + tz_pos_value = NULL; LOG_WARN("fail to insert new_tz_info to tz_info_name_map_", K(name_id_value), K(ret)); } else { - tz_info_map.id_map_.revert(tz_pos_value); - tz_info_map.name_map_.revert(name_id_value); - LOG_INFO("succ to add new time zone info", KPC(name_id_value), KPC(tz_pos_value)); + tz_info_map.id_map_->revert(tz_pos_value); + tz_info_map.name_map_->revert(name_id_value); + LOG_INFO("succ to add new time zone info", K(&tz_info_map), K(&(tz_info_map.name_map_)), + KPC(name_id_value), KPC(tz_pos_value)); + tz_pos_value = NULL; + name_id_value = NULL; } } else if (OB_FAIL(stored_tz_info->compare_upgrade(new_tz_info, is_equal))) { LOG_WARN("fail to compare_upgrade", KPC(stored_tz_info), K(new_tz_info), K(ret)); @@ -309,7 +365,8 @@ int ObTimeZoneInfoManager::calc_default_tran_type(const ObIArray types_with_null; bool is_tran_time_null = false; - while(OB_SUCC(ret) && OB_SUCC(result.next())) { tz_tran_type.reset(); is_tran_time_null = false; @@ -381,7 +437,7 @@ int ObTimeZoneInfoManager::fill_tz_info_map(sqlclient::ObMySQLResult &result, } else { tz_info.reset(); types_with_null.reset(); - tz_info_map.id_map_.revert(stored_tz_info); + tz_info_map.id_map_->revert(stored_tz_info); stored_tz_info = NULL; } } @@ -432,74 +488,78 @@ int ObTimeZoneInfoManager::fill_tz_info_map(sqlclient::ObMySQLResult &result, } } - tz_info_map.id_map_.revert(stored_tz_info); + tz_info_map.id_map_->revert(stored_tz_info); stored_tz_info = NULL; return ret; } -int ObTimeZoneInfoManager::fill_tz_info_map(ObRequestTZInfoResult &tz_result) -{ - int ret = OB_SUCCESS; - ObString tz_name_str; - ObTimeZoneInfoPos *stored_tz_info = NULL; - - for(int64_t i = 0 ; OB_SUCC(ret) && i < tz_result.tz_array_.count(); ++i) { - if (NULL != stored_tz_info) { - tz_info_map_.id_map_.revert(stored_tz_info); - stored_tz_info = NULL; - } - ObTimeZoneInfoPos &new_tz_info = tz_result.tz_array_.at(i); - if (OB_FAIL(tz_info_map_.id_map_.get(new_tz_info.get_tz_id(), stored_tz_info))) { - if (OB_ENTRY_NOT_EXIST == ret) { - ret = OB_SUCCESS; - } else { - LOG_WARN("fail to get stored_tz_info, should not happened", - "tz_id", new_tz_info.get_tz_id(), "tz_name", new_tz_info.get_tz_name(), K(ret)); - } - } - - if (OB_SUCC(ret)) { - if (OB_FAIL(set_tz_info_map(stored_tz_info, new_tz_info, tz_info_map_))) { - LOG_WARN("fail to set tz_info map", K(ret)); - } - } - } - - if (NULL != stored_tz_info) { - tz_info_map_.id_map_.revert(stored_tz_info); - stored_tz_info = NULL; - } - - if (OB_SUCC(ret)) { - (void)print_tz_info_map(); - last_version_ = tz_result.last_version_; - } - return ret; -} - -int ObTimeZoneInfoManager::print_tz_info_map() -{ - int ret = OB_SUCCESS; - // SpinRLockGuard rd_guard(rw_lock_); - LOG_INFO("dump current time zone info", "version", last_version_); - if (OB_FAIL(tz_info_map_.print_tz_info_map())) { - LOG_WARN("fail to print_tz_info_map", K(ret)); - } - return ret; -} int ObTimeZoneInfoManager::find_time_zone_info(const common::ObString &tz_name, ObTimeZoneInfoPos &tz_info) { int ret = OB_SUCCESS; if (OB_FAIL(tz_info_map_.get_tz_info_by_name(tz_name, tz_info))) { LOG_WARN("fail to get time zone info", K(tz_name), K(ret)); - } - - if (OB_ENTRY_NOT_EXIST == ret) { - ret = OB_ERR_UNKNOWN_TIME_ZONE; + if (OB_ENTRY_NOT_EXIST == ret) { + ret = OB_ERR_UNKNOWN_TIME_ZONE; + } } return ret; } +bool ObTimeZoneInfoManager::cmp_tz_info_map(ObTZInfoMap &map1, ObTZInfoMap &map2) +{ + bool same = true; + bool locked = false; + if (!(locked = sys_rwlock_.try_rdlock())) { + same = false; + } else if (map1.id_map_->size() != map2.id_map_->size() || map1.name_map_->size() != map2.name_map_->size()) { + same = false; + LOG_INFO("tz info map not same", K(map1.id_map_->size()), K(map2.id_map_->size()), + K(map1.name_map_->size()), K(map2.name_map_->size())); + } + if (same) { + // compare id map. + ObTZInfoIDPosMap::Iterator iter1(*map1.id_map_); + ObTZInfoIDPosMap::Iterator iter2(*map2.id_map_); + ObTimeZoneInfoPos *tz_pos1 = NULL; + ObTimeZoneInfoPos *tz_pos2 = NULL; + tz_pos1 = iter1.next(tz_pos1); + tz_pos2 = iter2.next(tz_pos2); + while (tz_pos1 != NULL && tz_pos2 != NULL && same) { + same = (*tz_pos1) == (*tz_pos2); + iter1.revert(tz_pos1); + iter2.revert(tz_pos2); + tz_pos1 = iter1.next(tz_pos1); + tz_pos2 = iter2.next(tz_pos2); + } + iter1.revert(tz_pos1); + iter2.revert(tz_pos2); + same = same && (NULL == tz_pos1) && (NULL == tz_pos2); + } + if (same) { + // compare name map. + ObTZInfoNameIDMap::Iterator iter1(*map1.name_map_); + ObTZInfoNameIDMap::Iterator iter2(*map2.name_map_); + ObTZNameIDInfo *tz_id_info1 = NULL; + ObTZNameIDInfo *tz_id_info2 = NULL; + tz_id_info1 = iter1.next(tz_id_info1); + tz_id_info2 = iter2.next(tz_id_info2); + while (tz_id_info1 != NULL && tz_id_info2 != NULL && same) { + same = (*tz_id_info1) == (*tz_id_info2); + iter1.revert(tz_id_info1); + iter2.revert(tz_id_info2); + tz_id_info1 = iter1.next(tz_id_info1); + tz_id_info2 = iter2.next(tz_id_info2); + } + iter1.revert(tz_id_info1); + iter2.revert(tz_id_info2); + same = same && (NULL == tz_id_info1) && (NULL == tz_id_info2); + } + if (locked) { + sys_rwlock_.unlock(); + } + return same; +} + OB_SERIALIZE_MEMBER(ObRequestTZInfoArg, obs_addr_, tenant_id_); diff --git a/src/share/ob_time_zone_info_manager.h b/src/share/ob_time_zone_info_manager.h index 4b79ae66a..b93346725 100644 --- a/src/share/ob_time_zone_info_manager.h +++ b/src/share/ob_time_zone_info_manager.h @@ -108,15 +108,11 @@ private: }; public: -ObTimeZoneInfoManager(obrpc::ObCommonRpcProxy &rs_rpc_proxy, - common::ObMySQLProxy &sql_proxy, - rootserver::ObRootService &root_service, - ObTZInfoMap &tz_info_map, +ObTimeZoneInfoManager(common::ObMySQLProxy &sql_proxy, int64_t tenant_id) - : rs_rpc_proxy_(rs_rpc_proxy), - sql_proxy_(sql_proxy), - root_service_(root_service), - tz_info_map_(tz_info_map), + : sql_proxy_(sql_proxy), + tz_info_map_(), + tz_info_map_buf_(), inited_(false), is_usable_(false), last_version_(-1), @@ -134,19 +130,19 @@ ObTimeZoneInfoManager(obrpc::ObCommonRpcProxy &rs_rpc_proxy, int update_time_zone_info(int64_t tz_info_version); int get_time_zone(); int find_time_zone_info(const common::ObString &tz_name, ObTimeZoneInfoPos &tz_info); - void free_tz_info_pos(ObTimeZoneInfoPos *&tz_info) { tz_info_map_.free_tz_info_pos(tz_info); } int64_t get_version() const { return last_version_; } - const ObTZInfoMap *get_tz_info_map() const { return &tz_info_map_; } - const ObTZInfoNameIDMap *get_tz_info_name_map() const { return &tz_info_map_.name_map_; } + ObTZInfoMap *get_tz_info_map() { return &tz_info_map_; } static const char *FETCH_TZ_INFO_SQL; static const char *FETCH_TENANT_TZ_INFO_SQL; static const char *FETCH_LATEST_TZ_VERSION_SQL; - static int fill_tz_info_map(common::sqlclient::ObMySQLResult &result, ObTZInfoMap &tz_info_map); + static int fill_tz_info_map(common::sqlclient::ObMySQLResult &result, ObTZInfoMap &tz_info_map, + uint64_t tenant_id = common::OB_SERVER_TENANT_ID); static int set_tz_info_map( ObTimeZoneInfoPos *&stored_tz_info, ObTimeZoneInfoPos &new_tz_info, ObTZInfoMap &tz_info_map); + static bool cmp_tz_info_map(ObTZInfoMap &tz_info_map1, ObTZInfoMap &tz_info_map2); private: int fetch_time_zone_info_from_tenant_table(const int64_t current_tz_version); @@ -154,14 +150,14 @@ private: ObTimeZoneInfoPos &type_info); static int prepare_tz_info(const common::ObIArray &types_with_null, ObTimeZoneInfoPos &type_info); - int fill_tz_info_map(ObRequestTZInfoResult &tz_result); - int print_tz_info_map(); private: - obrpc::ObCommonRpcProxy &rs_rpc_proxy_; + static ObTZInfoMap shared_tz_info_map_; + static int64_t loaded_tz_info_count_; + static SpinRWLock sys_rwlock_; common::ObMySQLProxy &sql_proxy_; - rootserver::ObRootService &root_service_; - ObTZInfoMap &tz_info_map_; + ObTZInfoMap tz_info_map_; + ObTZInfoMap tz_info_map_buf_; bool inited_; //is_usable_ == true时，server才可以对外提供服务；设置其为true的情况 diff --git a/src/share/object/ob_obj_cast.cpp b/src/share/object/ob_obj_cast.cpp index 84b41bcef..f2bda09b3 100644 --- a/src/share/object/ob_obj_cast.cpp +++ b/src/share/object/ob_obj_cast.cpp @@ -23,7 +23,7 @@ #include "share/object/ob_obj_cast_util.h" #include "share/rc/ob_tenant_base.h" #include "common/sql_mode/ob_sql_mode_utils.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "lib/json_type/ob_json_tree.h" #include "lib/json_type/ob_json_bin.h" #include "lib/json_type/ob_json_base.h" @@ -1548,7 +1548,7 @@ static int common_get_srs_item(omt::ObSrsCacheGuard &srs_guard, // do nothing } else if (0 == srid) { // do nothing - } else if (OB_FAIL(OTSRS_MGR.get_tenant_srs_guard(tenant_id, srs_guard))) { + } else if (OB_FAIL(OTSRS_MGR->get_tenant_srs_guard(srs_guard))) { LOG_WARN("fail to get srs guard", K(ret), K(tenant_id)); } else if (OB_FAIL(srs_guard.get_srs_item(srid, srs))) { LOG_WARN("fail to get srs", K(ret), K(srid)); diff --git a/src/share/rc/ob_tenant_base.h b/src/share/rc/ob_tenant_base.h index cd5372b37..80c7eba99 100755 --- a/src/share/rc/ob_tenant_base.h +++ b/src/share/rc/ob_tenant_base.h @@ -33,10 +33,13 @@ namespace common { class ObTenantIOManager; template class ObServerObjectPool; class ObDetectManager; + class ObOptStatMonitorManager; } namespace omt { class ObPxPools; class ObTenant; + class ObSharedTimer; + class ObTenantSrs; } namespace obmysql { class ObMySQLRequestManager; @@ -214,6 +217,7 @@ using ObPartTransCtxObjPool = common::ObServerObjectPool; #define MTL_MEMBERS \ MTL_LIST( \ + omt::ObSharedTimer*, \ storage::ObTenantMetaMemMgr*, \ ObPartTransCtxObjPool*, \ ObTableScanIteratorObjPool*, \ @@ -302,7 +306,9 @@ using ObTableScanIteratorObjPool = common::ObServerObjectPool RefObjVersion; + typedef common::ObSEArray RefObjVersion; ObDependencyObjItem() : error_ret_(common::OB_SUCCESS), ref_obj_op_(INVALID_OP), @@ -348,7 +348,7 @@ OB_INLINE ret_type get_##name() const { return name##_; } }; - typedef common::ObSEArray DependencyObjKeyItemPairs; + typedef common::ObSEArray DependencyObjKeyItemPairs; struct ObGetDependencyObjOp { public: diff --git a/src/share/stat/ob_opt_stat_monitor_manager.cpp b/src/share/stat/ob_opt_stat_monitor_manager.cpp index 3a64f5eda..4e81766c8 100644 --- a/src/share/stat/ob_opt_stat_monitor_manager.cpp +++ b/src/share/stat/ob_opt_stat_monitor_manager.cpp @@ -24,9 +24,13 @@ #include "lib/mysqlclient/ob_mysql_proxy.h" #include "observer/ob_sql_client_decorator.h" #include "storage/ob_locality_manager.h" +#include "lib/rc/ob_rc.h" +#include "observer/ob_server.h" namespace oceanbase { +using namespace observer; + namespace common { #define INSERT_COLUMN_USAGE "INSERT INTO __all_column_usage(tenant_id," \ @@ -68,107 +72,84 @@ namespace common "updates = updates + values(updates)," \ "deletes = deletes + values(deletes);" -int ObOptStatMonitorFlushAllTask::init(int tg_id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(is_inited_)) { - ret = OB_INIT_TWICE; - LOG_WARN("ObOptStatMonitorFlushAllTask init twice", K(ret)); - } else { - is_inited_ = true; - disable_timeout_check(); - if (OB_FAIL(TG_SCHEDULE(tg_id, *this, FLUSH_INTERVAL, true /*schedule repeatly*/))) { - LOG_WARN("fail to schedule ObOptStatMonitorFlushAllTask", K(ret)); - } - } - return ret; -} - void ObOptStatMonitorFlushAllTask::runTimerTask() { int ret = OB_SUCCESS; - if (OB_FAIL(ObOptStatMonitorManager::get_instance().update_opt_stat_monitoring_info(false))) { - LOG_WARN("failed to update column usage table", K(ret)); + if (OB_NOT_NULL(optstat_monitor_mgr_) && optstat_monitor_mgr_->inited_) { + LOG_INFO("run opt stat monitor flush all task", K(optstat_monitor_mgr_->tenant_id_)); + share::schema::ObMultiVersionSchemaService &schema_service = share::schema::ObMultiVersionSchemaService::get_instance(); + share::schema::ObSchemaGetterGuard schema_guard; + bool in_restore = false; + if (OB_FAIL(schema_service.get_tenant_schema_guard(optstat_monitor_mgr_->tenant_id_, schema_guard))) { + LOG_WARN("failed to get schema guard", K(ret)); + } else if (OB_FAIL(schema_guard.check_tenant_is_restore(optstat_monitor_mgr_->tenant_id_, in_restore))) { + LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (in_restore || GCTX.is_standby_cluster()) { + // do nothing + } else if (OB_FAIL(optstat_monitor_mgr_->update_column_usage_info(false))) { + LOG_WARN("failed to update column usage info", K(ret)); + } else if (OB_FAIL(optstat_monitor_mgr_->update_dml_stat_info())) { + LOG_WARN("failed to failed to update dml stat info", K(ret)); + } else {/*do nothing*/} } } -int ObOptStatMonitorCheckTask::init(int tg_id) -{ - int ret = OB_SUCCESS; - if (OB_UNLIKELY(is_inited_)) { - ret = OB_INIT_TWICE; - LOG_WARN("ObOptStatMonitorCheckTask init twice", K(ret)); - } else { - is_inited_ = true; - disable_timeout_check(); - if (OB_FAIL(TG_SCHEDULE(tg_id, *this, CHECK_INTERVAL, true /*schedule repeatly*/))) { - LOG_WARN("fail to schedule ObOptStatMonitorCheckTask", K(ret)); - } - } - return ret; -} - void ObOptStatMonitorCheckTask::runTimerTask() { int ret = OB_SUCCESS; - if (OB_FAIL(ObOptStatMonitorManager::get_instance().update_opt_stat_monitoring_info(true))) { - LOG_WARN("failed to update column usage table", K(ret)); + if (OB_NOT_NULL(optstat_monitor_mgr_) && optstat_monitor_mgr_->inited_) { + LOG_INFO("run opt stat monitor check task", K(optstat_monitor_mgr_->tenant_id_)); + share::schema::ObMultiVersionSchemaService &schema_service = share::schema::ObMultiVersionSchemaService::get_instance(); + share::schema::ObSchemaGetterGuard schema_guard; + bool in_restore = false; + if (OB_FAIL(schema_service.get_tenant_schema_guard(optstat_monitor_mgr_->tenant_id_, schema_guard))) { + LOG_WARN("failed to get schema guard", K(ret)); + } else if (OB_FAIL(schema_guard.check_tenant_is_restore(optstat_monitor_mgr_->tenant_id_, in_restore))) { + LOG_WARN("failed to check tenant is restore", K(ret)); + } else if (in_restore || GCTX.is_standby_cluster()) { + // do nothing + } else if (OB_FAIL(optstat_monitor_mgr_->update_column_usage_info(true))) { + LOG_WARN("failed to update column usage info", K(ret)); + } else if (OB_FAIL(optstat_monitor_mgr_->update_dml_stat_info())) { + LOG_WARN("failed to failed to update dml stat info", K(ret)); + } else {/*do nothing*/} } } -int ObOptStatMonitorManager::init(ObMySQLProxy *mysql_proxy) +int ObOptStatMonitorManager::init(uint64_t tenant_id) { int ret = OB_SUCCESS; - ObMemAttr attr(OB_SERVER_TENANT_ID, "DmlStatsHashMap"); - SET_USE_500(attr); if (inited_) { ret = OB_INIT_TWICE; LOG_WARN("column usage manager has already been initialized.", K(ret)); - } else if (OB_ISNULL(mysql_proxy)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get null mysql proxy", K(ret)); - } else if (OB_FAIL(column_usage_maps_.create(100, SET_USE_500("ColUsagHashMap")))) { - LOG_WARN("failed to create column usage maps", K(ret)); - } else if (OB_FAIL(dml_stat_maps_.create(100, attr))) { - LOG_WARN("failed to create dml stats maps", K(ret)); - } else if (OB_FAIL(flush_all_task_.init(lib::TGDefIDs::ServerGTimer))) { - LOG_WARN("failed to init column usage task", K(ret)); - } else if (OB_FAIL(check_task_.init(lib::TGDefIDs::ServerGTimer))) { - LOG_WARN("failed to init column usage task", K(ret)); + } else if (OB_FAIL(column_usage_map_.create(10000, "ColUsagHashMap", "ColUsagNode", tenant_id))) { + LOG_WARN("failed to column usage map", K(ret)); + } else if (OB_FAIL(dml_stat_map_.create(10000, "DmlStatHashMap", "DmlStatNode", tenant_id))) { + LOG_WARN("failed to create dml stat map", K(ret)); } else { inited_ = true; - mysql_proxy_ = mysql_proxy; + mysql_proxy_ = &ObServer::get_instance().get_mysql_proxy(); + tenant_id_ = tenant_id; + destroyed_ = false; + } + if (OB_FAIL(ret) && !inited_) { + destroy(); } return ret; } void ObOptStatMonitorManager::destroy() { - if (inited_) { + if (!destroyed_) { + destroyed_ = true; inited_ = false; - for (auto iter = column_usage_maps_.begin(); iter != column_usage_maps_.end(); ++iter) { - if (OB_ISNULL(iter->second)) { - BACKTRACE_RET(ERROR, OB_ERR_UNEXPECTED, true, "column usage map is null"); - } else { - iter->second->destroy(); - } - } - for (auto iter = dml_stat_maps_.begin(); iter != dml_stat_maps_.end(); ++iter) { - if (OB_ISNULL(iter->second)) { - BACKTRACE_RET(ERROR, OB_ERR_UNEXPECTED, true, "dml stats map is null"); - } else { - iter->second->destroy(); - } - } + TG_DESTROY(tg_id_); + SpinWLockGuard guard(lock_); + column_usage_map_.destroy(); + dml_stat_map_.destroy(); } } -ObOptStatMonitorManager &ObOptStatMonitorManager::get_instance() -{ - static ObOptStatMonitorManager instance_; - return instance_; -} - int ObOptStatMonitorManager::flush_database_monitoring_info(sql::ObExecContext &ctx, const bool is_flush_col_usage, const bool is_flush_dml_stat, @@ -220,217 +201,116 @@ int ObOptStatMonitorManager::flush_database_monitoring_info(sql::ObExecContext & return ret; } -int ObOptStatMonitorManager::erase_opt_stat_monitoring_info_map(uint64_t tenant_id) +int ObOptStatMonitorManager::update_local_cache(common::ObIArray &args) { int ret = OB_SUCCESS; - if (OB_FAIL(erase_column_usage_map(tenant_id))) { - LOG_WARN("failed to erase column usage map", K(ret), K(tenant_id)); - } else if (OB_FAIL(erase_dml_stat_map(tenant_id))) { - LOG_WARN("failed to erase dml stat map", K(ret), K(tenant_id)); - } else {/*do nothing*/} - return ret; -} - - -int ObOptStatMonitorManager::erase_column_usage_map(uint64_t tenant_id) -{ - int ret = OB_SUCCESS; - ColumnUsageMap *col_map = nullptr; - if (OB_FAIL(column_usage_maps_.get_refactored(tenant_id, col_map))) { - LOG_WARN("failed to get column usage map", K(ret)); - } else if (OB_FAIL(column_usage_maps_.erase_refactored(tenant_id))) { - LOG_WARN("failed to erase column usage map", K(ret)); - } else { - col_map->destroy(); - ob_free(col_map); - } - return ret; -} - -int ObOptStatMonitorManager::erase_dml_stat_map(uint64_t tenant_id) -{ - int ret = OB_SUCCESS; - DmlStatMap *dml_stat_map= nullptr; - if (OB_FAIL(dml_stat_maps_.get_refactored(tenant_id, dml_stat_map))) { - LOG_WARN("failed to get dml stat map", K(ret)); - } else if (OB_FAIL(dml_stat_maps_.erase_refactored(tenant_id))) { - LOG_WARN("failed to erase dml stat map", K(ret)); - } else { - dml_stat_map->destroy(); - ob_free(dml_stat_map); - } - return ret; -} - -int ObOptStatMonitorManager::update_local_cache(uint64_t tenant_id, - common::ObIArray &args) -{ - int ret = OB_SUCCESS; - ReadMapAtomicOp atomic_op(&args); - ObMemAttr attr(OB_SERVER_TENANT_ID, "ColUsagHashMap"); - SET_USE_500(attr); if (GCTX.is_standby_cluster()) { // standby cluster can't write __all_column_usage, so do not need to update local update - } else if (OB_FAIL(column_usage_maps_.read_atomic(tenant_id, atomic_op))) { - if (OB_HASH_NOT_EXIST == ret) {//not exists such tenant id map, need alloc new map - ColumnUsageMap *col_map = NULL; - ColumnUsageMap *tmp_col_map = NULL; - void *buff = ob_malloc(sizeof(ColumnUsageMap), attr); - if (OB_ISNULL(buff)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("alloc memory failed", K(ret)); - } else if (OB_FALSE_IT(col_map = new(buff)ColumnUsageMap())) { - } else if (OB_FAIL(col_map->create(10000, attr))) { - LOG_WARN("failed to create column usage map", K(ret)); - } else if (OB_FAIL(column_usage_maps_.set_refactored(tenant_id, col_map))) { - // set refacter failed, may created by other thread - if (OB_SUCCESS == column_usage_maps_.get_refactored(tenant_id, tmp_col_map)) { - LOG_TRACE("get column usage map succeed", K(tenant_id), K(tmp_col_map)); + } else { + SpinRLockGuard guard(lock_); + for (int64_t i = 0; OB_SUCC(ret) && i < args.count(); ++i) { + ColumnUsageArg &arg = args.at(i); + StatKey col_key(arg.table_id_, arg.column_id_); + int64_t flags = 0; + if (OB_FAIL(column_usage_map_.get_refactored(col_key, flags))) { + if (OB_LIKELY(ret == OB_HASH_NOT_EXIST)) { + if (OB_FAIL(column_usage_map_.set_refactored(col_key, arg.flags_))) { + // other thread set the refactor, try update again + if (OB_FAIL(column_usage_map_.get_refactored(col_key, flags))) { + LOG_WARN("failed to get refactored", K(ret)); + } else if ((~flags) & arg.flags_) { + UpdateValueAtomicOp atomic_op(arg.flags_); + if (OB_FAIL(column_usage_map_.atomic_refactored(col_key, atomic_op))) { + LOG_WARN("failed to atomic refactored", K(ret)); + } + } + } } else { - LOG_WARN("get column usage map failed", K(tenant_id), K(tmp_col_map)); + LOG_WARN("failed to get refactored", K(ret)); + } + } else if ((~flags) & arg.flags_) { + UpdateValueAtomicOp atomic_op(arg.flags_); + if (OB_FAIL(column_usage_map_.atomic_refactored(col_key, atomic_op))) { + LOG_WARN("failed to atomic refactored", K(ret)); } } - if (OB_FAIL(ret) && OB_NOT_NULL(col_map)) {//free unused memory - col_map->~ColumnUsageMap(); - ob_free(buff); - buff = NULL; - col_map = tmp_col_map; - } - if (OB_NOT_NULL(col_map)) { - //arrive at here, indicates get a valid dml_stat_map, we need reset error code, and atomic update values - if (OB_FAIL(column_usage_maps_.read_atomic(tenant_id, atomic_op))) { - LOG_WARN("failed to atomic refactored", K(ret), K(tenant_id)); - } - } - } else { - LOG_WARN("failed to atomic refactored", K(ret)); } } return ret; } -int ObOptStatMonitorManager::update_local_cache(uint64_t tenant_id, ObOptDmlStat &dml_stat) +int ObOptStatMonitorManager::update_local_cache(ObOptDmlStat &dml_stat) { int ret = OB_SUCCESS; - ReadMapAtomicOp atomic_op(dml_stat); - ObMemAttr attr(OB_SERVER_TENANT_ID, "DmlStatsHashMap"); - SET_USE_500(attr); if (GCTX.is_standby_cluster()) { // standby cluster can't write __all_monitor_modified, so do not need to update local update - } else if (OB_FAIL(dml_stat_maps_.read_atomic(tenant_id, atomic_op))) { - if (OB_LIKELY(OB_HASH_NOT_EXIST == ret)) {//not exists such tenant id map, need alloc new map - ret = OB_SUCCESS; - DmlStatMap *dml_stat_map = NULL; - DmlStatMap *tmp_dml_stat_map = NULL; - void *buff = ob_malloc(sizeof(DmlStatMap), attr); - if (OB_ISNULL(buff)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("alloc memory failed", K(ret)); - } else if (OB_FALSE_IT(dml_stat_map = new(buff)DmlStatMap())) { - } else if (OB_FAIL(dml_stat_map->create(10000, attr))) { - LOG_WARN("failed to create column usage map", K(ret)); - } else if (OB_FAIL(dml_stat_maps_.set_refactored(tenant_id, dml_stat_map))) { - // set refacter failed, may created by other thread - if (OB_SUCCESS == dml_stat_maps_.get_refactored(tenant_id, tmp_dml_stat_map)) { - LOG_TRACE("get dml stats succeed", K(tenant_id), K(tmp_dml_stat_map)); - } else { - LOG_WARN("get dml stats failed", K(tenant_id), K(tmp_dml_stat_map), K(ret)); - } - } - if (OB_FAIL(ret) && OB_NOT_NULL(dml_stat_map)) {//free unused memory - dml_stat_map->~DmlStatMap(); - ob_free(buff); - buff = NULL; - dml_stat_map = tmp_dml_stat_map; - } - if (OB_NOT_NULL(dml_stat_map)) { - //arrive at here, indicates get a valid dml_stat_map, we need reset error code, and atomic update values - if (OB_FAIL(dml_stat_maps_.read_atomic(tenant_id, atomic_op))) { - LOG_WARN("failed to atomic refactored", K(ret), K(tenant_id)); + } else { + SpinRLockGuard guard(lock_); + StatKey key(dml_stat.table_id_, dml_stat.tablet_id_); + ObOptDmlStat tmp_dml_stat; + if (OB_FAIL(dml_stat_map_.get_refactored(key, tmp_dml_stat))) { + if (OB_LIKELY(ret == OB_HASH_NOT_EXIST)) { + if (OB_FAIL(dml_stat_map_.set_refactored(key, dml_stat))) { + // other thread set the refactor, try update again + if (OB_FAIL(dml_stat_map_.get_refactored(key, tmp_dml_stat))) { + LOG_WARN("failed to get refactored", K(ret)); + } else { + UpdateValueAtomicOp atomic_op(dml_stat); + if (OB_FAIL(dml_stat_map_.atomic_refactored(key, atomic_op))) { + LOG_WARN("failed to atomic refactored", K(ret)); + } + } } + } else { + LOG_WARN("failed to get refactored", K(ret)); } } else { - LOG_WARN("failed to atomic refactored", K(ret)); + UpdateValueAtomicOp atomic_op(dml_stat); + if (OB_FAIL(dml_stat_map_.atomic_refactored(key, atomic_op))) { + LOG_WARN("failed to atomic refactored", K(ret)); + } else {/*do nothing*/} } } return ret; } -int ObOptStatMonitorManager::update_opt_stat_monitoring_info(const bool with_check) -{ - int ret = OB_SUCCESS; - if (OB_FAIL(update_column_usage_table(with_check))) { - LOG_WARN("failed to update column usage table", K(ret)); - } else if (OB_FAIL(update_dml_stat_info(with_check))) { - LOG_WARN("failed to update dml stat info", K(ret)); - } else {/*do nothing*/} - return ret; -} - int ObOptStatMonitorManager::update_opt_stat_monitoring_info(const obrpc::ObFlushOptStatArg &arg) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!arg.is_valid())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(ret), K(arg)); - } else if (arg.is_flush_col_usage_ && OB_FAIL(update_tenant_column_usage_info(arg.tenant_id_))) { + } else if (arg.is_flush_col_usage_ && OB_FAIL(update_column_usage_info(false))) { LOG_WARN("failed to update tenant column usage info", K(ret)); - } else if (arg.is_flush_dml_stat_ && OB_FAIL(update_tenant_dml_stat_info(arg.tenant_id_))) { + } else if (arg.is_flush_dml_stat_ && OB_FAIL(update_dml_stat_info())) { LOG_WARN("failed to update tenant column usage info", K(ret)); } else { /*do nothing*/ } return ret; } -int ObOptStatMonitorManager::update_column_usage_table(const bool with_check) +int ObOptStatMonitorManager::update_column_usage_info(const bool with_check) { int ret = OB_SUCCESS; - ObSEArray tenant_ids; - GetAllKeyOp get_all_key_op(&tenant_ids, with_check); + bool is_writeable = false; + ObArray col_stat_keys; + ObArray col_flags; if (!inited_) { ret = OB_NOT_INIT; LOG_WARN("opt stat monitor is not inited", K(ret)); - } else if (OB_FAIL(column_usage_maps_.foreach_refactored(get_all_key_op))) { - LOG_WARN("failed to foreach refactored", K(ret)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < tenant_ids.count(); ++i) { - if (OB_FAIL(update_tenant_column_usage_info(tenant_ids.at(i)))) { - LOG_WARN("failed to update tenant column usage info", K(ret)); - } - } - return ret; -} - -int ObOptStatMonitorManager::update_tenant_column_usage_info(uint64_t tenant_id) -{ - int ret = OB_SUCCESS; - SwapMapAtomicOp swap_map_op; - ColumnUsageMap *col_map = NULL; - bool is_writeable = false; - if (OB_FAIL(check_table_writeable(tenant_id, is_writeable))) { + } else if (OB_FAIL(check_table_writeable(is_writeable))) { LOG_WARN("failed to check tabke writeable", K(ret)); } else if (!is_writeable) { // do nothing - } else if (OB_FAIL(column_usage_maps_.atomic_refactored(tenant_id, swap_map_op))) { - if (OB_LIKELY(OB_HASH_NOT_EXIST == ret)) { - // tenant may not exsits on this server - ret = OB_SUCCESS; - } else { - LOG_WARN("failed to atomic refactored", K(ret)); - } - } else if (OB_ISNULL(col_map = swap_map_op.get_column_usage_map())) { - // NULL means column usage map is empty, do nothing - } else { + } else if (OB_FAIL(get_col_usage_info(with_check, col_stat_keys, col_flags))) { + LOG_WARN("failed to get col usage info", K(ret)); + } else if (!col_stat_keys.empty() && col_stat_keys.count() == col_flags.count()) { ObSqlString value_sql; int count = 0; - for (auto iter = col_map->begin(); OB_SUCC(ret) && iter != col_map->end(); ++iter) { - if (OB_FAIL(get_column_usage_sql(tenant_id, - iter->first, - iter->second, - 0 != count, // need_add_comma - value_sql))) { + for (int64_t i = 0; OB_SUCC(ret) && i < col_stat_keys.count(); ++i) { + if (OB_FAIL(get_column_usage_sql(col_stat_keys.at(i), col_flags.at(i), 0 != count, value_sql))) { LOG_WARN("failed to get column usage sql", K(ret)); } else if (UPDATE_OPT_STAT_BATCH_CNT == ++count) { - if (OB_FAIL(exec_insert_column_usage_sql(tenant_id, value_sql))) { + if (OB_FAIL(exec_insert_column_usage_sql(value_sql))) { LOG_WARN("failed to exec insert sql", K(ret)); } else { count = 0; @@ -439,68 +319,36 @@ int ObOptStatMonitorManager::update_tenant_column_usage_info(uint64_t tenant_id) } } if (OB_SUCC(ret) && count != 0) { - if (OB_FAIL(exec_insert_column_usage_sql(tenant_id, value_sql))) { + if (OB_FAIL(exec_insert_column_usage_sql(value_sql))) { LOG_WARN("failed to exec insert sql", K(ret)); } } - - if (OB_NOT_NULL(col_map)) { - col_map->~ColumnUsageMap(); - ob_free(col_map); - col_map = NULL; - } } return ret; } -int ObOptStatMonitorManager::update_dml_stat_info(const bool with_check) +int ObOptStatMonitorManager::update_dml_stat_info() { int ret = OB_SUCCESS; - ObSEArray tenant_ids; - GetAllKeyOp get_all_key_op(&tenant_ids, with_check); + bool is_writeable = false; + ObArray dml_stats; if (!inited_) { ret = OB_NOT_INIT; LOG_WARN("opt stat monitor is not inited", K(ret)); - } else if (OB_FAIL(dml_stat_maps_.foreach_refactored(get_all_key_op))) { - LOG_WARN("failed to foreach refactored", K(ret)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < tenant_ids.count(); ++i) { - if (OB_FAIL(update_tenant_dml_stat_info(tenant_ids.at(i)))) { - LOG_WARN("failed to update tenant dml stat info", K(ret)); - } - } - return ret; -} - -int ObOptStatMonitorManager::update_tenant_dml_stat_info(uint64_t tenant_id) -{ - int ret = OB_SUCCESS; - SwapMapAtomicOp swap_map_op; - DmlStatMap *dml_stat_map = NULL; - bool is_writeable = false; - if (OB_FAIL(check_table_writeable(tenant_id, is_writeable))) { + } else if (OB_FAIL(check_table_writeable(is_writeable))) { LOG_WARN("failed to check tabke writeable", K(ret)); } else if (!is_writeable) { // do nothing - } else if (OB_FAIL(dml_stat_maps_.atomic_refactored(tenant_id, swap_map_op))) { - if (OB_LIKELY(OB_HASH_NOT_EXIST == ret)) { - // tenant may not exsits on this server - ret = OB_SUCCESS; - } else { - LOG_WARN("failed to atomic refactored", K(ret)); - } - } else if (OB_ISNULL(dml_stat_map = swap_map_op.get_dml_stat_map())) { - // NULL means dml stats map is empty, do nothing - } else { + } else if (OB_FAIL(get_dml_stats(dml_stats))) { + LOG_WARN("failed to swap get dml stat", K(ret)); + } else if (!dml_stats.empty()) { ObSqlString value_sql; int count = 0; - for (auto iter = dml_stat_map->begin(); OB_SUCC(ret) && iter != dml_stat_map->end(); ++iter) { - if (OB_FAIL(get_dml_stat_sql(tenant_id, iter->second, - 0 != count, // need_add_comma - value_sql))) { + for (int64_t i = 0; OB_SUCC(ret) && i < dml_stats.count(); ++i) { + if (OB_FAIL(get_dml_stat_sql(dml_stats.at(i), 0 != count, value_sql))) { LOG_WARN("failed to get dml stat sql", K(ret)); } else if (UPDATE_OPT_STAT_BATCH_CNT == ++count) { - if (OB_FAIL(exec_insert_monitor_modified_sql(tenant_id, value_sql))) { + if (OB_FAIL(exec_insert_monitor_modified_sql(value_sql))) { LOG_WARN("failed to exec insert sql", K(ret)); } else { count = 0; @@ -509,19 +357,12 @@ int ObOptStatMonitorManager::update_tenant_dml_stat_info(uint64_t tenant_id) } } if (OB_SUCC(ret) && count != 0) { - if (OB_FAIL(exec_insert_monitor_modified_sql(tenant_id, value_sql))) { + if (OB_FAIL(exec_insert_monitor_modified_sql(value_sql))) { LOG_WARN("failed to exec insert sql", K(ret)); } } - - if (OB_NOT_NULL(dml_stat_map)) { - dml_stat_map->~DmlStatMap(); - ob_free(dml_stat_map); - dml_stat_map = NULL; - } - if (OB_SUCC(ret)) { - if (OB_FAIL(clean_useless_dml_stat_info(tenant_id))) { + if (OB_FAIL(clean_useless_dml_stat_info())) { LOG_WARN("failed to clean useless dml stat info", K(ret)); } else {/*do nohting*/} } @@ -529,8 +370,7 @@ int ObOptStatMonitorManager::update_tenant_dml_stat_info(uint64_t tenant_id) return ret; } -int ObOptStatMonitorManager::get_column_usage_sql(const uint64_t tenant_id, - const StatKey &col_key, +int ObOptStatMonitorManager::get_column_usage_sql(const StatKey &col_key, const int64_t flags, const bool need_add_comma, ObSqlString &sql_string) @@ -538,8 +378,8 @@ int ObOptStatMonitorManager::get_column_usage_sql(const uint64_t tenant_id, int ret = OB_SUCCESS; share::ObDMLSqlSplicer dml_splicer; uint64_t table_id = col_key.first; - uint64_t ext_tenant_id = share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id); - uint64_t pure_table_id = share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, table_id); + uint64_t ext_tenant_id = share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id_, tenant_id_); + uint64_t pure_table_id = share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id_, table_id); int64_t equality_preds = flags & EQUALITY_PREDS ? 1 : 0; int64_t equijoin_preds = flags & EQUIJOIN_PREDS ? 1 : 0; int64_t nonequijion_preds = flags & NONEQUIJOIN_PREDS ? 1 : 0; @@ -571,7 +411,7 @@ int ObOptStatMonitorManager::get_column_usage_sql(const uint64_t tenant_id, return ret; } -int ObOptStatMonitorManager::exec_insert_column_usage_sql(uint64_t tenant_id,ObSqlString &values_sql) +int ObOptStatMonitorManager::exec_insert_column_usage_sql(ObSqlString &values_sql) { int ret = OB_SUCCESS; ObSqlString insert_sql; @@ -582,9 +422,9 @@ int ObOptStatMonitorManager::exec_insert_column_usage_sql(uint64_t tenant_id,ObS LOG_WARN("failed to append format", K(ret)); } else if (OB_FAIL(insert_sql.append(ON_DUPLICATE_UPDATE))) { LOG_WARN("failed to append string", K(ret)); - } else if (OB_FAIL(mysql_proxy_->write(tenant_id, insert_sql.ptr(), affected_rows))) { + } else if (OB_FAIL(mysql_proxy_->write(tenant_id_, insert_sql.ptr(), affected_rows))) { LOG_WARN("fail to exec sql", K(insert_sql), K(ret)); - } else if (OB_FAIL(mysql_proxy_->write(tenant_id, "commit;", affected_rows))) { + } else if (OB_FAIL(mysql_proxy_->write(tenant_id_, "commit;", affected_rows))) { LOG_WARN("fail to exec sql", K(ret)); } return ret; @@ -691,7 +531,7 @@ int ObOptStatMonitorManager::construct_get_column_usage_sql(ObIArraycheck_tenant_is_restore(NULL, tenant_id, in_restore))) { + } else if (OB_FAIL(GCTX.schema_service_->check_tenant_is_restore(NULL, tenant_id_, in_restore))) { LOG_WARN("failed to check tenant is restore", K(ret)); } else if (OB_UNLIKELY(in_restore) || GCTX.is_standby_cluster()) { is_writeable = false; @@ -707,107 +547,6 @@ int ObOptStatMonitorManager::check_table_writeable(const uint64_t tenant_id, boo return ret; } -int ObOptStatMonitorManager::GetAllKeyOp::operator()(common::hash::HashMapPair &entry) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(key_array_)) { - ret = OB_NOT_INIT; - LOG_WARN("key_array not inited", K(ret)); - } else if (with_check_ && entry.second->size() < 10000) { - // do nothing - } else if (OB_FAIL(key_array_->push_back(entry.first))) { - LOG_WARN("fail to push back key", K(ret)); - } - return ret; -} - -int ObOptStatMonitorManager::GetAllKeyOp::operator()(common::hash::HashMapPair &entry) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(key_array_)) { - ret = OB_NOT_INIT; - LOG_WARN("key_array not inited", K(ret)); - } else if (OB_FAIL(key_array_->push_back(entry.first))) { - LOG_WARN("fail to push back key", K(ret)); - } - return ret; -} - -// get old ColumnUsageMap, allocate a new one -int ObOptStatMonitorManager::SwapMapAtomicOp::operator() (common::hash::HashMapPair &entry) -{ - int ret = OB_SUCCESS; - column_usage_map_ = NULL; - ObMemAttr attr(OB_SERVER_TENANT_ID, "DmlStatsHashMap"); - SET_USE_500(attr); - if (OB_ISNULL(entry.second)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret)); - } else if (entry.second->size() == 0) { - // do nothing - } else { - ColumnUsageMap *col_map = NULL; - void *buff = ob_malloc(sizeof(ColumnUsageMap), attr); - if (OB_ISNULL(buff)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("alloc memory failed", K(ret)); - } else if (NULL == (col_map = new(buff)ColumnUsageMap())) { - ret = OB_NOT_INIT; - LOG_WARN("fail to constructor column usage map", K(ret)); - } else if (OB_FAIL(col_map->create(10000, attr))) { - LOG_WARN("failed to create column usage map", K(ret)); - } else { - column_usage_map_ = entry.second; - entry.second = col_map; - } - - if (OB_FAIL(ret) && OB_NOT_NULL(col_map)) { - col_map->~ColumnUsageMap(); - ob_free(buff); - buff = NULL; - col_map = NULL; - } - } - return ret; -} - -// get old DmlStatMap, allocate a new one -int ObOptStatMonitorManager::SwapMapAtomicOp::operator() (common::hash::HashMapPair &entry) -{ - int ret = OB_SUCCESS; - dml_stat_map_ = NULL; - ObMemAttr attr(OB_SERVER_TENANT_ID, "DmlStatMap"); - SET_USE_500(attr); - if (OB_ISNULL(entry.second)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret)); - } else if (entry.second->size() == 0) { - // do nothing - } else { - DmlStatMap *dml_stat_map = NULL; - void *buff = ob_malloc(sizeof(DmlStatMap), attr); - if (OB_ISNULL(buff)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("alloc memory failed", K(ret)); - } else if (NULL == (dml_stat_map = new(buff)DmlStatMap())) { - ret = OB_NOT_INIT; - LOG_WARN("fail to constructor DmlStatMap", K(ret)); - } else if (OB_FAIL(dml_stat_map->create(10000, attr))) { - LOG_WARN("failed to create column usage map", K(ret)); - } else { - dml_stat_map_ = entry.second; - entry.second = dml_stat_map; - } - if (OB_FAIL(ret) && OB_NOT_NULL(dml_stat_map)) { - dml_stat_map->~DmlStatMap(); - ob_free(buff); - buff = NULL; - dml_stat_map = NULL; - } - } - return ret; -} - int ObOptStatMonitorManager::UpdateValueAtomicOp::operator() (common::hash::HashMapPair &entry) { entry.second |= flags_; @@ -829,80 +568,7 @@ int ObOptStatMonitorManager::UpdateValueAtomicOp::operator() (common::hash::Hash return ret; } -int ObOptStatMonitorManager::ReadMapAtomicOp::operator() (common::hash::HashMapPair &entry) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(entry.second) || OB_ISNULL(col_usage_args_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(entry.second), K(col_usage_args_)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < col_usage_args_->count(); ++i) { - ColumnUsageArg &arg = col_usage_args_->at(i); - StatKey col_key(arg.table_id_, arg.column_id_); - int64_t flags = 0; - if (OB_FAIL(entry.second->get_refactored(col_key, flags))) { - if (OB_LIKELY(ret == OB_HASH_NOT_EXIST)) { - if (OB_FAIL(entry.second->set_refactored(col_key, arg.flags_))) { - // other thread set the refactor, try update again - if (OB_FAIL(entry.second->get_refactored(col_key, flags))) { - LOG_WARN("failed to get refactored", K(ret)); - } else if ((~flags) & arg.flags_) { - UpdateValueAtomicOp atomic_op(arg.flags_); - if (OB_FAIL(entry.second->atomic_refactored(col_key, atomic_op))) { - LOG_WARN("failed to atomic refactored", K(ret)); - } - } - } - } else { - LOG_WARN("failed to get refactored", K(ret)); - } - } else if ((~flags) & arg.flags_) { - UpdateValueAtomicOp atomic_op(arg.flags_); - if (OB_FAIL(entry.second->atomic_refactored(col_key, atomic_op))) { - LOG_WARN("failed to atomic refactored", K(ret)); - } - } - } - } - return ret; -} - -int ObOptStatMonitorManager::ReadMapAtomicOp::operator() (common::hash::HashMapPair &entry) -{ - int ret = OB_SUCCESS; - StatKey key(dml_stat_.table_id_, dml_stat_.tablet_id_); - UpdateValueAtomicOp atomic_op(dml_stat_); - ObOptDmlStat tmp_dml_stat; - if (OB_ISNULL(entry.second)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret)); - } else if (OB_FAIL(entry.second->get_refactored(key, tmp_dml_stat))) { - if (OB_LIKELY(ret == OB_HASH_NOT_EXIST)) { - if (OB_FAIL(entry.second->set_refactored(key, dml_stat_))) { - // other thread set the refactor, try update again - if (OB_FAIL(entry.second->get_refactored(key, tmp_dml_stat))) { - LOG_WARN("failed to get refactored", K(ret)); - } else { - UpdateValueAtomicOp atomic_op(dml_stat_); - if (OB_FAIL(entry.second->atomic_refactored(key, atomic_op))) { - LOG_WARN("failed to atomic refactored", K(ret)); - } - } - } - } else { - LOG_WARN("failed to get refactored", K(ret)); - } - } else { - UpdateValueAtomicOp atomic_op(dml_stat_); - if (OB_FAIL(entry.second->atomic_refactored(key, atomic_op))) { - LOG_WARN("failed to atomic refactored", K(ret)); - } else {/*do nothing*/} - } - return ret; -} - -int ObOptStatMonitorManager::exec_insert_monitor_modified_sql(uint64_t tenant_id, - ObSqlString &values_sql) +int ObOptStatMonitorManager::exec_insert_monitor_modified_sql(ObSqlString &values_sql) { int ret = OB_SUCCESS; ObSqlString insert_sql; @@ -913,24 +579,23 @@ int ObOptStatMonitorManager::exec_insert_monitor_modified_sql(uint64_t tenant_id LOG_WARN("failed to append format", K(ret)); } else if (OB_FAIL(insert_sql.append(ON_DUPLICATE_UPDATE_MONITOR_MODIFIED))) { LOG_WARN("failed to append string", K(ret)); - } else if (OB_FAIL(mysql_proxy_->write(tenant_id, insert_sql.ptr(), affected_rows))) { + } else if (OB_FAIL(mysql_proxy_->write(tenant_id_, insert_sql.ptr(), affected_rows))) { LOG_WARN("fail to exec sql", K(insert_sql), K(ret)); } else { - LOG_TRACE("succeed to exec insert monitor modified sql", K(tenant_id), K(values_sql)); + LOG_TRACE("succeed to exec insert monitor modified sql", K(tenant_id_), K(values_sql)); } return ret; } -int ObOptStatMonitorManager::get_dml_stat_sql(const uint64_t tenant_id, - const ObOptDmlStat &dml_stat, +int ObOptStatMonitorManager::get_dml_stat_sql(const ObOptDmlStat &dml_stat, const bool need_add_comma, ObSqlString &sql_string) { int ret = OB_SUCCESS; share::ObDMLSqlSplicer dml_splicer; uint64_t table_id = dml_stat.table_id_; - uint64_t ext_tenant_id = share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id, tenant_id); - uint64_t pure_table_id = share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id, table_id); + uint64_t ext_tenant_id = share::schema::ObSchemaUtils::get_extract_tenant_id(tenant_id_, tenant_id_); + uint64_t pure_table_id = share::schema::ObSchemaUtils::get_extract_schema_id(tenant_id_, table_id); if (OB_FAIL(dml_splicer.add_pk_column("tenant_id", ext_tenant_id)) || OB_FAIL(dml_splicer.add_pk_column("table_id", pure_table_id)) || OB_FAIL(dml_splicer.add_pk_column("tablet_id", dml_stat.tablet_id_)) || @@ -948,16 +613,17 @@ int ObOptStatMonitorManager::get_dml_stat_sql(const uint64_t tenant_id, return ret; } -int ObOptStatMonitorManager::generate_opt_stat_monitoring_info_rows(observer::ObOptDmlStatMapsGetter &getter) +int ObOptStatMonitorManager::generate_opt_stat_monitoring_info_rows(observer::ObOptDmlStatMapGetter &getter) { int ret = OB_SUCCESS; - if (OB_FAIL(dml_stat_maps_.foreach_refactored(getter))) { + SpinRLockGuard guard(lock_); + if (OB_FAIL(dml_stat_map_.foreach_refactored(getter))) { LOG_WARN("fail to generate opt stat monitoring info rows", K(ret)); } else {/*do nothing*/} return ret; } -int ObOptStatMonitorManager::clean_useless_dml_stat_info(uint64_t tenant_id) +int ObOptStatMonitorManager::clean_useless_dml_stat_info() { int ret = OB_SUCCESS; ObSqlString delete_table_sql; @@ -965,7 +631,7 @@ int ObOptStatMonitorManager::clean_useless_dml_stat_info(uint64_t tenant_id) int64_t affected_rows1 = 0; int64_t affected_rows2 = 0; const char* all_table_name = NULL; - if (OB_FAIL(ObSchemaUtils::get_all_table_name(tenant_id, all_table_name))) { + if (OB_FAIL(ObSchemaUtils::get_all_table_name(tenant_id_, all_table_name))) { LOG_WARN("failed to get all table name", K(ret)); } else if (OB_FAIL(delete_table_sql.append_fmt("DELETE FROM %s m WHERE (NOT EXISTS (SELECT 1 " \ "FROM %s t, %s db WHERE t.tenant_id = db.tenant_id AND t.database_id = db.database_id "\ @@ -985,12 +651,12 @@ int ObOptStatMonitorManager::clean_useless_dml_stat_info(uint64_t tenant_id) all_table_name, share::OB_ALL_DATABASE_TNAME, share::OB_ALL_PART_TNAME, share::OB_ALL_SUB_PART_TNAME, OB_MAX_INNER_TABLE_ID))) { LOG_WARN("failed to append fmt", K(ret)); - } else if (OB_FAIL(mysql_proxy_->write(tenant_id, delete_table_sql.ptr(), affected_rows1))) { + } else if (OB_FAIL(mysql_proxy_->write(tenant_id_, delete_table_sql.ptr(), affected_rows1))) { LOG_WARN("failed to execute sql", K(ret), K(delete_table_sql)); - } else if (OB_FAIL(mysql_proxy_->write(tenant_id, delete_part_sql.ptr(), affected_rows2))) { + } else if (OB_FAIL(mysql_proxy_->write(tenant_id_, delete_part_sql.ptr(), affected_rows2))) { LOG_WARN("failed to execute sql", K(ret), K(delete_part_sql)); } else { - LOG_TRACE("succeed to clean useless monitor modified_data", K(tenant_id), K(delete_table_sql), + LOG_TRACE("succeed to clean useless monitor modified_data", K(tenant_id_), K(delete_table_sql), K(affected_rows1), K(delete_part_sql), K(affected_rows2)); } @@ -998,22 +664,39 @@ int ObOptStatMonitorManager::clean_useless_dml_stat_info(uint64_t tenant_id) } int ObOptStatMonitorManager::update_dml_stat_info_from_direct_load(const ObIArray &dml_stats) +{ + int ret = OB_SUCCESS; + ObOptStatMonitorManager *optstat_monitor_mgr = NULL; + LOG_TRACE("begin to update dml stat info from direct load", K(dml_stats)); + if (dml_stats.empty() || OB_ISNULL(dml_stats.at(0))) { + //do nothing + } else if (OB_UNLIKELY(dml_stats.at(0)->tenant_id_ != MTL_ID())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected error", K(ret), K(dml_stats), K(MTL_ID())); + } else if (OB_ISNULL(optstat_monitor_mgr = MTL(ObOptStatMonitorManager*))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(optstat_monitor_mgr)); + } else if (OB_FAIL(optstat_monitor_mgr->update_dml_stat_info(dml_stats))) { + LOG_WARN("failed to update dml stat info", K(ret)); + } else {/*do nothing*/} + return ret; +} + +int ObOptStatMonitorManager::update_dml_stat_info(const ObIArray &dml_stats) { int ret = OB_SUCCESS; ObSqlString value_sql; int count = 0; - uint64_t tenant_id = 0; LOG_TRACE("begin to update dml stat info from direct load", K(dml_stats)); for (int64_t i = 0; OB_SUCC(ret) && i < dml_stats.count(); ++i) { if (OB_ISNULL(dml_stats.at(i))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpcted error", K(ret), K(dml_stats.at(i))); } else { - tenant_id = dml_stats.at(i)->tenant_id_; - if (OB_FAIL(get_dml_stat_sql(tenant_id, *dml_stats.at(i), 0 != count, value_sql))) { + if (OB_FAIL(get_dml_stat_sql(*dml_stats.at(i), 0 != count, value_sql))) { LOG_WARN("failed to get dml stat sql", K(ret)); } else if (UPDATE_OPT_STAT_BATCH_CNT == ++count) { - if (OB_FAIL(exec_insert_monitor_modified_sql(tenant_id, value_sql))) { + if (OB_FAIL(exec_insert_monitor_modified_sql(value_sql))) { LOG_WARN("failed to exec insert sql", K(ret)); } else { count = 0; @@ -1023,12 +706,102 @@ int ObOptStatMonitorManager::update_dml_stat_info_from_direct_load(const ObIArra } } if (OB_SUCC(ret) && count != 0) { - if (OB_FAIL(exec_insert_monitor_modified_sql(tenant_id, value_sql))) { + if (OB_FAIL(exec_insert_monitor_modified_sql(value_sql))) { LOG_WARN("failed to exec insert sql", K(ret)); } } return ret; } +int ObOptStatMonitorManager::mtl_init(ObOptStatMonitorManager* &optstat_monitor_mgr) +{ + int ret = OB_SUCCESS; + uint64_t tenant_id = lib::current_resource_owner_id(); + if (OB_LIKELY(nullptr != optstat_monitor_mgr)) { + if (OB_FAIL(optstat_monitor_mgr->init(tenant_id))) { + LOG_WARN("failed to init event list", K(ret)); + } + } + return ret; +} + +int ObOptStatMonitorManager::mtl_start(ObOptStatMonitorManager* &optstat_monitor_mgr) +{ + int ret = OB_SUCCESS; + if (OB_LIKELY(nullptr != optstat_monitor_mgr)) { + if (OB_FAIL(TG_SCHEDULE(MTL(omt::ObSharedTimer*)->get_tg_id(), optstat_monitor_mgr->get_flush_all_task(), + ObOptStatMonitorFlushAllTask::FLUSH_INTERVAL, true))) { + LOG_WARN("failed to scheduler flush all task", K(ret)); + } else if (OB_FAIL(TG_SCHEDULE(MTL(omt::ObSharedTimer*)->get_tg_id(), optstat_monitor_mgr->get_check_task(), + ObOptStatMonitorCheckTask::CHECK_INTERVAL, true))) { + LOG_WARN("failed to scheduler check task", K(ret)); + } else { + optstat_monitor_mgr->get_flush_all_task().disable_timeout_check(); + optstat_monitor_mgr->get_flush_all_task().optstat_monitor_mgr_ = optstat_monitor_mgr; + optstat_monitor_mgr->get_check_task().disable_timeout_check(); + optstat_monitor_mgr->get_check_task().optstat_monitor_mgr_ = optstat_monitor_mgr; + } + } + return ret; +} + +void ObOptStatMonitorManager::mtl_stop(ObOptStatMonitorManager* &optstat_monitor_mgr) +{ + if (OB_LIKELY(nullptr != optstat_monitor_mgr)) { + TG_CANCEL_TASK(MTL(omt::ObSharedTimer*)->get_tg_id(), optstat_monitor_mgr->get_flush_all_task()); + TG_CANCEL_TASK(MTL(omt::ObSharedTimer*)->get_tg_id(), optstat_monitor_mgr->get_check_task()); + } +} + +void ObOptStatMonitorManager::mtl_wait(ObOptStatMonitorManager* &optstat_monitor_mgr) +{ + if (OB_LIKELY(nullptr != optstat_monitor_mgr)) { + TG_WAIT_TASK(MTL(omt::ObSharedTimer*)->get_tg_id(), optstat_monitor_mgr->get_flush_all_task()); + TG_WAIT_TASK(MTL(omt::ObSharedTimer*)->get_tg_id(), optstat_monitor_mgr->get_check_task()); + } +} + +int ObOptStatMonitorManager::get_col_usage_info(const bool with_check, + ObIArray &col_stat_keys, + ObIArray &col_flags) +{ + int ret = OB_SUCCESS; + SpinWLockGuard guard(lock_); + if (column_usage_map_.empty() || + (with_check && column_usage_map_.size() < 10000)) { + //do nothing + } else { + for (auto iter = column_usage_map_.begin(); OB_SUCC(ret) && iter != column_usage_map_.end(); ++iter) { + if (OB_FAIL(col_stat_keys.push_back(iter->first)) || + OB_FAIL(col_flags.push_back(iter->second))) { + LOG_WARN("failed to push back", K(ret)); + } + } + if (OB_SUCC(ret)) { + column_usage_map_.reuse(); + } + } + return ret; +} + +int ObOptStatMonitorManager::get_dml_stats(ObIArray &dml_stats) +{ + int ret = OB_SUCCESS; + SpinWLockGuard guard(lock_); + if (dml_stat_map_.empty()) { + //do nothing + } else { + for (auto iter = dml_stat_map_.begin(); OB_SUCC(ret) && iter != dml_stat_map_.end(); ++iter) { + if (OB_FAIL(dml_stats.push_back(iter->second))) { + LOG_WARN("failed to get dml stat sql", K(ret)); + } + } + if (OB_SUCC(ret)) { + dml_stat_map_.reuse(); + } + } + return ret; +} + } } diff --git a/src/share/stat/ob_opt_stat_monitor_manager.h b/src/share/stat/ob_opt_stat_monitor_manager.h index 9560c8b6a..ecb47062c 100644 --- a/src/share/stat/ob_opt_stat_monitor_manager.h +++ b/src/share/stat/ob_opt_stat_monitor_manager.h @@ -23,7 +23,7 @@ namespace oceanbase namespace observer { -class ObOptDmlStatMapsGetter; +class ObOptDmlStatMapGetter; } namespace common @@ -31,10 +31,8 @@ namespace common typedef std::pair StatKey; typedef common::hash::ObHashMap ColumnUsageMap; -typedef common::hash::ObHashMap ColumnUsageMaps; typedef common::hash::ObHashMap DmlStatMap; -typedef common::hash::ObHashMap DmlStatMaps; class ObMySQLProxy; struct ObColumnStatParam; @@ -50,64 +48,29 @@ struct ColumnUsageArg class ObOptStatMonitorFlushAllTask : public common::ObTimerTask { public: - ObOptStatMonitorFlushAllTask() - : is_inited_(false) {} + ObOptStatMonitorFlushAllTask() : optstat_monitor_mgr_(NULL) {} virtual ~ObOptStatMonitorFlushAllTask() {} - int init(int tg_id); virtual void runTimerTask() override; -private: const static int64_t FLUSH_INTERVAL = 24L * 3600L * 1000L * 1000L; // 24h - bool is_inited_; + ObOptStatMonitorManager *optstat_monitor_mgr_; }; class ObOptStatMonitorCheckTask : public common::ObTimerTask { public: - ObOptStatMonitorCheckTask() - : is_inited_(false) {} + ObOptStatMonitorCheckTask() : optstat_monitor_mgr_(NULL) {} virtual ~ObOptStatMonitorCheckTask() {} - int init(int tg_id); virtual void runTimerTask() override; const static int64_t CHECK_INTERVAL = 900L * 1000L * 1000L; // 15min -private: - bool is_inited_; + ObOptStatMonitorManager *optstat_monitor_mgr_; }; class ObOptStatMonitorManager { -public: - // A callback struct used to get all key of hash map - struct GetAllKeyOp - { - public: - GetAllKeyOp(common::ObIArray *key_array, const bool with_check) - : key_array_(key_array), with_check_(with_check) {} - virtual ~GetAllKeyOp() {}; - int operator()(common::hash::HashMapPair &entry); - int operator()(common::hash::HashMapPair &entry); - private: - DISALLOW_COPY_AND_ASSIGN(GetAllKeyOp); - common::ObIArray *key_array_; - bool with_check_; - }; + friend class ObOptStatMonitorFlushAllTask; + friend class ObOptStatMonitorCheckTask; - // A callback struct used to get ColumnUsageMap or ObOptDmlStat, and allocate a new one - struct SwapMapAtomicOp - { - public: - SwapMapAtomicOp() : column_usage_map_(NULL), dml_stat_map_(NULL) {}; - virtual ~SwapMapAtomicOp() {}; - int operator() (common::hash::HashMapPair &entry); - int operator() (common::hash::HashMapPair &entry); - ColumnUsageMap *get_column_usage_map() { return column_usage_map_; } - DmlStatMap *get_dml_stat_map() { return dml_stat_map_; } - private: - DISALLOW_COPY_AND_ASSIGN(SwapMapAtomicOp); - ColumnUsageMap *column_usage_map_; - DmlStatMap *dml_stat_map_; - }; - - // A callback struct used to update ColumnUsageMap value or DmlStatMap value + // A callback struct used to update ColumnUsageMap value or DmlStatMap value struct UpdateValueAtomicOp { public: @@ -122,85 +85,75 @@ public: ObOptDmlStat dml_stat_; }; - // A callback struct used to read ColumnUsageMaps value or DmlStatMaps value - struct ReadMapAtomicOp - { - public: - ReadMapAtomicOp(common::ObIArray *col_usage_args) : - col_usage_args_(col_usage_args), dml_stat_() {}; - ReadMapAtomicOp(ObOptDmlStat &dml_stat) : col_usage_args_(NULL), dml_stat_(dml_stat) {}; - virtual ~ReadMapAtomicOp() {}; - int operator() (common::hash::HashMapPair &entry); - int operator() (common::hash::HashMapPair &entry); - private: - DISALLOW_COPY_AND_ASSIGN(ReadMapAtomicOp); - common::ObIArray *col_usage_args_; - ObOptDmlStat dml_stat_; - }; - public: ObOptStatMonitorManager() - : inited_(false), mysql_proxy_(NULL) {} - virtual ~ObOptStatMonitorManager() { destroy(); } - - int init(ObMySQLProxy *mysql_proxy); + : inited_(false), + tenant_id_(0), + tg_id_(-1), + destroyed_(false), + mysql_proxy_(NULL) + {} + virtual ~ObOptStatMonitorManager() { if (inited_) { destroy(); } } void destroy(); - static ObOptStatMonitorManager &get_instance(); + static int mtl_init(ObOptStatMonitorManager* &optstat_monitor_mgr); + static int mtl_start(ObOptStatMonitorManager* &optstat_monitor_mgr); + static void mtl_stop(ObOptStatMonitorManager* &optstat_monitor_mgr); + static void mtl_wait(ObOptStatMonitorManager* &optstat_monitor_mgr); public: static int flush_database_monitoring_info(sql::ObExecContext &ctx, const bool is_flush_col_usage = true, const bool is_flush_dml_stat = true, const bool ignore_failed = true); - int update_opt_stat_monitoring_info(const bool with_check); int update_opt_stat_monitoring_info(const obrpc::ObFlushOptStatArg &arg); - int update_local_cache(uint64_t tenant_id, common::ObIArray &args); - int update_local_cache(uint64_t tenant_id, ObOptDmlStat &dml_stat); - int update_column_usage_table(const bool with_check); - int update_dml_stat_info(const bool with_check); - int update_tenant_column_usage_info(uint64_t tenant_id); - int update_tenant_dml_stat_info(uint64_t tenant_id); - int erase_opt_stat_monitoring_info_map(uint64_t tenant_id); - int erase_column_usage_map(uint64_t tenant_id); - int erase_dml_stat_map(uint64_t tenant_id); - int get_column_usage_sql(const uint64_t tenant_id, - const StatKey &col_key, + int update_local_cache(common::ObIArray &args); + int update_local_cache(ObOptDmlStat &dml_stat); + int update_column_usage_info(const bool with_check); + int update_dml_stat_info(); + int update_dml_stat_info(const ObIArray &dml_stats); + int get_column_usage_sql(const StatKey &col_key, const int64_t flags, const bool need_add_comma, ObSqlString &sql_string); - int get_dml_stat_sql(const uint64_t tenant_id, - const ObOptDmlStat &dml_stat, + int get_dml_stat_sql(const ObOptDmlStat &dml_stat, const bool need_add_comma, ObSqlString &sql_string); - int exec_insert_column_usage_sql(uint64_t tenant_id, ObSqlString &values_sql); - int exec_insert_monitor_modified_sql(uint64_t tenant_id, - ObSqlString &values_sql); - int get_column_usage_from_table(sql::ObExecContext &ctx, - ObIArray &column_params, - uint64_t tenant_id, - uint64_t table_id); - int construct_get_column_usage_sql(ObIArray &column_params, - const uint64_t tenant_id, - const uint64_t table_id, - ObSqlString &select_sql); + int exec_insert_column_usage_sql(ObSqlString &values_sql); + int exec_insert_monitor_modified_sql(ObSqlString &values_sql); + static int get_column_usage_from_table(sql::ObExecContext &ctx, + ObIArray &column_params, + uint64_t tenant_id, + uint64_t table_id); + static int construct_get_column_usage_sql(ObIArray &column_params, + const uint64_t tenant_id, + const uint64_t table_id, + ObSqlString &select_sql); - int check_table_writeable(const uint64_t tenant_id, bool &is_writeable); - - int generate_opt_stat_monitoring_info_rows(observer::ObOptDmlStatMapsGetter &getter); - - int clean_useless_dml_stat_info(uint64_t tenant_id); - - int update_dml_stat_info_from_direct_load(const ObIArray &dml_stats); + int check_table_writeable(bool &is_writeable); + int generate_opt_stat_monitoring_info_rows(observer::ObOptDmlStatMapGetter &getter); + int clean_useless_dml_stat_info(); + static int update_dml_stat_info_from_direct_load(const ObIArray &dml_stats); + int get_col_usage_info(const bool with_check, + ObIArray &col_stat_keys, + ObIArray &col_flags); + int get_dml_stats(ObIArray &dml_stats); + ObOptStatMonitorFlushAllTask &get_flush_all_task() { return flush_all_task_; } + ObOptStatMonitorCheckTask &get_check_task() { return check_task_; } + int init(uint64_t tenant_id); private: DISALLOW_COPY_AND_ASSIGN(ObOptStatMonitorManager); const static int64_t UPDATE_OPT_STAT_BATCH_CNT = 200; const static int64_t info_count = 8; + bool inited_; + uint64_t tenant_id_; + int tg_id_; + bool destroyed_; + ObMySQLProxy *mysql_proxy_; + ColumnUsageMap column_usage_map_; + DmlStatMap dml_stat_map_; + common::SpinRWLock lock_; ObOptStatMonitorFlushAllTask flush_all_task_; ObOptStatMonitorCheckTask check_task_; - ColumnUsageMaps column_usage_maps_; - DmlStatMaps dml_stat_maps_; - bool inited_; - ObMySQLProxy *mysql_proxy_; }; // end of class ObOptStatMonitorManager } // end of namespace common diff --git a/src/share/transfer/ob_transfer_task_operator.cpp b/src/share/transfer/ob_transfer_task_operator.cpp index 3e2a5092a..d30436ed5 100644 --- a/src/share/transfer/ob_transfer_task_operator.cpp +++ b/src/share/transfer/ob_transfer_task_operator.cpp @@ -994,6 +994,48 @@ int ObTransferTaskOperator::get_history_task( return ret; } +int ObTransferTaskOperator::get_max_task_id_from_history( + common::ObISQLClient &sql_proxy, + const uint64_t tenant_id, + ObTransferTaskID &max_task_id) +{ + int ret = OB_SUCCESS; + max_task_id.reset(); + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id)); + } else { + SMART_VAR(ObISQLClient::ReadResult, result) { + ObSqlString sql; + common::sqlclient::ObMySQLResult *res = NULL; + int64_t max_task_id_int64 = ObTransferTaskID::INVALID_ID; + if (OB_FAIL(sql.assign_fmt("SELECT max(task_id) as max_task_id FROM %s", + OB_ALL_TRANSFER_TASK_HISTORY_TNAME))) { + LOG_WARN("fail to assign sql", KR(ret), K(sql)); + } else if (OB_FAIL(sql_proxy.read(result, tenant_id, sql.ptr()))) { + LOG_WARN("execute sql failed", KR(ret), K(tenant_id), K(sql)); + } else if (OB_ISNULL(res = result.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get mysql result failed", KR(ret), K(tenant_id), K(sql)); + } else if (OB_FAIL(res->next())) { + LOG_WARN("next failed", KR(ret), K(sql)); + } else if (OB_FAIL(res->get_int("max_task_id", max_task_id_int64))) { + if (OB_ERR_NULL_VALUE == ret) { + max_task_id.reset(); // return INVALID_ID when history is empty + LOG_TRACE("transfer history is empty", KR(ret), K(tenant_id), K(max_task_id)); + ret = OB_SUCCESS; + } else { + LOG_WARN("get max task id failed", KR(ret), K(max_task_id), K(sql)); + } + } else { + max_task_id = max_task_id_int64; + LOG_TRACE("get max transfer task_id from history success", KR(ret), K(tenant_id), K(max_task_id)); + } + } + } + return ret; +} + int ObTransferTaskOperator::update_comment( common::ObISQLClient &sql_proxy, const uint64_t tenant_id, diff --git a/src/share/transfer/ob_transfer_task_operator.h b/src/share/transfer/ob_transfer_task_operator.h index c9bdb1ca7..e2a9614e2 100644 --- a/src/share/transfer/ob_transfer_task_operator.h +++ b/src/share/transfer/ob_transfer_task_operator.h @@ -376,6 +376,22 @@ public: int64_t &create_time, int64_t &finish_time); + /* + * get max transfer task id from __all_transfer_task_history + * + * @param [in] sql_proxy: sql client + * @param [in] tenant_id: target tenant_id + * @param [out] max_task_id: max transfer task id + * return INVALID_ID(-1) when history is empty + * @return + * - OB_SUCCESS: successful + * - other: failed + */ + static int get_max_task_id_from_history( + common::ObISQLClient &sql_proxy, + const uint64_t tenant_id, + ObTransferTaskID &max_task_id); + /* * update comment in __all_transfer_task according to task_id * diff --git a/src/share/unit/ob_unit_config.cpp b/src/share/unit/ob_unit_config.cpp index 5863b3adc..2289e6764 100644 --- a/src/share/unit/ob_unit_config.cpp +++ b/src/share/unit/ob_unit_config.cpp @@ -127,7 +127,7 @@ int ObUnitConfig::gen_sys_tenant_unit_config(const bool is_hidden_sys) const char *name = is_hidden_sys ? HIDDEN_SYS_UNIT_CONFIG_NAME : SYS_UNIT_CONFIG_NAME; const uint64_t unit_config_id = is_hidden_sys ? HIDDEN_SYS_UNIT_CONFIG_ID : SYS_UNIT_CONFIG_ID; - if (OB_FAIL(ur.gen_sys_tenant_default_unit_resource())) { + if (OB_FAIL(ur.gen_sys_tenant_default_unit_resource(is_hidden_sys))) { LOG_WARN("generate sys tenant default unit resource fail", KR(ret), K(ur)); } else if (OB_FAIL(init(unit_config_id, name, ur))) { LOG_WARN("init unit config fail", KR(ret), K(unit_config_id), K(name), K(ur)); diff --git a/src/share/unit/ob_unit_resource.cpp b/src/share/unit/ob_unit_resource.cpp index f62391590..0c9af33ae 100644 --- a/src/share/unit/ob_unit_resource.cpp +++ b/src/share/unit/ob_unit_resource.cpp @@ -70,7 +70,7 @@ void ObUnitResource::reset() max_cpu_ = 0; min_cpu_ = 0; memory_size_ = 0; - log_disk_size_ = 0; + log_disk_size_ = INVALID_LOG_DISK_SIZE; max_iops_ = 0; min_iops_ = 0; iops_weight_ = INVALID_IOPS_WEIGHT; @@ -164,7 +164,11 @@ int ObUnitResource::init_and_check_log_disk_(const ObUnitResource &user_spec) const int64_t unit_min_log_disk_size = UNIT_MIN_LOG_DISK_SIZE; // user specify log_disk_size if (user_spec.is_log_disk_size_valid()) { - if (user_spec.log_disk_size() < unit_min_log_disk_size) { + if (0 == user_spec.log_disk_size()) { + // log_disk_size is only allowed to be 0 for hidden SYS + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Log_disk_size can only be specified as 0 for hidden SYS, not for normal unit.", KR(ret), K(user_spec)); + } else if (user_spec.log_disk_size() < unit_min_log_disk_size) { ret = OB_RESOURCE_UNIT_VALUE_BELOW_LIMIT; LOG_WARN("log_disk_size is below limit", KR(ret), K(user_spec), K(unit_min_log_disk_size)); @@ -374,7 +378,11 @@ int ObUnitResource::update_and_check_log_disk_(const ObUnitResource &user_spec) if (! user_spec.is_log_disk_size_valid()) { // not specified, need not update } else { - if (user_spec.log_disk_size() < unit_min_log_disk_size) { + if (0 == user_spec.log_disk_size()) { + // log_disk_size is only allowed to be 0 for hidden SYS + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Log_disk_size can only be specified as 0 for hidden SYS, not for normal unit.", KR(ret), K(user_spec)); + } else if (user_spec.log_disk_size() < unit_min_log_disk_size) { ret = OB_RESOURCE_UNIT_VALUE_BELOW_LIMIT; LOG_WARN("log_disk_size is below limit", KR(ret), K(user_spec), K(unit_min_log_disk_size)); LOG_USER_ERROR(OB_RESOURCE_UNIT_VALUE_BELOW_LIMIT, "LOG_DISK_SIZE", to_cstring(unit_min_log_disk_size)); @@ -471,7 +479,7 @@ int ObUnitResource::update_and_check_valid_for_unit(const ObUnitResource &user_s ret = update_and_check_mem_(user_spec); } - // check MEMORY + // check LOG_DISK_SIZE if (OB_SUCCESS == ret) { ret = update_and_check_log_disk_(user_spec); } @@ -565,7 +573,8 @@ DEF_TO_STRING(ObUnitResource) "min_cpu:%.6g, max_cpu:%.6g, memory_size:\"%.9gGB\", " "log_disk_size:\"%.9gGB\", min_iops:%ld, max_iops:%ld, iops_weight:%ld", min_cpu_, max_cpu_, (double)memory_size_/1024/1024/1024, - (double)log_disk_size_/1024/1024/1024, min_iops_, max_iops_, iops_weight_); + is_log_disk_size_valid() ? (double)log_disk_size_/1024/1024/1024 : log_disk_size_, + min_iops_, max_iops_, iops_weight_); J_OBJ_END(); return pos; } @@ -651,30 +660,29 @@ int ObUnitResource::divide_meta_tenant(ObUnitResource &meta_resource) return ret; } -int ObUnitResource::gen_sys_tenant_default_unit_resource() +int ObUnitResource::gen_sys_tenant_default_unit_resource(const bool is_hidden_sys) { int ret = OB_SUCCESS; reset(); + memory_size_ = is_hidden_sys ? GMEMCONF.get_hidden_sys_memory() : + max(UNIT_MIN_MEMORY, GCONF.__min_full_resource_pool_memory); + max_cpu_ = GCONF.get_sys_tenant_default_max_cpu(); + min_cpu_ = GCONF.get_sys_tenant_default_min_cpu(); + // for hidden SYS tenant, log_disk_size is 0 + // for real SYS tenant, log_disk_size is determined by real_memory_size (including extra_memory) + int64_t real_memory_size = memory_size_ + GMEMCONF.get_extra_memory(); + log_disk_size_ = is_hidden_sys ? 0 : max(real_memory_size, UNIT_MIN_LOG_DISK_SIZE); + max_iops_ = get_default_iops(); + min_iops_ = max_iops_; + iops_weight_ = get_default_iops_weight(min_cpu_); - if (OB_FAIL(get_sys_tenant_default_memory(memory_size_))) { - LOG_ERROR("generate sys tenant default memory fail", KR(ret)); - } else { - max_cpu_ = GCONF.get_sys_tenant_default_max_cpu(); - min_cpu_ = GCONF.get_sys_tenant_default_min_cpu(); - // SYS tenant log_disk_size keep same with memory size - log_disk_size_ = max(memory_size_, UNIT_MIN_LOG_DISK_SIZE); - max_iops_ = get_default_iops(); - min_iops_ = max_iops_; - iops_weight_ = get_default_iops_weight(min_cpu_); - - if (OB_UNLIKELY(! is_valid_for_unit())) { - ret = OB_RESOURCE_UNIT_VALUE_INVALID; - LOG_ERROR("sys tenant default unit resource is not valid for unit", KR(ret), KPC(this)); - } + if (OB_UNLIKELY(! is_valid_for_unit())) { + ret = OB_RESOURCE_UNIT_VALUE_INVALID; + LOG_ERROR("sys tenant default unit resource is not valid for unit", KR(ret), K(is_hidden_sys), KPC(this)); } - LOG_INFO("gen_sys_tenant_default_unit_resource", KR(ret), KPC(this), K(lbt())); + LOG_INFO("gen_sys_tenant_default_unit_resource", KR(ret), K(is_hidden_sys), KPC(this), K(lbt())); if (OB_FAIL(ret)) { reset(); @@ -682,63 +690,5 @@ int ObUnitResource::gen_sys_tenant_default_unit_resource() return ret; } -int ObUnitResource::get_sys_tenant_default_memory(int64_t &memory_size) -{ - int ret = OB_SUCCESS; - const int64_t unit_min_memory = UNIT_MIN_MEMORY; - const int64_t __min_full_resource_pool_memory = GCONF.__min_full_resource_pool_memory; - const int64_t sys_tenant_memory = GCONF._hidden_sys_tenant_memory; - const int64_t system_memory = GMEMCONF.get_reserved_server_memory(); - const int64_t server_memory_limit = GMEMCONF.get_server_memory_limit(); - const int64_t server_avail_memory = server_memory_limit - system_memory; - - memory_size = 0; - - if (OB_UNLIKELY(server_avail_memory < UNIT_MIN_MEMORY)) { - ret = OB_INVALID_CONFIG; - LOG_ERROR("server available memory is little than unit min memory, can not create sys tenant. " - "try adjust config 'memory_limit' or 'system_memory'.", - KR(ret), K(unit_min_memory), K(server_avail_memory), K(system_memory), K(server_memory_limit)); - } else if (OB_UNLIKELY(server_avail_memory < __min_full_resource_pool_memory)) { - ret = OB_INVALID_CONFIG; - LOG_ERROR("server available memory is little than __min_full_resource_pool_memory, can not create sys tenant. " - "try adjust config '__min_full_resource_pool_memory', 'memory_limit' or 'system_memory'.", - KR(ret), K(__min_full_resource_pool_memory), K(server_avail_memory), K(system_memory), K(server_memory_limit)); - } else if (OB_UNLIKELY(server_avail_memory < sys_tenant_memory)) { - ret = OB_INVALID_CONFIG; - LOG_ERROR("server available memory is little than sys_tenant_memory, can not create sys tenant. " - "try adjust config 'sys_tenant_memory', 'memory_limit' or 'system_memory'.", - KR(ret), K(sys_tenant_memory), K(server_avail_memory), K(system_memory), K(server_memory_limit)); - } else if (OB_UNLIKELY(0 != sys_tenant_memory && sys_tenant_memory < __min_full_resource_pool_memory)) { - ret = OB_INVALID_CONFIG; - LOG_ERROR("sys_tenant_memory is little than __min_full_resource_pool_memory, can not create sys tenant. " - "try adjust config 'sys_tenant_memory' or '__min_full_resource_pool_memory'.", - KR(ret), K(sys_tenant_memory), K(__min_full_resource_pool_memory)); - } else { - if (0 == sys_tenant_memory) { - // SYS tenant MEMORY is auto computed by memory_limit - memory_size = GMEMCONF.get_capacity_default_memory(ObServerMemoryConfig::HIDDEN_SYS_MEMORY, - server_memory_limit); - - // SYS tenant MEMORY is restricted by UNIT_MIN_MEMORY - memory_size = max(memory_size, UNIT_MIN_MEMORY); - - // SYS tenant MEMORY is restricted by __min_full_resource_pool_memory - memory_size = max(memory_size, __min_full_resource_pool_memory); - } else { - memory_size = sys_tenant_memory; - } - LOG_INFO("get_sys_tenant_default_memory", - "sys_tenant_default_memory_G", memory_size/GB, - "sys_tenant_memory_G", sys_tenant_memory/GB, - "server_avail_memory_G", server_avail_memory/GB, - "unit_min_memory_G", unit_min_memory/GB, - "__min_full_resource_pool_memory_G", __min_full_resource_pool_memory/GB, - "server_memory_limit_G", server_memory_limit/GB, - "system_memory_G", system_memory/GB); - } - return ret; -} - }//end namespace share }//end namespace oceanbase diff --git a/src/share/unit/ob_unit_resource.h b/src/share/unit/ob_unit_resource.h index f086c0684..e3151d31f 100644 --- a/src/share/unit/ob_unit_resource.h +++ b/src/share/unit/ob_unit_resource.h @@ -83,6 +83,7 @@ public: // default factor of mapping MEMORY_SIZE to LOG_DISK_SIZE // MEMORY_SIZE * FACTOR = LOG_DISK_SIZE static const int64_t MEMORY_TO_LOG_DISK_FACTOR = 3; + static const int64_t INVALID_LOG_DISK_SIZE = -1; ////////////////////////// IOPS //////////////////////////// // IOPS is shared by META and USER tenant. @@ -260,8 +261,8 @@ public: bool is_memory_size_valid_for_user_tenant() const { return memory_size_ >= USER_TENANT_MIN_MEMORY; } int64_t log_disk_size() const { return log_disk_size_; } - bool is_log_disk_size_valid() const { return log_disk_size_ > 0; } - bool is_log_disk_size_valid_for_unit() const { return log_disk_size_ >= UNIT_MIN_LOG_DISK_SIZE; } + bool is_log_disk_size_valid() const { return log_disk_size_ >= 0; } + bool is_log_disk_size_valid_for_unit() const { return 0 == log_disk_size_ || log_disk_size_ >= UNIT_MIN_LOG_DISK_SIZE; } bool is_log_disk_size_valid_for_meta_tenant() const { return log_disk_size_ >= META_TENANT_MIN_LOG_DISK_SIZE; } bool is_log_disk_size_valid_for_user_tenant() const { return log_disk_size_ >= USER_TENANT_MIN_LOG_DISK_SIZE; } @@ -298,10 +299,9 @@ public: int divide_meta_tenant(ObUnitResource &meta_resource); // generate sys tenant default unit resource based on configuration - int gen_sys_tenant_default_unit_resource(); + int gen_sys_tenant_default_unit_resource(const bool is_hidden_sys = false); /////////////////////////////////// static functions /////////////////////////////////// - static int get_sys_tenant_default_memory(int64_t &memory_size); // get default LOG_DISK_SIZE based on MEMORY_SIZE static int64_t get_default_log_disk_size(const int64_t memory_size) diff --git a/src/sql/das/ob_das_utils.cpp b/src/sql/das/ob_das_utils.cpp index 17d91284d..332dd9a48 100644 --- a/src/sql/das/ob_das_utils.cpp +++ b/src/sql/das/ob_das_utils.cpp @@ -20,7 +20,7 @@ #include "share/schema/ob_multi_version_schema_service.h" #include "share/location_cache/ob_location_service.h" #include "observer/ob_server_struct.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "lib/geo/ob_s2adapter.h" #include "lib/geo/ob_geo_utils.h" namespace oceanbase @@ -311,13 +311,13 @@ int ObDASUtils::generate_spatial_index_rows( if (OB_FAIL(ObGeoTypeUtil::get_srid_from_wkb(wkb_str, srid))) { LOG_WARN("failed to get srid", K(ret), K(wkb_str)); } else if (srid != 0 && - OB_FAIL(OTSRS_MGR.get_tenant_srs_guard(MTL_ID(), srs_guard))) { + OB_FAIL(OTSRS_MGR->get_tenant_srs_guard(srs_guard))) { LOG_WARN("failed to get srs guard", K(ret), K(MTL_ID()), K(srid)); } else if (srid != 0 && OB_FAIL(srs_guard.get_srs_item(srid, srs_item))) { LOG_WARN("failed to get srs item", K(ret), K(MTL_ID()), K(srid)); } else if (((srid == 0) || !(srs_item->is_geographical_srs())) && - OB_FAIL(OTSRS_MGR.get_srs_bounds(srid, srs_item, srs_bound))) { + OB_FAIL(OTSRS_MGR->get_srs_bounds(srid, srs_item, srs_bound))) { LOG_WARN("failed to get srs bound", K(ret), K(srid)); } else { ObS2Adapter s2object(&allocator, srid != 0 ? srs_item->is_geographical_srs() : false); diff --git a/src/sql/engine/expr/ob_expr_priv_st_asewkb.cpp b/src/sql/engine/expr/ob_expr_priv_st_asewkb.cpp index 42d419546..970b24710 100644 --- a/src/sql/engine/expr/ob_expr_priv_st_asewkb.cpp +++ b/src/sql/engine/expr/ob_expr_priv_st_asewkb.cpp @@ -15,7 +15,7 @@ #include "sql/engine/expr/ob_expr_priv_st_asewkb.h" #include "lib/geo/ob_geo_utils.h" #include "sql/engine/expr/ob_geo_expr_utils.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" using namespace oceanbase::common; using namespace oceanbase::sql; diff --git a/src/sql/engine/expr/ob_expr_priv_st_geogfromtext.cpp b/src/sql/engine/expr/ob_expr_priv_st_geogfromtext.cpp index 0c1483669..2050bc2a9 100644 --- a/src/sql/engine/expr/ob_expr_priv_st_geogfromtext.cpp +++ b/src/sql/engine/expr/ob_expr_priv_st_geogfromtext.cpp @@ -15,7 +15,7 @@ #include "sql/engine/expr/ob_expr_priv_st_geogfromtext.h" #include "lib/geo/ob_geo_common.h" #include "lib/geo/ob_wkt_parser.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "sql/engine/expr/ob_geo_expr_utils.h" #include "lib/geo/ob_geo_reverse_coordinate_visitor.h" #include "lib/geo/ob_geo_func_common.h" @@ -106,7 +106,7 @@ int ObExprPrivSTGeogFromText::eval_priv_st_geogfromtext_common(const ObExpr &exp } if (OB_FAIL(ret)) { - } else if (OB_FAIL(OTSRS_MGR.get_tenant_srs_guard(session->get_effective_tenant_id(), srs_guard))) { + } else if (OB_FAIL(OTSRS_MGR->get_tenant_srs_guard(srs_guard))) { LOG_WARN("failed to get srs guard", K(ret)); } else if (OB_FAIL(srs_guard.get_srs_item(srid, srs_item))) { LOG_WARN("failed to get srs item", K(ret)); diff --git a/src/sql/engine/expr/ob_expr_priv_st_point.cpp b/src/sql/engine/expr/ob_expr_priv_st_point.cpp index fc37c9fd7..88b3aa45b 100644 --- a/src/sql/engine/expr/ob_expr_priv_st_point.cpp +++ b/src/sql/engine/expr/ob_expr_priv_st_point.cpp @@ -13,7 +13,7 @@ #define USING_LOG_PREFIX SQL_ENG #include "sql/engine/expr/ob_expr_priv_st_point.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "sql/session/ob_sql_session_info.h" #include "share/object/ob_obj_cast_util.h" #include "lib/geo/ob_geo_func_common.h" @@ -180,7 +180,7 @@ int ObExprPrivSTPoint::eval_priv_st_point(const ObExpr &expr, LOG_USER_ERROR(OB_OPERATE_OVERFLOW, "SRID", N_PRIV_ST_POINT); LOG_WARN("srid input value out of range", K(ret), K(datum_srid->get_int())); } else if (0 != (srid = datum_srid->get_uint32())) { - if (OB_FAIL(OTSRS_MGR.get_tenant_srs_guard(MTL_ID(), srs_guard))) { + if (OB_FAIL(OTSRS_MGR->get_tenant_srs_guard(srs_guard))) { LOG_WARN("fail to get srs guard", K(ret)); } else if (OB_FAIL(srs_guard.get_srs_item(srid, srs_item))) { LOG_WARN("fail to get srs item", K(ret)); diff --git a/src/sql/engine/expr/ob_expr_priv_st_setsrid.cpp b/src/sql/engine/expr/ob_expr_priv_st_setsrid.cpp index 888dafd83..0ddd97cff 100644 --- a/src/sql/engine/expr/ob_expr_priv_st_setsrid.cpp +++ b/src/sql/engine/expr/ob_expr_priv_st_setsrid.cpp @@ -17,7 +17,7 @@ #include "lib/geo/ob_geo_func_common.h" #include "lib/geo/ob_geo_utils.h" #include "lib/geo/ob_geo.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "sql/engine/expr/ob_geo_expr_utils.h" #include "sql/session/ob_sql_session_info.h" #include "share/object/ob_obj_cast_util.h" diff --git a/src/sql/engine/expr/ob_expr_priv_st_transform.cpp b/src/sql/engine/expr/ob_expr_priv_st_transform.cpp index c0de0e260..7f3846626 100644 --- a/src/sql/engine/expr/ob_expr_priv_st_transform.cpp +++ b/src/sql/engine/expr/ob_expr_priv_st_transform.cpp @@ -15,7 +15,7 @@ #include "lib/geo/ob_geo_func_register.h" #include "ob_expr_priv_st_transform.h" #include "lib/geo/ob_srs_info.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "lib/geo/ob_geo_normalize_visitor.h" #include "lib/geo/ob_geo_wkb_size_visitor.h" #include "lib/geo/ob_geo_wkb_visitor.h" @@ -115,7 +115,7 @@ int ObExprPrivSTTransform::eval_priv_st_transform(const ObExpr &expr, ObEvalCtx ret = OB_ERR_GIS_INVALID_DATA; LOG_USER_ERROR(OB_ERR_GIS_INVALID_DATA, N_PRIV_ST_TRANSFORM); LOG_WARN("get srid from wkb failed", K(wkb), K(ret)); - } else if (OB_FAIL(OTSRS_MGR.get_tenant_srs_guard(session->get_effective_tenant_id(), srs_guard))) { + } else if (OB_FAIL(OTSRS_MGR->get_tenant_srs_guard(srs_guard))) { LOG_WARN("get tenant srs guard failed", K(session->get_effective_tenant_id()), K(src_srid), K(ret)); } else if (src_srid != 0 && OB_FAIL(srs_guard.get_srs_item(src_srid, src_srs_item))) { LOG_WARN("failed to get srs item", K(ret), K(src_srid)); diff --git a/src/sql/engine/expr/ob_expr_spatial_cellid.cpp b/src/sql/engine/expr/ob_expr_spatial_cellid.cpp index d2ac25da6..07c1caef2 100644 --- a/src/sql/engine/expr/ob_expr_spatial_cellid.cpp +++ b/src/sql/engine/expr/ob_expr_spatial_cellid.cpp @@ -15,7 +15,7 @@ #include "sql/engine/expr/ob_expr_spatial_cellid.h" #include "lib/geo/ob_geo_utils.h" #include "lib/geo/ob_s2adapter.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" using namespace oceanbase::common; using namespace oceanbase::sql; diff --git a/src/sql/engine/expr/ob_expr_spatial_mbr.cpp b/src/sql/engine/expr/ob_expr_spatial_mbr.cpp index 13cda89e4..ca657416b 100644 --- a/src/sql/engine/expr/ob_expr_spatial_mbr.cpp +++ b/src/sql/engine/expr/ob_expr_spatial_mbr.cpp @@ -15,7 +15,7 @@ #include "sql/engine/expr/ob_expr_spatial_mbr.h" #include "lib/geo/ob_geo_utils.h" #include "lib/geo/ob_s2adapter.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" using namespace oceanbase::common; using namespace oceanbase::sql; diff --git a/src/sql/engine/expr/ob_expr_st_area.cpp b/src/sql/engine/expr/ob_expr_st_area.cpp index 67e7384de..e59b06b97 100644 --- a/src/sql/engine/expr/ob_expr_st_area.cpp +++ b/src/sql/engine/expr/ob_expr_st_area.cpp @@ -18,7 +18,7 @@ #include "lib/geo/ob_geo_utils.h" #include "sql/session/ob_sql_session_info.h" #include "sql/engine/ob_exec_context.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "sql/engine/expr/ob_geo_expr_utils.h" using namespace oceanbase::common; diff --git a/src/sql/engine/expr/ob_expr_st_asewkt.cpp b/src/sql/engine/expr/ob_expr_st_asewkt.cpp index 9e6679642..364b8955f 100644 --- a/src/sql/engine/expr/ob_expr_st_asewkt.cpp +++ b/src/sql/engine/expr/ob_expr_st_asewkt.cpp @@ -13,7 +13,7 @@ #define USING_LOG_PREFIX SQL_ENG #include "sql/engine/expr/ob_expr_st_asewkt.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "sql/engine/expr/ob_geo_expr_utils.h" #include "lib/geo/ob_geo_utils.h" #include "lib/geo/ob_geo_reverse_coordinate_visitor.h" diff --git a/src/sql/engine/expr/ob_expr_st_astext.cpp b/src/sql/engine/expr/ob_expr_st_astext.cpp index 9b73e1644..6cb62529c 100755 --- a/src/sql/engine/expr/ob_expr_st_astext.cpp +++ b/src/sql/engine/expr/ob_expr_st_astext.cpp @@ -13,7 +13,7 @@ #define USING_LOG_PREFIX SQL_ENG #include "sql/engine/expr/ob_expr_st_astext.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "sql/engine/expr/ob_geo_expr_utils.h" #include "lib/geo/ob_geo_utils.h" #include "lib/geo/ob_geo_reverse_coordinate_visitor.h" diff --git a/src/sql/engine/expr/ob_expr_st_bestsrid.cpp b/src/sql/engine/expr/ob_expr_st_bestsrid.cpp index e586ff04c..d8fd883bf 100644 --- a/src/sql/engine/expr/ob_expr_st_bestsrid.cpp +++ b/src/sql/engine/expr/ob_expr_st_bestsrid.cpp @@ -17,7 +17,7 @@ #include "lib/geo/ob_geo_ibin.h" #include "sql/session/ob_sql_session_info.h" #include "sql/engine/ob_exec_context.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "ob_expr_st_bestsrid.h" #include "lib/geo/ob_geo_utils.h" #include "sql/engine/expr/ob_geo_expr_utils.h" diff --git a/src/sql/engine/expr/ob_expr_st_buffer.h b/src/sql/engine/expr/ob_expr_st_buffer.h index 6515c8368..0679c353d 100644 --- a/src/sql/engine/expr/ob_expr_st_buffer.h +++ b/src/sql/engine/expr/ob_expr_st_buffer.h @@ -15,7 +15,7 @@ #define OCEANBASE_SQL_OB_EXPR_ST_BUFFER #include "sql/engine/expr/ob_expr_operator.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "sql/engine/expr/ob_geo_expr_utils.h" namespace oceanbase diff --git a/src/sql/engine/expr/ob_expr_st_contains.cpp b/src/sql/engine/expr/ob_expr_st_contains.cpp index bd98539eb..07e1fa6e9 100644 --- a/src/sql/engine/expr/ob_expr_st_contains.cpp +++ b/src/sql/engine/expr/ob_expr_st_contains.cpp @@ -17,7 +17,7 @@ #include "lib/geo/ob_geo_ibin.h" #include "sql/session/ob_sql_session_info.h" #include "sql/engine/ob_exec_context.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "ob_expr_st_contains.h" #include "lib/geo/ob_geo_utils.h" #include "sql/engine/expr/ob_geo_expr_utils.h" diff --git a/src/sql/engine/expr/ob_expr_st_covers.cpp b/src/sql/engine/expr/ob_expr_st_covers.cpp index 3d10b727b..0285119e1 100644 --- a/src/sql/engine/expr/ob_expr_st_covers.cpp +++ b/src/sql/engine/expr/ob_expr_st_covers.cpp @@ -17,7 +17,7 @@ #include "lib/geo/ob_geo_ibin.h" #include "sql/session/ob_sql_session_info.h" #include "sql/engine/ob_exec_context.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "ob_expr_st_covers.h" #include "lib/geo/ob_geo_utils.h" #include "sql/engine/expr/ob_geo_expr_utils.h" diff --git a/src/sql/engine/expr/ob_expr_st_distance.cpp b/src/sql/engine/expr/ob_expr_st_distance.cpp index 553e1694a..c9104aaba 100644 --- a/src/sql/engine/expr/ob_expr_st_distance.cpp +++ b/src/sql/engine/expr/ob_expr_st_distance.cpp @@ -17,7 +17,7 @@ #include "lib/geo/ob_geo_func_register.h" #include "lib/geo/ob_geo_ibin.h" #include "sql/engine/ob_exec_context.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "ob_expr_st_distance.h" #include "lib/geo/ob_geo_utils.h" #include "sql/engine/expr/ob_geo_expr_utils.h" diff --git a/src/sql/engine/expr/ob_expr_st_dwithin.cpp b/src/sql/engine/expr/ob_expr_st_dwithin.cpp index 1d6ddae43..3520a13d5 100644 --- a/src/sql/engine/expr/ob_expr_st_dwithin.cpp +++ b/src/sql/engine/expr/ob_expr_st_dwithin.cpp @@ -15,7 +15,7 @@ #include "lib/geo/ob_geo_func_register.h" #include "lib/geo/ob_geo_ibin.h" #include "sql/engine/ob_exec_context.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "sql/engine/expr/ob_expr_st_dwithin.h" #include "lib/geo/ob_geo_utils.h" #include "sql/engine/expr/ob_geo_expr_utils.h" diff --git a/src/sql/engine/expr/ob_expr_st_geomfromewkb.cpp b/src/sql/engine/expr/ob_expr_st_geomfromewkb.cpp index 4946156ed..c82408eb4 100644 --- a/src/sql/engine/expr/ob_expr_st_geomfromewkb.cpp +++ b/src/sql/engine/expr/ob_expr_st_geomfromewkb.cpp @@ -20,7 +20,7 @@ #include "lib/geo/ob_geo_wkb_check_visitor.h" #include "lib/geo/ob_geo_to_tree_visitor.h" #include "lib/geo/ob_geo_func_common.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" using namespace oceanbase::common; using namespace oceanbase::sql; diff --git a/src/sql/engine/expr/ob_expr_st_geomfromewkt.cpp b/src/sql/engine/expr/ob_expr_st_geomfromewkt.cpp index c11f1c9b7..6b377c9b2 100644 --- a/src/sql/engine/expr/ob_expr_st_geomfromewkt.cpp +++ b/src/sql/engine/expr/ob_expr_st_geomfromewkt.cpp @@ -14,7 +14,7 @@ #define USING_LOG_PREFIX SQL_ENG #include "sql/engine/expr/ob_expr_st_geomfromewkt.h" #include "lib/geo/ob_wkt_parser.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "sql/engine/expr/ob_geo_expr_utils.h" #include "lib/geo/ob_geo_coordinate_range_visitor.h" #include "lib/geo/ob_geo_reverse_coordinate_visitor.h" @@ -93,7 +93,7 @@ int ObExprPrivSTGeomFromEwkt::eval_st_geomfromewkt(const ObExpr &expr, ObEvalCtx ret = OB_OPERATE_OVERFLOW; LOG_WARN("srid input value out of range", K(ret), K(datum->get_int())); } else if (0 != srid) { - if (OB_FAIL(OTSRS_MGR.get_tenant_srs_guard(session->get_effective_tenant_id(), srs_guard))) { + if (OB_FAIL(OTSRS_MGR->get_tenant_srs_guard(srs_guard))) { LOG_WARN("failed to get srs guard", K(ret)); } else if (OB_FAIL(srs_guard.get_srs_item(srid, srs_item))) { LOG_WARN("failed to get srs item", K(ret)); diff --git a/src/sql/engine/expr/ob_expr_st_geomfromtext.cpp b/src/sql/engine/expr/ob_expr_st_geomfromtext.cpp index 3be9e656f..18ec1e83e 100644 --- a/src/sql/engine/expr/ob_expr_st_geomfromtext.cpp +++ b/src/sql/engine/expr/ob_expr_st_geomfromtext.cpp @@ -14,7 +14,7 @@ #define USING_LOG_PREFIX SQL_ENG #include "sql/engine/expr/ob_expr_st_geomfromtext.h" #include "lib/geo/ob_wkt_parser.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "sql/engine/expr/ob_geo_expr_utils.h" #include "lib/geo/ob_geo_coordinate_range_visitor.h" #include "lib/geo/ob_geo_reverse_coordinate_visitor.h" @@ -133,7 +133,7 @@ int ObExprSTGeomFromText::eval_st_geomfromtext_common(const ObExpr &expr, LOG_USER_ERROR(OB_OPERATE_OVERFLOW, "SRID", func_name); LOG_WARN("srid input value out of range", K(ret), K(datum->get_int())); } else if (0 != (srid = datum->get_uint32())) { - if (OB_FAIL(OTSRS_MGR.get_tenant_srs_guard(session->get_effective_tenant_id(), srs_guard))) { + if (OB_FAIL(OTSRS_MGR->get_tenant_srs_guard(srs_guard))) { LOG_WARN("failed to get srs guard", K(ret)); } else if (OB_FAIL(srs_guard.get_srs_item(srid, srs_item))) { LOG_WARN("failed to get srs item", K(ret)); diff --git a/src/sql/engine/expr/ob_expr_st_geomfromwkb.cpp b/src/sql/engine/expr/ob_expr_st_geomfromwkb.cpp index 520191457..486e7a6c9 100644 --- a/src/sql/engine/expr/ob_expr_st_geomfromwkb.cpp +++ b/src/sql/engine/expr/ob_expr_st_geomfromwkb.cpp @@ -19,7 +19,7 @@ #include "lib/geo/ob_geo_to_tree_visitor.h" #include "lib/geo/ob_geo_wkb_check_visitor.h" #include "lib/geo/ob_geo_func_common.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" using namespace oceanbase::common; using namespace oceanbase::sql; @@ -121,7 +121,7 @@ int ObIExprSTGeomFromWKB::eval_geom_wkb(const ObExpr &expr, ObEvalCtx &ctx, ObDa ret = OB_OPERATE_OVERFLOW; LOG_WARN("srid input value out of range", K(ret), K(datum->get_int())); } else if (0 != (srid = datum->get_uint32())) { - if (OB_FAIL(OTSRS_MGR.get_tenant_srs_guard(session->get_effective_tenant_id(), srs_guard))) { + if (OB_FAIL(OTSRS_MGR->get_tenant_srs_guard(srs_guard))) { LOG_WARN("failed to get srs guard", K(ret)); } else if (OB_FAIL(srs_guard.get_srs_item(srid, srs_item))) { LOG_WARN("failed to get srs item", K(ret)); diff --git a/src/sql/engine/expr/ob_expr_st_intersects.cpp b/src/sql/engine/expr/ob_expr_st_intersects.cpp index cc0144d08..2b765c188 100644 --- a/src/sql/engine/expr/ob_expr_st_intersects.cpp +++ b/src/sql/engine/expr/ob_expr_st_intersects.cpp @@ -16,7 +16,7 @@ #include "lib/geo/ob_geo_func_register.h" #include "lib/geo/ob_geo_ibin.h" #include "sql/engine/ob_exec_context.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "ob_expr_st_intersects.h" #include "lib/geo/ob_geo_utils.h" #include "sql/engine/expr/ob_geo_expr_utils.h" diff --git a/src/sql/engine/expr/ob_expr_st_isvalid.cpp b/src/sql/engine/expr/ob_expr_st_isvalid.cpp index 1917f8f9b..0c58c8db1 100644 --- a/src/sql/engine/expr/ob_expr_st_isvalid.cpp +++ b/src/sql/engine/expr/ob_expr_st_isvalid.cpp @@ -17,7 +17,7 @@ #include "lib/geo/ob_geo_func_isvalid.h" #include "ob_expr_st_isvalid.h" #include "lib/geo/ob_srs_info.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "sql/engine/expr/ob_geo_expr_utils.h" diff --git a/src/sql/engine/expr/ob_expr_st_srid.cpp b/src/sql/engine/expr/ob_expr_st_srid.cpp index 1702b612e..42da87a5b 100644 --- a/src/sql/engine/expr/ob_expr_st_srid.cpp +++ b/src/sql/engine/expr/ob_expr_st_srid.cpp @@ -17,7 +17,7 @@ #include "lib/geo/ob_geo_func_common.h" #include "lib/geo/ob_geo_utils.h" #include "lib/geo/ob_geo.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "sql/engine/expr/ob_geo_expr_utils.h" using namespace oceanbase::common; @@ -114,7 +114,7 @@ int ObExprSTSRID::eval_st_srid_common(const ObExpr &expr, ObEvalCtx &ctx, ObDatu LOG_USER_ERROR(OB_OPERATE_OVERFLOW, "SRID", func_name); LOG_WARN("srid input value out of range", K(ret), K(datum->get_int())); } else if (0 != (srid = datum->get_uint32())) { - if (OB_FAIL(OTSRS_MGR.get_tenant_srs_guard(session->get_effective_tenant_id(), srs_guard))) { + if (OB_FAIL(OTSRS_MGR->get_tenant_srs_guard(srs_guard))) { LOG_WARN("failed to get srs guard", K(ret)); } else if (OB_FAIL(srs_guard.get_srs_item(srid, srs))) { LOG_WARN("failed to get srs item", K(ret)); diff --git a/src/sql/engine/expr/ob_expr_st_transform.cpp b/src/sql/engine/expr/ob_expr_st_transform.cpp index 7c358f2d0..8d0190b54 100644 --- a/src/sql/engine/expr/ob_expr_st_transform.cpp +++ b/src/sql/engine/expr/ob_expr_st_transform.cpp @@ -15,7 +15,7 @@ #include "lib/geo/ob_geo_func_register.h" #include "ob_expr_st_transform.h" #include "lib/geo/ob_srs_info.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "lib/geo/ob_geo_normalize_visitor.h" #include "lib/geo/ob_geo_wkb_size_visitor.h" #include "lib/geo/ob_geo_wkb_visitor.h" diff --git a/src/sql/engine/expr/ob_expr_st_within.cpp b/src/sql/engine/expr/ob_expr_st_within.cpp index aadd99b8d..4cde43799 100644 --- a/src/sql/engine/expr/ob_expr_st_within.cpp +++ b/src/sql/engine/expr/ob_expr_st_within.cpp @@ -17,7 +17,7 @@ #include "lib/geo/ob_geo_ibin.h" #include "sql/session/ob_sql_session_info.h" #include "sql/engine/ob_exec_context.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "ob_expr_st_within.h" #include "lib/geo/ob_geo_utils.h" #include "sql/engine/expr/ob_geo_expr_utils.h" diff --git a/src/sql/engine/expr/ob_expr_st_x.cpp b/src/sql/engine/expr/ob_expr_st_x.cpp index ebfbf9c24..b82917bc5 100644 --- a/src/sql/engine/expr/ob_expr_st_x.cpp +++ b/src/sql/engine/expr/ob_expr_st_x.cpp @@ -17,7 +17,7 @@ #include "lib/geo/ob_geo.h" #include "lib/geo/ob_geo_utils.h" #include "lib/geo/ob_srs_info.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "sql/session/ob_sql_session_info.h" #include "sql/engine/ob_exec_context.h" #include "sql/engine/expr/ob_geo_expr_utils.h" diff --git a/src/sql/engine/expr/ob_geo_expr_utils.cpp b/src/sql/engine/expr/ob_geo_expr_utils.cpp index 4806e29ff..9379306c1 100644 --- a/src/sql/engine/expr/ob_geo_expr_utils.cpp +++ b/src/sql/engine/expr/ob_geo_expr_utils.cpp @@ -14,7 +14,7 @@ #define USING_LOG_PREFIX SQL_ENG #include "sql/engine/expr/ob_geo_expr_utils.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "lib/geo/ob_geo_func_register.h" #include "lib/geo/ob_geo_coordinate_range_visitor.h" #include "lib/geo/ob_geo_wkb_size_visitor.h" @@ -40,7 +40,7 @@ int ObGeoExprUtils::get_srs_item(uint64_t tenant_id, { int ret = OB_SUCCESS; - if (0 != srid && OB_FAIL(OTSRS_MGR.get_tenant_srs_guard(tenant_id, srs_guard))) { + if (0 != srid && OB_FAIL(OTSRS_MGR->get_tenant_srs_guard(srs_guard))) { LOG_WARN("fail to get srs guard", K(ret), K(tenant_id)); } else if (0 != srid && OB_FAIL(srs_guard.get_srs_item(srid, srs))) { LOG_WARN("fail to get srs", K(ret), K(srid)); diff --git a/src/sql/engine/expr/ob_geo_expr_utils.h b/src/sql/engine/expr/ob_geo_expr_utils.h index a93649a28..d460014bb 100644 --- a/src/sql/engine/expr/ob_geo_expr_utils.h +++ b/src/sql/engine/expr/ob_geo_expr_utils.h @@ -23,7 +23,7 @@ #include "sql/session/ob_sql_session_info.h" #include "sql/engine/ob_exec_context.h" #include "share/ob_i_sql_expression.h" // for ObExprCtx -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "sql/engine/expr/ob_expr_lob_utils.h" namespace oceanbase diff --git a/src/sql/engine/table/ob_table_scan_op.cpp b/src/sql/engine/table/ob_table_scan_op.cpp index 7f304c00f..d985ace92 100644 --- a/src/sql/engine/table/ob_table_scan_op.cpp +++ b/src/sql/engine/table/ob_table_scan_op.cpp @@ -27,7 +27,7 @@ #include "observer/ob_server.h" #include "observer/virtual_table/ob_virtual_data_access_service.h" #include "sql/engine/expr/ob_expr_lob_utils.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "share/external_table/ob_external_table_file_mgr.h" #include "share/external_table/ob_external_table_utils.h" #include "lib/container/ob_array_wrap.h" @@ -2946,13 +2946,13 @@ int ObTableScanOp::inner_get_next_spatial_index_row() } else if (OB_FAIL(ObGeoTypeUtil::get_srid_from_wkb(geo_wkb, srid))) { LOG_WARN("failed to get srid", K(ret), K(geo_wkb)); } else if (srid != 0 && - OB_FAIL(OTSRS_MGR.get_tenant_srs_guard(tenant_id, srs_guard))) { + OB_FAIL(OTSRS_MGR->get_tenant_srs_guard(srs_guard))) { LOG_WARN("failed to get srs guard", K(ret), K(tenant_id), K(srid)); } else if (srid != 0 && OB_FAIL(srs_guard.get_srs_item(srid, srs_item))) { LOG_WARN("failed to get srs item", K(ret), K(tenant_id), K(srid)); } else if (((srid == 0) || !(srs_item->is_geographical_srs())) && - OB_FAIL(OTSRS_MGR.get_srs_bounds(srid, srs_item, srs_bound))) { + OB_FAIL(OTSRS_MGR->get_srs_bounds(srid, srs_item, srs_bound))) { LOG_WARN("failed to get srs bound", K(ret), K(srid)); } else if (OB_FAIL(ObGeoTypeUtil::get_cellid_mbr_from_geom(geo_wkb, srs_item, srs_bound, cellids, mbr_val))) { diff --git a/src/sql/executor/ob_maintain_dependency_info_task.cpp b/src/sql/executor/ob_maintain_dependency_info_task.cpp index bd3998191..52d785d99 100644 --- a/src/sql/executor/ob_maintain_dependency_info_task.cpp +++ b/src/sql/executor/ob_maintain_dependency_info_task.cpp @@ -195,7 +195,7 @@ int ObMaintainDepInfoTaskQueue::init(const int64_t thread_cnt, const int64_t que { int ret = OB_SUCCESS; auto attr = SET_USE_500("DepInfoTaskQ"); - if (OB_FAIL(ObAsyncTaskQueue::init(thread_cnt, queue_size, "MaintainDepInfoTaskQueue"))) { + if (OB_FAIL(ObAsyncTaskQueue::init(thread_cnt, queue_size, "MaintainDepInfoTaskQueue", OB_MALLOC_MIDDLE_BLOCK_SIZE))) { LOG_WARN("failed to init base queue", K(ret)); } else if (OB_FAIL(view_info_set_.create(INIT_BKT_SIZE, attr, attr))) { LOG_WARN("failed to init view set", K(ret)); diff --git a/src/sql/executor/ob_remote_executor_processor.cpp b/src/sql/executor/ob_remote_executor_processor.cpp index ef89fc839..9361821f3 100644 --- a/src/sql/executor/ob_remote_executor_processor.cpp +++ b/src/sql/executor/ob_remote_executor_processor.cpp @@ -834,10 +834,19 @@ int ObRpcRemoteExecuteP::init() int ret = OB_SUCCESS; if (OB_FAIL(base_init())) { LOG_WARN("init remote base execute context failed", K(ret)); - } else if (OB_FAIL(result_.init())) { - LOG_WARN("fail to init result", K(ret)); - } else { - arg_.set_deserialize_param(exec_ctx_, phy_plan_); + } + if (OB_SUCC(ret)) { + uint64_t mtl_id = MTL_ID(); + mtl_id = (mtl_id == OB_INVALID_TENANT_ID ? + OB_SERVER_TENANT_ID : + mtl_id); + + result_.set_tenant_id(mtl_id); + if (OB_FAIL(result_.init())) { + LOG_WARN("fail to init result", K(ret)); + } else { + arg_.set_deserialize_param(exec_ctx_, phy_plan_); + } } return ret; } @@ -1026,15 +1035,19 @@ int ObRpcRemoteSyncExecuteP::init() ObRemoteTask &task = arg_; if (OB_FAIL(base_init())) { LOG_WARN("init remote base execute context failed", K(ret)); - } else if (OB_FAIL(result_.init())) { - LOG_WARN("fail to init result", K(ret)); - } else if (OB_FAIL(exec_ctx_.create_physical_plan_ctx())) { - LOG_WARN("create physical plan ctx failed", K(ret)); - } else { - ObPhysicalPlanCtx *plan_ctx = exec_ctx_.get_physical_plan_ctx(); - plan_ctx->get_remote_sql_info().ps_params_ = &plan_ctx->get_param_store_for_update(); - task.set_remote_sql_info(&plan_ctx->get_remote_sql_info()); - task.set_exec_ctx(&exec_ctx_); + } + if (OB_SUCC(ret)) { + result_.set_tenant_id(MTL_ID()); + if (OB_FAIL(result_.init())) { + LOG_WARN("fail to init result", K(ret)); + } else if (OB_FAIL(exec_ctx_.create_physical_plan_ctx())) { + LOG_WARN("create physical plan ctx failed", K(ret)); + } else { + ObPhysicalPlanCtx *plan_ctx = exec_ctx_.get_physical_plan_ctx(); + plan_ctx->get_remote_sql_info().ps_params_ = &plan_ctx->get_param_store_for_update(); + task.set_remote_sql_info(&plan_ctx->get_remote_sql_info()); + task.set_exec_ctx(&exec_ctx_); + } } return ret; } diff --git a/src/sql/ob_sql_utils.cpp b/src/sql/ob_sql_utils.cpp index 527160e50..e81227856 100644 --- a/src/sql/ob_sql_utils.cpp +++ b/src/sql/ob_sql_utils.cpp @@ -56,7 +56,7 @@ #ifdef OB_BUILD_SPM #include "sql/spm/ob_spm_controller.h" #endif -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "sql/executor/ob_maintain_dependency_info_task.h" #include "sql/resolver/ddl/ob_create_view_resolver.h" extern "C" { @@ -4204,7 +4204,7 @@ int ObSqlGeoUtils::check_srid_by_srs(uint64_t tenant_id, uint64_t srid) ret = OB_OPERATE_OVERFLOW; LOG_USER_ERROR(OB_OPERATE_OVERFLOW, "srid", "UINT32_MAX"); } else if (srid != 0 && - OB_FAIL(OTSRS_MGR.get_tenant_srs_guard(tenant_id, srs_guard))) { + OB_FAIL(OTSRS_MGR->get_tenant_srs_guard(srs_guard))) { LOG_WARN("failed to get srs guard", K(tenant_id), K(srid), K(ret)); } else if (OB_FAIL(srs_guard.get_srs_item(srid, srs))) { LOG_WARN("get srs failed", K(srid), K(ret)); diff --git a/src/sql/optimizer/ob_optimizer.cpp b/src/sql/optimizer/ob_optimizer.cpp index 86fdce201..a087c6ad0 100644 --- a/src/sql/optimizer/ob_optimizer.cpp +++ b/src/sql/optimizer/ob_optimizer.cpp @@ -1089,16 +1089,21 @@ int ObOptimizer::add_column_usage_arg(const ObDMLStmt &stmt, int ObOptimizer::update_column_usage_infos() { int ret = OB_SUCCESS; - const ObSQLSessionInfo *session = ctx_.get_session_info(); + ObSQLSessionInfo *session = ctx_.get_session_info(); if (OB_ISNULL(session)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret)); + LOG_WARN("get unexpected null", K(ret), K(session)); } else { - ret = ObOptStatMonitorManager::get_instance().update_local_cache( - session->get_effective_tenant_id(), - ctx_.get_column_usage_infos()); + MTL_SWITCH(session->get_effective_tenant_id()) { + ObOptStatMonitorManager *optstat_monitor_mgr = NULL; + if (OB_ISNULL(optstat_monitor_mgr = MTL(ObOptStatMonitorManager*))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(optstat_monitor_mgr)); + } else if (OB_FAIL(optstat_monitor_mgr->update_local_cache(ctx_.get_column_usage_infos()))) { + LOG_WARN("failed to update local cache", K(ret)); + } else {/*do nothiing*/} + } } - return ret; } diff --git a/src/sql/rewrite/ob_query_range.cpp b/src/sql/rewrite/ob_query_range.cpp index 9531b684f..4d05fad9a 100644 --- a/src/sql/rewrite/ob_query_range.cpp +++ b/src/sql/rewrite/ob_query_range.cpp @@ -21,7 +21,7 @@ #include "sql/engine/expr/ob_expr_like.h" #include "common/ob_smart_call.h" #include "sql/optimizer/ob_optimizer_util.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "sql/engine/expr/ob_geo_expr_utils.h" //if cnd is true get full range key part which is always true @@ -8759,12 +8759,12 @@ int ObQueryRange::get_geo_intersects_keypart(uint32_t input_srid, double distance = NAN; // todo : fix me, get effective tenant_id - if ((input_srid != 0) && OB_FAIL(OTSRS_MGR.get_tenant_srs_guard(lib::current_resource_owner_id(), srs_guard))) { - LOG_WARN("get tenant srs guard failed", K(lib::current_resource_owner_id()), K(input_srid), K(ret)); + if ((input_srid != 0) && OB_FAIL(OTSRS_MGR->get_tenant_srs_guard(srs_guard))) { + LOG_WARN("get tenant srs guard failed", K(input_srid), K(ret)); } else if ((input_srid != 0) && OB_FAIL(srs_guard.get_srs_item(input_srid, srs_item))) { LOG_WARN("get tenant srs failed", K(input_srid), K(ret)); } else if (((input_srid == 0) || !(srs_item->is_geographical_srs())) && - OB_FAIL(OTSRS_MGR.get_srs_bounds(input_srid, srs_item, srs_bound))) { + OB_FAIL(OTSRS_MGR->get_srs_bounds(input_srid, srs_item, srs_bound))) { LOG_WARN("failed to get srs item", K(ret)); } else if (op_type == ObGeoRelationType::T_DWITHIN) { distance = out_key_part->geo_keypart_->distance_.get_double(); @@ -8903,12 +8903,12 @@ int ObQueryRange::get_geo_coveredby_keypart(uint32_t input_srid, ObExecContext *exec_ctx = NULL; ObString buffer_geo; - if ((input_srid != 0) && OB_FAIL(OTSRS_MGR.get_tenant_srs_guard(lib::current_resource_owner_id(), srs_guard))) { - LOG_WARN("get tenant srs guard failed", K(lib::current_resource_owner_id()), K(input_srid), K(ret)); + if ((input_srid != 0) && OB_FAIL(OTSRS_MGR->get_tenant_srs_guard(srs_guard))) { + LOG_WARN("get tenant srs guard failed", K(input_srid), K(ret)); } else if ((input_srid != 0) && OB_FAIL(srs_guard.get_srs_item(input_srid, srs_item))) { LOG_WARN("get tenant srs failed", K(input_srid), K(ret)); } else if (((input_srid == 0) || !(srs_item->is_geographical_srs())) && - OB_FAIL(OTSRS_MGR.get_srs_bounds(input_srid, srs_item, srs_bound))) { + OB_FAIL(OTSRS_MGR->get_srs_bounds(input_srid, srs_item, srs_bound))) { LOG_WARN("failed to get srs item", K(ret)); } if (s2object == NULL && OB_SUCC(ret)) { diff --git a/src/storage/blocksstable/ob_tmp_file.cpp b/src/storage/blocksstable/ob_tmp_file.cpp index 8c75bd4a5..32c848aab 100644 --- a/src/storage/blocksstable/ob_tmp_file.cpp +++ b/src/storage/blocksstable/ob_tmp_file.cpp @@ -1072,7 +1072,8 @@ int ObTmpFile::aio_write(const ObTmpFileIOInfo &io_info, ObTmpFileIOHandle &hand } else { alloc_size = size; } - if (OB_ISNULL(buff = allocator_->alloc(sizeof(ObTmpFileExtent)))) { + lib::ObMemAttr attr(tenant_id_, "TmpFileExtent"); + if (OB_ISNULL(buff = allocator_->alloc(sizeof(ObTmpFileExtent), attr))) { ret = OB_ALLOCATE_MEMORY_FAILED; STORAGE_LOG(WARN, "fail to alloc a buf", K(ret)); } else if (OB_ISNULL(extent = new (buff) ObTmpFileExtent(this))) { @@ -1322,7 +1323,7 @@ int ObTmpFileManager::init() if (IS_INIT) { ret = OB_INIT_TWICE; STORAGE_LOG(WARN, "ObTmpFileManager has not been inited", K(ret)); - } else if (OB_FAIL(files_.init(DEFAULT_BUCKET_NUM, attr, TOTAL_LIMIT, HOLD_LIMIT, BLOCK_SIZE))) { + } else if (OB_FAIL(files_.init(DEFAULT_BUCKET_NUM, attr, *lib::ObMallocAllocator::get_instance()))) { STORAGE_LOG(WARN, "fail to init map for temporary files", K(ret)); } else if (OB_FAIL(OB_TMP_FILE_STORE.init())) { STORAGE_LOG(WARN, "fail to init the block manager for temporary files", K(ret)); diff --git a/src/storage/blocksstable/ob_tmp_file.h b/src/storage/blocksstable/ob_tmp_file.h index 72ffd8b3a..fb9987e4b 100644 --- a/src/storage/blocksstable/ob_tmp_file.h +++ b/src/storage/blocksstable/ob_tmp_file.h @@ -408,9 +408,6 @@ private: private: static const int64_t DEFAULT_BUCKET_NUM = 10243L; - static const int64_t TOTAL_LIMIT = 15 * 1024L * 1024L * 1024L; - static const int64_t HOLD_LIMIT = 8 * 1024L * 1024L; - static const int64_t BLOCK_SIZE = common::OB_MALLOC_NORMAL_BLOCK_SIZE; bool is_inited_; int64_t next_fd_; int64_t next_dir_; diff --git a/src/storage/ddl/ob_ddl_redo_log_writer.cpp b/src/storage/ddl/ob_ddl_redo_log_writer.cpp index c75d01ba2..7c979351b 100755 --- a/src/storage/ddl/ob_ddl_redo_log_writer.cpp +++ b/src/storage/ddl/ob_ddl_redo_log_writer.cpp @@ -747,7 +747,8 @@ int ObDDLRedoLogWriter::write( return ret; } -int ObDDLRedoLogWriter::write_ddl_start_log(ObTabletHandle &tablet_handle, +int ObDDLRedoLogWriter::write_ddl_start_log(ObLSHandle &ls_handle, + ObTabletHandle &tablet_handle, ObDDLKvMgrHandle &ddl_kv_mgr_handle, const ObDDLStartLog &log, ObLogHandler *log_handler, @@ -799,6 +800,8 @@ int ObDDLRedoLogWriter::write_ddl_start_log(ObTabletHandle &tablet_handle, LOG_WARN("fail to seriaize ddl start log", K(ret)); } else if (OB_FAIL(log.serialize(buffer, buffer_size, pos))) { LOG_WARN("fail to seriaize ddl start log", K(ret)); + } else if (OB_FAIL(ls_handle.get_ls()->get_ddl_log_handler()->add_tablet(log.get_table_key().get_tablet_id()))) { + LOG_WARN("add tablet failed", K(ret)); } else if (OB_FAIL(log_handler->append(buffer, buffer_size, SCN::min_scn(), @@ -1180,7 +1183,7 @@ int ObDDLSSTableRedoWriter::start_ddl_redo(const ObITable::TableKey &table_key, LOG_WARN("get tablet handle failed", K(ret), K(ls_id_), K(tablet_id_)); } else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle, true/*try_create*/))) { LOG_WARN("create ddl kv mgr failed", K(ret)); - } else if (OB_FAIL(ObDDLRedoLogWriter::get_instance().write_ddl_start_log(tablet_handle, ddl_kv_mgr_handle, log, ls->get_log_handler(), tmp_scn))) { + } else if (OB_FAIL(ObDDLRedoLogWriter::get_instance().write_ddl_start_log(ls_handle, tablet_handle, ddl_kv_mgr_handle, log, ls->get_log_handler(), tmp_scn))) { LOG_WARN("fail to write ddl start log", K(ret), K(table_key)); } else if (FALSE_IT(set_start_scn(tmp_scn))) { } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->register_to_tablet(get_start_scn(), ddl_kv_mgr_handle))) { diff --git a/src/storage/ddl/ob_ddl_redo_log_writer.h b/src/storage/ddl/ob_ddl_redo_log_writer.h index 9aa2e30fd..9013a538f 100644 --- a/src/storage/ddl/ob_ddl_redo_log_writer.h +++ b/src/storage/ddl/ob_ddl_redo_log_writer.h @@ -224,7 +224,8 @@ public: const blocksstable::MacroBlockId ¯o_block_id, char *buffer, ObDDLRedoLogHandle &handle); - int write_ddl_start_log(ObTabletHandle &tablet_handle, + int write_ddl_start_log(ObLSHandle &ls_handle, + ObTabletHandle &tablet_handle, ObDDLKvMgrHandle &ddl_kv_mgr_handle, const ObDDLStartLog &log, logservice::ObLogHandler *log_handler, diff --git a/src/storage/ddl/ob_ddl_replay_executor.cpp b/src/storage/ddl/ob_ddl_replay_executor.cpp index 5b337ee87..baae97595 100755 --- a/src/storage/ddl/ob_ddl_replay_executor.cpp +++ b/src/storage/ddl/ob_ddl_replay_executor.cpp @@ -128,7 +128,8 @@ int ObDDLStartReplayExecutor::do_replay_(ObTabletHandle &handle) LOG_WARN("skip replay ddl start", K(ret), "ls_id", ls_->get_ls_id(), K(handle)); } else if (OB_FAIL(handle.get_obj()->get_ddl_kv_mgr(ddl_kv_mgr_handle, true/*try_create*/))) { LOG_WARN("create ddl kv mgr failed", K(ret), K(handle), KPC_(log), K_(scn)); - } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->ddl_start(*handle.get_obj(), + } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->ddl_start(*ls_, + *handle.get_obj(), table_key, scn_, log_->get_data_format_version(), diff --git a/src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp b/src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp index 23a9f623a..00241861b 100644 --- a/src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp +++ b/src/storage/ddl/ob_tablet_ddl_kv_mgr.cpp @@ -137,7 +137,8 @@ int ObTabletDDLKvMgr::ddl_start_nolock(const ObITable::TableKey &table_key, // ddl start from checkpoint // keep ddl sstable table -int ObTabletDDLKvMgr::ddl_start(ObTablet &tablet, +int ObTabletDDLKvMgr::ddl_start(ObLS &ls, + ObTablet &tablet, const ObITable::TableKey &table_key, const SCN &start_scn, const int64_t data_format_version, @@ -154,6 +155,8 @@ int ObTabletDDLKvMgr::ddl_start(ObTablet &tablet, ObLatchWGuard state_guard(state_lock_, ObLatchIds::TABLET_DDL_KV_MGR_LOCK); if (OB_FAIL(ddl_start_nolock(table_key, start_scn, data_format_version, execution_id, checkpoint_scn))) { LOG_WARN("failed to ddl start", K(ret)); + } else if (OB_FAIL(ls.get_ddl_log_handler()->add_tablet(tablet_id_))) { + LOG_WARN("add tablet failed", K(ret)); } else { // save variables under lock saved_start_scn = start_scn_; @@ -583,7 +586,8 @@ int ObTabletDDLKvMgr::online() table_key.version_range_.base_version_ = 0; table_key.version_range_.snapshot_version_ = tablet_meta.ddl_snapshot_version_; const SCN &start_scn = tablet_meta.ddl_start_scn_; - if (OB_FAIL(ddl_start(*tablet_handle.get_obj(), + if (OB_FAIL(ddl_start(*ls_handle.get_ls(), + *tablet_handle.get_obj(), table_key, start_scn, tablet_meta.ddl_data_format_version_, diff --git a/src/storage/ddl/ob_tablet_ddl_kv_mgr.h b/src/storage/ddl/ob_tablet_ddl_kv_mgr.h index 3f770532c..f8a5cf383 100644 --- a/src/storage/ddl/ob_tablet_ddl_kv_mgr.h +++ b/src/storage/ddl/ob_tablet_ddl_kv_mgr.h @@ -37,7 +37,7 @@ public: ~ObTabletDDLKvMgr(); int init(const share::ObLSID &ls_id, const common::ObTabletID &tablet_id); // init before memtable mgr int ddl_start_nolock(const ObITable::TableKey &table_key, const share::SCN &start_scn, const int64_t data_format_version, const int64_t execution_id, const share::SCN &checkpoint_scn); - int ddl_start(ObTablet &tablet, const ObITable::TableKey &table_key, const share::SCN &start_scn, const int64_t data_format_version, const int64_t execution_id, const share::SCN &checkpoint_scn); + int ddl_start(ObLS &ls, ObTablet &tablet, const ObITable::TableKey &table_key, const share::SCN &start_scn, const int64_t data_format_version, const int64_t execution_id, const share::SCN &checkpoint_scn); int ddl_commit(ObTablet &tablet, const share::SCN &start_scn, const share::SCN &commit_scn); // schedule build a major sstable int schedule_ddl_dump_task(ObTablet &tablet, const share::SCN &start_scn, const share::SCN &rec_scn); int schedule_ddl_merge_task(ObTablet &tablet, const share::SCN &start_scn, const share::SCN &commit_scn); // try wait build major sstable diff --git a/src/storage/ls/ob_ls_ddl_log_handler.cpp b/src/storage/ls/ob_ls_ddl_log_handler.cpp index cca723d31..e1895d440 100644 --- a/src/storage/ls/ob_ls_ddl_log_handler.cpp +++ b/src/storage/ls/ob_ls_ddl_log_handler.cpp @@ -32,6 +32,134 @@ using namespace share; namespace storage { +ObActiveDDLKVMgr::ObActiveDDLKVMgr() + : lock_(), active_ddl_tablets_() +{ +} + +int ObActiveDDLKVMgr::add_tablet(const ObTabletID &tablet_id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!tablet_id.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(ret), K(tablet_id)); + } else { + ObSpinLockGuard guard(lock_); + if (!has_exist_in_array(active_ddl_tablets_, tablet_id)) { + if (OB_FAIL(active_ddl_tablets_.push_back(tablet_id))) { + LOG_WARN("push back tablet id failed", K(ret)); + } + } + } + if (OB_SUCC(ret)) { + FLOG_INFO("add tablet to active ddl kv mgr", K(tablet_id)); + } + return ret; +} + +int ObActiveDDLKVMgr::del_tablets(const common::ObIArray &tablet_ids) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(tablet_ids.count() < 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(ret), K(tablet_ids.count())); + } else if (tablet_ids.count() == 0) { + // do nothing + } else { + ObSpinLockGuard guard(lock_); + ObArray tmp_active_tablet_ids; + for (int64_t i = 0; OB_SUCC(ret) && i < active_ddl_tablets_.count(); ++i) { + if (!has_exist_in_array(tablet_ids, active_ddl_tablets_.at(i))) { + if (OB_FAIL(tmp_active_tablet_ids.push_back(active_ddl_tablets_.at(i)))) { + LOG_WARN("push back active tablet id failed", K(ret)); + } + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(active_ddl_tablets_.assign(tmp_active_tablet_ids))) { + LOG_WARN("assign active ddl tablet ids failed", K(ret)); + } else { + FLOG_INFO("del tablets from active ddl kv mgr", K_(active_ddl_tablets), K(tablet_ids)); + } + } + } + return ret; +} + +int ObActiveDDLKVMgr::get_tablets(common::ObIArray &tablet_ids) +{ + int ret = OB_SUCCESS; + ObSpinLockGuard guard(lock_); + if (OB_FAIL(tablet_ids.assign(active_ddl_tablets_))) { + LOG_WARN("assign tablet ids failed", K(ret)); + } + return ret; +} + +int ObActiveDDLKVIterator::init(ObLS *ls, ObActiveDDLKVMgr &mgr) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(is_inited_)) { + ret = OB_INIT_TWICE; + LOG_WARN("ObActiveDDLKVIterator has been inited twice", K(ret)); + } else if (OB_ISNULL(ls)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(ret), KP(ls)); + } else if (OB_FAIL(mgr.get_tablets(active_ddl_tablets_))) { + LOG_WARN("get tablets failed", K(ret)); + } else { + ls_ = ls; + mgr_ = &mgr; + idx_ = 0; + is_inited_ = true; + } + return ret; +} + +int ObActiveDDLKVIterator::get_next_ddl_kv_mgr(ObDDLKvMgrHandle &handle) +{ + int ret = OB_SUCCESS; + handle.reset(); + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("ObActiveDDLKVIterator has not been inited", K(ret)); + } else { + ObTabletHandle tablet_handle; + while (OB_SUCC(ret) && !handle.is_valid()) { + if (idx_ >= active_ddl_tablets_.count()) { + ret = OB_ITER_END; + } else { + ObTabletID &tablet_id = active_ddl_tablets_.at(idx_); + if (OB_FAIL(ls_->get_tablet(tablet_id, tablet_handle))) { + if (OB_TABLET_NOT_EXIST == ret) { + if (OB_FAIL(to_del_tablets_.push_back(tablet_id))) { + LOG_WARN("push back to delete tablet id failed", K(ret)); + } + } else { + LOG_WARN("failed to get tablet", K(ret), K(ls_->get_ls_id()), K(tablet_id)); + } + } else if (tablet_handle.get_obj()->get_tablet_meta().ddl_commit_scn_.is_valid_and_not_min()) { + if (OB_FAIL(to_del_tablets_.push_back(tablet_id))) { + LOG_WARN("push back to deleted tablet failed", K(ret)); + } + } else if (OB_FAIL(tablet_handle.get_obj()->get_ddl_kv_mgr(handle))) { + LOG_WARN("get ddl kv mgr failed", K(ret)); + } + } + if (OB_SUCC(ret)) { + ++idx_; + } + } + } + if (OB_ITER_END == ret) { + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(mgr_->del_tablets(to_del_tablets_))) { + LOG_WARN("del tablets failed", K(tmp_ret)); + } + } + return ret; +} + int ObLSDDLLogHandler::init(ObLS *ls) { int ret = OB_SUCCESS; @@ -262,23 +390,24 @@ SCN ObLSDDLLogHandler::get_rec_scn() ObLSTabletIterator tablet_iter(ObMDSGetTabletMode::READ_WITHOUT_CHECK); SCN rec_scn = SCN::max_scn(); bool has_ddl_kv = false; - if (OB_FAIL(ls_->get_tablet_svr()->build_tablet_iter(tablet_iter))) { - LOG_WARN("failed to build ls tablet iter", K(ret), K(ls_)); + ObActiveDDLKVIterator active_ddl_kv_mgr_iter; + if (OB_FAIL(active_ddl_kv_mgr_iter.init(ls_, active_ddl_kv_mgr_))) { + LOG_WARN("initialize active ddl kv mgr iterator failed", K(ret)); } else { + ObDDLKvMgrHandle ddl_kv_mgr_handle; while (OB_SUCC(ret)) { - ObDDLKvMgrHandle ddl_kv_mgr_handle; - if (OB_FAIL(tablet_iter.get_next_ddl_kv_mgr(ddl_kv_mgr_handle))) { + if (OB_FAIL(active_ddl_kv_mgr_iter.get_next_ddl_kv_mgr(ddl_kv_mgr_handle))) { if (OB_ITER_END == ret) { ret = OB_SUCCESS; break; } else { - LOG_WARN("failed to get ddl kv mgr", K(ret), K(ddl_kv_mgr_handle)); + LOG_WARN("get next ddl kv mgr failed", K(ret)); } } else if (OB_UNLIKELY(!ddl_kv_mgr_handle.is_valid())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid ddl kv mgr handle", K(ret), K(ddl_kv_mgr_handle)); } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->get_rec_scn(rec_scn))) { - LOG_WARN("failed to get rec scn", K(ret)); + LOG_WARN("get rec scn failed", K(ret)); } } } @@ -369,5 +498,32 @@ int ObLSDDLLogHandler::replay_ddl_start_log_(const char *log_buf, return ret; } +int ObLSDDLLogHandler::add_tablet(const ObTabletID &tablet_id) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(active_ddl_kv_mgr_.add_tablet(tablet_id))) { + LOG_WARN("add tablet failed", K(ret)); + } + return ret; +} + +int ObLSDDLLogHandler::del_tablets(const common::ObIArray &tablet_ids) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(active_ddl_kv_mgr_.del_tablets(tablet_ids))) { + LOG_WARN("del tablets failed", K(ret)); + } + return ret; +} + +int ObLSDDLLogHandler::get_tablets(common::ObIArray &tablet_ids) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(active_ddl_kv_mgr_.get_tablets(tablet_ids))) { + LOG_WARN("get tablets failed", K(ret)); + } + return ret; +} + } } diff --git a/src/storage/ls/ob_ls_ddl_log_handler.h b/src/storage/ls/ob_ls_ddl_log_handler.h index 23128577e..9f008bc9b 100644 --- a/src/storage/ls/ob_ls_ddl_log_handler.h +++ b/src/storage/ls/ob_ls_ddl_log_handler.h @@ -24,6 +24,37 @@ namespace storage class ObLS; +class ObActiveDDLKVMgr final +{ +public: + ObActiveDDLKVMgr(); + ~ObActiveDDLKVMgr() = default; + int add_tablet(const ObTabletID &tablet_id); + int del_tablets(const common::ObIArray &tablet_ids); + int get_tablets(common::ObIArray &tablet_ids); +private: + ObSpinLock lock_; + ObArray active_ddl_tablets_; +}; + +class ObActiveDDLKVIterator final +{ +public: + ObActiveDDLKVIterator() + : active_ddl_tablets_(), to_del_tablets_(), mgr_(nullptr), idx_(0), ls_(nullptr), is_inited_(false) + {} + ~ObActiveDDLKVIterator() = default; + int init(ObLS *ls, ObActiveDDLKVMgr &mgr); + int get_next_ddl_kv_mgr(ObDDLKvMgrHandle &handle); +private: + common::ObArray active_ddl_tablets_; + common::ObArray to_del_tablets_; + ObActiveDDLKVMgr *mgr_; + int64_t idx_; + ObLS *ls_; + bool is_inited_; +}; + class ObLSDDLLogHandler : public logservice::ObIReplaySubHandler, public logservice::ObIRoleChangeSubHandler, public logservice::ObICheckpointSubHandler @@ -55,6 +86,12 @@ public: // for checkpoint int flush(share::SCN &rec_scn) override final; share::SCN get_rec_scn() override final; + + // manage active ddl kv mgr for ls + int add_tablet(const ObTabletID &tablet_id); + int del_tablets(const common::ObIArray &tablet_ids); + int get_tablets(common::ObIArray &tablet_ids); + private: int replay_ddl_redo_log_(const char *log_buf, const int64_t buf_size, int64_t pos, const share::SCN &scn); int replay_ddl_commit_log_(const char *log_buf, const int64_t buf_size, int64_t pos, const share::SCN &scn); @@ -67,6 +104,7 @@ private: common::TCRWLock online_lock_; ObDDLRedoLogReplayer ddl_log_replayer_; share::SCN last_rec_scn_; + ObActiveDDLKVMgr active_ddl_kv_mgr_; }; } // storage diff --git a/src/storage/meta_mem/ob_storage_meta_cache.cpp b/src/storage/meta_mem/ob_storage_meta_cache.cpp index 625de5847..6c0e229c1 100644 --- a/src/storage/meta_mem/ob_storage_meta_cache.cpp +++ b/src/storage/meta_mem/ob_storage_meta_cache.cpp @@ -25,6 +25,7 @@ #include "share/ob_tablet_autoincrement_param.h" #include "storage/tablet/ob_tablet.h" #include "storage/blocksstable/ob_storage_cache_suite.h" +#include "storage/meta_mem/ob_tenant_meta_mem_mgr.h" namespace oceanbase { @@ -486,13 +487,8 @@ int ObStorageMetaHandle::wait(const int64_t timeout_ms) int ObStorageMetaCache::init(const char *cache_name, const int64_t priority) { int ret = OB_SUCCESS; - const int64_t mem_limit = 4 * 1024 * 1024 * 1024LL; if (OB_FAIL((common::ObKVCache::init(cache_name, priority)))) { LOG_WARN("fail to init storage meta kv cache", K(ret), K(priority)); - } else if (OB_FAIL(allocator_.init(mem_limit, OB_MALLOC_BIG_BLOCK_SIZE, OB_MALLOC_BIG_BLOCK_SIZE))) { - LOG_WARN("fail to init io allocator", K(ret)); - } else { - allocator_.set_label("StorMetaCacheIO"); } return ret; } @@ -635,7 +631,7 @@ int ObStorageMetaCache::get_meta( if (OB_UNLIKELY(!key.is_valid() || type >= ObStorageMetaValue::MAX)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(key), K(type)); - } else if (OB_FAIL(meta_handle.cache_handle_.new_value(allocator_))) { + } else if (OB_FAIL(meta_handle.cache_handle_.new_value(MTL(ObTenantMetaMemMgr *)->get_meta_cache_io_allocator()))) { LOG_WARN("fail to new cache handle value", K(ret)); } else if (OB_FAIL(get(key, meta_handle.cache_handle_.get_cache_value()->value_, meta_handle.cache_handle_.get_cache_value()->cache_handle_))) { @@ -720,7 +716,7 @@ int ObStorageMetaCache::prefetch( ObStorageMetaIOCallback callback(type, key, meta_handle.cache_handle_, - &allocator_, + &(MTL(ObTenantMetaMemMgr *)->get_meta_cache_io_allocator()), tablet); if (OB_FAIL(read_io(key.get_meta_addr(), callback, meta_handle))) { LOG_WARN("fail to read storage meta from io", K(ret), K(key), K(meta_handle)); @@ -739,13 +735,13 @@ int ObStorageMetaCache::get_meta_and_bypass_cache( if (OB_UNLIKELY(!key.is_valid() || type >= ObStorageMetaValue::MAX)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(key), K(type)); - } else if (OB_FAIL(handle.cache_handle_.new_value(allocator_))) { + } else if (OB_FAIL(handle.cache_handle_.new_value(MTL(ObTenantMetaMemMgr *)->get_meta_cache_io_allocator()))) { LOG_WARN("fail to new cache handle value", K(ret)); } else { ObStorageMetaIOCallback callback(type, key, handle.cache_handle_, - &allocator_, + &(MTL(ObTenantMetaMemMgr *)->get_meta_cache_io_allocator()), nullptr/*tablet*/, &allocator/*bypass_cache*/); if (OB_FAIL(read_io(key.get_meta_addr(), callback, handle))) { @@ -782,7 +778,6 @@ int ObStorageMetaCache::read_io( } ObStorageMetaCache::ObStorageMetaCache() - : allocator_() { } diff --git a/src/storage/meta_mem/ob_storage_meta_cache.h b/src/storage/meta_mem/ob_storage_meta_cache.h index b31ac8121..1fc50b0b7 100644 --- a/src/storage/meta_mem/ob_storage_meta_cache.h +++ b/src/storage/meta_mem/ob_storage_meta_cache.h @@ -275,7 +275,6 @@ private: ObStorageMetaIOCallback &callback, ObStorageMetaHandle &handle); private: - common::ObConcurrentFIFOAllocator allocator_; DISALLOW_COPY_AND_ASSIGN(ObStorageMetaCache); }; diff --git a/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp b/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp index 87b2c4b58..b576f8826 100755 --- a/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp +++ b/src/storage/meta_mem/ob_tenant_meta_mem_mgr.cpp @@ -127,9 +127,7 @@ void ObTenantMetaMemMgr::TabletPersistTask::runTimerTask() } ObTenantMetaMemMgr::ObTenantMetaMemMgr(const uint64_t tenant_id) - : cmp_ret_(OB_SUCCESS), - compare_(cmp_ret_), - wash_lock_(common::ObLatchIds::TENANT_META_MEM_MGR_LOCK), + : wash_lock_(common::ObLatchIds::TENANT_META_MEM_MGR_LOCK), wash_func_(*this), tenant_id_(tenant_id), bucket_lock_(), @@ -154,6 +152,7 @@ ObTenantMetaMemMgr::ObTenantMetaMemMgr(const uint64_t tenant_id) tx_data_memtable_pool_(tenant_id, MAX_TX_DATA_MEMTABLE_CNT_IN_OBJ_POOL, "TxDataMemObj", ObCtxIds::DEFAULT_CTX_ID), tx_ctx_memtable_pool_(tenant_id, MAX_TX_CTX_MEMTABLE_CNT_IN_OBJ_POOL, "TxCtxMemObj", ObCtxIds::DEFAULT_CTX_ID), lock_memtable_pool_(tenant_id, MAX_LOCK_MEMTABLE_CNT_IN_OBJ_POOL, "LockMemObj", ObCtxIds::DEFAULT_CTX_ID), + meta_cache_io_allocator_(), is_inited_(false) { for (int64_t i = 0; i < ObITable::TableType::MAX_TABLE_TYPE; i++) { @@ -185,6 +184,7 @@ int ObTenantMetaMemMgr::init() lib::ObMemAttr mem_attr(tenant_id_, "MetaAllocator", ObCtxIds::META_OBJ_CTX_ID); lib::ObMemAttr map_attr(tenant_id_, "TabletMap"); lib::ObMemAttr other_attr(tenant_id_, "T3MOtherMem"); + const int64_t mem_limit = 4 * 1024 * 1024 * 1024LL; const int64_t bucket_num = cal_adaptive_bucket_num(); const int64_t pin_set_bucket_num = common::hash::cal_next_prime(DEFAULT_BUCKET_NUM); if (OB_UNLIKELY(is_inited_)) { @@ -204,6 +204,8 @@ int ObTenantMetaMemMgr::init() LOG_WARN("fail to create thread for t3m", K(ret)); } else if (OB_FAIL(TG_CREATE_TENANT(lib::TGDefIDs::TenantMetaMemMgr, persist_tg_id_))) { LOG_WARN("fail to create thread for t3m", K(ret)); + } else if (OB_FAIL(meta_cache_io_allocator_.init(OB_MALLOC_MIDDLE_BLOCK_SIZE, "StorMetaCacheIO", tenant_id_, mem_limit))) { + LOG_WARN("fail to init storage meta cache io allocator", K(ret), K_(tenant_id), K(mem_limit)); } else { init_pool_arr(); is_inited_ = true; @@ -319,6 +321,7 @@ void ObTenantMetaMemMgr::destroy() for (int64_t i = 0; i <= ObITable::TableType::REMOTE_LOGICAL_MINOR_SSTABLE; i++) { pool_arr_[i] = nullptr; } + meta_cache_io_allocator_.destroy(); is_inited_ = false; } diff --git a/src/storage/meta_mem/ob_tenant_meta_mem_mgr.h b/src/storage/meta_mem/ob_tenant_meta_mem_mgr.h index a78a9fb23..e76b16cf8 100644 --- a/src/storage/meta_mem/ob_tenant_meta_mem_mgr.h +++ b/src/storage/meta_mem/ob_tenant_meta_mem_mgr.h @@ -269,6 +269,7 @@ public: int get_tablet_pointer_initial_state(const ObTabletMapKey &key, bool &initial_state); int get_tablet_ddl_kv_mgr(const ObTabletMapKey &key, ObDDLKvMgrHandle &ddl_kv_mgr_handle); ObFullTabletCreator &get_mstx_tablet_creator() { return full_tablet_creator_; } + common::ObIAllocator &get_meta_cache_io_allocator() { return meta_cache_io_allocator_; } OB_INLINE int64_t get_total_tablet_cnt() const { return tablet_map_.count(); } int has_meta_wait_gc(bool &is_wait); @@ -476,8 +477,6 @@ private: void batch_gc_memtable_(); private: - int cmp_ret_; - HeapCompare compare_; common::SpinRWLock wash_lock_; TryWashTabletFunc wash_func_; const uint64_t tenant_id_; @@ -511,6 +510,8 @@ private: TabletBufferList normal_tablet_header_; TabletBufferList large_tablet_header_; + common::ObConcurrentFIFOAllocator meta_cache_io_allocator_; + bool is_inited_; }; diff --git a/src/storage/ob_resource_map.h b/src/storage/ob_resource_map.h index ee582d432..455338a75 100644 --- a/src/storage/ob_resource_map.h +++ b/src/storage/ob_resource_map.h @@ -18,6 +18,7 @@ #include "lib/hash/ob_hashmap.h" #include "lib/hash/ob_hashutils.h" #include "lib/lock/ob_bucket_lock.h" +#include "src/share/rc/ob_tenant_base.h" namespace oceanbase { @@ -162,6 +163,7 @@ public: virtual ~ObResourceMap(); int init(const int64_t bucket_num, const ObMemAttr &attr, const int64_t total_limit, const int64_t hold_limit, const int64_t page_size); + int init(const int64_t bucket_num, const ObMemAttr &attr, common::ObIAllocator &allocator); int get(const Key &key, ObResourceHandle &handle); template > int set(const Key &key, Value &value, Callback callback = ObResourceDefaultCallback()); @@ -171,7 +173,7 @@ public: int foreach(Callback &callback); void destroy(); int dec_handle_ref(ObResourceValueStore *ptr); - common::ObIAllocator &get_allocator() { return allocator_; } + common::ObIAllocator &get_allocator() { return *allocator_; } int inc_handle_ref(ObResourceValueStore *ptr); protected: int get_without_lock( @@ -182,7 +184,8 @@ protected: typedef ObResourceValueStore ValueStore; typedef common::hash::ObHashMap MAP; MAP map_; - common::ObConcurrentFIFOAllocator allocator_; + common::ObConcurrentFIFOAllocator default_allocator_; + common::ObIAllocator *allocator_; common::ObBucketLock bucket_lock_; common::hash::hash_func hash_func_; bool is_inited_; @@ -190,7 +193,7 @@ protected: template ObResourceMap::ObResourceMap() - : map_(), allocator_(), bucket_lock_(), is_inited_(false) + : map_(), default_allocator_(), allocator_(&default_allocator_), bucket_lock_(), is_inited_(false) { } @@ -226,10 +229,38 @@ int ObResourceMap::init( ret = OB_ERR_UNEXPECTED; STORAGE_LOG(WARN, "lock buckets isn't equal to map buckets, which could cause concurrency issues", K(ret), K(bkt_num), K(map_.bucket_count())); - } else if (OB_FAIL(allocator_.init(page_size, "ResourceMap", tenant_id, total_limit))) { + } else if (OB_FAIL(default_allocator_.init(page_size, "ResourceMap", tenant_id, total_limit))) { STORAGE_LOG(WARN, "fail to init allocator", K(ret)); } else { - allocator_.set_attr(attr); + default_allocator_.set_attr(attr); + is_inited_ = true; + STORAGE_LOG(INFO, "init resource map success", K(ret), K(attr), K(bkt_num)); + } + return ret; +} + +template +int ObResourceMap::init(const int64_t bucket_num, const ObMemAttr &attr, common::ObIAllocator &allocator) +{ + int ret = OB_SUCCESS; + const uint64_t tenant_id = attr.tenant_id_; + const int64_t bkt_num = common::hash::cal_next_prime(bucket_num); + if (OB_UNLIKELY(is_inited_)) { + ret = common::OB_INIT_TWICE; + STORAGE_LOG(WARN, "ObResourceMap has already been inited", K(ret)); + } else if (OB_UNLIKELY(bucket_num <= 0 || OB_INVALID_TENANT_ID == tenant_id)) { + ret = common::OB_INVALID_ARGUMENT; + STORAGE_LOG(WARN, "invalid argument", K(ret), K(bucket_num), K(tenant_id)); + } else if (OB_FAIL(bucket_lock_.init(bkt_num, ObLatchIds::DEFAULT_BUCKET_LOCK, ObMemAttr(tenant_id, "ResourMapLock")))) { + STORAGE_LOG(WARN, "fail to init bucket lock", K(ret), K(bkt_num)); + } else if (OB_FAIL(map_.create(bkt_num, attr, attr))) { + STORAGE_LOG(WARN, "fail to create map", K(ret)); + } else if (OB_UNLIKELY(bkt_num != map_.bucket_count())) { + ret = OB_ERR_UNEXPECTED; + STORAGE_LOG(WARN, "lock buckets isn't equal to map buckets, which could cause concurrency issues", K(ret), + K(bkt_num), K(map_.bucket_count())); + } else { + allocator_ = &allocator; is_inited_ = true; STORAGE_LOG(INFO, "init resource map success", K(ret), K(attr), K(bkt_num)); } @@ -289,11 +320,12 @@ int ObResourceMap::set(const Key &key, Value &value, Callback callba if (OB_FAIL(hash_func_(key, hash_val))) { STORAGE_LOG(WARN, "fail to do hash", K(ret)); } else { + lib::ObMemAttr attr(MTL_ID(), "ResourceMapSet"); common::ObBucketHashWLockGuard guard(bucket_lock_, hash_val); if (OB_UNLIKELY(!is_inited_)) { ret = common::OB_NOT_INIT; STORAGE_LOG(WARN, "ObResourceMap has not been inited", K(ret)); - } else if (OB_ISNULL(buf = static_cast(allocator_.alloc(buf_size)))) { + } else if (OB_ISNULL(buf = static_cast(allocator_->alloc(buf_size, attr)))) { ret = common::OB_ALLOCATE_MEMORY_FAILED; STORAGE_LOG(WARN, "fail to allocate memory", K(ret), K(buf_size)); } else { @@ -382,7 +414,7 @@ void ObResourceMap::free_resource(ValueStore *value_store) ptr->~Value(); ptr = NULL; } - allocator_.free(buf); + allocator_->free(buf); buf = NULL; } } @@ -404,7 +436,8 @@ void ObResourceMap::destroy() } bucket_lock_.destroy(); map_.destroy(); - allocator_.destroy(); + default_allocator_.destroy(); + allocator_ = &default_allocator_; is_inited_ = false; } diff --git a/src/storage/tablet/ob_tablet.cpp b/src/storage/tablet/ob_tablet.cpp index f4679edb3..c0722048b 100755 --- a/src/storage/tablet/ob_tablet.cpp +++ b/src/storage/tablet/ob_tablet.cpp @@ -4146,13 +4146,25 @@ int ObTablet::start_ddl_if_need() } else if (OB_FAIL(get_ddl_kv_mgr(ddl_kv_mgr_handle, true/*try_create*/))) { LOG_WARN("create ddl kv mgr failed", K(ret)); } else { + ObLS *ls = nullptr; + ObLSService *ls_service = nullptr; + ObLSHandle ls_handle; ObITable::TableKey table_key; table_key.table_type_ = ObITable::TableType::MAJOR_SSTABLE; table_key.tablet_id_ = tablet_meta_.tablet_id_; table_key.version_range_.base_version_ = 0; table_key.version_range_.snapshot_version_ = tablet_meta_.ddl_snapshot_version_; const SCN &start_scn = tablet_meta_.ddl_start_scn_; - if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->ddl_start(*this, + if (OB_ISNULL(ls_service = MTL(ObLSService*))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get ObLSService from MTL", K(ret), KP(ls_service)); + } else if (OB_FAIL(ls_service->get_ls(tablet_meta_.ls_id_, ls_handle, ObLSGetMod::TABLET_MOD))) { + LOG_WARN("failed to get ls", K(ret)); + } else if (OB_ISNULL(ls = ls_handle.get_ls())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls should not be NULL", K(ret), KP(ls)); + } else if (OB_FAIL(ddl_kv_mgr_handle.get_obj()->ddl_start(*ls, + *this, table_key, start_scn, tablet_meta_.ddl_data_format_version_, diff --git a/src/storage/tx_storage/ob_access_service.cpp b/src/storage/tx_storage/ob_access_service.cpp index bc888c22e..b3391ccac 100755 --- a/src/storage/tx_storage/ob_access_service.cpp +++ b/src/storage/tx_storage/ob_access_service.cpp @@ -1247,10 +1247,12 @@ int ObAccessService::audit_tablet_opt_dml_stat( } else { dml_stat.delete_row_count_ = affected_rows; } - if (OB_FAIL(ObOptStatMonitorManager::get_instance().update_local_cache(tenant_id_, dml_stat))) { - LOG_WARN("failed to update dml stat local cache", K(ret)); - } else { - LOG_TRACE("succeed to update dml stat local cache", K(dml_stat)); + if (MTL(ObOptStatMonitorManager*) != NULL) { + if (OB_FAIL(MTL(ObOptStatMonitorManager*)->update_local_cache(dml_stat))) { + LOG_WARN("failed to update local cache", K(ret)); + } else { + LOG_TRACE("succeed to update dml stat local cache", K(dml_stat)); + } } //last_access_ts = ObClockGenerator::getClock(); } diff --git a/src/storage/tx_storage/ob_tenant_memory_printer.cpp b/src/storage/tx_storage/ob_tenant_memory_printer.cpp index d375a75da..f89cb46bf 100644 --- a/src/storage/tx_storage/ob_tenant_memory_printer.cpp +++ b/src/storage/tx_storage/ob_tenant_memory_printer.cpp @@ -19,6 +19,7 @@ #include "share/allocator/ob_memstore_allocator_mgr.h" // ObMemstoreAllocatorMgr #include "storage/tx_storage/ob_tenant_freezer.h" // ObTenantFreezer #include "storage/tx_storage/ob_tenant_memory_printer.h" +#include "deps/oblib/src/lib/alloc/malloc_hook.h" namespace oceanbase { @@ -68,8 +69,9 @@ int ObTenantMemoryPrinter::print_tenant_usage() } else { if (OB_FAIL(databuff_printf(print_buf, BUF_LEN, pos, "=== TENANTS MEMORY INFO ===\n" - "all_tenants_memstore_used=% '15ld\n", - allocator_mgr->get_all_tenants_memstore_used()))) { + "all_tenants_memstore_used=% '15ld, divisive_memory_used=% '15ld\n", + allocator_mgr->get_all_tenants_memstore_used(), + get_divisive_mem_size()))) { LOG_WARN("print failed", K(ret)); } else if (OB_FAIL(ObVirtualTenantManager::get_instance().print_tenant_usage(print_buf, BUF_LEN, diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result index f5031bf58..734296a9c 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result @@ -983,6 +983,8 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 21447 DBA_OB_TENANT_EVENT_HISTORY 1 201001 1 21448 CDB_OB_TENANT_EVENT_HISTORY 1 201001 1 21449 GV$OB_FLT_TRACE_CONFIG 1 201001 1 +21477 GV$OB_TENANT_RUNTIME_INFO 1 201001 1 +21478 V$OB_TENANT_RUNTIME_INFO 1 201001 1 check sys table count and table_id range success check count and table_id range for virtual table success select * from information_schema.CHARACTER_SETS limit 1; diff --git a/unittest/share/CMakeLists.txt b/unittest/share/CMakeLists.txt index 42ea8d565..b4a9722e0 100644 --- a/unittest/share/CMakeLists.txt +++ b/unittest/share/CMakeLists.txt @@ -39,7 +39,7 @@ storage_unittest(test_dag_net_in_dag_scheduler scheduler/test_dag_net_in_dag_sch #ob_unittest(test_ob_log) #ob_unittest(test_truncated_string) #ob_unittest(test_latch) -#ob_unittest(test_debug_sync) +ob_unittest(test_debug_sync) #ob_unittest(test_locality_priority) #ob_unittest(test_primary_zone_util) #ob_unittest(test_time_zone_info_manager) diff --git a/unittest/share/cache/test_cache_working_set.cpp b/unittest/share/cache/test_cache_working_set.cpp index d11b27430..2a24d5de9 100644 --- a/unittest/share/cache/test_cache_working_set.cpp +++ b/unittest/share/cache/test_cache_working_set.cpp @@ -158,7 +158,6 @@ TEST_F(TestCacheWorkingSet, cache_size_increase) // close background wash timer task // ObKVGlobalCache::get_instance().wash_timer_.stop(); // ObKVGlobalCache::get_instance().wash_timer_.wait(); - ObKVGlobalCache::get_instance().wash_task_.cancelCallBack(); Key key; Value value; diff --git a/unittest/share/location_cache/test_tablet_ls_map.cpp b/unittest/share/location_cache/test_tablet_ls_map.cpp index d58bd9643..156c30b99 100644 --- a/unittest/share/location_cache/test_tablet_ls_map.cpp +++ b/unittest/share/location_cache/test_tablet_ls_map.cpp @@ -92,7 +92,7 @@ public: { assert(NULL != tablet_ls_map); int ret = OB_SUCCESS; - int tid = get_tid(); + uint64_t tid = get_thread_idx(); LOG_INFO("thread start", K(tid), K(tablet_ls_map->size())); // generate data diff --git a/unittest/share/test_debug_sync.cpp b/unittest/share/test_debug_sync.cpp index 770061ab1..e953f766f 100644 --- a/unittest/share/test_debug_sync.cpp +++ b/unittest/share/test_debug_sync.cpp @@ -428,11 +428,32 @@ TEST(debug_sync, ObDebugSync) } } +TEST(debug_sync, debug_sync_action_overflow) +{ + ObDSActionArray dsa; + ObDebugSyncAction action; + action.sync_point_ = NOW; + action.signal_ = "a"; + action.wait_ = "b"; + action.execute_ = 1; + ASSERT_TRUE(action.is_valid()); + for (int i = 0; i < ObDSActionArray::MAX_DEBUG_SYNC_CACHED_POINT; i++) { + action.sync_point_ = (oceanbase::common::ObDebugSyncPoint)(i + 1); + ASSERT_EQ(OB_SUCCESS, dsa.add_action(action)); + ASSERT_TRUE(dsa.is_active((oceanbase::common::ObDebugSyncPoint)(i + 1 ))); + } + action.sync_point_ = (oceanbase::common::ObDebugSyncPoint)(ObDSActionArray::MAX_DEBUG_SYNC_CACHED_POINT + 1); + ASSERT_EQ(OB_SIZE_OVERFLOW, dsa.add_action(action)); + ASSERT_FALSE(dsa.is_active((oceanbase::common::ObDebugSyncPoint)(ObDSActionArray::MAX_DEBUG_SYNC_CACHED_POINT + 1))); +} + } } int main(int argc, char **argv) { + oceanbase::common::ObLogger::get_logger().set_log_level("INFO"); + OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc,argv); return RUN_ALL_TESTS(); } diff --git a/unittest/share/test_geo_bin.cpp b/unittest/share/test_geo_bin.cpp index 57aaa5760..e8fe5bf01 100644 --- a/unittest/share/test_geo_bin.cpp +++ b/unittest/share/test_geo_bin.cpp @@ -17,7 +17,7 @@ #include "lib/geo/ob_geo_reverse_coordinate_visitor.h" #include "lib/geo/ob_srs_info.h" #include "lib/geo/ob_geo_utils.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "share/schema/ob_multi_version_schema_service.h" #include "lib/random/ob_random.h" #undef private @@ -1988,9 +1988,9 @@ TEST_F(TestGeoBin, wkb_size_visitor_geom_collection) ASSERT_EQ(data.length(), visitor.geo_size()); } -int mock_get_tenant_srs_item(ObIAllocator &allocator, omt::ObTenantSrsMgr &srs_mgr, uint64_t tenant_id, uint64_t srs_id, const ObSrsItem *&srs_item) +int mock_get_tenant_srs_item(ObIAllocator &allocator, uint64_t tenant_id, uint64_t srs_id, const ObSrsItem *&srs_item) { - UNUSEDx(srs_mgr, tenant_id); + UNUSEDx( tenant_id); int ret = OB_SUCCESS; ObGeographicRs rs; rs.rs_name.assign_ptr("ED50", strlen("ED50")); @@ -2027,17 +2027,7 @@ int mock_get_tenant_srs_item(ObIAllocator &allocator, omt::ObTenantSrsMgr &srs_m void get_srs_item(ObIAllocator &allocator, uint64_t srs_id, const ObSrsItem *&srs_item) { - static bool is_inited = false; - common::ObMySQLProxy sql_proxy; - omt::ObTenantSrsMgr &tenant_srs_mgr_instance = OTSRS_MGR; - ObAddr tmp_addr(ObAddr::IPV4, "127.0.0.1", 80); - ObMultiVersionSchemaService &tmp_schema_service = ObMultiVersionSchemaService::get_instance(); - if (!is_inited) { - ASSERT_EQ(OB_SUCCESS, tenant_srs_mgr_instance.init(&sql_proxy, tmp_addr, &tmp_schema_service)); - is_inited = true; - ASSERT_EQ(OB_SUCCESS, mock_get_tenant_srs_item(allocator, tenant_srs_mgr_instance, OB_SYS_TENANT_ID, srs_id, srs_item)); - } - + ASSERT_EQ(OB_SUCCESS, mock_get_tenant_srs_item(allocator, OB_SYS_TENANT_ID, srs_id, srs_item)); } TEST_F(TestGeoBin, coordinate_range_visitor_point) diff --git a/unittest/share/test_geo_srs.cpp b/unittest/share/test_geo_srs.cpp index 32aae959d..06d385b0c 100644 --- a/unittest/share/test_geo_srs.cpp +++ b/unittest/share/test_geo_srs.cpp @@ -4,7 +4,7 @@ #include "lib/geo/ob_srs_info.h" #include "lib/geo/ob_srs_wkt_parser.h" #include "lib/random/ob_random.h" -#include "observer/omt/ob_tenant_srs_mgr.h" +#include "observer/omt/ob_tenant_srs.h" #include "share/schema/ob_multi_version_schema_service.h" #undef private #include diff --git a/unittest/sql/test_sql_utils.cpp b/unittest/sql/test_sql_utils.cpp index 5fc15e498..01e1a4687 100644 --- a/unittest/sql/test_sql_utils.cpp +++ b/unittest/sql/test_sql_utils.cpp @@ -189,7 +189,7 @@ void TestSqlUtils::init() //common::ObDITenantCache::get_instance().init(100000, 4); schema_service_ = new MockSchemaService(); ASSERT_TRUE(schema_service_); - ObVirtualTenantManager::get_instance().init(10); + ObVirtualTenantManager::get_instance().init(); ObVirtualTenantManager::get_instance().add_tenant(sys_tenant_id_); ObVirtualTenantManager::get_instance().set_tenant_mem_limit(sys_tenant_id_, 1024L * 1024L * 1024L, 1024L * 1024L * 1024L); if (OB_SUCCESS != (ret = ObPreProcessSysVars::init_sys_var())) {