diff --git a/cmake/Env.cmake b/cmake/Env.cmake index 866e87eee..008b9bbbb 100644 --- a/cmake/Env.cmake +++ b/cmake/Env.cmake @@ -54,8 +54,8 @@ ob_define(THIN_LTO_OPT "") ob_define(THIN_LTO_CONCURRENCY_LINK "") if(ENABLE_THIN_LTO) - set(THIN_LTO_OPT "-flto=thin") - set(THIN_LTO_CONCURRENCY_LINK "-Wl,--thinlto-jobs=32") + set(THIN_LTO_OPT "-flto=thin -fwhole-program-vtables") + set(THIN_LTO_CONCURRENCY_LINK "-Wl,--thinlto-jobs=32,--lto-whole-program-visibility") endif() diff --git a/deps/oblib/src/lib/alloc/memory_dump.cpp b/deps/oblib/src/lib/alloc/memory_dump.cpp index 4d439cdab..03afe6f5b 100644 --- a/deps/oblib/src/lib/alloc/memory_dump.cpp +++ b/deps/oblib/src/lib/alloc/memory_dump.cpp @@ -22,6 +22,7 @@ #include "lib/thread/thread_mgr.h" #include "lib/utility/ob_print_utils.h" #include "rpc/obrpc/ob_rpc_packet.h" +#include "common/ob_clock_generator.h" namespace oceanbase { @@ -177,14 +178,10 @@ void ObMemoryDump::destroy() int ObMemoryDump::push(void *task) { int ret = OB_SUCCESS; - const bool enable_dump = lib::is_trace_log_enabled(); if (!is_inited_) { ret = OB_NOT_INIT; } else if (NULL == task) { ret = OB_INVALID_ARGUMENT; - } else if (!enable_dump) { - // do nothing - free_task(task); } else { ret = queue_.push(task); if (OB_SIZE_OVERFLOW == ret) { @@ -210,14 +207,13 @@ void ObMemoryDump::run1() SANITY_DISABLE_CHECK_RANGE(); // prevent sanity_check_range int ret = OB_SUCCESS; lib::set_thread_name("MemoryDump"); - static int64_t last_dump_ts = ObTimeUtility::current_time(); - const bool enable_dump = lib::is_trace_log_enabled(); - while (!has_set_stop() && enable_dump) { + static int64_t last_dump_ts = common::ObClockGenerator::getClock(); + while (!has_set_stop()) { void *task = NULL; if (OB_SUCC(queue_.pop(task, 100 * 1000))) { handle(task); } else if (OB_ENTRY_NOT_EXIST == ret) { - int64_t current_ts = ObTimeUtility::current_time(); + int64_t current_ts = common::ObClockGenerator::getClock(); if (current_ts - last_dump_ts > STAT_LABEL_INTERVAL) { auto *task = alloc_task(); if (OB_ISNULL(task)) { @@ -230,6 +226,8 @@ void ObMemoryDump::run1() } } last_dump_ts = current_ts; + } else { + ob_usleep(current_ts - last_dump_ts); } } } @@ -384,13 +382,9 @@ int label_stat(AChunk *chunk, ABlock *block, AObject *object, } else { hold = align_up2(chunk->alloc_bytes_ + ACHUNK_HEADER_SIZE, get_page_size()); } - char label[AOBJECT_LABEL_SIZE + 1]; - STRNCPY(label, object->label_, sizeof(label)); - label[sizeof(label) - 1] = '\0'; - int len = strlen(label); - ObString str(len, label); LabelItem *litem = nullptr; - LabelInfoItem *linfoitem = lmap.get(str); + auto key = std::make_pair(*(uint64_t*)object->label_, *((uint64_t*)object->label_ + 1)); + LabelInfoItem *linfoitem = lmap.get(key); int64_t bt_size = object->on_malloc_sample_ ? AOBJECT_BACKTRACE_SIZE : 0; if (NULL != linfoitem) { // exist @@ -412,15 +406,15 @@ int label_stat(AChunk *chunk, ABlock *block, AObject *object, LOG_WARN("label cnt too large", K(ret), K(item_cap), K(item_used)); } else { litem = &items[item_used++]; - MEMCPY(litem->str_, label, len); - litem->str_[len] = '\0'; - litem->str_len_ = len; + STRNCPY(litem->str_, object->label_, sizeof(litem->str_)); + litem->str_[sizeof(litem->str_) - 1] = '\0'; + litem->str_len_ = strlen(litem->str_); litem->hold_ = hold; litem->used_ = (object->alloc_bytes_ - bt_size); litem->count_ = 1; litem->block_cnt_ = 1; litem->chunk_cnt_ = 1; - ret = lmap.set_refactored(ObString(litem->str_len_, litem->str_), LabelInfoItem(litem, chunk, block)); + ret = lmap.set_refactored(key, LabelInfoItem(litem, chunk, block)); } } } diff --git a/deps/oblib/src/lib/alloc/memory_dump.h b/deps/oblib/src/lib/alloc/memory_dump.h index 3da8bb35f..d8258fab5 100644 --- a/deps/oblib/src/lib/alloc/memory_dump.h +++ b/deps/oblib/src/lib/alloc/memory_dump.h @@ -104,7 +104,7 @@ struct LabelInfoItem void *block_; }; -typedef common::hash::ObHashMap LabelMap; +typedef common::hash::ObHashMap, LabelInfoItem, hash::NoPthreadDefendMode> LabelMap; using lib::AChunk; using lib::ABlock; diff --git a/deps/oblib/src/lib/alloc/object_set.cpp b/deps/oblib/src/lib/alloc/object_set.cpp index 0121cacd9..8f6688237 100644 --- a/deps/oblib/src/lib/alloc/object_set.cpp +++ b/deps/oblib/src/lib/alloc/object_set.cpp @@ -91,7 +91,7 @@ AObject *ObjectSet::alloc_object( STRNCPY(&obj->label_[0], attr.label_.str_, sizeof(obj->label_)); obj->label_[sizeof(obj->label_) - 1] = '\0'; } else { - obj->label_[0] = '\0'; + MEMSET(obj->label_, '\0', sizeof(obj->label_)); } allocs_++; alloc_bytes_ += size; diff --git a/deps/oblib/src/lib/lock/ob_bucket_lock.h b/deps/oblib/src/lib/lock/ob_bucket_lock.h index aaace4fc9..8c9af0725 100644 --- a/deps/oblib/src/lib/lock/ob_bucket_lock.h +++ b/deps/oblib/src/lib/lock/ob_bucket_lock.h @@ -17,6 +17,7 @@ #include "lib/allocator/ob_mod_define.h" #include "lib/container/ob_array.h" #include "lib/time/ob_tsc_timestamp.h" +#include "common/ob_clock_generator.h" namespace oceanbase { @@ -83,7 +84,7 @@ public: if (OB_UNLIKELY(OB_SUCCESS != (ret_ = lock_.rdlock(index_)))) { COMMON_LOG_RET(WARN, ret_, "Fail to read lock bucket, ", K_(index), K_(ret)); } else { - lock_start_ts_ = ObTimeUtility::fast_current_time(); + lock_start_ts_ = common::ObClockGenerator::getClock(); } }; ~ObBucketRLockGuard() @@ -92,7 +93,7 @@ public: if (OB_UNLIKELY(OB_SUCCESS != (ret_ = lock_.unlock(index_)))) { COMMON_LOG_RET(WARN, ret_, "Fail to unlock bucket, ", K_(ret)); } else { - const int64_t lock_end_ts = ObTimeUtility::fast_current_time(); + const int64_t lock_end_ts = common::ObClockGenerator::getClock(); if (lock_end_ts - lock_start_ts_ > 5 * 1000 * 1000) { STORAGE_LOG(INFO, "bucket lock handle cost too much time", K_(lock_start_ts), @@ -124,7 +125,7 @@ public: if (OB_UNLIKELY(OB_SUCCESS != (ret_ = lock_.wrlock(index_)))) { COMMON_LOG_RET(WARN, ret_, "Fail to write lock bucket, ", K_(index), K_(ret)); } else { - lock_start_ts_ = OB_TSC_TIMESTAMP.current_time(); + lock_start_ts_ = common::ObClockGenerator::getClock(); } }; ~ObBucketWLockGuard() @@ -133,7 +134,7 @@ public: if (OB_UNLIKELY(OB_SUCCESS != (ret_ = lock_.unlock(index_)))) { COMMON_LOG_RET(WARN, ret_, "Fail to unlock bucket, ", K_(index), K_(ret)); } else { - const int64_t lock_end_ts = OB_TSC_TIMESTAMP.current_time(); + const int64_t lock_end_ts = common::ObClockGenerator::getClock(); if (lock_end_ts - lock_start_ts_ > 5 * 1000 * 1000) { STORAGE_LOG(INFO, "bucket lock handle cost too much time", K_(lock_start_ts), @@ -254,7 +255,7 @@ public: } } } else { - lock_start_ts_ = ObTimeUtility::current_time(); + lock_start_ts_ = common::ObClockGenerator::getClock(); } }; ~ObBucketTryRLockAllGuard() @@ -263,7 +264,7 @@ public: if (OB_UNLIKELY(OB_SUCCESS != (ret_ = lock_.unlock_all()))) { COMMON_LOG_RET(WARN, ret_, "Fail to unlock all buckets, ", K_(ret)); } else { - const int64_t lock_end_ts = ObTimeUtility::current_time(); + const int64_t lock_end_ts = common::ObClockGenerator::getClock(); if (lock_end_ts - lock_start_ts_ > 5 * 1000 * 1000) { STORAGE_LOG(INFO, "bucket lock handle cost too much time", K_(lock_start_ts), diff --git a/deps/oblib/src/lib/ob_define.h b/deps/oblib/src/lib/ob_define.h index fded32473..6bdda5dba 100644 --- a/deps/oblib/src/lib/ob_define.h +++ b/deps/oblib/src/lib/ob_define.h @@ -802,26 +802,15 @@ const double MONITOR_MEM_FACTOR = 0.01; const double KVCACHE_FACTOR = TENANT_RESERVE_MEM_RATIO; const double MIN_TENANT_QUOTA = .5; -const double EXT_LOG_TENANT_CPU = 4.; -const int64_t EXT_LOG_TENANT_MEMORY_LIMIT = 4L << 30; -const double OB_MONITOR_CPU = 1.; -const double OB_DTL_CPU = 5.; -const double OB_DIAG_CPU = 1.0; -const double OB_DATA_CPU = 2.5; -const double OB_RS_CPU = 1.0; -const double OB_SVR_BLACKLIST_CPU = 1.0; -const int64_t OB_RS_MEMORY = 2L << 30; +const double OB_DTL_CPU = (sysconf(_SC_NPROCESSORS_ONLN) <= 4) ? 1. : 5.; +const double OB_DATA_CPU = (sysconf(_SC_NPROCESSORS_ONLN) <= 4) ? 1. : 2.5; const uint64_t OB_INVALID_TENANT_ID = 0; const uint64_t OB_SYS_TENANT_ID = 1; const uint64_t OB_GTS_TENANT_ID = 2; const uint64_t OB_SERVER_TENANT_ID = 500; -const uint64_t OB_ELECT_TENANT_ID = 501; -const uint64_t OB_EXT_LOG_TENANT_ID = 506; -// const uint64_t OB_MONITOR_TENANT_ID = 507; const uint64_t OB_DTL_TENANT_ID = 508; const uint64_t OB_DATA_TENANT_ID = 509; -const uint64_t OB_RS_TENANT_ID = 510; const uint64_t OB_GTS_SOURCE_TENANT_ID = 511; const uint64_t OB_SVR_BLACKLIST_TENANT_ID = 512; const uint64_t OB_MAX_RESERVED_TENANT_ID = 1000; @@ -1851,9 +1840,7 @@ OB_INLINE bool is_valid_cluster_id(const int64_t cluster_id) OB_INLINE bool is_virtual_tenant_for_memory(const uint64_t tenant_id) { - return is_virtual_tenant_id(tenant_id) && - (OB_EXT_LOG_TENANT_ID == tenant_id || - OB_RS_TENANT_ID == tenant_id); + return is_virtual_tenant_id(tenant_id); } enum ObNameCaseMode diff --git a/deps/oblib/src/lib/ob_name_id_def.h b/deps/oblib/src/lib/ob_name_id_def.h index d3d4de657..8971740bd 100644 --- a/deps/oblib/src/lib/ob_name_id_def.h +++ b/deps/oblib/src/lib/ob_name_id_def.h @@ -844,6 +844,33 @@ DEF_NAME(id, "id") DEF_NAME(update_cluster_info, "update_cluster_info") DEF_NAME(update_master_key_info, "update_master_key_info") DEF_NAME(do_heartbeat_event, "do_heartbeat_event") + + //ObTenantInfoLoader + DEF_NAME(ob_tenant_info_loader, "ob_tenant_info_loader") + DEF_NAME(cost_time_us, "cost_time_us") + DEF_NAME(broadcast_times, "broadcast_times") + DEF_NAME(is_inited, "is_inited") + DEF_NAME(tenant_info_cache, "tenant_info_cache") + DEF_NAME(tenant_info, "tenant_info") + DEF_NAME(ora_rowscn, "ora_rowscn") + DEF_NAME(switchover_epoch, "switchover_epoch") + DEF_NAME(sync_scn, "sync_scn") + DEF_NAME(replayable_scn, "replayable_scn") + DEF_NAME(standby_scn, "standby_scn") + DEF_NAME(recovery_until_scn, "recovery_until_scn") + DEF_NAME(log_mode, "log_mode") + DEF_NAME(tenant_role, "tenant_role") + DEF_NAME(switchover_status, "switchover_status") + DEF_NAME(is_sys_ls_leader, "is_sys_ls_leader") + DEF_NAME(broadcast_cost_time, "broadcast_cost_time") + DEF_NAME(broadcast_per_sec, "broadcast_per_sec") + DEF_NAME(rpc_update_times, "rpc_update_times") + DEF_NAME(rpc_update_per_sec, "rpc_update_per_sec") + DEF_NAME(last_rpc_update_time_us, "last_rpc_update_time_us") + DEF_NAME(sql_update_cost_time, "sql_update_cost_time") + DEF_NAME(sql_update_times, "sql_update_times") + DEF_NAME(last_sql_update_time, "last_sql_update_time") + // << add pair events BEFORE this line DEF_NAME(NAME_COUNT, "invalid") #endif diff --git a/deps/oblib/src/lib/ob_running_mode.cpp b/deps/oblib/src/lib/ob_running_mode.cpp index 450ffc695..f2befd16f 100644 --- a/deps/oblib/src/lib/ob_running_mode.cpp +++ b/deps/oblib/src/lib/ob_running_mode.cpp @@ -18,5 +18,6 @@ namespace lib { const int64_t ObRunningModeConfig::MIN_MEM = 1L << 30; // The minimum value for memory_limit. const int64_t ObRunningModeConfig::MINI_MEM_LOWER = 4L << 30; const int64_t ObRunningModeConfig::MINI_MEM_UPPER = 12L << 30; +const int64_t ObRunningModeConfig::MINI_CPU_UPPER = 8; } //end of namespace lib } //end of namespace oceanbase diff --git a/deps/oblib/src/lib/ob_running_mode.h b/deps/oblib/src/lib/ob_running_mode.h index 169a4633e..9b085645c 100644 --- a/deps/oblib/src/lib/ob_running_mode.h +++ b/deps/oblib/src/lib/ob_running_mode.h @@ -23,7 +23,9 @@ struct ObRunningModeConfig static const int64_t MIN_MEM; static const int64_t MINI_MEM_LOWER; static const int64_t MINI_MEM_UPPER; + static const int64_t MINI_CPU_UPPER; bool mini_mode_ = false; + bool mini_cpu_mode_ = false; int64_t memory_limit_ = 0; static ObRunningModeConfig &instance(); private: @@ -41,10 +43,16 @@ inline bool is_mini_mode() return ObRunningModeConfig::instance().mini_mode_; } -inline void update_mini_mode(int64_t memory_limit) +inline bool is_mini_cpu_mode() +{ + return ObRunningModeConfig::instance().mini_cpu_mode_; +} + +inline void update_mini_mode(int64_t memory_limit, int64_t cpu_cnt) { ObRunningModeConfig::instance().memory_limit_ = memory_limit; ObRunningModeConfig::instance().mini_mode_ = (memory_limit < lib::ObRunningModeConfig::MINI_MEM_UPPER); + ObRunningModeConfig::instance().mini_cpu_mode_ = (cpu_cnt <= lib::ObRunningModeConfig::MINI_CPU_UPPER); } } //lib diff --git a/deps/oblib/src/lib/queue/ob_dedup_queue.h b/deps/oblib/src/lib/queue/ob_dedup_queue.h index 1e7f781e5..8f4a7ac83 100644 --- a/deps/oblib/src/lib/queue/ob_dedup_queue.h +++ b/deps/oblib/src/lib/queue/ob_dedup_queue.h @@ -192,8 +192,8 @@ private: typedef hash::HashMapTypes::pair_type TaskMapKVPair; static const int32_t DEFAULT_THREAD_NUM = 4; static const int32_t MAX_THREAD_NUM = 64; - static const int32_t QUEUE_WAIT_TIME_MS = 10; //10ms - static const int32_t MAX_QUEUE_WAIT_TIME_MS = 100; //100ms + static const int32_t QUEUE_WAIT_TIME_MS = 50; //50ms + static const int32_t MAX_QUEUE_WAIT_TIME_MS = 500; //500ms static const int64_t GC_BATCH_NUM = 512; static const int64_t DEFALT_THREAD_DEAD_THRESHOLD = 30000000L; //30s static const int64_t THREAD_CHECK_INTERVAL = 10000000L; //10s diff --git a/deps/oblib/src/lib/rc/context.h b/deps/oblib/src/lib/rc/context.h index e30a4f46a..294737253 100644 --- a/deps/oblib/src/lib/rc/context.h +++ b/deps/oblib/src/lib/rc/context.h @@ -30,6 +30,7 @@ #include "lib/thread_local/ob_tsi_factory.h" #include "lib/list/ob_dlist.h" #include "lib/hash/ob_hashmap.h" +#include "common/ob_clock_generator.h" #ifdef OB_USE_ASAN #include "lib/allocator/ob_asan_allocator.h" @@ -258,7 +259,7 @@ struct DynamicInfo { DynamicInfo() : tid_(GETTID()), cid_(0lu), - create_time_(common::ObTimeUtility::fast_current_time()) + create_time_(common::ObClockGenerator::getClock()) {} TO_STRING_KV(K(tid_), K(cid_), K(create_time_)); int64_t tid_; diff --git a/deps/oblib/src/lib/statistic_event/ob_stat_event.h b/deps/oblib/src/lib/statistic_event/ob_stat_event.h index 211a4a251..5792d2b04 100644 --- a/deps/oblib/src/lib/statistic_event/ob_stat_event.h +++ b/deps/oblib/src/lib/statistic_event/ob_stat_event.h @@ -13,173 +13,133 @@ #ifdef STAT_EVENT_ADD_DEF // NETWORK STAT_EVENT_ADD_DEF(RPC_PACKET_IN, "rpc packet in", ObStatClassIds::NETWORK, "rpc packet in", 10000, true, true) -STAT_EVENT_ADD_DEF(RPC_PACKET_IN_BYTES, "rpc packet in bytes", ObStatClassIds::NETWORK, "rpc packet in bytes", 10001, true, true) +STAT_EVENT_ADD_DEF(RPC_PACKET_IN_BYTES, "rpc packet in bytes", ObStatClassIds::NETWORK, "rpc packet in bytes", 10001, false, true) STAT_EVENT_ADD_DEF(RPC_PACKET_OUT, "rpc packet out", ObStatClassIds::NETWORK, "rpc packet out", 10002, true, true) -STAT_EVENT_ADD_DEF(RPC_PACKET_OUT_BYTES, "rpc packet out bytes", ObStatClassIds::NETWORK, "rpc packet out bytes", 10003, true, true) -STAT_EVENT_ADD_DEF(RPC_DELIVER_FAIL, "rpc deliver fail", ObStatClassIds::NETWORK, "rpc deliver fail", 10004, true, true) -STAT_EVENT_ADD_DEF(RPC_NET_DELAY, "rpc net delay", ObStatClassIds::NETWORK, "rpc net delay", 10005, true, true) -STAT_EVENT_ADD_DEF(RPC_NET_FRAME_DELAY, "rpc net frame delay", ObStatClassIds::NETWORK, "rpc net frame delay", 10006, true, true) -STAT_EVENT_ADD_DEF(MYSQL_PACKET_IN, "mysql packet in", ObStatClassIds::NETWORK, "mysql packet in", 10007, true, true) -STAT_EVENT_ADD_DEF(MYSQL_PACKET_IN_BYTES, "mysql packet in bytes", ObStatClassIds::NETWORK, "mysql packet in bytes", 10008, true, true) -STAT_EVENT_ADD_DEF(MYSQL_PACKET_OUT, "mysql packet out", ObStatClassIds::NETWORK, "mysql packet out", 10009, true, true) -STAT_EVENT_ADD_DEF(MYSQL_PACKET_OUT_BYTES, "mysql packet out bytes", ObStatClassIds::NETWORK, "mysql packet out bytes", 10010, true, true) -STAT_EVENT_ADD_DEF(MYSQL_DELIVER_FAIL, "mysql deliver fail", ObStatClassIds::NETWORK, "mysql deliver fail", 10011, true, true) -STAT_EVENT_ADD_DEF(RPC_COMPRESS_ORIGINAL_PACKET_CNT, "rpc compress original packet cnt", ObStatClassIds::NETWORK, "rpc compress original packet cnt", 10012, true, true) -STAT_EVENT_ADD_DEF(RPC_COMPRESS_COMPRESSED_PACKET_CNT, "rpc compress compressed packet cnt", ObStatClassIds::NETWORK, "rpc compress compressed pacekt cnt", 10013, true, true) -STAT_EVENT_ADD_DEF(RPC_COMPRESS_ORIGINAL_SIZE, "rpc compress original size", ObStatClassIds::NETWORK, "rpc compress original size", 10014, true, true) -STAT_EVENT_ADD_DEF(RPC_COMPRESS_COMPRESSED_SIZE, "rpc compress compressed size", ObStatClassIds::NETWORK, "rpc compress compressed size", 10015, true, true) +STAT_EVENT_ADD_DEF(RPC_PACKET_OUT_BYTES, "rpc packet out bytes", ObStatClassIds::NETWORK, "rpc packet out bytes", 10003, false, true) +STAT_EVENT_ADD_DEF(RPC_DELIVER_FAIL, "rpc deliver fail", ObStatClassIds::NETWORK, "rpc deliver fail", 10004, false, true) +STAT_EVENT_ADD_DEF(RPC_NET_DELAY, "rpc net delay", ObStatClassIds::NETWORK, "rpc net delay", 10005, false, true) +STAT_EVENT_ADD_DEF(RPC_NET_FRAME_DELAY, "rpc net frame delay", ObStatClassIds::NETWORK, "rpc net frame delay", 10006, false, true) +STAT_EVENT_ADD_DEF(MYSQL_PACKET_IN, "mysql packet in", ObStatClassIds::NETWORK, "mysql packet in", 10007, false, true) +STAT_EVENT_ADD_DEF(MYSQL_PACKET_IN_BYTES, "mysql packet in bytes", ObStatClassIds::NETWORK, "mysql packet in bytes", 10008, false, true) +STAT_EVENT_ADD_DEF(MYSQL_PACKET_OUT, "mysql packet out", ObStatClassIds::NETWORK, "mysql packet out", 10009, false, true) +STAT_EVENT_ADD_DEF(MYSQL_PACKET_OUT_BYTES, "mysql packet out bytes", ObStatClassIds::NETWORK, "mysql packet out bytes", 10010, false, true) +STAT_EVENT_ADD_DEF(MYSQL_DELIVER_FAIL, "mysql deliver fail", ObStatClassIds::NETWORK, "mysql deliver fail", 10011, false, true) +STAT_EVENT_ADD_DEF(RPC_COMPRESS_ORIGINAL_PACKET_CNT, "rpc compress original packet cnt", ObStatClassIds::NETWORK, "rpc compress original packet cnt", 10012, false, true) +STAT_EVENT_ADD_DEF(RPC_COMPRESS_COMPRESSED_PACKET_CNT, "rpc compress compressed packet cnt", ObStatClassIds::NETWORK, "rpc compress compressed pacekt cnt", 10013, false, true) +STAT_EVENT_ADD_DEF(RPC_COMPRESS_ORIGINAL_SIZE, "rpc compress original size", ObStatClassIds::NETWORK, "rpc compress original size", 10014, false, true) +STAT_EVENT_ADD_DEF(RPC_COMPRESS_COMPRESSED_SIZE, "rpc compress compressed size", ObStatClassIds::NETWORK, "rpc compress compressed size", 10015, false, true) -STAT_EVENT_ADD_DEF(RPC_STREAM_COMPRESS_ORIGINAL_PACKET_CNT, "rpc stream compress original packet cnt", ObStatClassIds::NETWORK, "rpc stream compress original packet cnt", 10016, true, true) -STAT_EVENT_ADD_DEF(RPC_STREAM_COMPRESS_COMPRESSED_PACKET_CNT, "rpc stream compress compressed packet cnt", ObStatClassIds::NETWORK, "rpc stream compress compressed pacekt cnt", 10017, true, true) -STAT_EVENT_ADD_DEF(RPC_STREAM_COMPRESS_ORIGINAL_SIZE, "rpc stream compress original size", ObStatClassIds::NETWORK, "rpc stream compress original size", 10018, true, true) -STAT_EVENT_ADD_DEF(RPC_STREAM_COMPRESS_COMPRESSED_SIZE, "rpc stream compress compressed size", ObStatClassIds::NETWORK, "rpc stream compress compressed size", 10019, true, true) +STAT_EVENT_ADD_DEF(RPC_STREAM_COMPRESS_ORIGINAL_PACKET_CNT, "rpc stream compress original packet cnt", ObStatClassIds::NETWORK, "rpc stream compress original packet cnt", 10016, false, true) +STAT_EVENT_ADD_DEF(RPC_STREAM_COMPRESS_COMPRESSED_PACKET_CNT, "rpc stream compress compressed packet cnt", ObStatClassIds::NETWORK, "rpc stream compress compressed pacekt cnt", 10017, false, true) +STAT_EVENT_ADD_DEF(RPC_STREAM_COMPRESS_ORIGINAL_SIZE, "rpc stream compress original size", ObStatClassIds::NETWORK, "rpc stream compress original size", 10018, false, true) +STAT_EVENT_ADD_DEF(RPC_STREAM_COMPRESS_COMPRESSED_SIZE, "rpc stream compress compressed size", ObStatClassIds::NETWORK, "rpc stream compress compressed size", 10019, false, true) // QUEUE // STAT_EVENT_ADD_DEF(REQUEST_QUEUED_COUNT, "REQUEST_QUEUED_COUNT", QUEUE, "REQUEST_QUEUED_COUNT") -STAT_EVENT_ADD_DEF(REQUEST_ENQUEUE_COUNT, "request enqueue count", ObStatClassIds::QUEUE, "request enqueue count", 20000, true, true) -STAT_EVENT_ADD_DEF(REQUEST_DEQUEUE_COUNT, "request dequeue count", ObStatClassIds::QUEUE, "request dequeue count", 20001, true, true) -STAT_EVENT_ADD_DEF(REQUEST_QUEUE_TIME, "request queue time", ObStatClassIds::QUEUE, "request queue time", 20002, true, true) +STAT_EVENT_ADD_DEF(REQUEST_ENQUEUE_COUNT, "request enqueue count", ObStatClassIds::QUEUE, "request enqueue count", 20000, false, true) +STAT_EVENT_ADD_DEF(REQUEST_DEQUEUE_COUNT, "request dequeue count", ObStatClassIds::QUEUE, "request dequeue count", 20001, false, true) +STAT_EVENT_ADD_DEF(REQUEST_QUEUE_TIME, "request queue time", ObStatClassIds::QUEUE, "request queue time", 20002, false, true) // TRANS -STAT_EVENT_ADD_DEF(TRANS_COMMIT_LOG_SYNC_TIME, "trans commit log sync time", ObStatClassIds::TRANS, "trans commit log sync time", 30000, true, true) -STAT_EVENT_ADD_DEF(TRANS_COMMIT_LOG_SYNC_COUNT, "trans commit log sync count", ObStatClassIds::TRANS, "trans commit log sync count", 30001, true, true) -STAT_EVENT_ADD_DEF(TRANS_COMMIT_LOG_SUBMIT_COUNT, "trans commit log submit count", ObStatClassIds::TRANS, "trans commit log submit count", 30002, true, true) -STAT_EVENT_ADD_DEF(TRANS_SYSTEM_TRANS_COUNT, "trans system trans count", ObStatClassIds::TRANS, "trans system trans count", 30003, true, true) -STAT_EVENT_ADD_DEF(TRANS_USER_TRANS_COUNT, "trans user trans count", ObStatClassIds::TRANS, "trans user trans count", 30004, true, true) -STAT_EVENT_ADD_DEF(TRANS_START_COUNT, "trans start count", ObStatClassIds::TRANS, "trans start count", 30005, true, true) -STAT_EVENT_ADD_DEF(TRANS_TOTAL_USED_TIME, "trans total used time", ObStatClassIds::TRANS, "trans total used time", 30006, true, true) -STAT_EVENT_ADD_DEF(TRANS_COMMIT_COUNT, "trans commit count", ObStatClassIds::TRANS, "trans commit count", 30007, true, true) -STAT_EVENT_ADD_DEF(TRANS_COMMIT_TIME, "trans commit time", ObStatClassIds::TRANS, "trans commit time", 30008, true, true) -STAT_EVENT_ADD_DEF(TRANS_ROLLBACK_COUNT, "trans rollback count", ObStatClassIds::TRANS, "trans rollback count", 30009, true, true) -STAT_EVENT_ADD_DEF(TRANS_ROLLBACK_TIME, "trans rollback time", ObStatClassIds::TRANS, "trans rollback time", 30010, true, true) -STAT_EVENT_ADD_DEF(TRANS_TIMEOUT_COUNT, "trans timeout count", ObStatClassIds::TRANS, "trans timeout count", 30011, true, true) -STAT_EVENT_ADD_DEF(TRANS_LOCAL_COUNT, "trans local trans count", ObStatClassIds::TRANS, "trans local trans count", 30012, true, true) -STAT_EVENT_ADD_DEF(TRANS_DIST_COUNT, "trans distribute trans count", ObStatClassIds::TRANS, "trans dist trans count", 30013, true, true) +STAT_EVENT_ADD_DEF(TRANS_COMMIT_LOG_SYNC_TIME, "trans commit log sync time", ObStatClassIds::TRANS, "trans commit log sync time", 30000, false, true) +STAT_EVENT_ADD_DEF(TRANS_COMMIT_LOG_SYNC_COUNT, "trans commit log sync count", ObStatClassIds::TRANS, "trans commit log sync count", 30001, false, true) +STAT_EVENT_ADD_DEF(TRANS_COMMIT_LOG_SUBMIT_COUNT, "trans commit log submit count", ObStatClassIds::TRANS, "trans commit log submit count", 30002, false, true) +STAT_EVENT_ADD_DEF(TRANS_SYSTEM_TRANS_COUNT, "trans system trans count", ObStatClassIds::TRANS, "trans system trans count", 30003, false, true) +STAT_EVENT_ADD_DEF(TRANS_USER_TRANS_COUNT, "trans user trans count", ObStatClassIds::TRANS, "trans user trans count", 30004, false, true) +STAT_EVENT_ADD_DEF(TRANS_START_COUNT, "trans start count", ObStatClassIds::TRANS, "trans start count", 30005, false, true) +STAT_EVENT_ADD_DEF(TRANS_TOTAL_USED_TIME, "trans total used time", ObStatClassIds::TRANS, "trans total used time", 30006, false, true) +STAT_EVENT_ADD_DEF(TRANS_COMMIT_COUNT, "trans commit count", ObStatClassIds::TRANS, "trans commit count", 30007, false, true) +STAT_EVENT_ADD_DEF(TRANS_COMMIT_TIME, "trans commit time", ObStatClassIds::TRANS, "trans commit time", 30008, false, true) +STAT_EVENT_ADD_DEF(TRANS_ROLLBACK_COUNT, "trans rollback count", ObStatClassIds::TRANS, "trans rollback count", 30009, false, true) +STAT_EVENT_ADD_DEF(TRANS_ROLLBACK_TIME, "trans rollback time", ObStatClassIds::TRANS, "trans rollback time", 30010, false, true) +STAT_EVENT_ADD_DEF(TRANS_TIMEOUT_COUNT, "trans timeout count", ObStatClassIds::TRANS, "trans timeout count", 30011, false, true) +STAT_EVENT_ADD_DEF(TRANS_LOCAL_COUNT, "trans local trans count", ObStatClassIds::TRANS, "trans local trans count", 30012, false, true) +STAT_EVENT_ADD_DEF(TRANS_DIST_COUNT, "trans distribute trans count", ObStatClassIds::TRANS, "trans dist trans count", 30013, false, true) //STAT_EVENT_ADD_DEF(TRANS_DISTRIBUTED_STMT_COUNT, "trans distributed stmt count", TRANS, "trans distributed stmt count", 30014) //STAT_EVENT_ADD_DEF(TRANS_LOCAL_STMT_COUNT, "trans local stmt count", TRANS, "trans local stmt count", 30015) //STAT_EVENT_ADD_DEF(TRANS_REMOTE_STMT_COUNT, "trans remote stmt count", TRANS, "trans remote stmt count", 30016) -STAT_EVENT_ADD_DEF(TRANS_READONLY_COUNT, "trans without participant count", ObStatClassIds::TRANS, "trans without participant count", 30017, true, true) -STAT_EVENT_ADD_DEF(REDO_LOG_REPLAY_COUNT, "redo log replay count", ObStatClassIds::TRANS, "redo log replay count", 30018, true, true) -STAT_EVENT_ADD_DEF(REDO_LOG_REPLAY_TIME, "redo log replay time", ObStatClassIds::TRANS, "redo log replay time", 30019, true, true) -STAT_EVENT_ADD_DEF(PREPARE_LOG_REPLAY_COUNT, "prepare log replay count", ObStatClassIds::TRANS, "prepare log replay count", 30020, true, true) -STAT_EVENT_ADD_DEF(PREPARE_LOG_REPLAY_TIME, "prepare log replay time", ObStatClassIds::TRANS, "prepare log replay time", 30021, true, true) -STAT_EVENT_ADD_DEF(COMMIT_LOG_REPLAY_COUNT, "commit log replay count", ObStatClassIds::TRANS, "commit log replay count", 30022, true, true) -STAT_EVENT_ADD_DEF(COMMIT_LOG_REPLAY_TIME, "commit log replay time", ObStatClassIds::TRANS, "commit log replay time", 30023, true, true) -STAT_EVENT_ADD_DEF(ABORT_LOG_REPLAY_COUNT, "abort log replay count", ObStatClassIds::TRANS, "abort log replay count", 30024, true, true) -STAT_EVENT_ADD_DEF(ABORT_LOG_REPLAY_TIME, "abort log replay time", ObStatClassIds::TRANS, "abort log replay time", 30025, true, true) -STAT_EVENT_ADD_DEF(CLEAR_LOG_REPLAY_COUNT, "clear log replay count", ObStatClassIds::TRANS, "clear log replay count", 30026, true, true) -STAT_EVENT_ADD_DEF(CLEAR_LOG_REPLAY_TIME, "clear log replay time", ObStatClassIds::TRANS, "clear log replay time", 30027, true, true) -STAT_EVENT_ADD_DEF(SP_REDO_LOG_CB_COUNT, "sp redo log cb count", ObStatClassIds::TRANS, "sp redo log cb count", 30028, true, true) -STAT_EVENT_ADD_DEF(SP_REDO_LOG_CB_TIME, "sp redo log cb time", ObStatClassIds::TRANS, "sp redo log cb time", 30029, true, true) -STAT_EVENT_ADD_DEF(SP_COMMIT_LOG_CB_COUNT, "sp commit log cb count", ObStatClassIds::TRANS, "sp commit log cb count", 30030, true, true) -STAT_EVENT_ADD_DEF(SP_COMMIT_LOG_CB_TIME, "sp commit log cb time", ObStatClassIds::TRANS, "sp commit log cb time", 30031, true, true) -STAT_EVENT_ADD_DEF(REDO_LOG_CB_COUNT, "redo log cb count", ObStatClassIds::TRANS, "redo log cb count", 30032, true, true) -STAT_EVENT_ADD_DEF(REDO_LOG_CB_TIME, "redo log cb time", ObStatClassIds::TRANS, "redo log cb time", 30033, true, true) -STAT_EVENT_ADD_DEF(PREPARE_LOG_CB_COUNT, "prepare log cb count", ObStatClassIds::TRANS, "prepare log cb count", 30034, true, true) -STAT_EVENT_ADD_DEF(PREPARE_LOG_CB_TIME, "prepare log cb time", ObStatClassIds::TRANS, "prepare log cb time", 30035, true, true) -STAT_EVENT_ADD_DEF(COMMIT_LOG_CB_COUNT, "commit log cb count", ObStatClassIds::TRANS, "commit log cb count", 30036, true, true) -STAT_EVENT_ADD_DEF(COMMIT_LOG_CB_TIME, "commit log cb time", ObStatClassIds::TRANS, "commit log cb time", 30037, true, true) -STAT_EVENT_ADD_DEF(ABORT_LOG_CB_COUNT, "abort log cb count", ObStatClassIds::TRANS, "abort log cb count", 30038, true, true) -STAT_EVENT_ADD_DEF(ABORT_LOG_CB_TIME, "abort log cb time", ObStatClassIds::TRANS, "abort log cb time", 30039, true, true) -STAT_EVENT_ADD_DEF(CLEAR_LOG_CB_COUNT, "clear log cb count", ObStatClassIds::TRANS, "clear log cb count", 30040, true, true) -STAT_EVENT_ADD_DEF(CLEAR_LOG_CB_TIME, "clear log cb time", ObStatClassIds::TRANS, "clear log cb time", 30041, true, true) -STAT_EVENT_ADD_DEF(TRANS_MT_END_COUNT, "trans memtable end count", ObStatClassIds::TRANS, "trans memtable end count", 30042, true, true) -STAT_EVENT_ADD_DEF(TRANS_MT_END_TIME, "trans memtable end time", ObStatClassIds::TRANS, "trans memtable end time", 30043, true, true) -STAT_EVENT_ADD_DEF(TRANS_CALLBACK_SQL_COUNT, "trans callback sql count", ObStatClassIds::TRANS, "trans callback sql count", 30044, true, true) -STAT_EVENT_ADD_DEF(TRANS_CALLBACK_SQL_TIME, "trans callback sql time", ObStatClassIds::TRANS, "trans callback sql time", 30045, true, true) -STAT_EVENT_ADD_DEF(TRANS_STRONG_CONSISTENCY_STMT_TIMEOUT_COUNT, "strong consistency stmt timeout count", ObStatClassIds::TRANS, "strong consistency stmt timeout count", 30046, true, true) -STAT_EVENT_ADD_DEF(TRANS_SLAVE_READ_STMT_TIMEOUT_COUNT, "slave read stmt timeout count", ObStatClassIds::TRANS, "slave read stmt timeout count", 30047, true, true) -STAT_EVENT_ADD_DEF(TRANS_SLAVE_READ_STMT_RETRY_COUNT, "slave read stmt retry count", ObStatClassIds::TRANS, "slave read stmt retry count", 30048, true, true) -STAT_EVENT_ADD_DEF(TRANS_FILL_REDO_LOG_COUNT, "trans fill redo log count", ObStatClassIds::TRANS, "trans fill redo log count", 30049, true, true) -STAT_EVENT_ADD_DEF(TRANS_FILL_REDO_LOG_TIME, "trans fill redo log time", ObStatClassIds::TRANS, "trans fill redo log time", 30050, true, true) -STAT_EVENT_ADD_DEF(TRANS_SUBMIT_LOG_COUNT, "trans submit log count", ObStatClassIds::TRANS, "trans submit log count", 30051, true, true) -STAT_EVENT_ADD_DEF(TRANS_SUBMIT_LOG_TIME, "trans submit log time", ObStatClassIds::TRANS, "trans submit log time", 30052, true, true) -STAT_EVENT_ADD_DEF(GTS_REQUEST_TOTAL_COUNT, "gts request total count", ObStatClassIds::TRANS, "gts request total count", 30053, true, true) -STAT_EVENT_ADD_DEF(GTS_ACQUIRE_TOTAL_TIME, "gts acquire total time", ObStatClassIds::TRANS, "gts acquire total time", 30054, true, true) -STAT_EVENT_ADD_DEF(GTS_ACQUIRE_TOTAL_COUNT, "gts acquire total count", ObStatClassIds::TRANS, "gts acquire total count", 30055, true, true) -STAT_EVENT_ADD_DEF(GTS_ACQUIRE_TOTAL_WAIT_COUNT, "gts acquire total wait count", ObStatClassIds::TRANS, "gts acquire total wait count", 30056, true, true) -STAT_EVENT_ADD_DEF(TRANS_STMT_TOTAL_COUNT, "trans stmt total count", ObStatClassIds::TRANS, "trans stmt total count", 30057, true, true) -STAT_EVENT_ADD_DEF(TRANS_STMT_INTERVAL_TIME, "trans stmt interval time", ObStatClassIds::TRANS, "trans stmt interval time", 30058, true, true) -STAT_EVENT_ADD_DEF(TRANS_BATCH_COMMIT_COUNT, "trans batch commit count", ObStatClassIds::TRANS, "trans batch commit count", 30059, true, true) -STAT_EVENT_ADD_DEF(TRANS_RELOCATE_ROW_COUNT, "trans relocate row count", ObStatClassIds::TRANS, "trans relocate row count", 30060, true, true) -STAT_EVENT_ADD_DEF(TRANS_RELOCATE_TOTAL_TIME, "trans relocate total time", ObStatClassIds::TRANS, "trans relocate total time", 30061, true, true) -STAT_EVENT_ADD_DEF(TRANS_MULTI_PARTITION_UPDATE_STMT_COUNT, "trans multi partition update stmt count", ObStatClassIds::TRANS, "trans multi partition update stmt count", 30062, true, true) -STAT_EVENT_ADD_DEF(GTS_WAIT_ELAPSE_TOTAL_TIME, "gts wait elapse total time", ObStatClassIds::TRANS, "gts wait elapse total time", 30063, true, true) -STAT_EVENT_ADD_DEF(GTS_WAIT_ELAPSE_TOTAL_COUNT, "gts wait elapse total count", ObStatClassIds::TRANS, "gts wait elapse total count", 30064, true, true) -STAT_EVENT_ADD_DEF(GTS_WAIT_ELAPSE_TOTAL_WAIT_COUNT, "gts wait elapse total wait count", ObStatClassIds::TRANS, "gts wait elapse total wait count", 30065, true, true) -STAT_EVENT_ADD_DEF(GTS_RPC_COUNT, "gts rpc count", ObStatClassIds::TRANS, "gts rpc count", 30066, true, true) -STAT_EVENT_ADD_DEF(GTS_TRY_ACQUIRE_TOTAL_COUNT, "gts try acquire total count", ObStatClassIds::TRANS, "gts try acquire total count", 30067, true, true) -STAT_EVENT_ADD_DEF(GTS_TRY_WAIT_ELAPSE_TOTAL_COUNT, "gts try wait elapse total count", ObStatClassIds::TRANS, "gts try wait elapse total count", 30068, true, true) -STAT_EVENT_ADD_DEF(HA_GTS_HANDLE_GET_REQUEST_COUNT, "ha gts handle get request count", ObStatClassIds::TRANS, "ha gts handle get request count", 30069, true, true) -STAT_EVENT_ADD_DEF(HA_GTS_SEND_GET_RESPONSE_COUNT, "ha gts send get response count", ObStatClassIds::TRANS, "ha gts send get response count", 30070, true, true) -STAT_EVENT_ADD_DEF(HA_GTS_HANDLE_PING_REQUEST_COUNT, "ha gts handle ping request count", ObStatClassIds::TRANS, "ha gts handle ping request count", 30071, true, true) -STAT_EVENT_ADD_DEF(HA_GTS_HANDLE_PING_RESPONSE_COUNT, "ha gts handle ping response count", ObStatClassIds::TRANS, "ha gts handle ping response count", 30072, true, true) -STAT_EVENT_ADD_DEF(HA_GTS_SOURCE_REQUEST_COUNT, "ha gts source request count", ObStatClassIds::TRANS, "ha gts source request count", 30073, true, true) -STAT_EVENT_ADD_DEF(HA_GTS_SOURCE_REQUEST_COST_TIME, "ha gts source request cost time", ObStatClassIds::TRANS, "ha gts source request cost time", 30074, true, true) -STAT_EVENT_ADD_DEF(HA_GTS_CACHE_HIT_COUNT, "ha gts cache hit count", ObStatClassIds::TRANS, "ha gts cache hit count", 30075, true, true) -STAT_EVENT_ADD_DEF(HA_GTS_CACHE_MISS_COUNT, "ha gts cache miss count", ObStatClassIds::TRANS, "ha gts cache miss count", 30076, true, true) -STAT_EVENT_ADD_DEF(TRANS_ELR_ENABLE_COUNT, "trans early lock release enable count", ObStatClassIds::TRANS, "trans early lock releaes enable count", 30077, true, true) -STAT_EVENT_ADD_DEF(TRANS_ELR_UNABLE_COUNT, "trans early lock release unable count", ObStatClassIds::TRANS, "trans early lock releaes unable count", 30078, true, true) -STAT_EVENT_ADD_DEF(READ_ELR_ROW_COUNT, "read elr row count", ObStatClassIds::TRANS, "read elr row count", 30079, true, true) -STAT_EVENT_ADD_DEF(TRANS_LOCAL_TOTAL_USED_TIME, "local trans total used time", ObStatClassIds::TRANS, "local trans total used time", 30080, true, true) -STAT_EVENT_ADD_DEF(TRANS_DIST_TOTAL_USED_TIME, "distributed trans total used time", ObStatClassIds::TRANS, "distributed trans total used time", 30081, true, true) -STAT_EVENT_ADD_DEF(TX_DATA_HIT_MINI_CACHE_COUNT, "tx data hit mini cache count", ObStatClassIds::TRANS, "tx data hit mini cache count", 30082, true, true) -STAT_EVENT_ADD_DEF(TX_DATA_HIT_KV_CACHE_COUNT, "tx data hit kv cache count", ObStatClassIds::TRANS, "tx data hit kv cache count", 30083, true, true) -STAT_EVENT_ADD_DEF(TX_DATA_READ_TX_CTX_COUNT, "tx data read tx ctx count", ObStatClassIds::TRANS, "tx data read tx ctx count", 30084, true, true) -STAT_EVENT_ADD_DEF(TX_DATA_READ_TX_DATA_MEMTABLE_COUNT, "tx data read tx data memtable count", ObStatClassIds::TRANS, "tx data read tx data memtable count", 30085, true, true) -STAT_EVENT_ADD_DEF(TX_DATA_READ_TX_DATA_SSTABLE_COUNT, "tx data read tx data sstable count", ObStatClassIds::TRANS, "tx data read tx data sstable count", 30086, true, true) +STAT_EVENT_ADD_DEF(TRANS_READONLY_COUNT, "trans without participant count", ObStatClassIds::TRANS, "trans without participant count", 30017, false, true) +STAT_EVENT_ADD_DEF(REDO_LOG_REPLAY_COUNT, "redo log replay count", ObStatClassIds::TRANS, "redo log replay count", 30018, false, true) +STAT_EVENT_ADD_DEF(REDO_LOG_REPLAY_TIME, "redo log replay time", ObStatClassIds::TRANS, "redo log replay time", 30019, false, true) +STAT_EVENT_ADD_DEF(PREPARE_LOG_REPLAY_COUNT, "prepare log replay count", ObStatClassIds::TRANS, "prepare log replay count", 30020, false, true) +STAT_EVENT_ADD_DEF(PREPARE_LOG_REPLAY_TIME, "prepare log replay time", ObStatClassIds::TRANS, "prepare log replay time", 30021, false, true) +STAT_EVENT_ADD_DEF(COMMIT_LOG_REPLAY_COUNT, "commit log replay count", ObStatClassIds::TRANS, "commit log replay count", 30022, false, true) +STAT_EVENT_ADD_DEF(COMMIT_LOG_REPLAY_TIME, "commit log replay time", ObStatClassIds::TRANS, "commit log replay time", 30023, false, true) +STAT_EVENT_ADD_DEF(ABORT_LOG_REPLAY_COUNT, "abort log replay count", ObStatClassIds::TRANS, "abort log replay count", 30024, false, true) +STAT_EVENT_ADD_DEF(ABORT_LOG_REPLAY_TIME, "abort log replay time", ObStatClassIds::TRANS, "abort log replay time", 30025, false, true) +STAT_EVENT_ADD_DEF(CLEAR_LOG_REPLAY_COUNT, "clear log replay count", ObStatClassIds::TRANS, "clear log replay count", 30026, false, true) +STAT_EVENT_ADD_DEF(CLEAR_LOG_REPLAY_TIME, "clear log replay time", ObStatClassIds::TRANS, "clear log replay time", 30027, false, true) +STAT_EVENT_ADD_DEF(GTS_REQUEST_TOTAL_COUNT, "gts request total count", ObStatClassIds::TRANS, "gts request total count", 30053, false, true) +STAT_EVENT_ADD_DEF(GTS_ACQUIRE_TOTAL_TIME, "gts acquire total time", ObStatClassIds::TRANS, "gts acquire total time", 30054, false, true) +STAT_EVENT_ADD_DEF(GTS_ACQUIRE_TOTAL_WAIT_COUNT, "gts acquire total wait count", ObStatClassIds::TRANS, "gts acquire total wait count", 30056, false, true) +STAT_EVENT_ADD_DEF(GTS_WAIT_ELAPSE_TOTAL_TIME, "gts wait elapse total time", ObStatClassIds::TRANS, "gts wait elapse total time", 30063, false, true) +STAT_EVENT_ADD_DEF(GTS_WAIT_ELAPSE_TOTAL_COUNT, "gts wait elapse total count", ObStatClassIds::TRANS, "gts wait elapse total count", 30064, false, true) +STAT_EVENT_ADD_DEF(GTS_WAIT_ELAPSE_TOTAL_WAIT_COUNT, "gts wait elapse total wait count", ObStatClassIds::TRANS, "gts wait elapse total wait count", 30065, false, true) +STAT_EVENT_ADD_DEF(GTS_RPC_COUNT, "gts rpc count", ObStatClassIds::TRANS, "gts rpc count", 30066, false, true) +STAT_EVENT_ADD_DEF(GTS_TRY_ACQUIRE_TOTAL_COUNT, "gts try acquire total count", ObStatClassIds::TRANS, "gts try acquire total count", 30067, false, true) +STAT_EVENT_ADD_DEF(GTS_TRY_WAIT_ELAPSE_TOTAL_COUNT, "gts try wait elapse total count", ObStatClassIds::TRANS, "gts try wait elapse total count", 30068, false, true) +STAT_EVENT_ADD_DEF(TRANS_ELR_ENABLE_COUNT, "trans early lock release enable count", ObStatClassIds::TRANS, "trans early lock releaes enable count", 30077, false, true) +STAT_EVENT_ADD_DEF(TRANS_ELR_UNABLE_COUNT, "trans early lock release unable count", ObStatClassIds::TRANS, "trans early lock releaes unable count", 30078, false, true) +STAT_EVENT_ADD_DEF(READ_ELR_ROW_COUNT, "read elr row count", ObStatClassIds::TRANS, "read elr row count", 30079, false, true) +STAT_EVENT_ADD_DEF(TRANS_LOCAL_TOTAL_USED_TIME, "local trans total used time", ObStatClassIds::TRANS, "local trans total used time", 30080, false, true) +STAT_EVENT_ADD_DEF(TRANS_DIST_TOTAL_USED_TIME, "distributed trans total used time", ObStatClassIds::TRANS, "distributed trans total used time", 30081, false, true) +STAT_EVENT_ADD_DEF(TX_DATA_HIT_MINI_CACHE_COUNT, "tx data hit mini cache count", ObStatClassIds::TRANS, "tx data hit mini cache count", 30082, false, true) +STAT_EVENT_ADD_DEF(TX_DATA_HIT_KV_CACHE_COUNT, "tx data hit kv cache count", ObStatClassIds::TRANS, "tx data hit kv cache count", 30083, false, true) +STAT_EVENT_ADD_DEF(TX_DATA_READ_TX_CTX_COUNT, "tx data read tx ctx count", ObStatClassIds::TRANS, "tx data read tx ctx count", 30084, false, true) +STAT_EVENT_ADD_DEF(TX_DATA_READ_TX_DATA_MEMTABLE_COUNT, "tx data read tx data memtable count", ObStatClassIds::TRANS, "tx data read tx data memtable count", 30085, false, true) +STAT_EVENT_ADD_DEF(TX_DATA_READ_TX_DATA_SSTABLE_COUNT, "tx data read tx data sstable count", ObStatClassIds::TRANS, "tx data read tx data sstable count", 30086, false, true) // SQL //STAT_EVENT_ADD_DEF(PLAN_CACHE_HIT, "PLAN_CACHE_HIT", SQL, "PLAN_CACHE_HIT") //STAT_EVENT_ADD_DEF(PLAN_CACHE_MISS, "PLAN_CACHE_MISS", SQL, "PLAN_CACHE_MISS") -STAT_EVENT_ADD_DEF(SQL_SELECT_COUNT, "sql select count", ObStatClassIds::SQL, "sql select count", 40000, true, true) -STAT_EVENT_ADD_DEF(SQL_SELECT_TIME, "sql select time", ObStatClassIds::SQL, "sql select time", 40001, true, true) -STAT_EVENT_ADD_DEF(SQL_INSERT_COUNT, "sql insert count", ObStatClassIds::SQL, "sql insert count", 40002, true, true) -STAT_EVENT_ADD_DEF(SQL_INSERT_TIME, "sql insert time", ObStatClassIds::SQL, "sql insert time", 40003, true, true) -STAT_EVENT_ADD_DEF(SQL_REPLACE_COUNT, "sql replace count", ObStatClassIds::SQL, "sql replace count", 40004, true, true) -STAT_EVENT_ADD_DEF(SQL_REPLACE_TIME, "sql replace time", ObStatClassIds::SQL, "sql replace time", 40005, true, true) -STAT_EVENT_ADD_DEF(SQL_UPDATE_COUNT, "sql update count", ObStatClassIds::SQL, "sql update count", 40006, true, true) -STAT_EVENT_ADD_DEF(SQL_UPDATE_TIME, "sql update time", ObStatClassIds::SQL, "sql update time", 40007, true, true) -STAT_EVENT_ADD_DEF(SQL_DELETE_COUNT, "sql delete count", ObStatClassIds::SQL, "sql delete count", 40008, true, true) -STAT_EVENT_ADD_DEF(SQL_DELETE_TIME, "sql delete time", ObStatClassIds::SQL, "sql delete time", 40009, true, true) -STAT_EVENT_ADD_DEF(SQL_OTHER_COUNT, "sql other count", ObStatClassIds::SQL, "sql other count", 40018, true, true) -STAT_EVENT_ADD_DEF(SQL_OTHER_TIME, "sql other time", ObStatClassIds::SQL, "sql other time", 40019, true, true) -STAT_EVENT_ADD_DEF(SQL_PS_PREPARE_COUNT, "ps prepare count", ObStatClassIds::SQL, "ps prepare count", 40020, true, true) -STAT_EVENT_ADD_DEF(SQL_PS_PREPARE_TIME, "ps prepare time", ObStatClassIds::SQL, "ps prepare time", 40021, true, true) -STAT_EVENT_ADD_DEF(SQL_PS_EXECUTE_COUNT, "ps execute count", ObStatClassIds::SQL, "ps execute count", 40022, true, true) -STAT_EVENT_ADD_DEF(SQL_PS_CLOSE_COUNT, "ps close count", ObStatClassIds::SQL, "ps close count", 40023, true, true) -STAT_EVENT_ADD_DEF(SQL_PS_CLOSE_TIME, "ps close time", ObStatClassIds::SQL, "ps close time", 40024, true, true) +STAT_EVENT_ADD_DEF(SQL_SELECT_COUNT, "sql select count", ObStatClassIds::SQL, "sql select count", 40000, false, true) +STAT_EVENT_ADD_DEF(SQL_SELECT_TIME, "sql select time", ObStatClassIds::SQL, "sql select time", 40001, false, true) +STAT_EVENT_ADD_DEF(SQL_INSERT_COUNT, "sql insert count", ObStatClassIds::SQL, "sql insert count", 40002, false, true) +STAT_EVENT_ADD_DEF(SQL_INSERT_TIME, "sql insert time", ObStatClassIds::SQL, "sql insert time", 40003, false, true) +STAT_EVENT_ADD_DEF(SQL_REPLACE_COUNT, "sql replace count", ObStatClassIds::SQL, "sql replace count", 40004, false, true) +STAT_EVENT_ADD_DEF(SQL_REPLACE_TIME, "sql replace time", ObStatClassIds::SQL, "sql replace time", 40005, false, true) +STAT_EVENT_ADD_DEF(SQL_UPDATE_COUNT, "sql update count", ObStatClassIds::SQL, "sql update count", 40006, false, true) +STAT_EVENT_ADD_DEF(SQL_UPDATE_TIME, "sql update time", ObStatClassIds::SQL, "sql update time", 40007, false, true) +STAT_EVENT_ADD_DEF(SQL_DELETE_COUNT, "sql delete count", ObStatClassIds::SQL, "sql delete count", 40008, false, true) +STAT_EVENT_ADD_DEF(SQL_DELETE_TIME, "sql delete time", ObStatClassIds::SQL, "sql delete time", 40009, false, true) +STAT_EVENT_ADD_DEF(SQL_OTHER_COUNT, "sql other count", ObStatClassIds::SQL, "sql other count", 40018, false, true) +STAT_EVENT_ADD_DEF(SQL_OTHER_TIME, "sql other time", ObStatClassIds::SQL, "sql other time", 40019, false, true) +STAT_EVENT_ADD_DEF(SQL_PS_PREPARE_COUNT, "ps prepare count", ObStatClassIds::SQL, "ps prepare count", 40020, false, true) +STAT_EVENT_ADD_DEF(SQL_PS_PREPARE_TIME, "ps prepare time", ObStatClassIds::SQL, "ps prepare time", 40021, false, true) +STAT_EVENT_ADD_DEF(SQL_PS_EXECUTE_COUNT, "ps execute count", ObStatClassIds::SQL, "ps execute count", 40022, false, true) +STAT_EVENT_ADD_DEF(SQL_PS_CLOSE_COUNT, "ps close count", ObStatClassIds::SQL, "ps close count", 40023, false, true) +STAT_EVENT_ADD_DEF(SQL_PS_CLOSE_TIME, "ps close time", ObStatClassIds::SQL, "ps close time", 40024, false, true) -STAT_EVENT_ADD_DEF(SQL_OPEN_CURSORS_CURRENT, "opened cursors current", ObStatClassIds::SQL, "opened cursors current", 40030, true, true) -STAT_EVENT_ADD_DEF(SQL_OPEN_CURSORS_CUMULATIVE, "opened cursors cumulative", ObStatClassIds::SQL, "opened cursors cumulative", 40031, true, true) +STAT_EVENT_ADD_DEF(SQL_OPEN_CURSORS_CURRENT, "opened cursors current", ObStatClassIds::SQL, "opened cursors current", 40030, false, true) +STAT_EVENT_ADD_DEF(SQL_OPEN_CURSORS_CUMULATIVE, "opened cursors cumulative", ObStatClassIds::SQL, "opened cursors cumulative", 40031, false, true) -STAT_EVENT_ADD_DEF(SQL_LOCAL_COUNT, "sql local count", ObStatClassIds::SQL, "sql local count", 40010, true, true) -STAT_EVENT_ADD_DEF(SQL_REMOTE_COUNT, "sql remote count", ObStatClassIds::SQL, "sql remote count", 40011, true, true) -STAT_EVENT_ADD_DEF(SQL_DISTRIBUTED_COUNT, "sql distributed count", ObStatClassIds::SQL, "sql distributed count", 40012, true, true) +STAT_EVENT_ADD_DEF(SQL_LOCAL_COUNT, "sql local count", ObStatClassIds::SQL, "sql local count", 40010, false, true) +STAT_EVENT_ADD_DEF(SQL_REMOTE_COUNT, "sql remote count", ObStatClassIds::SQL, "sql remote count", 40011, false, true) +STAT_EVENT_ADD_DEF(SQL_DISTRIBUTED_COUNT, "sql distributed count", ObStatClassIds::SQL, "sql distributed count", 40012, false, true) STAT_EVENT_ADD_DEF(ACTIVE_SESSIONS, "active sessions", ObStatClassIds::SQL, "active sessions", 40013, false, true) -STAT_EVENT_ADD_DEF(SQL_SINGLE_QUERY_COUNT, "single query count", ObStatClassIds::SQL, "single query count", 40014, true, true) -STAT_EVENT_ADD_DEF(SQL_MULTI_QUERY_COUNT, "multiple query count", ObStatClassIds::SQL, "multiple query count", 40015, true, true) -STAT_EVENT_ADD_DEF(SQL_MULTI_ONE_QUERY_COUNT, "multiple query with one stmt count", ObStatClassIds::SQL, "multiple query with one stmt count", 40016, true, true) +STAT_EVENT_ADD_DEF(SQL_SINGLE_QUERY_COUNT, "single query count", ObStatClassIds::SQL, "single query count", 40014, false, true) +STAT_EVENT_ADD_DEF(SQL_MULTI_QUERY_COUNT, "multiple query count", ObStatClassIds::SQL, "multiple query count", 40015, false, true) +STAT_EVENT_ADD_DEF(SQL_MULTI_ONE_QUERY_COUNT, "multiple query with one stmt count", ObStatClassIds::SQL, "multiple query with one stmt count", 40016, false, true) -STAT_EVENT_ADD_DEF(SQL_INNER_SELECT_COUNT, "sql inner select count", ObStatClassIds::SQL, "sql inner select count", 40100, true, true) -STAT_EVENT_ADD_DEF(SQL_INNER_SELECT_TIME, "sql inner select time", ObStatClassIds::SQL, "sql inner select time", 40101, true, true) -STAT_EVENT_ADD_DEF(SQL_INNER_INSERT_COUNT, "sql inner insert count", ObStatClassIds::SQL, "sql inner insert count", 40102, true, true) -STAT_EVENT_ADD_DEF(SQL_INNER_INSERT_TIME, "sql inner insert time", ObStatClassIds::SQL, "sql inner insert time", 40103, true, true) -STAT_EVENT_ADD_DEF(SQL_INNER_REPLACE_COUNT, "sql inner replace count", ObStatClassIds::SQL, "sql inner replace count", 40104, true, true) -STAT_EVENT_ADD_DEF(SQL_INNER_REPLACE_TIME, "sql inner replace time", ObStatClassIds::SQL, "sql inner replace time", 40105, true, true) -STAT_EVENT_ADD_DEF(SQL_INNER_UPDATE_COUNT, "sql inner update count", ObStatClassIds::SQL, "sql inner update count", 40106, true, true) -STAT_EVENT_ADD_DEF(SQL_INNER_UPDATE_TIME, "sql inner update time", ObStatClassIds::SQL, "sql inner update time", 40107, true, true) -STAT_EVENT_ADD_DEF(SQL_INNER_DELETE_COUNT, "sql inner delete count", ObStatClassIds::SQL, "sql inner delete count", 40108, true, true) -STAT_EVENT_ADD_DEF(SQL_INNER_DELETE_TIME, "sql inner delete time", ObStatClassIds::SQL, "sql inner delete time", 40109, true, true) -STAT_EVENT_ADD_DEF(SQL_INNER_OTHER_COUNT, "sql inner other count", ObStatClassIds::SQL, "sql inner other count", 40110, true, true) -STAT_EVENT_ADD_DEF(SQL_INNER_OTHER_TIME, "sql inner other time", ObStatClassIds::SQL, "sql inner other time", 40111, true, true) -STAT_EVENT_ADD_DEF(SQL_USER_LOGONS_CUMULATIVE, "user logons cumulative", ObStatClassIds::SQL, "user logons cumulative", 40112, true, true) -STAT_EVENT_ADD_DEF(SQL_USER_LOGOUTS_CUMULATIVE, "user logouts cumulative", ObStatClassIds::SQL, "user logouts cumulative", 40113, true, true) -STAT_EVENT_ADD_DEF(SQL_USER_LOGONS_FAILED_CUMULATIVE, "user logons failed cumulative", ObStatClassIds::SQL, "user logons failed cumulative", 40114, true, true) -STAT_EVENT_ADD_DEF(SQL_USER_LOGONS_COST_TIME_CUMULATIVE, "user logons time cumulative", ObStatClassIds::SQL, "user logons time cumulative", 40115, true, true) -STAT_EVENT_ADD_DEF(SQL_LOCAL_TIME, "sql local execute time", ObStatClassIds::SQL, "sql local execute time", 40116, true, true) -STAT_EVENT_ADD_DEF(SQL_REMOTE_TIME, "sql remote execute time", ObStatClassIds::SQL, "sql remote execute time", 40117, true, true) -STAT_EVENT_ADD_DEF(SQL_DISTRIBUTED_TIME, "sql distributed execute time", ObStatClassIds::SQL, "sql distributed execute time", 40118, true, true) +STAT_EVENT_ADD_DEF(SQL_INNER_SELECT_COUNT, "sql inner select count", ObStatClassIds::SQL, "sql inner select count", 40100, false, true) +STAT_EVENT_ADD_DEF(SQL_INNER_SELECT_TIME, "sql inner select time", ObStatClassIds::SQL, "sql inner select time", 40101, false, true) +STAT_EVENT_ADD_DEF(SQL_INNER_INSERT_COUNT, "sql inner insert count", ObStatClassIds::SQL, "sql inner insert count", 40102, false, true) +STAT_EVENT_ADD_DEF(SQL_INNER_INSERT_TIME, "sql inner insert time", ObStatClassIds::SQL, "sql inner insert time", 40103, false, true) +STAT_EVENT_ADD_DEF(SQL_INNER_REPLACE_COUNT, "sql inner replace count", ObStatClassIds::SQL, "sql inner replace count", 40104, false, true) +STAT_EVENT_ADD_DEF(SQL_INNER_REPLACE_TIME, "sql inner replace time", ObStatClassIds::SQL, "sql inner replace time", 40105, false, true) +STAT_EVENT_ADD_DEF(SQL_INNER_UPDATE_COUNT, "sql inner update count", ObStatClassIds::SQL, "sql inner update count", 40106, false, true) +STAT_EVENT_ADD_DEF(SQL_INNER_UPDATE_TIME, "sql inner update time", ObStatClassIds::SQL, "sql inner update time", 40107, false, true) +STAT_EVENT_ADD_DEF(SQL_INNER_DELETE_COUNT, "sql inner delete count", ObStatClassIds::SQL, "sql inner delete count", 40108, false, true) +STAT_EVENT_ADD_DEF(SQL_INNER_DELETE_TIME, "sql inner delete time", ObStatClassIds::SQL, "sql inner delete time", 40109, false, true) +STAT_EVENT_ADD_DEF(SQL_INNER_OTHER_COUNT, "sql inner other count", ObStatClassIds::SQL, "sql inner other count", 40110, false, true) +STAT_EVENT_ADD_DEF(SQL_INNER_OTHER_TIME, "sql inner other time", ObStatClassIds::SQL, "sql inner other time", 40111, false, true) +STAT_EVENT_ADD_DEF(SQL_USER_LOGONS_CUMULATIVE, "user logons cumulative", ObStatClassIds::SQL, "user logons cumulative", 40112, false, true) +STAT_EVENT_ADD_DEF(SQL_USER_LOGOUTS_CUMULATIVE, "user logouts cumulative", ObStatClassIds::SQL, "user logouts cumulative", 40113, false, true) +STAT_EVENT_ADD_DEF(SQL_USER_LOGONS_FAILED_CUMULATIVE, "user logons failed cumulative", ObStatClassIds::SQL, "user logons failed cumulative", 40114, false, true) +STAT_EVENT_ADD_DEF(SQL_USER_LOGONS_COST_TIME_CUMULATIVE, "user logons time cumulative", ObStatClassIds::SQL, "user logons time cumulative", 40115, false, true) +STAT_EVENT_ADD_DEF(SQL_LOCAL_TIME, "sql local execute time", ObStatClassIds::SQL, "sql local execute time", 40116, false, true) +STAT_EVENT_ADD_DEF(SQL_REMOTE_TIME, "sql remote execute time", ObStatClassIds::SQL, "sql remote execute time", 40117, false, true) +STAT_EVENT_ADD_DEF(SQL_DISTRIBUTED_TIME, "sql distributed execute time", ObStatClassIds::SQL, "sql distributed execute time", 40118, false, true) // CACHE STAT_EVENT_ADD_DEF(ROW_CACHE_HIT, "row cache hit", ObStatClassIds::CACHE, "row cache hit", 50000, true, true) @@ -190,53 +150,53 @@ STAT_EVENT_ADD_DEF(BLOOM_FILTER_FILTS, "bloom filter filts", ObStatClassIds::CAC STAT_EVENT_ADD_DEF(BLOOM_FILTER_PASSES, "bloom filter passes", ObStatClassIds::CACHE, "bloom filter passes", 50007, true, true) STAT_EVENT_ADD_DEF(BLOCK_CACHE_HIT, "block cache hit", ObStatClassIds::CACHE, "block cache hit", 50008, true, true) STAT_EVENT_ADD_DEF(BLOCK_CACHE_MISS, "block cache miss", ObStatClassIds::CACHE, "block cache miss", 50009, true, true) -STAT_EVENT_ADD_DEF(LOCATION_CACHE_HIT, "location cache hit", ObStatClassIds::CACHE, "location cache hit", 50010, true, true) -STAT_EVENT_ADD_DEF(LOCATION_CACHE_MISS, "location cache miss", ObStatClassIds::CACHE, "location cache miss", 50011, true, true) -STAT_EVENT_ADD_DEF(LOCATION_CACHE_WAIT, "location cache wait", ObStatClassIds::CACHE, "location cache wait", 50012, true, true) -STAT_EVENT_ADD_DEF(LOCATION_CACHE_PROXY_HIT, "location cache get hit from proxy virtual table", ObStatClassIds::CACHE, "location cache get hit from proxy virtual table", 50013, true, true) -STAT_EVENT_ADD_DEF(LOCATION_CACHE_PROXY_MISS, "location cache get miss from proxy virtual table", ObStatClassIds::CACHE, "location cache get miss from proxy virtual table", 50014, true, true) -STAT_EVENT_ADD_DEF(LOCATION_CACHE_CLEAR_LOCATION, "location cache nonblock renew", ObStatClassIds::CACHE, "location cache nonblock renew", 50015, true, true) -STAT_EVENT_ADD_DEF(LOCATION_CACHE_CLEAR_LOCATION_IGNORED, "location cache nonblock renew ignored", ObStatClassIds::CACHE, "location cache nonblock renew ignored", 50016, true, true) -STAT_EVENT_ADD_DEF(LOCATION_CACHE_NONBLOCK_HIT, "location nonblock get hit", ObStatClassIds::CACHE, "location nonblock get hit", 50017, true, true) -STAT_EVENT_ADD_DEF(LOCATION_CACHE_NONBLOCK_MISS, "location nonblock get miss", ObStatClassIds::CACHE, "location nonblock get miss", 50018, true, true) +STAT_EVENT_ADD_DEF(LOCATION_CACHE_HIT, "location cache hit", ObStatClassIds::CACHE, "location cache hit", 50010, false, true) +STAT_EVENT_ADD_DEF(LOCATION_CACHE_MISS, "location cache miss", ObStatClassIds::CACHE, "location cache miss", 50011, false, true) +STAT_EVENT_ADD_DEF(LOCATION_CACHE_WAIT, "location cache wait", ObStatClassIds::CACHE, "location cache wait", 50012, false, true) +STAT_EVENT_ADD_DEF(LOCATION_CACHE_PROXY_HIT, "location cache get hit from proxy virtual table", ObStatClassIds::CACHE, "location cache get hit from proxy virtual table", 50013, false, true) +STAT_EVENT_ADD_DEF(LOCATION_CACHE_PROXY_MISS, "location cache get miss from proxy virtual table", ObStatClassIds::CACHE, "location cache get miss from proxy virtual table", 50014, false, true) +STAT_EVENT_ADD_DEF(LOCATION_CACHE_CLEAR_LOCATION, "location cache nonblock renew", ObStatClassIds::CACHE, "location cache nonblock renew", 50015, false, true) +STAT_EVENT_ADD_DEF(LOCATION_CACHE_CLEAR_LOCATION_IGNORED, "location cache nonblock renew ignored", ObStatClassIds::CACHE, "location cache nonblock renew ignored", 50016, false, true) +STAT_EVENT_ADD_DEF(LOCATION_CACHE_NONBLOCK_HIT, "location nonblock get hit", ObStatClassIds::CACHE, "location nonblock get hit", 50017, false, true) +STAT_EVENT_ADD_DEF(LOCATION_CACHE_NONBLOCK_MISS, "location nonblock get miss", ObStatClassIds::CACHE, "location nonblock get miss", 50018, false, true) //STAT_EVENT_ADD_DEF(LOCATION_CACHE_RPC_CHECK_LEADER, "location check leader count", ObStatClassIds::CACHE, "location check leader count", 50019, true, true) //STAT_EVENT_ADD_DEF(LOCATION_CACHE_RPC_CHECK_MEMBER, "location check member count", ObStatClassIds::CACHE, "location check member count", 50020, true, true) -STAT_EVENT_ADD_DEF(LOCATION_CACHE_RPC_CHECK, "location cache rpc renew count", ObStatClassIds::CACHE, "location cache rpc renew count", 50021, true, true) -STAT_EVENT_ADD_DEF(LOCATION_CACHE_RENEW, "location cache renew", ObStatClassIds::CACHE, "location cache renew", 50022, true, true) -STAT_EVENT_ADD_DEF(LOCATION_CACHE_RENEW_IGNORED, "location cache renew ignored", ObStatClassIds::CACHE, "location cache renew ignored", 50023, true, true) +STAT_EVENT_ADD_DEF(LOCATION_CACHE_RPC_CHECK, "location cache rpc renew count", ObStatClassIds::CACHE, "location cache rpc renew count", 50021, false, true) +STAT_EVENT_ADD_DEF(LOCATION_CACHE_RENEW, "location cache renew", ObStatClassIds::CACHE, "location cache renew", 50022, false, true) +STAT_EVENT_ADD_DEF(LOCATION_CACHE_RENEW_IGNORED, "location cache renew ignored", ObStatClassIds::CACHE, "location cache renew ignored", 50023, false, true) //STAT_EVENT_ADD_DEF(MMAP_COUNT, "mmap count", ObStatClassIds::CACHE, "mmap count", 50024, true, true) //STAT_EVENT_ADD_DEF(MUNMAP_COUNT, "munmap count", ObStatClassIds::CACHE, "munmap count", 50025, true, true) //STAT_EVENT_ADD_DEF(MMAP_SIZE, "mmap size", ObStatClassIds::CACHE, "mmap size", 50026, true, true) //STAT_EVENT_ADD_DEF(MUNMAP_SIZE, "munmap size", ObStatClassIds::CACHE, "munmap size", 50027, true, true) -STAT_EVENT_ADD_DEF(KVCACHE_SYNC_WASH_TIME, "kvcache sync wash time", ObStatClassIds::CACHE, "kvcache sync wash time", 50028, true, true) -STAT_EVENT_ADD_DEF(KVCACHE_SYNC_WASH_COUNT, "kvcache sync wash count", ObStatClassIds::CACHE, "kvcache sync wash count", 50029, true, true) -STAT_EVENT_ADD_DEF(LOCATION_CACHE_RPC_RENEW_FAIL, "location cache rpc renew fail count", ObStatClassIds::CACHE, "location cache rpc renew fail count", 50030, true, true) -STAT_EVENT_ADD_DEF(LOCATION_CACHE_SQL_RENEW, "location cache sql renew count", ObStatClassIds::CACHE, "location cache sql renew count", 50031, true, true) -STAT_EVENT_ADD_DEF(LOCATION_CACHE_IGNORE_RPC_RENEW, "location cache ignore rpc renew count", ObStatClassIds::CACHE, "location cache ignore rpc renew count", 50032, true, true) +STAT_EVENT_ADD_DEF(KVCACHE_SYNC_WASH_TIME, "kvcache sync wash time", ObStatClassIds::CACHE, "kvcache sync wash time", 50028, false, true) +STAT_EVENT_ADD_DEF(KVCACHE_SYNC_WASH_COUNT, "kvcache sync wash count", ObStatClassIds::CACHE, "kvcache sync wash count", 50029, false, true) +STAT_EVENT_ADD_DEF(LOCATION_CACHE_RPC_RENEW_FAIL, "location cache rpc renew fail count", ObStatClassIds::CACHE, "location cache rpc renew fail count", 50030, false, true) +STAT_EVENT_ADD_DEF(LOCATION_CACHE_SQL_RENEW, "location cache sql renew count", ObStatClassIds::CACHE, "location cache sql renew count", 50031, false, true) +STAT_EVENT_ADD_DEF(LOCATION_CACHE_IGNORE_RPC_RENEW, "location cache ignore rpc renew count", ObStatClassIds::CACHE, "location cache ignore rpc renew count", 50032, false, true) STAT_EVENT_ADD_DEF(FUSE_ROW_CACHE_HIT, "fuse row cache hit", ObStatClassIds::CACHE, "fuse row cache hit", 50033, true, true) STAT_EVENT_ADD_DEF(FUSE_ROW_CACHE_MISS, "fuse row cache miss", ObStatClassIds::CACHE, "fuse row cache miss", 50034, true, true) -STAT_EVENT_ADD_DEF(SCHEMA_CACHE_HIT, "schema cache hit", ObStatClassIds::CACHE, "schema cache hit", 50035, true, true) -STAT_EVENT_ADD_DEF(SCHEMA_CACHE_MISS, "schema cache miss", ObStatClassIds::CACHE, "schema cache miss", 50036, true, true) -STAT_EVENT_ADD_DEF(TABLET_LS_CACHE_HIT, "tablet ls cache hit", ObStatClassIds::CACHE, "tablet ls cache hit", 50037, true, true) -STAT_EVENT_ADD_DEF(TABLET_LS_CACHE_MISS, "tablet ls cache miss", ObStatClassIds::CACHE, "tablet ls cache miss", 50038, true, true) +STAT_EVENT_ADD_DEF(SCHEMA_CACHE_HIT, "schema cache hit", ObStatClassIds::CACHE, "schema cache hit", 50035, false, true) +STAT_EVENT_ADD_DEF(SCHEMA_CACHE_MISS, "schema cache miss", ObStatClassIds::CACHE, "schema cache miss", 50036, false, true) +STAT_EVENT_ADD_DEF(TABLET_LS_CACHE_HIT, "tablet ls cache hit", ObStatClassIds::CACHE, "tablet ls cache hit", 50037, false, true) +STAT_EVENT_ADD_DEF(TABLET_LS_CACHE_MISS, "tablet ls cache miss", ObStatClassIds::CACHE, "tablet ls cache miss", 50038, false, true) // obsoleted part from INDEX_CLOG_CACHE_HIT to USER_TABLE_STAT_CACHE_MISS -STAT_EVENT_ADD_DEF(INDEX_CLOG_CACHE_HIT, "index clog cache hit", ObStatClassIds::CACHE, "index clog cache hit", 50039, true, true) -STAT_EVENT_ADD_DEF(INDEX_CLOG_CACHE_MISS, "index clog cache miss", ObStatClassIds::CACHE, "index clog cache miss", 50040, true, true) -STAT_EVENT_ADD_DEF(USER_TAB_COL_STAT_CACHE_HIT, "user tab col stat cache hit", ObStatClassIds::CACHE, "user tab col stat cache hit", 50041, true, true) -STAT_EVENT_ADD_DEF(USER_TAB_COL_STAT_CACHE_MISS, "user tab col stat cache miss", ObStatClassIds::CACHE, "user tab col stat cache miss", 50042, true, true) -STAT_EVENT_ADD_DEF(USER_TABLE_STAT_CACHE_HIT, "user table stat cache hit", ObStatClassIds::CACHE, "user table stat cache hit", 50043, true, true) -STAT_EVENT_ADD_DEF(USER_TABLE_STAT_CACHE_MISS, "user table stat cache miss", ObStatClassIds::CACHE, "user table stat cache miss", 50044, true, true) +STAT_EVENT_ADD_DEF(INDEX_CLOG_CACHE_HIT, "index clog cache hit", ObStatClassIds::CACHE, "index clog cache hit", 50039, false, true) +STAT_EVENT_ADD_DEF(INDEX_CLOG_CACHE_MISS, "index clog cache miss", ObStatClassIds::CACHE, "index clog cache miss", 50040, false, true) +STAT_EVENT_ADD_DEF(USER_TAB_COL_STAT_CACHE_HIT, "user tab col stat cache hit", ObStatClassIds::CACHE, "user tab col stat cache hit", 50041, false, true) +STAT_EVENT_ADD_DEF(USER_TAB_COL_STAT_CACHE_MISS, "user tab col stat cache miss", ObStatClassIds::CACHE, "user tab col stat cache miss", 50042, false, true) +STAT_EVENT_ADD_DEF(USER_TABLE_STAT_CACHE_HIT, "user table stat cache hit", ObStatClassIds::CACHE, "user table stat cache hit", 50043, false, true) +STAT_EVENT_ADD_DEF(USER_TABLE_STAT_CACHE_MISS, "user table stat cache miss", ObStatClassIds::CACHE, "user table stat cache miss", 50044, false, true) // obsoleted part from CLOG_CACHE_HIT to USER_TABLE_STAT_CACHE_MISS -STAT_EVENT_ADD_DEF(OPT_TABLE_STAT_CACHE_HIT, "opt table stat cache hit", ObStatClassIds::CACHE, "opt table stat cache hit", 50045, true, true) -STAT_EVENT_ADD_DEF(OPT_TABLE_STAT_CACHE_MISS, "opt table stat cache miss", ObStatClassIds::CACHE, "opt table stat cache miss", 50046, true, true) -STAT_EVENT_ADD_DEF(OPT_COLUMN_STAT_CACHE_HIT, "opt column stat cache hit", ObStatClassIds::CACHE, "opt column stat cache hit", 50047, true, true) -STAT_EVENT_ADD_DEF(OPT_COLUMN_STAT_CACHE_MISS, "opt column stat cache miss", ObStatClassIds::CACHE, "opt column stat cache miss", 50048, true, true) -STAT_EVENT_ADD_DEF(TMP_PAGE_CACHE_HIT, "tmp page cache hit", ObStatClassIds::CACHE, "tmp page cache hit", 50049, true, true) -STAT_EVENT_ADD_DEF(TMP_PAGE_CACHE_MISS, "tmp page cache miss", ObStatClassIds::CACHE, "tmp page cache miss", 50050, true, true) -STAT_EVENT_ADD_DEF(TMP_BLOCK_CACHE_HIT, "tmp block cache hit", ObStatClassIds::CACHE, "tmp block cache hit", 50051, true, true) -STAT_EVENT_ADD_DEF(TMP_BLOCK_CACHE_MISS, "tmp block cache miss", ObStatClassIds::CACHE, "tmp block cache miss", 50052, true, true) -STAT_EVENT_ADD_DEF(SECONDARY_META_CACHE_HIT, "secondary meta cache hit", ObStatClassIds::CACHE, "secondary meta cache hit", 50053, true, true) -STAT_EVENT_ADD_DEF(SECONDARY_META_CACHE_MISS, "secondary meta cache miss", ObStatClassIds::CACHE, "secondary meta cache miss", 50054, true, true) +STAT_EVENT_ADD_DEF(OPT_TABLE_STAT_CACHE_HIT, "opt table stat cache hit", ObStatClassIds::CACHE, "opt table stat cache hit", 50045, false, true) +STAT_EVENT_ADD_DEF(OPT_TABLE_STAT_CACHE_MISS, "opt table stat cache miss", ObStatClassIds::CACHE, "opt table stat cache miss", 50046, false, true) +STAT_EVENT_ADD_DEF(OPT_COLUMN_STAT_CACHE_HIT, "opt column stat cache hit", ObStatClassIds::CACHE, "opt column stat cache hit", 50047, false, true) +STAT_EVENT_ADD_DEF(OPT_COLUMN_STAT_CACHE_MISS, "opt column stat cache miss", ObStatClassIds::CACHE, "opt column stat cache miss", 50048, false, true) +STAT_EVENT_ADD_DEF(TMP_PAGE_CACHE_HIT, "tmp page cache hit", ObStatClassIds::CACHE, "tmp page cache hit", 50049, false, true) +STAT_EVENT_ADD_DEF(TMP_PAGE_CACHE_MISS, "tmp page cache miss", ObStatClassIds::CACHE, "tmp page cache miss", 50050, false, true) +STAT_EVENT_ADD_DEF(TMP_BLOCK_CACHE_HIT, "tmp block cache hit", ObStatClassIds::CACHE, "tmp block cache hit", 50051, false, true) +STAT_EVENT_ADD_DEF(TMP_BLOCK_CACHE_MISS, "tmp block cache miss", ObStatClassIds::CACHE, "tmp block cache miss", 50052, false, true) +STAT_EVENT_ADD_DEF(SECONDARY_META_CACHE_HIT, "secondary meta cache hit", ObStatClassIds::CACHE, "secondary meta cache hit", 50053, false, true) +STAT_EVENT_ADD_DEF(SECONDARY_META_CACHE_MISS, "secondary meta cache miss", ObStatClassIds::CACHE, "secondary meta cache miss", 50054, false, true) // STORAGE @@ -259,7 +219,6 @@ STAT_EVENT_ADD_DEF(MEMSTORE_GET_FAIL_COUNT, "memstore get fail count", ObStatCla STAT_EVENT_ADD_DEF(MEMSTORE_APPLY_COUNT, "memstore apply count", ObStatClassIds::STORAGE, "memstore apply count", 60012, true, true) STAT_EVENT_ADD_DEF(MEMSTORE_APPLY_SUCC_COUNT, "memstore apply succ count", ObStatClassIds::STORAGE, "memstore apply succ count", 60013, true, true) STAT_EVENT_ADD_DEF(MEMSTORE_APPLY_FAIL_COUNT, "memstore apply fail count", ObStatClassIds::STORAGE, "memstore apply fail count", 60014, true, true) -STAT_EVENT_ADD_DEF(MEMSTORE_ROW_COUNT, "memstore row count", ObStatClassIds::STORAGE, "memstore row count", 60015, true, true) STAT_EVENT_ADD_DEF(MEMSTORE_GET_TIME, "memstore get time", ObStatClassIds::STORAGE, "memstore get time", 60016, true, true) STAT_EVENT_ADD_DEF(MEMSTORE_SCAN_TIME, "memstore scan time", ObStatClassIds::STORAGE, "memstore scan time", 60017, true, true) STAT_EVENT_ADD_DEF(MEMSTORE_APPLY_TIME, "memstore apply time", ObStatClassIds::STORAGE, "memstore apply time", 60018, true, true) @@ -269,91 +228,56 @@ STAT_EVENT_ADD_DEF(MEMSTORE_WRITE_LOCK_SUCC_COUNT, "memstore write lock succ cou STAT_EVENT_ADD_DEF(MEMSTORE_WRITE_LOCK_FAIL_COUNT, "memstore write lock fail count", ObStatClassIds::STORAGE, "memstore write lock fail count", 60022, true, true) STAT_EVENT_ADD_DEF(MEMSTORE_WAIT_WRITE_LOCK_TIME, "memstore wait write lock time", ObStatClassIds::STORAGE, "memstore wait write lock time", 60023, true, true) STAT_EVENT_ADD_DEF(MEMSTORE_WAIT_READ_LOCK_TIME, "memstore wait read lock time", ObStatClassIds::STORAGE, "memstore wait read lock time", 60024, true, true) -STAT_EVENT_ADD_DEF(IO_READ_MICRO_INDEX_COUNT, "io read micro index count", ObStatClassIds::STORAGE, "io read micro index count", 60025, true, true) -STAT_EVENT_ADD_DEF(IO_READ_MICRO_INDEX_BYTES, "io read micro index bytes", ObStatClassIds::STORAGE, "io read micro index bytes", 60026, true, true) -STAT_EVENT_ADD_DEF(IO_READ_PREFETCH_MICRO_COUNT, "io prefetch micro block count", ObStatClassIds::STORAGE, "io prefetch micro block count", 60027, true, true) -STAT_EVENT_ADD_DEF(IO_READ_PREFETCH_MICRO_BYTES, "io prefetch micro block bytes", ObStatClassIds::STORAGE, "io prefetch micro block bytes", 60028, true, true) -STAT_EVENT_ADD_DEF(IO_READ_UNCOMP_MICRO_COUNT, "io read uncompress micro block count", ObStatClassIds::STORAGE, "io read uncompress micro block count", 60029, true, true) -STAT_EVENT_ADD_DEF(IO_READ_UNCOMP_MICRO_BYTES, "io read uncompress micro block bytes", ObStatClassIds::STORAGE, "io read uncompress micro block bytes", 60030, true, true) -STAT_EVENT_ADD_DEF(STORAGE_READ_ROW_COUNT, "storage read row count", ObStatClassIds::STORAGE, "storage read row count", 60031, true, true) -STAT_EVENT_ADD_DEF(STORAGE_DELETE_ROW_COUNT, "storage delete row count", ObStatClassIds::STORAGE, "storage delete row count", 60032, true, true) -STAT_EVENT_ADD_DEF(STORAGE_INSERT_ROW_COUNT, "storage insert row count", ObStatClassIds::STORAGE, "storage insert row count", 60033, true, true) -STAT_EVENT_ADD_DEF(STORAGE_UPDATE_ROW_COUNT, "storage update row count", ObStatClassIds::STORAGE, "storage update row count", 60034, true, true) -STAT_EVENT_ADD_DEF(MEMSTORE_MUTATOR_REPLAY_TIME, "memstore mutator replay time", ObStatClassIds::STORAGE, "memstore mutator replay time", 60035, true, true) -STAT_EVENT_ADD_DEF(MEMSTORE_MUTATOR_REPLAY_COUNT, "memstore mutator replay count", ObStatClassIds::STORAGE, "memstore mutator replay count", 60036, true, true) -STAT_EVENT_ADD_DEF(MEMSTORE_ROW_PURGE_COUNT, "memstore row purge count", ObStatClassIds::STORAGE, "memstore row purge count", 60037, true, true) -STAT_EVENT_ADD_DEF(MEMSTORE_ROW_COMPACTION_COUNT, "memstore row compaction count", ObStatClassIds::STORAGE, "memstore row compaction count", 60038, true, true) +STAT_EVENT_ADD_DEF(IO_READ_MICRO_INDEX_COUNT, "io read micro index count", ObStatClassIds::STORAGE, "io read micro index count", 60025, false, true) +STAT_EVENT_ADD_DEF(IO_READ_MICRO_INDEX_BYTES, "io read micro index bytes", ObStatClassIds::STORAGE, "io read micro index bytes", 60026, false, true) +STAT_EVENT_ADD_DEF(IO_READ_PREFETCH_MICRO_COUNT, "io prefetch micro block count", ObStatClassIds::STORAGE, "io prefetch micro block count", 60027, false, true) +STAT_EVENT_ADD_DEF(IO_READ_PREFETCH_MICRO_BYTES, "io prefetch micro block bytes", ObStatClassIds::STORAGE, "io prefetch micro block bytes", 60028, false, true) +STAT_EVENT_ADD_DEF(IO_READ_UNCOMP_MICRO_COUNT, "io read uncompress micro block count", ObStatClassIds::STORAGE, "io read uncompress micro block count", 60029, false, true) +STAT_EVENT_ADD_DEF(IO_READ_UNCOMP_MICRO_BYTES, "io read uncompress micro block bytes", ObStatClassIds::STORAGE, "io read uncompress micro block bytes", 60030, false, true) +STAT_EVENT_ADD_DEF(STORAGE_READ_ROW_COUNT, "storage read row count", ObStatClassIds::STORAGE, "storage read row count", 60031, false, true) +STAT_EVENT_ADD_DEF(STORAGE_DELETE_ROW_COUNT, "storage delete row count", ObStatClassIds::STORAGE, "storage delete row count", 60032, false, true) +STAT_EVENT_ADD_DEF(STORAGE_INSERT_ROW_COUNT, "storage insert row count", ObStatClassIds::STORAGE, "storage insert row count", 60033, false, true) +STAT_EVENT_ADD_DEF(STORAGE_UPDATE_ROW_COUNT, "storage update row count", ObStatClassIds::STORAGE, "storage update row count", 60034, false, true) +STAT_EVENT_ADD_DEF(MEMSTORE_ROW_PURGE_COUNT, "memstore row purge count", ObStatClassIds::STORAGE, "memstore row purge count", 60037, false, true) +STAT_EVENT_ADD_DEF(MEMSTORE_ROW_COMPACTION_COUNT, "memstore row compaction count", ObStatClassIds::STORAGE, "memstore row compaction count", 60038, false, true) -STAT_EVENT_ADD_DEF(IO_READ_QUEUE_DELAY, "io read queue delay", ObStatClassIds::STORAGE, "io read queue delay", 60039, true, true) -STAT_EVENT_ADD_DEF(IO_WRITE_QUEUE_DELAY, "io write queue delay", ObStatClassIds::STORAGE, "io write queue delay", 60040, true, true) -STAT_EVENT_ADD_DEF(IO_READ_CB_QUEUE_DELAY, "io read callback queuing delay", ObStatClassIds::STORAGE, "io read callback queuing delay", 60041, true, true) -STAT_EVENT_ADD_DEF(IO_READ_CB_PROCESS_DELAY, "io read callback process delay", ObStatClassIds::STORAGE, "io read callback process delay", 60042, true, true) -STAT_EVENT_ADD_DEF(WARM_UP_REQUEST_COUNT, "warm up request count", ObStatClassIds::STORAGE, "warm up request count", 60043, true, true) -STAT_EVENT_ADD_DEF(WARM_UP_REQUEST_SCAN_COUNT, "warm up request scan count", ObStatClassIds::STORAGE, "warm up request scan count", 60044, true, true) -STAT_EVENT_ADD_DEF(WARM_UP_REQUEST_GET_COUNT, "warm up request get count", ObStatClassIds::STORAGE, "warm up request get count", 60045, true, true) -STAT_EVENT_ADD_DEF(WARM_UP_REQUEST_MULTI_GET_COUNT, "warm up request multi get count", ObStatClassIds::STORAGE, "warm up request multi get count", 60046, true, true) -STAT_EVENT_ADD_DEF(WARM_UP_REQUEST_SEND_COUNT, "warm up request send count", ObStatClassIds::STORAGE, "warm up request send count", 60047, true, true) -STAT_EVENT_ADD_DEF(WARM_UP_REQUEST_SEND_SIZE, "warm up request send size", ObStatClassIds::STORAGE, "warm up request send size", 60048, true, true) -STAT_EVENT_ADD_DEF(WARM_UP_REQUEST_IN_DROP_COUNT, "warm up request in drop count", ObStatClassIds::STORAGE, "warm up request in drop count", 60049, true, true) -STAT_EVENT_ADD_DEF(WARM_UP_REQUEST_OUT_DROP_COUNT, "warm up request out drop count", ObStatClassIds::STORAGE, "warm up request out drop count", 60050, true, true) -STAT_EVENT_ADD_DEF(BANDWIDTH_IN_THROTTLE, "bandwidth in throttle size", ObStatClassIds::STORAGE, "bandwidth in throttle size", 60051, true, true) -STAT_EVENT_ADD_DEF(BANDWIDTH_OUT_THROTTLE, "bandwidth out throttle size", ObStatClassIds::STORAGE, "bandwidth out throttle size", 60052, true, true) -STAT_EVENT_ADD_DEF(WARM_UP_REQUEST_IGNORE_COUNT, "warm up request ignored when send queue is full", ObStatClassIds::STORAGE, "warm up request ignored when send queue is full", 60053, true, true) -STAT_EVENT_ADD_DEF(WARM_UP_REQUEST_ROWKEY_CHECK_COUNT, "warm up request rowkey check count", ObStatClassIds::STORAGE, "warm up request rowkey check count", 60054, true, true) -STAT_EVENT_ADD_DEF(WARM_UP_REQUEST_MULTI_SCAN_COUNT, "warm up request multi scan count", ObStatClassIds::STORAGE, "warm up request multi scan count", 60055, true, true) +STAT_EVENT_ADD_DEF(IO_READ_QUEUE_DELAY, "io read queue delay", ObStatClassIds::STORAGE, "io read queue delay", 60039, false, true) +STAT_EVENT_ADD_DEF(IO_WRITE_QUEUE_DELAY, "io write queue delay", ObStatClassIds::STORAGE, "io write queue delay", 60040, false, true) +STAT_EVENT_ADD_DEF(IO_READ_CB_QUEUE_DELAY, "io read callback queuing delay", ObStatClassIds::STORAGE, "io read callback queuing delay", 60041, false, true) +STAT_EVENT_ADD_DEF(IO_READ_CB_PROCESS_DELAY, "io read callback process delay", ObStatClassIds::STORAGE, "io read callback process delay", 60042, false, true) +STAT_EVENT_ADD_DEF(BANDWIDTH_IN_THROTTLE, "bandwidth in throttle size", ObStatClassIds::STORAGE, "bandwidth in throttle size", 60051, false, true) +STAT_EVENT_ADD_DEF(BANDWIDTH_OUT_THROTTLE, "bandwidth out throttle size", ObStatClassIds::STORAGE, "bandwidth out throttle size", 60052, false, true) STAT_EVENT_ADD_DEF(MEMSTORE_READ_ROW_COUNT, "memstore read row count", ObStatClassIds::STORAGE, "memstore read row count", 60056, true, true) STAT_EVENT_ADD_DEF(SSSTORE_READ_ROW_COUNT, "ssstore read row count", ObStatClassIds::STORAGE, "ssstore read row count", 60057, true, true) -STAT_EVENT_ADD_DEF(MEMSTORE_BIG_GAP_ROW_COUNT, "memstore big gap row count", ObStatClassIds::STORAGE, "memstore big gap row count", 60058, true, true) -STAT_EVENT_ADD_DEF(MEMSTORE_SMALL_GAP_ROW_COUNT, "memstore small gap row count", ObStatClassIds::STORAGE, "memstore small gap row count", 60059, true, true) -STAT_EVENT_ADD_DEF(MEMSTORE_BIG_GAP_COUNT, "memstore big gap count", ObStatClassIds::STORAGE, "memstore big gap count", 60060, true, true) -STAT_EVENT_ADD_DEF(MEMSTORE_SMALL_GAP_COUNT, "memstore small gap count", ObStatClassIds::STORAGE, "memstore small gap count", 60061, true, true) -STAT_EVENT_ADD_DEF(SSSTORE_BIG_GAP_ROW_COUNT, "ssstore big gap row count", ObStatClassIds::STORAGE, "ssstore big gap row count", 60062, true, true) -STAT_EVENT_ADD_DEF(SSSTORE_SMALL_GAP_ROW_COUNT, "ssstore small gap row count", ObStatClassIds::STORAGE, "ssstore small gap row count", 60063, true, true) -STAT_EVENT_ADD_DEF(SSSTORE_BIG_GAP_COUNT, "ssstore big gap count", ObStatClassIds::STORAGE, "ssstore big gap count", 60064, true, true) -STAT_EVENT_ADD_DEF(SSSTORE_SMALL_GAP_COUNT, "ssstore small gap count", ObStatClassIds::STORAGE, "ssstore small gap count", 60065, true, true) -STAT_EVENT_ADD_DEF(MEMSTORE_RANGE_PURGE_COUNT, "memstore range purge count", ObStatClassIds::STORAGE, "memstore range purge count", 60067, true, true) -STAT_EVENT_ADD_DEF(MEMSTORE_WRITE_LOCK_WAKENUP_COUNT, "memstore write lock wakenup count in lock_wait_mgr", ObStatClassIds::STORAGE, "memstore write lock wakenup count in lock_wait_mgr", 60068, true, true) -STAT_EVENT_ADD_DEF(MEMSTORE_SDR_SUBMIT_COUNT, "memstore active purge submit sdr count", ObStatClassIds::STORAGE, "memstore active purge submit sdr count", 60069, true, true) -STAT_EVENT_ADD_DEF(MEMSTORE_SDR_SUCC_COUNT, "memstore active purge handle sdr succ count", ObStatClassIds::STORAGE, "memstore handle sdr succ count", 60070, true, true) -STAT_EVENT_ADD_DEF(MEMSTORE_SDR_FAIL_COUNT, "memstore active purge handle sdr fail count", ObStatClassIds::STORAGE, "memstore handle sdr fail count", 60071, true, true) -STAT_EVENT_ADD_DEF(MEMSTORE_SDR_RETRY_COUNT, "memstore active purge handle sdr retry count", ObStatClassIds::STORAGE, "memstore handle sdr retry count", 60072, true, true) -STAT_EVENT_ADD_DEF(MEMSTORE_SDR_DELAY, "memstore active purge handle sdr delay", ObStatClassIds::STORAGE, "memstore active purge handle sdr delay", 60073, true, true) +STAT_EVENT_ADD_DEF(MEMSTORE_WRITE_LOCK_WAKENUP_COUNT, "memstore write lock wakenup count in lock_wait_mgr", ObStatClassIds::STORAGE, "memstore write lock wakenup count in lock_wait_mgr", 60068, false, true) -STAT_EVENT_ADD_DEF(ROWID_HIT, "rowid hit count", ObStatClassIds::STORAGE, "rowid hit count", 60074, true, true) -STAT_EVENT_ADD_DEF(EXIST_ROW_EFFECT_READ, "exist row effect read", ObStatClassIds::STORAGE, "exist row effect read", 60075, true, true) -STAT_EVENT_ADD_DEF(EXIST_ROW_EMPTY_READ, "exist row empty read", ObStatClassIds::STORAGE, "exist row empty read", 60076, true, true) -STAT_EVENT_ADD_DEF(GET_ROW_EFFECT_READ, "get row effect read", ObStatClassIds::STORAGE, "get row effect read", 60077, true, true) -STAT_EVENT_ADD_DEF(GET_ROW_EMPTY_READ, "get row empty read", ObStatClassIds::STORAGE, "get row empty read", 60078, true, true) -STAT_EVENT_ADD_DEF(SCAN_ROW_EFFECT_READ, "scan row effect read", ObStatClassIds::STORAGE, "scan row effect read", 60079, true, true) -STAT_EVENT_ADD_DEF(SCAN_ROW_EMPTY_READ, "scan row empty read", ObStatClassIds::STORAGE, "scan row empty read", 60080, true, true) -STAT_EVENT_ADD_DEF(BANDWIDTH_IN_SLEEP_US, "bandwidth in sleep us", ObStatClassIds::STORAGE, "bandwidth in sleep us", 60081, true, true) -STAT_EVENT_ADD_DEF(BANDWIDTH_OUT_SLEEP_US, "bandwidth out sleep us", ObStatClassIds::STORAGE, "bandwidth out sleep us", 60082, true, true) +STAT_EVENT_ADD_DEF(EXIST_ROW_EFFECT_READ, "exist row effect read", ObStatClassIds::STORAGE, "exist row effect read", 60075, false, true) +STAT_EVENT_ADD_DEF(EXIST_ROW_EMPTY_READ, "exist row empty read", ObStatClassIds::STORAGE, "exist row empty read", 60076, false, true) +STAT_EVENT_ADD_DEF(GET_ROW_EFFECT_READ, "get row effect read", ObStatClassIds::STORAGE, "get row effect read", 60077, false, true) +STAT_EVENT_ADD_DEF(GET_ROW_EMPTY_READ, "get row empty read", ObStatClassIds::STORAGE, "get row empty read", 60078, false, true) +STAT_EVENT_ADD_DEF(SCAN_ROW_EFFECT_READ, "scan row effect read", ObStatClassIds::STORAGE, "scan row effect read", 60079, false, true) +STAT_EVENT_ADD_DEF(SCAN_ROW_EMPTY_READ, "scan row empty read", ObStatClassIds::STORAGE, "scan row empty read", 60080, false, true) +STAT_EVENT_ADD_DEF(BANDWIDTH_IN_SLEEP_US, "bandwidth in sleep us", ObStatClassIds::STORAGE, "bandwidth in sleep us", 60081, false, true) +STAT_EVENT_ADD_DEF(BANDWIDTH_OUT_SLEEP_US, "bandwidth out sleep us", ObStatClassIds::STORAGE, "bandwidth out sleep us", 60082, false, true) -STAT_EVENT_ADD_DEF(MEMSTORE_WRITE_LOCK_WAIT_TIMEOUT_COUNT, "memstore write lock wait timeout count", ObStatClassIds::STORAGE, "memstore write wait timeout count", 60083, true, true) +STAT_EVENT_ADD_DEF(MEMSTORE_WRITE_LOCK_WAIT_TIMEOUT_COUNT, "memstore write lock wait timeout count", ObStatClassIds::STORAGE, "memstore write wait timeout count", 60083, false, true) -STAT_EVENT_ADD_DEF(DATA_BLOCK_READ_CNT, "accessed data micro block count", ObStatClassIds::STORAGE, "accessed data micro block count", 60084, true, true) -STAT_EVENT_ADD_DEF(DATA_BLOCK_CACHE_HIT, "data micro block cache hit", ObStatClassIds::STORAGE, "data micro block cache hit", 60085, true, true) -STAT_EVENT_ADD_DEF(INDEX_BLOCK_READ_CNT, "accessed index micro block count", ObStatClassIds::STORAGE, "accessed index micro block count", 60086, true, true) -STAT_EVENT_ADD_DEF(INDEX_BLOCK_CACHE_HIT, "index micro block cache hit", ObStatClassIds::STORAGE, "index micro block cache hit", 60087, true, true) -STAT_EVENT_ADD_DEF(BLOCKSCAN_BLOCK_CNT, "blockscaned data micro block count", ObStatClassIds::STORAGE, "blockscaned data micro block count", 60088, true, true) -STAT_EVENT_ADD_DEF(BLOCKSCAN_ROW_CNT, "blockscaned row count", ObStatClassIds::STORAGE, "blockscaned row count", 60089, true, true) -STAT_EVENT_ADD_DEF(PUSHDOWN_STORAGE_FILTER_ROW_CNT, "storage filtered row count", ObStatClassIds::STORAGE, "storage filter row count", 60090, true, true) +STAT_EVENT_ADD_DEF(DATA_BLOCK_READ_CNT, "accessed data micro block count", ObStatClassIds::STORAGE, "accessed data micro block count", 60084, false, true) +STAT_EVENT_ADD_DEF(DATA_BLOCK_CACHE_HIT, "data micro block cache hit", ObStatClassIds::STORAGE, "data micro block cache hit", 60085, false, true) +STAT_EVENT_ADD_DEF(INDEX_BLOCK_READ_CNT, "accessed index micro block count", ObStatClassIds::STORAGE, "accessed index micro block count", 60086, false, true) +STAT_EVENT_ADD_DEF(INDEX_BLOCK_CACHE_HIT, "index micro block cache hit", ObStatClassIds::STORAGE, "index micro block cache hit", 60087, false, true) +STAT_EVENT_ADD_DEF(BLOCKSCAN_BLOCK_CNT, "blockscaned data micro block count", ObStatClassIds::STORAGE, "blockscaned data micro block count", 60088, false, true) +STAT_EVENT_ADD_DEF(BLOCKSCAN_ROW_CNT, "blockscaned row count", ObStatClassIds::STORAGE, "blockscaned row count", 60089, false, true) +STAT_EVENT_ADD_DEF(PUSHDOWN_STORAGE_FILTER_ROW_CNT, "storage filtered row count", ObStatClassIds::STORAGE, "storage filter row count", 60090, false, true) // backup & restore STAT_EVENT_ADD_DEF(BACKUP_IO_READ_COUNT, "backup io read count", ObStatClassIds::STORAGE, "backup io read count", 69000, true, true) STAT_EVENT_ADD_DEF(BACKUP_IO_READ_BYTES, "backup io read bytes", ObStatClassIds::STORAGE, "backup io read bytes", 69001, true, true) STAT_EVENT_ADD_DEF(BACKUP_IO_WRITE_COUNT, "backup io write count", ObStatClassIds::STORAGE, "backup io write count", 69002, true, true) STAT_EVENT_ADD_DEF(BACKUP_IO_WRITE_BYTES, "backup io write bytes", ObStatClassIds::STORAGE, "backup io write bytes", 69003, true, true) -STAT_EVENT_ADD_DEF(COS_IO_READ_COUNT, "cos io read count", ObStatClassIds::STORAGE, "cos io read count", 69004, true, true) -STAT_EVENT_ADD_DEF(COS_IO_READ_BYTES, "cos io read bytes", ObStatClassIds::STORAGE, "cos io read bytes", 69005, true, true) -STAT_EVENT_ADD_DEF(COS_IO_WRITE_COUNT, "cos io write count", ObStatClassIds::STORAGE, "cos io write count", 69006, true, true) -STAT_EVENT_ADD_DEF(COS_IO_WRITE_BYTES, "cos io write bytes", ObStatClassIds::STORAGE, "cos io write bytes", 69007, true, true) -STAT_EVENT_ADD_DEF(COS_IO_LS_COUNT, "cos io list count", ObStatClassIds::STORAGE, "cos io list count", 69008, true, true) -STAT_EVENT_ADD_DEF(COS_IO_LS_LIMIT_COUNT, "cos list io limit count", ObStatClassIds::STORAGE, "cos list io limit count", 69009, true, true) STAT_EVENT_ADD_DEF(BACKUP_DELETE_COUNT, "backup delete count", ObStatClassIds::STORAGE, "backup delete count", 69010, true, true) -STAT_EVENT_ADD_DEF(COS_DELETE_COUNT, "cos delete count", ObStatClassIds::STORAGE, "cos delete count", 69011, true, true) STAT_EVENT_ADD_DEF(BACKUP_IO_READ_DELAY, "backup io read delay", ObStatClassIds::STORAGE, "backup io read delay", 69012, true, true) STAT_EVENT_ADD_DEF(BACKUP_IO_WRITE_DELAY, "backup io write delay", ObStatClassIds::STORAGE, "backup io write delay", 69013, true, true) @@ -361,7 +285,6 @@ STAT_EVENT_ADD_DEF(COS_IO_READ_DELAY, "cos io read delay", ObStatClassIds::STORA STAT_EVENT_ADD_DEF(COS_IO_WRITE_DELAY, "cos io write delay", ObStatClassIds::STORAGE, "cos io write delay", 69015, true, true) STAT_EVENT_ADD_DEF(COS_IO_LS_DELAY, "cos io list delay", ObStatClassIds::STORAGE, "cos io list delay", 69016, true, true) STAT_EVENT_ADD_DEF(BACKUP_DELETE_DELAY, "backup delete delay", ObStatClassIds::STORAGE, "backup delete delay", 69017, true, true) -STAT_EVENT_ADD_DEF(COS_DELETE_DELAY, "cos delete delay", ObStatClassIds::STORAGE, "cos delete delay", 69018, true, true) STAT_EVENT_ADD_DEF(BACKUP_IO_LS_COUNT, "backup io list count", ObStatClassIds::STORAGE, "backup io list count", 69019, true, true) STAT_EVENT_ADD_DEF(BACKUP_IO_READ_FAIL_COUNT, "backup io read failed count", ObStatClassIds::STORAGE, "backup io read failed count", 69020, true, true) @@ -373,185 +296,41 @@ STAT_EVENT_ADD_DEF(BACKUP_TAGGING_COUNT, "backup io tagging count", ObStatClassI STAT_EVENT_ADD_DEF(BACKUP_TAGGING_FAIL_COUNT, "backup io tagging failed count", ObStatClassIds::STORAGE, "backup io tagging count", 69025, true, true) // DEBUG -STAT_EVENT_ADD_DEF(REFRESH_SCHEMA_COUNT, "refresh schema count", ObStatClassIds::DEBUG, "refresh schema count", 70000, true, true) -STAT_EVENT_ADD_DEF(REFRESH_SCHEMA_TIME, "refresh schema time", ObStatClassIds::DEBUG, "refresh schema time", 70001, true, true) -STAT_EVENT_ADD_DEF(INNER_SQL_CONNECTION_EXECUTE_COUNT, "inner sql connection execute count", ObStatClassIds::DEBUG, "inner sql connection execute count", 70002, true, true) -STAT_EVENT_ADD_DEF(INNER_SQL_CONNECTION_EXECUTE_TIME, "inner sql connection execute time", ObStatClassIds::DEBUG, "inner sql connection execute time", 70003, true, true) -STAT_EVENT_ADD_DEF(PARTITION_TABLE_OPERATOR_GET_COUNT, "partition table operator get count", ObStatClassIds::DEBUG, "partition table operator get count", 70004, true, true) -STAT_EVENT_ADD_DEF(PARTITION_TABLE_OPERATOR_GET_TIME, "partition table operator get time", ObStatClassIds::DEBUG, "partition table operator get time", 70005, true, true) -STAT_EVENT_ADD_DEF(LS_ALL_TABLE_OPERATOR_GET_COUNT, "log stream table operator get count", ObStatClassIds::DEBUG, "log stream table operator get count", 70006, true, true) -STAT_EVENT_ADD_DEF(LS_ALL_TABLE_OPERATOR_GET_TIME, "log stream table operator get time", ObStatClassIds::DEBUG, "log stream table operator get time", 70007, true, true) +STAT_EVENT_ADD_DEF(REFRESH_SCHEMA_COUNT, "refresh schema count", ObStatClassIds::DEBUG, "refresh schema count", 70000, false, true) +STAT_EVENT_ADD_DEF(REFRESH_SCHEMA_TIME, "refresh schema time", ObStatClassIds::DEBUG, "refresh schema time", 70001, false, true) +STAT_EVENT_ADD_DEF(INNER_SQL_CONNECTION_EXECUTE_COUNT, "inner sql connection execute count", ObStatClassIds::DEBUG, "inner sql connection execute count", 70002, false, true) +STAT_EVENT_ADD_DEF(INNER_SQL_CONNECTION_EXECUTE_TIME, "inner sql connection execute time", ObStatClassIds::DEBUG, "inner sql connection execute time", 70003, false, true) +STAT_EVENT_ADD_DEF(LS_ALL_TABLE_OPERATOR_GET_COUNT, "log stream table operator get count", ObStatClassIds::DEBUG, "log stream table operator get count", 70006, false, true) +STAT_EVENT_ADD_DEF(LS_ALL_TABLE_OPERATOR_GET_TIME, "log stream table operator get time", ObStatClassIds::DEBUG, "log stream table operator get time", 70007, false, true) //CLOG 80001 ~ 90000 -STAT_EVENT_ADD_DEF(CLOG_SW_SUBMITTED_LOG_COUNT, "submitted to sliding window log count", ObStatClassIds::CLOG, "submitted to sliding window log count", 80001, true, true) -STAT_EVENT_ADD_DEF(CLOG_SW_SUBMITTED_LOG_SIZE, "submitted to sliding window log size", ObStatClassIds::CLOG, "submitted to sliding window log size", 80002, true, true) -STAT_EVENT_ADD_DEF(CLOG_INDEX_LOG_FLUSHED_LOG_COUNT, "index log flushed count", ObStatClassIds::CLOG, "index log flushed count", 80003, true, true) -STAT_EVENT_ADD_DEF(CLOG_INDEX_LOG_FLUSHED_LOG_SIZE, "index log flushed clog size", ObStatClassIds::CLOG, "index log flushed clog size", 80004, true, true) -STAT_EVENT_ADD_DEF(CLOG_FLUSHED_COUNT, "clog flushed count", ObStatClassIds::CLOG, "clog flushed count", 80005, true, true) -STAT_EVENT_ADD_DEF(CLOG_FLUSHED_SIZE, "clog flushed size", ObStatClassIds::CLOG, "clog flushed size", 80006, true, true) -STAT_EVENT_ADD_DEF(CLOG_READ_SIZE, "clog read size", ObStatClassIds::CLOG, "clog read size", 80007, true, true) -STAT_EVENT_ADD_DEF(CLOG_READ_COUNT, "clog read count", ObStatClassIds::CLOG, "clog read count", 80008, true, true) -STAT_EVENT_ADD_DEF(CLOG_DISK_READ_SIZE, "clog disk read size", ObStatClassIds::CLOG, "clog disk read size", 80009, true, true) -STAT_EVENT_ADD_DEF(CLOG_DISK_READ_COUNT, "clog disk read count", ObStatClassIds::CLOG, "clog disk read count", 80010, true, true) -STAT_EVENT_ADD_DEF(CLOG_DISK_READ_TIME, "clog disk read time", ObStatClassIds::CLOG, "clog disk read time", 80011, true, true) -STAT_EVENT_ADD_DEF(CLOG_FETCH_LOG_SIZE, "clog fetch log size", ObStatClassIds::CLOG, "clog fetch log size", 80012, true, true) -STAT_EVENT_ADD_DEF(CLOG_FETCH_LOG_COUNT, "clog fetch log count", ObStatClassIds::CLOG, "clog fetch log count", 80013, true, true) -STAT_EVENT_ADD_DEF(CLOG_FETCH_LOG_BY_LOCATION_SIZE, "clog fetch log by localtion size", ObStatClassIds::CLOG, "clog fetch log by localtion size", 80014, true, true) -STAT_EVENT_ADD_DEF(CLOG_FETCH_LOG_BY_LOCATION_COUNT, "clog fetch log by location count", ObStatClassIds::CLOG, "clog fetch log by localtion size", 80015, true, true) -STAT_EVENT_ADD_DEF(CLOG_READ_REQUEST_SUCC_SIZE, "clog read request succ size", ObStatClassIds::CLOG, "clog read request succ size", 80016, true, true) -STAT_EVENT_ADD_DEF(CLOG_READ_REQUEST_SUCC_COUNT, "clog read request succ count", ObStatClassIds::CLOG, "clog read request succ count", 80017, true, true) -STAT_EVENT_ADD_DEF(CLOG_READ_REQUEST_FAIL_COUNT, "clog read request fail count", ObStatClassIds::CLOG, "clog read request fail count", 80018, true, true) -STAT_EVENT_ADD_DEF(CLOG_LEADER_CONFIRM_TIME, "clog leader confirm time", ObStatClassIds::CLOG, "clog leader confirm time", 80019, true, true) -STAT_EVENT_ADD_DEF(CLOG_FLUSH_TASK_GENERATE_COUNT, "clog flush task generate count", ObStatClassIds::CLOG, "clog flush task generate count", 80020, true, true) -STAT_EVENT_ADD_DEF(CLOG_FLUSH_TASK_RELEASE_COUNT, "clog flush task release count", ObStatClassIds::CLOG, "clog flush task release count", 80021, true, true) -STAT_EVENT_ADD_DEF(CLOG_RPC_DELAY_TIME, "clog rpc delay time", ObStatClassIds::CLOG, "clog rpc delay time", 80022, true, true) -STAT_EVENT_ADD_DEF(CLOG_RPC_COUNT, "clog rpc count", ObStatClassIds::CLOG, "clog rpc count", 80023, true, true) -STAT_EVENT_ADD_DEF(CLOG_NON_KV_CACHE_HIT_COUNT, "clog non kv cache hit count", ObStatClassIds::CLOG, "clog non kv cache hit count", 80024, true, true) -STAT_EVENT_ADD_DEF(CLOG_RPC_REQUEST_HANDLE_TIME, "clog rpc request handle time", ObStatClassIds::CLOG, "clog rpc request handle time", 80025, true, true) -STAT_EVENT_ADD_DEF(CLOG_RPC_REQUEST_COUNT, "clog rpc request count", ObStatClassIds::CLOG, "clog rpc request count", 80026, true, true) -STAT_EVENT_ADD_DEF(CLOG_CACHE_HIT_COUNT, "clog cache hit count", ObStatClassIds::CLOG, "clog cache hit count", 80027, true, true) -STAT_EVENT_ADD_DEF(CLOG_STATE_LOOP_COUNT, "clog state loop count", ObStatClassIds::CLOG, "clog state loop count", 80028, true, true) -STAT_EVENT_ADD_DEF(CLOG_STATE_LOOP_TIME, "clog state loop time", ObStatClassIds::CLOG, "clog state loop time", 80029, true, true) -STAT_EVENT_ADD_DEF(CLOG_REPLAY_LOOP_COUNT, "clog replay loop count", ObStatClassIds::CLOG, "clog state loop count", 80030, true, true) -STAT_EVENT_ADD_DEF(CLOG_REPLAY_LOOP_TIME, "clog replay loop time", ObStatClassIds::CLOG, "clog state loop time", 80031, true, true) -STAT_EVENT_ADD_DEF(CLOG_TO_LEADER_ACTIVE_COUNT, "clog to leader active count", ObStatClassIds::CLOG, "clog to leader active count", 80032, true, true) -STAT_EVENT_ADD_DEF(CLOG_TO_LEADER_ACTIVE_TIME, "clog to leader active time", ObStatClassIds::CLOG, "clog to leader active time", 80033, true, true) -STAT_EVENT_ADD_DEF(ON_SUCCESS_COUNT, "on success cb count", ObStatClassIds::CLOG, "on success cb count", 80034, true, true) -STAT_EVENT_ADD_DEF(ON_SUCCESS_TIME, "on success cb time", ObStatClassIds::CLOG, "on success cb time", 80035, true, true) -STAT_EVENT_ADD_DEF(ON_LEADER_REVOKE_COUNT, "on leader revoke count", ObStatClassIds::CLOG, "on leader revoke count", 80036, true, true) -STAT_EVENT_ADD_DEF(ON_LEADER_REVOKE_TIME, "on leader revoke time", ObStatClassIds::CLOG, "on leader revoke time", 80037, true, true) -STAT_EVENT_ADD_DEF(ON_LEADER_TAKEOVER_COUNT, "on leader takeover count", ObStatClassIds::CLOG, "on leader takeover count", 80038, true, true) -STAT_EVENT_ADD_DEF(ON_LEADER_TAKEOVER_TIME, "on leader takeover time", ObStatClassIds::CLOG, "on leader takeover time", 80039, true, true) -STAT_EVENT_ADD_DEF(CLOG_WRITE_COUNT, "clog write count", ObStatClassIds::CLOG, "clog write count", 80040, true, true) -STAT_EVENT_ADD_DEF(CLOG_WRITE_TIME, "clog write time", ObStatClassIds::CLOG, "clog write time", 80041, true, true) -STAT_EVENT_ADD_DEF(ILOG_WRITE_COUNT, "ilog write count", ObStatClassIds::CLOG, "ilog write count", 80042, true, true) -STAT_EVENT_ADD_DEF(ILOG_WRITE_TIME, "ilog write time", ObStatClassIds::CLOG, "ilog write time", 80043, true, true) -STAT_EVENT_ADD_DEF(CLOG_FLUSHED_TIME, "clog flushed time", ObStatClassIds::CLOG, "clog flushed time", 80044, true, true) -STAT_EVENT_ADD_DEF(CLOG_TASK_CB_COUNT, "clog task cb count", ObStatClassIds::CLOG, "clog task cb count", 80045, true, true) -STAT_EVENT_ADD_DEF(CLOG_CB_QUEUE_TIME, "clog cb queue time", ObStatClassIds::CLOG, "clog cb queue time", 80046, true, true) -STAT_EVENT_ADD_DEF(CLOG_ACK_COUNT, "clog ack count", ObStatClassIds::CLOG, "clog ack count", 80049, true, true) -STAT_EVENT_ADD_DEF(CLOG_ACK_TIME, "clog ack time", ObStatClassIds::CLOG, "clog ack time", 80050, true, true) -STAT_EVENT_ADD_DEF(CLOG_FIRST_ACK_COUNT, "clog first ack count", ObStatClassIds::CLOG, "clog first ack count", 80051, true, true) -STAT_EVENT_ADD_DEF(CLOG_FIRST_ACK_TIME, "clog first ack time", ObStatClassIds::CLOG, "clog first ack time", 80052, true, true) -STAT_EVENT_ADD_DEF(CLOG_LEADER_CONFIRM_COUNT, "clog leader confirm count", ObStatClassIds::CLOG, "clog leader confirm count", 80053, true, true) -STAT_EVENT_ADD_DEF(CLOG_BATCH_CB_COUNT, "clog batch cb count", ObStatClassIds::CLOG, "clog batch cb count", 80054, true, true) -STAT_EVENT_ADD_DEF(CLOG_BATCH_CB_QUEUE_TIME, "clog batch cb queue time", ObStatClassIds::CLOG, "clog batch cb queue time", 80055, true, true) -STAT_EVENT_ADD_DEF(CLOG_MEMSTORE_MUTATOR_TOTAL_SIZE, "clog memstore mutator total size", ObStatClassIds::CLOG, "clog memstore mutator total size", 80056, true, true) -STAT_EVENT_ADD_DEF(CLOG_TRANS_LOG_TOTAL_SIZE, "clog trans log total size", ObStatClassIds::CLOG, "clog trans log total size", 80057, true, true) -STAT_EVENT_ADD_DEF(CLOG_SUBMIT_LOG_TOTAL_SIZE, "clog submit log total size", ObStatClassIds::CLOG, "clog submit log total size", 80058, true, true) -STAT_EVENT_ADD_DEF(CLOG_BATCH_BUFFER_TOTAL_SIZE, "clog batch buffer total size", ObStatClassIds::CLOG, "clog batch buffer total size", 80059, true, true) -STAT_EVENT_ADD_DEF(ILOG_BATCH_BUFFER_TOTAL_SIZE, "ilog batch buffer total size", ObStatClassIds::CLOG, "ilog batch buffer total size", 80060, true, true) -STAT_EVENT_ADD_DEF(CLOG_FILE_TOTAL_SIZE, "clog file total size", ObStatClassIds::CLOG, "clog file total size", 80061, true, true) -STAT_EVENT_ADD_DEF(ILOG_FILE_TOTAL_SIZE, "ilog file total size", ObStatClassIds::CLOG, "ilog file total size", 80062, true, true) -STAT_EVENT_ADD_DEF(CLOG_BATCH_SUBMITTED_COUNT, "clog batch submitted count", ObStatClassIds::CLOG, "clog batch submitted count", 80063, true, true) -STAT_EVENT_ADD_DEF(CLOG_BATCH_COMMITTED_COUNT, "clog batch committed count", ObStatClassIds::CLOG, "clog batch committed count", 80064, true, true) +STAT_EVENT_ADD_DEF(CLOG_RPC_COUNT, "clog rpc count", ObStatClassIds::CLOG, "clog rpc count", 80023, false, true) +STAT_EVENT_ADD_DEF(CLOG_RPC_REQUEST_HANDLE_TIME, "clog rpc request handle time", ObStatClassIds::CLOG, "clog rpc request handle time", 80025, false, true) +STAT_EVENT_ADD_DEF(CLOG_RPC_REQUEST_COUNT, "clog rpc request count", ObStatClassIds::CLOG, "clog rpc request count", 80026, false, true) +STAT_EVENT_ADD_DEF(CLOG_WRITE_COUNT, "clog write count", ObStatClassIds::CLOG, "clog write count", 80040, false, true) +STAT_EVENT_ADD_DEF(CLOG_WRITE_TIME, "clog write time", ObStatClassIds::CLOG, "clog write time", 80041, false, true) +STAT_EVENT_ADD_DEF(CLOG_TRANS_LOG_TOTAL_SIZE, "clog trans log total size", ObStatClassIds::CLOG, "clog trans log total size", 80057, false, true) // CLOG.EXTLOG 81001 ~ 90000 -STAT_EVENT_ADD_DEF(CLOG_EXTLOG_FETCH_LOG_SIZE, "external log service fetch log size", ObStatClassIds::CLOG, "external log service fetch log size", 81001, true, true) -STAT_EVENT_ADD_DEF(CLOG_EXTLOG_FETCH_LOG_COUNT, "external log service fetch log count", ObStatClassIds::CLOG, "external log service fetch log count", 81002, true, true) -STAT_EVENT_ADD_DEF(CLOG_EXTLOG_FETCH_RPC_COUNT, "external log service fetch rpc count", ObStatClassIds::CLOG, "external log service fetch rpc count", 81003, true, true) -STAT_EVENT_ADD_DEF(CLOG_EXTLOG_HEARTBEAT_RPC_COUNT, "external log service heartbeat rpc count", ObStatClassIds::CLOG, "external log service heartbeat rpc count", 81004, true, true) -STAT_EVENT_ADD_DEF(CLOG_EXTLOG_HEARTBEAT_PARTITION_COUNT, "external log service heartbeat partition count", ObStatClassIds::CLOG, "external log service heartbeat partition count", 81005, true, true) +STAT_EVENT_ADD_DEF(CLOG_EXTLOG_FETCH_LOG_SIZE, "external log service fetch log size", ObStatClassIds::CLOG, "external log service fetch log size", 81001, false, true) +STAT_EVENT_ADD_DEF(CLOG_EXTLOG_FETCH_LOG_COUNT, "external log service fetch log count", ObStatClassIds::CLOG, "external log service fetch log count", 81002, false, true) +STAT_EVENT_ADD_DEF(CLOG_EXTLOG_FETCH_RPC_COUNT, "external log service fetch rpc count", ObStatClassIds::CLOG, "external log service fetch rpc count", 81003, false, true) //CLOG.REPLAY -STAT_EVENT_ADD_DEF(CLOG_SUCC_REPLAY_TRANS_LOG_COUNT, "replay engine success replay transaction log count", ObStatClassIds::CLOG, "replay engine success replay transaction log count", 82001, true, true) -STAT_EVENT_ADD_DEF(CLOG_SUCC_REPLAY_TRANS_LOG_TIME, "replay engine success replay transaction log time", ObStatClassIds::CLOG, "replay engine success replay transaction log time", 82002, true, true) -STAT_EVENT_ADD_DEF(CLOG_FAIL_REPLAY_TRANS_LOG_COUNT, "replay engine fail replay transaction log count", ObStatClassIds::CLOG, "replay engine fail replay transaction log count", 82003, true, true) -STAT_EVENT_ADD_DEF(CLOG_FAIL_REPLAY_TRANS_LOG_TIME, "replay engine fail replay transaction log time", ObStatClassIds::CLOG, "replay engine fail replay transaction log time", 82004, true, true) -STAT_EVENT_ADD_DEF(CLOG_SUCC_REPLAY_START_MEMBERSHIP_LOG_COUNT, "replay engine success replay start_membership log count", ObStatClassIds::CLOG, "replay engine success replay start_membership log count", 82005, true, true) -STAT_EVENT_ADD_DEF(CLOG_SUCC_REPLAY_START_MEMBERSHIP_LOG_TIME, "replay engine success replay start_membership log time", ObStatClassIds::CLOG, "replay engine success replay start_membership log time", 82006, true, true) -STAT_EVENT_ADD_DEF(CLOG_FAIL_REPLAY_START_MEMBERSHIP_LOG_COUNT, "replay engine fail replay start_membership log count", ObStatClassIds::CLOG, "replay engine fail replay start_membership log count", 82007, true, true) -STAT_EVENT_ADD_DEF(CLOG_FAIL_REPLAY_START_MEMBERSHIP_LOG_TIME, "replay engine fail replay start_membership log time", ObStatClassIds::CLOG, "replay engine fail replay start_membership log time", 82008, true, true) -STAT_EVENT_ADD_DEF(CLOG_SUCC_REPLAY_OFFLINE_COUNT, "replay engine success replay offline replica count", ObStatClassIds::CLOG, "replay engine success replay offline replica count", 82009, true, true) -STAT_EVENT_ADD_DEF(CLOG_FAIL_REPLAY_OFFLINE_COUNT, "replay engine fail replay offline replica count", ObStatClassIds::CLOG, "replay engine fail replay offline replica count", 82010, true, true) -STAT_EVENT_ADD_DEF(CLOG_REPLAY_RETRY_COUNT, "replay engine retry replay task count", ObStatClassIds::CLOG, "replay engine retry replay task count", 82011, true, true) -STAT_EVENT_ADD_DEF(CLOG_HANDLE_REPLAY_TASK_COUNT, "replay engine handle replay task count", ObStatClassIds::CLOG, "replay engine handle replay task count", 82012, true, true) -STAT_EVENT_ADD_DEF(CLOG_HANDLE_REPLAY_TIME, "replay engine total handle time", ObStatClassIds::CLOG, "replay engine total handle time", 82013, true, true) -STAT_EVENT_ADD_DEF(CLOG_REPLAY_TASK_SUBMIT_COUNT, "replay engine submitted replay task count", ObStatClassIds::CLOG, "replay engine submitted replay task count", 82014, true, true) -STAT_EVENT_ADD_DEF(CLOG_REPLAY_TASK_TRANS_SUBMIT_COUNT, "replay engine submitted transaction replay task count", ObStatClassIds::CLOG, "replay engine submitted transaction replay task count", 82015, true, true) -STAT_EVENT_ADD_DEF(CLOG_REPLAY_TASK_FREEZE_SUBMIT_COUNT, "replay engine submitted freeze replay task count", ObStatClassIds::CLOG, "replay engine submitted freeze replay task count", 82016, true, true) -STAT_EVENT_ADD_DEF(CLOG_REPLAY_TASK_OFFLINE_SUBMIT_COUNT, "replay engine submitted offline replay task count", ObStatClassIds::CLOG, "replay engine submitted offline replay task count", 82017, true, true) -STAT_EVENT_ADD_DEF(CLOG_REPLAY_TASK_SPLIT_SUBMIT_COUNT, "replay engine submitted split replay task count", ObStatClassIds::CLOG, "replay engine submitted split replay task count", 82018, true, true) -STAT_EVENT_ADD_DEF(CLOG_REPLAY_TASK_START_MEMBERSHIP_SUBMIT_COUNT, "replay engine submitted start membership replay task count", ObStatClassIds::CLOG, "replay engine submitted start membership replay task count", 82019, true, true) -STAT_EVENT_ADD_DEF(CLOG_SUCC_REPLAY_SPLIT_LOG_COUNT, "replay engine success replay split log count", ObStatClassIds::CLOG, "replay engine success replay split log count", 82020, true, true) -STAT_EVENT_ADD_DEF(CLOG_FAIL_REPLAY_SPLIT_LOG_COUNT, "replay engine fail replay split log count", ObStatClassIds::CLOG, "replay engine fail replay split log count", 82021, true, true) -STAT_EVENT_ADD_DEF(CLOG_REPLAY_TASK_META_SUBMIT_COUNT, "replay engine submitted partition meta replay task count", ObStatClassIds::CLOG, "replay engine submitted partition meta replay task count", 82022, true, true) -STAT_EVENT_ADD_DEF(CLOG_SUCC_REPLAY_META_LOG_COUNT, "replay engine success replay partition meta log count", ObStatClassIds::CLOG, "replay engine success replay partition meta log count", 82023, true, true) -STAT_EVENT_ADD_DEF(CLOG_FAIL_REPLAY_META_LOG_COUNT, "replay engine fail replay partition meta log count", ObStatClassIds::CLOG, "replay engine fail replay partition meta log count", 82024, true, true) -STAT_EVENT_ADD_DEF(CLOG_REPLAY_TASK_FLASHBACK_SUBMIT_COUNT, "replay engine submitted flashback replay task count", ObStatClassIds::CLOG, "replay engine submitted flashback replay task count", 82025, true, true) -STAT_EVENT_ADD_DEF(CLOG_SUCC_REPLAY_FLASHBACK_LOG_COUNT, "replay engine success replay flashback replica count", ObStatClassIds::CLOG, "replay engine success replay flashback replica count", 82026, true, true) -STAT_EVENT_ADD_DEF(CLOG_FAIL_REPLAY_FLASHBACK_LOG_COUNT, "replay engine fail replay flashback replica count", ObStatClassIds::CLOG, "replay engine fail replay flashback replica count", 82027, true, true) -STAT_EVENT_ADD_DEF(CLOG_REPLAY_TASK_ADD_PARTITION_TO_PG_SUBMIT_COUNT, "replay engine submitted add partition to pg replay task count", ObStatClassIds::CLOG, "replay engine submitted add partition to pg replay task count", 82028, true, true) -STAT_EVENT_ADD_DEF(CLOG_SUCC_REPLAY_ADD_PARTITION_TO_PG_LOG_COUNT, "replay engine success replay add partition to pg count", ObStatClassIds::CLOG, "replay engine success replay add partition to pg count", 82029, true, true) -STAT_EVENT_ADD_DEF(CLOG_FAIL_REPLAY_ADD_PARTITION_TO_PG_LOG_COUNT, "replay engine fail replay add partition to pg count", ObStatClassIds::CLOG, "replay engine fail replay add partition to pg count", 82030, true, true) -STAT_EVENT_ADD_DEF(CLOG_REPLAY_TASK_REMOVE_PG_PARTITION_SUBMIT_COUNT, "replay engine submitted remove pg partition replay task count", ObStatClassIds::CLOG, "replay engine submitted remove pg partition replay task count", 82031, true, true) -STAT_EVENT_ADD_DEF(CLOG_SUCC_REPLAY_REMOVE_PG_PARTITION_LOG_COUNT, "replay engine success replay remove pg partition count", ObStatClassIds::CLOG, "replay engine success replay remove pg partition count", 82032, true, true) -STAT_EVENT_ADD_DEF(CLOG_FAIL_REPLAY_REMOVE_PG_PARTITION_LOG_COUNT, "replay engine fail replay remove pg partition count", ObStatClassIds::CLOG, "replay engine fail replay remove pg partition count", 82033, true, true) -STAT_EVENT_ADD_DEF(CLOG_HANDLE_SUBMIT_TASK_COUNT, "replay engine handle submit task count", ObStatClassIds::CLOG, "replay engine handle submit task count", 82034, true, true) -STAT_EVENT_ADD_DEF(CLOG_HANDLE_SUBMIT_TASK_SIZE, "replay engine handle submit task size", ObStatClassIds::CLOG, "replay engine handle submit task size", 82035, true, true) -STAT_EVENT_ADD_DEF(CLOG_HANDLE_SUBMIT_TIME, "replay engine handle submit time", ObStatClassIds::CLOG, "replay engine handle submit time", 82036, true, true) //CLOG.PROXY -STAT_EVENT_ADD_DEF(CLOG_PROXY_ACTIVE_LOADER_COUNT, "active proxy loader count for principal server", ObStatClassIds::CLOG, "active proxy loader count for principal server", 83001, true, true) -STAT_EVENT_ADD_DEF(CLOG_PROXY_LOAD_REQUSET_COUNT, "load proxy request count", ObStatClassIds::CLOG, "load proxy request count", 83002, true, true) -STAT_EVENT_ADD_DEF(CLOG_PROXY_FAILED_LOADER_COUNT, "failed proxy loader count", ObStatClassIds::CLOG, "failed proxy loader count", 83003, true, true) -STAT_EVENT_ADD_DEF(CLOG_PROXY_EXPIRED_LOADER_COUNT, "expired proxy loader count", ObStatClassIds::CLOG, "expired proxy loader count", 83004, true, true) -STAT_EVENT_ADD_DEF(CLOG_PROXY_TASK_NOT_HANDLED, "proxy loading task count waiting in queue", ObStatClassIds::CLOG, "proxy loading task count waiting in queue", 83005, true, true) -STAT_EVENT_ADD_DEF(CLOG_PROXY_LOAD_ILOG_FILE_COUNT, "total file count in fill file id cache", ObStatClassIds::CLOG, "total file count in fill file id cache", 83006, true, true) -STAT_EVENT_ADD_DEF(CLOG_PROXY_LOAD_ILOG_FILE_TIME, "total time cost on fill file id cache", ObStatClassIds::CLOG, "total time cost on fill file id cache", 83007, true, true) // ELECTION -STAT_EVENT_ADD_DEF(ELECTION_CHANGE_LEAER_COUNT, "election change leader count", ObStatClassIds::ELECT, "election change leader count", 90001, false, true) -STAT_EVENT_ADD_DEF(ELECTION_LEADER_REVOKE_COUNT, "election leader revoke count", ObStatClassIds::ELECT, "election leader revoke count", 90002, false, true) //OBSERVER -STAT_EVENT_ADD_DEF(OBSERVER_PARTITION_TABLE_UPDATER_BATCH_COUNT, "observer partition table updater batch count", - ObStatClassIds::OBSERVER, "observer partition table updater batch count", 100001, false, true) -STAT_EVENT_ADD_DEF(OBSERVER_PARTITION_TABLE_UPDATE_COUNT, "observer partition table updater count", - ObStatClassIds::OBSERVER, "observer partition table updater count", 100002, false, true) -STAT_EVENT_ADD_DEF(OBSERVER_PARTITION_TABLE_UPDATER_PROCESS_TIME, "observer partition table updater process time", - ObStatClassIds::OBSERVER, "observer partition table updater process time", 100003, false, true) -STAT_EVENT_ADD_DEF(OBSERVER_PARTITION_TABLE_UPDATER_DROP_COUNT, "observer partition table updater drop task count", - ObStatClassIds::OBSERVER, "observer partition table updater drop task count", 100004, false, true) -STAT_EVENT_ADD_DEF(OBSERVER_PARTITION_TABLE_UPDATER_REPUT_COUNT, "observer partition table updater reput to queue count", - ObStatClassIds::OBSERVER, "observer partition table updater reput to queue count", 100005, false, true) -STAT_EVENT_ADD_DEF(OBSERVER_PARTITION_TABLE_UPDATER_FAIL_TIMES, "observer partition table updater execute fail times", - ObStatClassIds::OBSERVER, "observer partition table updater fail times", 100006, false, true) -STAT_EVENT_ADD_DEF(OBSERVER_PARTITION_TABLE_UPDATER_FINISH_COUNT, "observer partition table updater success execute count", - ObStatClassIds::OBSERVER, "observer partition table updater execute success count", 100007, false, true) -STAT_EVENT_ADD_DEF(OBSERVER_PARTITION_TABLE_UPDATER_TOTAL_COUNT, "observer partition table updater total task count", - ObStatClassIds::OBSERVER, "observer partition table updater total task count", 100008, false, true) // rootservice -STAT_EVENT_ADD_DEF(RS_PARTITION_MIGRATE_COUNT, "partition migrate count", ObStatClassIds::RS, "partition migrate count", 110001, false, true) -STAT_EVENT_ADD_DEF(RS_PARTITION_MIGRATE_TIME, "partition migrate time", ObStatClassIds::RS, "partition migrate time", 110002, false, true) -STAT_EVENT_ADD_DEF(RS_PARTITION_ADD_COUNT, "partition add count", ObStatClassIds::RS, "partition add count", 110003, false, true) -STAT_EVENT_ADD_DEF(RS_PARTITION_ADD_TIME, "partition add time", ObStatClassIds::RS, "partition add time", 110004, false, true) -STAT_EVENT_ADD_DEF(RS_PARTITION_REBUILD_COUNT, "partition rebuild count", ObStatClassIds::RS, "partition rebuild count", 110005, false, true) -STAT_EVENT_ADD_DEF(RS_PARTITION_REBUILD_TIME, "partition rebuild time", ObStatClassIds::RS, "partition rebuild time", 110006, false, true) -STAT_EVENT_ADD_DEF(RS_PARTITION_TRANSFORM_COUNT, "partition transform count", ObStatClassIds::RS, "partition transform count", 110007, false, true) -STAT_EVENT_ADD_DEF(RS_PARTITION_TRANSFORM_TIME, "partition transform time", ObStatClassIds::RS, "partition transform time", 110008, false, true) -STAT_EVENT_ADD_DEF(RS_PARTITION_REMOVE_COUNT, "partition remove count", ObStatClassIds::RS, "partition remove count", 110009, false, true) -STAT_EVENT_ADD_DEF(RS_PARTITION_REMOVE_TIME, "partition remove time", ObStatClassIds::RS, "partition remove time", 110010, false, true) -STAT_EVENT_ADD_DEF(RS_PARTITION_CHANGE_MEMBER_COUNT, "partition change member count", ObStatClassIds::RS, "partition change member count", 110011, false, true) -STAT_EVENT_ADD_DEF(RS_PARTITION_CHANGE_MEMBER_TIME, "partition change member time", ObStatClassIds::RS, "partition change member time", 110012, false, true) -STAT_EVENT_ADD_DEF(RS_SWITCH_LEADER_SUCC, "succ switch leader count", ObStatClassIds::RS, "succ switch leader count", 110013, false, true) -STAT_EVENT_ADD_DEF(RS_SWITCH_LEADER_FAIL, "failed switch leader count", ObStatClassIds::RS, "failed switch leader count", 110014, false, true) STAT_EVENT_ADD_DEF(RS_RPC_SUCC_COUNT, "success rpc process", ObStatClassIds::RS, "success rpc process", 110015, false, true) STAT_EVENT_ADD_DEF(RS_RPC_FAIL_COUNT, "failed rpc process", ObStatClassIds::RS, "failed rpc process", 110016, false, true) STAT_EVENT_ADD_DEF(RS_BALANCER_SUCC_COUNT, "balancer succ execute count", ObStatClassIds::RS, "balanacer succ execute count", 110017, false, true) STAT_EVENT_ADD_DEF(RS_BALANCER_FAIL_COUNT, "balancer failed execute count", ObStatClassIds::RS, "balanacer failed execute count", 110018, false, true) -STAT_EVENT_ADD_DEF(RS_TASK_FAIL_COUNT, "rebalance task failed execute count", ObStatClassIds::RS, "rebalance task failed execute count", 110019, false, true) -STAT_EVENT_ADD_DEF(RS_PARTITION_MODIFY_QUORUM_COUNT, "partition modify quorum count", ObStatClassIds::RS, "partition modify quorum count", 110020, false, true) -STAT_EVENT_ADD_DEF(RS_PARTITION_MODIFY_QUORUM_TIME, "partition modify quorum time", ObStatClassIds::RS, "partition modify quorum time", 110021, false, true) -STAT_EVENT_ADD_DEF(RS_COPY_GLOBAL_INDEX_SSTABLE_COUNT, "copy global index sstable count", ObStatClassIds::RS, "copy global index sstable count", 110022, false, true) -STAT_EVENT_ADD_DEF(RS_COPY_GLOBAL_INDEX_SSTABLE_TIME, "copy global index sstable time", ObStatClassIds::RS, "copy global index sstable time", 110023, false, true) -STAT_EVENT_ADD_DEF(RS_COPY_LOCAL_INDEX_SSTABLE_COUNT, "copy local index sstable count", ObStatClassIds::RS, "copy local index sstable count", 110024, false, true) -STAT_EVENT_ADD_DEF(RS_COPY_LOCAL_INDEX_SSTABLE_TIME, "copy local index sstable time", ObStatClassIds::RS, "copy local index sstable time", 110025, false, true) -STAT_EVENT_ADD_DEF(RS_STANDBY_CUTDATA_TASK_COUNT, "standby cutdata task count", ObStatClassIds::RS, "standby cutdata task count", 110026, false, true) -STAT_EVENT_ADD_DEF(RS_STANDBY_CUTDATA_TASK_TIME, "standby cutdata task count", ObStatClassIds::RS, "standby cutdata task count", 110027, false, true) // TABLE API (19xxxx) // -- retrieve 1900xx @@ -597,12 +376,7 @@ STAT_EVENT_ADD_DEF(TABLEAPI_BATCH_RETRIEVE_ROW, "batch retrieve rows", ObStatCla // -- complex_batch_operation 1907xx STAT_EVENT_ADD_DEF(TABLEAPI_BATCH_HYBRID_COUNT, "batch hybrid execute count", ObStatClassIds::TABLEAPI, "batch hybrid count", 190701, true, true) STAT_EVENT_ADD_DEF(TABLEAPI_BATCH_HYBRID_TIME, "batch hybrid execute time", ObStatClassIds::TABLEAPI, "batch hybrid time", 190702, true, true) -STAT_EVENT_ADD_DEF(TABLEAPI_BATCH_HYBRID_RETRIEVE_ROW, "batch hybrid retrieve rows", ObStatClassIds::TABLEAPI, "batch hybrid retrieve rows", 190703, true, true) -STAT_EVENT_ADD_DEF(TABLEAPI_BATCH_HYBRID_INSERT_ROW, "batch hybrid insert rows", ObStatClassIds::TABLEAPI, "batch hybrid insert rows", 190704, true, true) -STAT_EVENT_ADD_DEF(TABLEAPI_BATCH_HYBRID_DELETE_ROW, "batch hybrid delete rows", ObStatClassIds::TABLEAPI, "batch hybrid delete rows", 190705, true, true) -STAT_EVENT_ADD_DEF(TABLEAPI_BATCH_HYBRID_UPDATE_ROW, "batch hybrid update rows", ObStatClassIds::TABLEAPI, "batch hybrid update rows", 190706, true, true) STAT_EVENT_ADD_DEF(TABLEAPI_BATCH_HYBRID_INSERT_OR_UPDATE_ROW, "batch hybrid insert_or_update rows", ObStatClassIds::TABLEAPI, "batch hybrid insert_or_update rows", 190707, true, true) -STAT_EVENT_ADD_DEF(TABLEAPI_BATCH_HYBRID_REPLACE_ROW, "batch hybrid replace rows", ObStatClassIds::TABLEAPI, "batch hybrid replace rows", 190708, true, true) // -- other 1908xx STAT_EVENT_ADD_DEF(TABLEAPI_LOGIN_COUNT, "table api login count", ObStatClassIds::TABLEAPI, "login count", 190801, true, true) STAT_EVENT_ADD_DEF(TABLEAPI_TSC_VIOLATE_COUNT, "table api OB_TRANSACTION_SET_VIOLATION count", ObStatClassIds::TABLEAPI, "OB_TRANSACTION_SET_VIOLATION count", 190802, true, true) @@ -686,10 +460,8 @@ STAT_EVENT_SET_DEF(OPT_TAB_COL_STAT_CACHE_SIZE, "table col stat cache size", ObS STAT_EVENT_SET_DEF(INDEX_BLOCK_CACHE_SIZE, "index block cache size", ObStatClassIds::CACHE, "index block cache size", 120004, false, true) STAT_EVENT_SET_DEF(SYS_BLOCK_CACHE_SIZE, "sys block cache size", ObStatClassIds::CACHE, "sys block cache size", 120005, false, true) STAT_EVENT_SET_DEF(USER_BLOCK_CACHE_SIZE, "user block cache size", ObStatClassIds::CACHE, "user block cache size", 120006, false, true) -STAT_EVENT_SET_DEF(SYS_ROW_CACHE_SIZE, "sys row cache size", ObStatClassIds::CACHE, "sys row cache size", 120007, false, true) STAT_EVENT_SET_DEF(USER_ROW_CACHE_SIZE, "user row cache size", ObStatClassIds::CACHE, "user row cache size", 120008, false, true) STAT_EVENT_SET_DEF(BLOOM_FILTER_CACHE_SIZE, "bloom filter cache size", ObStatClassIds::CACHE, "bloom filter cache size", 120009, false, true) -STAT_EVENT_SET_DEF(BLOOM_FILTER_CACHE_STRATEGY, "bloom filter cache strategy", ObStatClassIds::CACHE, "bloom filter cache strategy", 120010, false, true) // STORAGE STAT_EVENT_SET_DEF(ACTIVE_MEMSTORE_USED, "active memstore used", ObStatClassIds::STORAGE, "active memstore used", 130000, false, true) @@ -717,9 +489,6 @@ STAT_EVENT_SET_DEF(WORKER_TIME, "worker time", ObStatClassIds::RESOURCE, "worker STAT_EVENT_SET_DEF(CPU_TIME, "cpu time", ObStatClassIds::RESOURCE, "cpu time", 140013, false, true) //CLOG -STAT_EVENT_SET_DEF(CLOG_DISK_FREE_SIZE, "clog disk free size", ObStatClassIds::CLOG, "clog disk free size", 150001, false, true) -STAT_EVENT_SET_DEF(CLOG_DISK_FREE_RATIO, "clog disk free ratio", ObStatClassIds::CLOG, "clog disk free ratio", 150002, false, true) -STAT_EVENT_SET_DEF(CLOG_LAST_CHECK_LOG_FILE_COLLECT_TIME, "clog last check log file collect time", ObStatClassIds::CLOG, "clog last check log file collect time", 150003, false, true) // DEBUG STAT_EVENT_SET_DEF(OBLOGGER_WRITE_SIZE, "oblogger log bytes", ObStatClassIds::DEBUG, "oblogger log bytes", 160001, false, true) @@ -728,18 +497,6 @@ STAT_EVENT_SET_DEF(ROOTSERVICE_WRITE_SIZE, "rootservice log bytes", ObStatClassI STAT_EVENT_SET_DEF(OBLOGGER_TOTAL_WRITE_COUNT, "oblogger total log count", ObStatClassIds::DEBUG, "oblogger total log count", 160004, false, true) STAT_EVENT_SET_DEF(ELECTION_TOTAL_WRITE_COUNT, "election total log count", ObStatClassIds::DEBUG, "election total log count", 160005, false, true) STAT_EVENT_SET_DEF(ROOTSERVICE_TOTAL_WRITE_COUNT, "rootservice total log count", ObStatClassIds::DEBUG, "rootservice total log count", 160006, false, true) -STAT_EVENT_SET_DEF(ASYNC_TINY_LOG_WRITE_COUNT, "async tiny log write count", ObStatClassIds::DEBUG, "async tiny log write count", 160007, false, true) -STAT_EVENT_SET_DEF(ASYNC_NORMAL_LOG_WRITE_COUNT, "async normal log write count", ObStatClassIds::DEBUG, "async normal log write count", 160008, false, true) -STAT_EVENT_SET_DEF(ASYNC_LARGE_LOG_WRITE_COUNT, "async large log write count", ObStatClassIds::DEBUG, "async large log write count", 160009, false, true) -STAT_EVENT_SET_DEF(ASYNC_TINY_LOG_DROPPED_COUNT, "async tiny log dropped count", ObStatClassIds::DEBUG, "async tiny log dropped count", 160010, false, true) -STAT_EVENT_SET_DEF(ASYNC_NORMAL_LOG_DROPPED_COUNT, "async normal log dropped count", ObStatClassIds::DEBUG, "async normal log dropped count", 160011, false, true) -STAT_EVENT_SET_DEF(ASYNC_LARGE_LOG_DROPPED_COUNT, "async large log dropped count", ObStatClassIds::DEBUG, "async large log dropped count", 160012, false, true) -STAT_EVENT_SET_DEF(ASYNC_ACTIVE_TINY_LOG_COUNT, "async active tiny log count", ObStatClassIds::DEBUG, "async active tiny log count", 160013, false, true) -STAT_EVENT_SET_DEF(ASYNC_ACTIVE_NORMAL_LOG_COUNT, "async active normal log count", ObStatClassIds::DEBUG, "async active normal log count", 160014, false, true) -STAT_EVENT_SET_DEF(ASYNC_ACTIVE_LARGE_LOG_COUNT, "async active large log count", ObStatClassIds::DEBUG, "async active large log count", 160015, false, true) -STAT_EVENT_SET_DEF(ASYNC_RELEASED_TINY_LOG_COUNT, "async released tiny log count", ObStatClassIds::DEBUG, "async released tiny log count", 160016, false, true) -STAT_EVENT_SET_DEF(ASYNC_RELEASED_NORMAL_LOG_COUNT, "async released normal log count", ObStatClassIds::DEBUG, "async released normal log count", 160017, false, true) -STAT_EVENT_SET_DEF(ASYNC_RELEASED_LARGE_LOG_COUNT, "async released large log count", ObStatClassIds::DEBUG, "async released large log count", 160018, false, true) STAT_EVENT_SET_DEF(ASYNC_ERROR_LOG_DROPPED_COUNT, "async error log dropped count", ObStatClassIds::DEBUG, "async error log dropped count", 160019, false, true) STAT_EVENT_SET_DEF(ASYNC_WARN_LOG_DROPPED_COUNT, "async warn log dropped count", ObStatClassIds::DEBUG, "async warn log dropped count", 160020, false, true) STAT_EVENT_SET_DEF(ASYNC_INFO_LOG_DROPPED_COUNT, "async info log dropped count", ObStatClassIds::DEBUG, "async info log dropped count", 160021, false, true) @@ -758,18 +515,6 @@ STAT_EVENT_SET_DEF(ASYNC_DATA_MAINTAIN_LOG_DROPPED_COUNT, "async data maintain l STAT_EVENT_SET_DEF(ASYNC_ROOT_SERVICE_LOG_DROPPED_COUNT, "async root service log dropped count", ObStatClassIds::DEBUG, "async root service log dropped count", 160034, false, true) STAT_EVENT_SET_DEF(ASYNC_SCHEMA_LOG_DROPPED_COUNT, "async schema log dropped count", ObStatClassIds::DEBUG, "async schema log dropped count", 160035, false, true) STAT_EVENT_SET_DEF(ASYNC_FORCE_ALLOW_LOG_DROPPED_COUNT, "async force allow log dropped count", ObStatClassIds::DEBUG, "async force allow log dropped count", 160036, false, true) -STAT_EVENT_SET_DEF(ASYNC_TINY_LOG_WRITE_COUNT_FOR_ERR, "async tiny log write count for error log", ObStatClassIds::DEBUG, "async tiny log write count for error log", 160037, false, true) -STAT_EVENT_SET_DEF(ASYNC_NORMAL_LOG_WRITE_COUNT_FOR_ERR, "async normal log write count for error log", ObStatClassIds::DEBUG, "async normal log write count for error log", 160038, false, true) -STAT_EVENT_SET_DEF(ASYNC_LARGE_LOG_WRITE_COUNT_FOR_ERR, "async large log write count for err/or log", ObStatClassIds::DEBUG, "async large log write count for error log", 160039, false, true) -STAT_EVENT_SET_DEF(ASYNC_TINY_LOG_DROPPED_COUNT_FOR_ERR, "async tiny log dropped count for error log", ObStatClassIds::DEBUG, "async tiny log dropped count for error log", 160040, false, true) -STAT_EVENT_SET_DEF(ASYNC_NORMAL_LOG_DROPPED_COUNT_FOR_ERR, "async normal log dropped count for error log", ObStatClassIds::DEBUG, "async normal log dropped count for error log", 160041, false, true) -STAT_EVENT_SET_DEF(ASYNC_LARGE_LOG_DROPPED_COUNT_FOR_ERR, "async large log dropped count for error log", ObStatClassIds::DEBUG, "async large log dropped count for error log", 160042, false, true) -STAT_EVENT_SET_DEF(ASYNC_ACTIVE_TINY_LOG_COUNT_FOR_ERR, "async active tiny log count for error log", ObStatClassIds::DEBUG, "async active tiny log count for error log", 160043, false, true) -STAT_EVENT_SET_DEF(ASYNC_ACTIVE_NORMAL_LOG_COUNT_FOR_ERR, "async active normal log count for error log", ObStatClassIds::DEBUG, "async active normal log count for error log", 160044, false, true) -STAT_EVENT_SET_DEF(ASYNC_ACTIVE_LARGE_LOG_COUNT_FOR_ERR, "async active large log count for error log", ObStatClassIds::DEBUG, "async active large log count for error log", 160045, false, true) -STAT_EVENT_SET_DEF(ASYNC_RELEASED_TINY_LOG_COUNT_FOR_ERR, "async released tiny log count for error log", ObStatClassIds::DEBUG, "async released tiny log count for error log", 160046, false, true) -STAT_EVENT_SET_DEF(ASYNC_RELEASED_NORMAL_LOG_COUNT_FOR_ERR, "async released normal log count for error log", ObStatClassIds::DEBUG, "async released normal log count for error log", 160047, false, true) -STAT_EVENT_SET_DEF(ASYNC_RELEASED_LARGE_LOG_COUNT_FOR_ERR, "async released large log count for error log", ObStatClassIds::DEBUG, "async released large log countfor error log", 160048, false, true) //OBSERVER @@ -779,10 +524,6 @@ STAT_EVENT_SET_DEF(OBSERVER_PARTITION_TABLE_UPATER_CORE_QUEUE_SIZE, "observer pa // rootservice STAT_EVENT_SET_DEF(RS_START_SERVICE_TIME, "rootservice start time", ObStatClassIds::RS, "rootservice start time", 180001, false, true) -STAT_EVENT_SET_DEF(RS_TASK_QUEUE_SIZE_HIGH_WAIT, "rootservice waiting task count with high priority", ObStatClassIds::RS, "rootservice waiting task count with high priority", 180002, false, true) -STAT_EVENT_SET_DEF(RS_TASK_QUEUE_SIZE_HIGH_SCHED, "rootservice scheduled task count with high priority", ObStatClassIds::RS, "rootservice scheduled task count with high priority", 180003, false, true) -STAT_EVENT_SET_DEF(RS_TASK_QUEUE_SIZE_LOW_WAIT, "rootservice waiting task count with low priority", ObStatClassIds::RS, "rootservice waiting task count with low priority", 180004, false, true) -STAT_EVENT_SET_DEF(RS_TASK_QUEUE_SIZE_LOW_SCHED, "rootservice scheduled task count with low priority", ObStatClassIds::RS, "rootservice scheduled task count with low priority", 180005, false, true) // END STAT_EVENT_SET_DEF(STAT_EVENT_SET_END, "event set end", ObStatClassIds::DEBUG, "event set end", 300000, false, false) #endif diff --git a/deps/oblib/src/lib/trace/ob_trace.cpp b/deps/oblib/src/lib/trace/ob_trace.cpp index b6aaaddae..94e29bb0f 100644 --- a/deps/oblib/src/lib/trace/ob_trace.cpp +++ b/deps/oblib/src/lib/trace/ob_trace.cpp @@ -18,6 +18,8 @@ #include "lib/oblog/ob_log_level.h" #include "lib/time/ob_time_utility.h" +#include "lib/time/ob_tsc_timestamp.h" +#include "common/ob_clock_generator.h" namespace oceanbase { @@ -126,7 +128,7 @@ uint64_t UUID::gen_rand() UUID UUID::gen() { UUID ret; - ret.high_ = ObTimeUtility::fast_current_time(); + ret.high_ = common::ObClockGenerator::getClock(); ret.low_ = gen_rand(); return ret; } @@ -418,7 +420,7 @@ ObSpanCtx* ObTrace::begin_span(uint32_t span_type, uint8_t level, bool is_follow new_span->span_id_.high_ = 0; new_span->source_span_ = last_active_span_; new_span->is_follow_ = is_follow; - new_span->start_ts_ = ObTimeUtility::fast_current_time(); + new_span->start_ts_ = OB_TSC_TIMESTAMP.current_time(); new_span->end_ts_ = 0; new_span->tags_ = nullptr; last_active_span_ = new_span; @@ -444,7 +446,7 @@ void ObTrace::end_span(ObSpanCtx* span) if (!trace_id_.is_inited() || OB_ISNULL(span) || !span->span_id_.is_inited()) { // do nothing } else { - span->end_ts_ = ObTimeUtility::fast_current_time(); + span->end_ts_ = OB_TSC_TIMESTAMP.current_time(); last_active_span_ = span->source_span_; } } diff --git a/deps/oblib/src/lib/trace/ob_trace_event.h b/deps/oblib/src/lib/trace/ob_trace_event.h index 684c2d259..7db64e98a 100644 --- a/deps/oblib/src/lib/trace/ob_trace_event.h +++ b/deps/oblib/src/lib/trace/ob_trace_event.h @@ -111,7 +111,7 @@ private: // template // EVENT_COUNT is generally sufficient // After BUF_SIZE exceeds the defined value, the kv pair in REC_TRACE_EXT will not be printed -typedef ObTraceEventRecorderBase<189, 1600> ObTraceEventRecorder; +typedef ObTraceEventRecorderBase<189, 1200> ObTraceEventRecorder; inline ObTraceEventRecorder *get_trace_recorder() { diff --git a/deps/oblib/src/lib/worker.cpp b/deps/oblib/src/lib/worker.cpp index cd7703524..8e7ade4d4 100644 --- a/deps/oblib/src/lib/worker.cpp +++ b/deps/oblib/src/lib/worker.cpp @@ -17,6 +17,7 @@ #include "lib/oblog/ob_log.h" #include "lib/time/ob_time_utility.h" #include "lib/allocator/ob_malloc.h" +#include "common/ob_clock_generator.h" using namespace oceanbase::common; using namespace oceanbase::lib; @@ -95,5 +96,5 @@ int64_t Worker::get_timeout_remain() const bool Worker::is_timeout() const { - return ObTimeUtility::current_time() >= timeout_ts_; + return common::ObClockGenerator::getClock() >= timeout_ts_; } diff --git a/deps/oblib/src/rpc/frame/ob_req_queue_thread.cpp b/deps/oblib/src/rpc/frame/ob_req_queue_thread.cpp index 7b80e0414..04ce2b6ff 100644 --- a/deps/oblib/src/rpc/frame/ob_req_queue_thread.cpp +++ b/deps/oblib/src/rpc/frame/ob_req_queue_thread.cpp @@ -142,7 +142,7 @@ int ObReqQueue::process_task(void *task) void ObReqQueue::loop() { int ret = OB_SUCCESS; - int64_t timeout = 300 * 1000; + int64_t timeout = 3000 * 1000; void *task = NULL; if (OB_ISNULL(qhandler_)) { ret = OB_INVALID_ARGUMENT; diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h index f392372b2..fef61f449 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h @@ -667,7 +667,7 @@ PCODE_DEF(OB_FLASHBACK_CLOG, 0x741) PCODE_DEF(OB_DUMP_SINGLE_TX_DATA, 0x742) PCODE_DEF(OB_GET_LS_REPLAYED_SCN, 0x743) PCODE_DEF(OB_REFRESH_TENANT_INFO, 0x744) -PCODE_DEF(OB_REFRESH_TENANT_INFO_CONTENT, 0x745) +PCODE_DEF(OB_UPDATE_TENANT_INFO_CACHE, 0x745) // BatchRpc PCODE_DEF(OB_BATCH, 0x750) @@ -967,6 +967,3 @@ PCODE_DEF(OB_LOB_QUERY, 0x1561) // 1581-1590 for workload repository // PCODE_DEF(OB_WR_ASYNC_SNAPSHOT_TASK, 0x1581) - -// session info verification -// PCODE_DEF(OB_SESS_INFO_VERIFICATION, 0x1592) diff --git a/deps/oblib/unittest/lib/stat/test_diagnose_info.cpp b/deps/oblib/unittest/lib/stat/test_diagnose_info.cpp index 0d103a9e8..8f15238ec 100644 --- a/deps/oblib/unittest/lib/stat/test_diagnose_info.cpp +++ b/deps/oblib/unittest/lib/stat/test_diagnose_info.cpp @@ -41,19 +41,19 @@ namespace common TEST(ObDiagnoseSessionInfo, guard) { - EVENT_INC(ELECTION_CHANGE_LEAER_COUNT); + EVENT_INC(SYS_TIME_MODEL_ENCRYPT_CPU); EXPECT_EQ(1, GET_TSI(ObSessionDIBuffer)->get_tenant_id()); - EXPECT_EQ(1, TENANT_EVENT_GET(ObStatEventIds::ELECTION_CHANGE_LEAER_COUNT)); + EXPECT_EQ(1, TENANT_EVENT_GET(ObStatEventIds::SYS_TIME_MODEL_ENCRYPT_CPU)); { ObTenantStatEstGuard tenant_guard(2); - EVENT_INC(ELECTION_CHANGE_LEAER_COUNT); + EVENT_INC(SYS_TIME_MODEL_ENCRYPT_CPU); EXPECT_EQ(2, GET_TSI(ObSessionDIBuffer)->get_tenant_id()); - EXPECT_EQ(1, TENANT_EVENT_GET(ObStatEventIds::ELECTION_CHANGE_LEAER_COUNT)); + EXPECT_EQ(1, TENANT_EVENT_GET(ObStatEventIds::SYS_TIME_MODEL_ENCRYPT_CPU)); { ObTenantStatEstGuard tenant_guard(3); - EVENT_INC(ELECTION_CHANGE_LEAER_COUNT); + EVENT_INC(SYS_TIME_MODEL_ENCRYPT_CPU); EXPECT_EQ(3, GET_TSI(ObSessionDIBuffer)->get_tenant_id()); - EXPECT_EQ(1, TENANT_EVENT_GET(ObStatEventIds::ELECTION_CHANGE_LEAER_COUNT)); + EXPECT_EQ(1, TENANT_EVENT_GET(ObStatEventIds::SYS_TIME_MODEL_ENCRYPT_CPU)); } EXPECT_EQ(2, GET_TSI(ObSessionDIBuffer)->get_tenant_id()); } diff --git a/mittest/mtlenv/storage/test_memtable_v2.cpp b/mittest/mtlenv/storage/test_memtable_v2.cpp index 5937b5522..e472e992b 100644 --- a/mittest/mtlenv/storage/test_memtable_v2.cpp +++ b/mittest/mtlenv/storage/test_memtable_v2.cpp @@ -21,6 +21,7 @@ #include "storage/memtable/mvcc/ob_mvcc_row.h" #include "storage/init_basic_struct.h" #include "observer/ob_safe_destroy_thread.h" +#include "share/ob_master_key_getter.h" namespace oceanbase { @@ -113,13 +114,15 @@ public: tenant_id_(TENANT_ID), rowkey_cnt_(1), value_cnt_(1), + encrypt_index_(500006), /*table_id*/ iter_param_(), columns_(), allocator_(), allocator2_(), read_info_(), trans_version_range_(), - query_flag_() + query_flag_(), + encrypt_meta_(NULL) { columns_.reset(); read_info_.reset(); @@ -163,6 +166,9 @@ public: // is_sstable_contain_lock is_sstable_contains_lock_ = false; + // mock master key getter + ASSERT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().init(NULL)); + TRANS_LOG(INFO, "setup success"); } @@ -182,6 +188,9 @@ public: // reset allocator allocator_.reset(); allocator2_.reset(); + + ObMasterKeyGetter::instance().destroy(); + TRANS_LOG(INFO, "teardown success"); } @@ -424,10 +433,11 @@ public: snapshot_scn.convert_for_tx(snapshot); start_stmt(wtx, snapshot_scn, expire_time); EXPECT_EQ(expect_ret, (ret = memtable->set(*wtx, - tablet_id_.id(), + encrypt_index_, read_info_, columns_, - write_row))); + write_row, + encrypt_meta_))); TRANS_LOG(INFO, "======================= end write tx ======================", K(ret), K(wtx->mvcc_acc_ctx_.tx_id_), K(*wtx), K(snapshot), K(expire_time), K(write_row)); @@ -721,6 +731,80 @@ public: encrypt_info)); } + void serialize_encrypted_redo_log(ObStoreCtx *store_ctx, + char *redo_log_buffer) + { + int64_t mutator_size = 0; + int64_t pos = 0; + ObRedoLogSubmitHelper helper; + ObPartTransCtx *tx_ctx = store_ctx->mvcc_acc_ctx_.tx_ctx_; + ObTxRedoLog redo_log(1000 /*fake log_no*/, + 1000 /*fake cluster_version_*/); + + redo_log.set_mutator_buf(redo_log_buffer); + redo_log.set_mutator_size(REDO_BUFFER_SIZE, false /*after_fill*/); + + ObIMemtableCtx *mem_ctx = store_ctx->mvcc_acc_ctx_.mem_ctx_; + EXPECT_EQ(OB_SUCCESS, mem_ctx->fill_redo_log(redo_log.get_mutator_buf(), + redo_log.get_mutator_size(), + mutator_size, + helper)); + + redo_log.set_mutator_size(mutator_size, true /*after_fill*/); + EXPECT_EQ(OB_SUCCESS, redo_log.serialize(redo_log_buffer, REDO_BUFFER_SIZE, pos)); + } + + void deserialize_redo_log_extract_encryption(char *redo_log_buffer, + ObTxRedoLog &redo_log, + ObMemtableMutatorIterator &mmi, + ObCLogEncryptInfo &encrypt_info) + { + int64_t pos = 0; + EXPECT_EQ(OB_SUCCESS, redo_log.deserialize(redo_log_buffer, REDO_BUFFER_SIZE, pos)); + + //mock replay iterator + //deserialize encrypt info + mmi.reset(); + pos = 0; + EXPECT_EQ(OB_SUCCESS, mmi.deserialize(redo_log.get_replay_mutator_buf(), + redo_log.get_mutator_size(), + pos, + encrypt_info)); + EXPECT_EQ(redo_log.get_mutator_size(), pos); + + //decrypt table key + ObTxEncryptMap *encrypt_map = encrypt_info.encrypt_map_; + char decrypted_table_key[OB_ENCRYPTED_TABLE_KEY_LEN] = {0}; + int64_t out_len = 0; + if (OB_NOT_NULL(encrypt_map) && encrypt_map->begin() != encrypt_map->end()) { + int64_t master_key_len = 0; + ObEncryptMeta &meta = encrypt_map->begin()->meta_; + meta.tenant_id_ = 1004; + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::get_master_key(meta.tenant_id_, + meta.master_key_version_, + meta.master_key_.ptr(), + OB_MAX_MASTER_KEY_LENGTH, + master_key_len)); + meta.master_key_.get_content().set_length(master_key_len); + EXPECT_EQ(OB_SUCCESS, ObAesEncryption::aes_decrypt(meta.master_key_.ptr(), + meta.master_key_.size(), + meta.encrypted_table_key_.ptr(), + meta.encrypted_table_key_.size(), + OB_ENCRYPTED_TABLE_KEY_LEN, + NULL, + 0, + static_cast(meta.encrypt_algorithm_), + decrypted_table_key, + out_len)); + meta.table_key_.set_content(decrypted_table_key, out_len); + + EXPECT_EQ(true, meta.is_valid()); + } else { + ob_abort(); + } + + } + void replay_tx(ObStoreCtx *store_ctx, ObMemtable *memtable, const int64_t replay_log_ts, @@ -728,22 +812,50 @@ public: { int ret = OB_SUCCESS; bool can_continue = true; + ObEncryptRowBuf unused_row_buf; + transaction::ObCLogEncryptInfo unused_encrypt_info; + unused_encrypt_info.init(); while (can_continue) { - if (OB_FAIL(mmi.iterate_next_row())) { + if (OB_FAIL(mmi.iterate_next_row(unused_row_buf, unused_encrypt_info))) { if (OB_ITER_END != ret) { TRANS_LOG(ERROR, "get row head failed", K(ret)); } can_continue = false; } else { - ObEncryptRowBuf row_buf; TRANS_LOG(INFO, "TEST_MEMTABLE V2: replay row", K(*store_ctx)); share::SCN replay_scn; replay_scn.convert_for_tx(replay_log_ts); store_ctx->mvcc_acc_ctx_.mem_ctx_->set_redo_scn(replay_scn); EXPECT_EQ(OB_SUCCESS, memtable->replay_row(*store_ctx, - &mmi, - row_buf)); + &mmi)); + } + } + } + + void replay_tx_with_encryption(ObStoreCtx *store_ctx, + ObMemtable *memtable, + const int64_t replay_log_ts, + ObMemtableMutatorIterator &mmi, + const ObCLogEncryptInfo &encrypt_info) + { + int ret = OB_SUCCESS; + bool can_continue = true; + ObEncryptRowBuf row_buf; + while (can_continue) { + if (OB_FAIL(mmi.iterate_next_row(row_buf, encrypt_info))) { + if (OB_ITER_END != ret) { + TRANS_LOG(ERROR, "get row head failed", K(ret)); + } + can_continue = false; + } else { + TRANS_LOG(INFO, "TEST_MEMTABLE V2: replay row", + K(*store_ctx)); + share::SCN replay_scn; + replay_scn.convert_for_tx(replay_log_ts); + store_ctx->mvcc_acc_ctx_.mem_ctx_->set_redo_scn(replay_scn); + EXPECT_EQ(OB_SUCCESS, memtable->replay_row(*store_ctx, + &mmi)); } } } @@ -953,6 +1065,58 @@ public: EXPECT_EQ(total_trans_node_cnt, row->total_trans_node_cnt_); TRANS_LOG(INFO, "=============== VERIFY MVCC ROW END ===============", K(*row)); } + + void create_and_store_encrypt_info(ObSerializeEncryptMeta &encrypt_meta) + { + //create fake master key + char cur_master_key[] = "abcdef"; + uint64_t tenant_id = 1004; + uint64_t master_key_id = 1; + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().set_master_key(tenant_id, master_key_id, cur_master_key, strlen(cur_master_key))); + + TRANS_LOG(INFO, "create and set fake master key success", K(cur_master_key)); + + //create encrypted_table_key + char origin_table_key[OB_ORIGINAL_TABLE_KEY_LEN] = {0}; + char encrypt_table_key[OB_ENCRYPTED_TABLE_KEY_LEN] = {0}; + + int64_t encrypt_out_len = 0; + int algorithm = ObAesOpMode::ob_invalid_mode + 3; + EXPECT_EQ(OB_SUCCESS, ObKeyGenerator::generate_encrypt_key(origin_table_key, OB_ORIGINAL_TABLE_KEY_LEN)); + EXPECT_EQ(OB_SUCCESS, ObAesEncryption::aes_encrypt(cur_master_key, strlen(cur_master_key), + origin_table_key, OB_ORIGINAL_TABLE_KEY_LEN, + OB_ENCRYPTED_TABLE_KEY_LEN, NULL, 0, + static_cast(algorithm), + encrypt_table_key, encrypt_out_len)); + encrypt_table_key[encrypt_out_len] = '\0'; + EXPECT_STRNE(origin_table_key, encrypt_table_key); + + TRANS_LOG(INFO, "create encrypted table key success", K(origin_table_key), K(encrypt_table_key)); + + //create encrypt_meta + char master_key[OB_MAX_MASTER_KEY_LENGTH] = {0}; + int64_t master_key_len = 0; + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::get_master_key(tenant_id, master_key_id, master_key, OB_MAX_MASTER_KEY_LENGTH, master_key_len)); + EXPECT_STREQ(master_key, cur_master_key); + + char random_string[OB_CLOG_ENCRYPT_RANDOM_LEN] = {0}; + EXPECT_EQ(OB_SUCCESS, ObKeyGenerator::generate_encrypt_key(random_string, OB_CLOG_ENCRYPT_RANDOM_LEN)); + + encrypt_meta.master_key_.set_content(master_key, master_key_len); + encrypt_meta.table_key_.set_content(origin_table_key, OB_ORIGINAL_TABLE_KEY_LEN); + encrypt_meta.encrypted_table_key_.set_content(encrypt_table_key, OB_ENCRYPTED_TABLE_KEY_LEN); + encrypt_meta.random_.set_content(random_string, OB_CLOG_ENCRYPT_RANDOM_LEN); + encrypt_meta.tenant_id_ = tenant_id; + encrypt_meta.master_key_version_ = master_key_id; + encrypt_meta.encrypt_algorithm_ = algorithm; + + TRANS_LOG(INFO, "create encrypt_meta success", K(encrypt_meta)); + + //store encrypt-info + encrypt_meta_ = &encrypt_meta; + TRANS_LOG(INFO, "store encrypt meta success"); + } + private: static const int64_t READ_TX_ID = 987654321; static const int64_t UNUSED_VALUE = -1; @@ -965,6 +1129,7 @@ private: const int64_t tenant_id_; const int64_t rowkey_cnt_; const int64_t value_cnt_; + uint64_t encrypt_index_; ObTableIterParam iter_param_; ObSEArray columns_; @@ -975,6 +1140,7 @@ private: ObVersionRange trans_version_range_; ObQueryFlag query_flag_; char redo_log_buffer_[REDO_BUFFER_SIZE]; + ObSerializeEncryptMeta *encrypt_meta_; }; ObLSTxCtxMgr TestMemtableV2::ls_tx_ctx_mgr_; @@ -2027,6 +2193,141 @@ TEST_F(TestMemtableV2, test_replay) fmemtable->destroy(); } +TEST_F(TestMemtableV2, test_replay_with_clog_encryption) +{ + ObMemtable *lmemtable = create_memtable(); + ObMemtable *fmemtable = create_memtable(); + + TRANS_LOG(INFO, "######## CASE1: txn1 write row in lmemtable"); + ObDatumRowkey rowkey; + ObStoreRow write_row; + ObDatumRowkey rowkey2; + ObStoreRow write_row2; + + EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ + 2, /*value*/ + rowkey, + write_row)); + EXPECT_EQ(OB_SUCCESS, mock_row(1, /*key*/ + 3, /*value*/ + rowkey2, + write_row2)); + + ObTransID write_tx_id = ObTransID(1); + ObStoreCtx *wtx = start_tx(write_tx_id); + + //create encrypt_meta and store encrypt_meta in encrypt_meta_ + ObSerializeEncryptMeta encrypt_meta; + create_and_store_encrypt_info(encrypt_meta); + + //encrypt_meta_ will be added to memtable during ObMemtable::set + write_tx(wtx, + lmemtable, + 1000, /*snapshot version*/ + write_row); + const int64_t wtx_seq_no1 = ObSequence::get_max_seq_no(); + write_tx(wtx, + lmemtable, + 1200, /*snapshot version*/ + write_row2); + const int64_t wtx_seq_no2 = ObSequence::get_max_seq_no(); + + //use encrypt_meta_ in memtable during submitting log + char *redo_log_buffer = new char[REDO_BUFFER_SIZE]; + serialize_encrypted_redo_log(wtx, redo_log_buffer); + + commit_txn(wtx, + 2000,/*commit_version*/ + false/*need_write_back*/); + + TRANS_LOG(INFO, "######## CASE2: txn2 replay row in fmemtable"); + + ObTxRedoLogTempRef temp_ref; + ObTxRedoLog redo_log(temp_ref); + + //get encrypt_info during deserializing log + ObMemtableMutatorIterator mmi; + ObCLogEncryptInfo encrypt_info; + encrypt_info.init(); + deserialize_redo_log_extract_encryption(redo_log_buffer, redo_log, mmi, encrypt_info); + + ObTransID replay_tx_id = ObTransID(2); + ObStoreCtx *ptx = start_tx(replay_tx_id, true); + replay_tx_with_encryption(ptx, + fmemtable, + 1300, /*replay_log_ts*/ + mmi, + encrypt_info); + read_row(ptx, + fmemtable, + rowkey, + 1500, /*snapshot version*/ + 1, /*key*/ + 3 /*value*/); + + ObMvccRowCallback *first_cb = (ObMvccRowCallback *)(get_tx_last_cb(ptx)->prev_); + verify_cb(get_tx_last_cb(ptx), + fmemtable, + wtx_seq_no2, + 1, /*key*/ + true, /*is_link*/ + false,/*need_fill_redo*/ + 1300 /*log_ts*/); + verify_cb(first_cb, + fmemtable, + wtx_seq_no1, + 1, /*key*/ + true, /*is_link*/ + false,/*need_fill_redo*/ + 1300 /*log_ts*/); + verify_tnode(get_tx_last_tnode(ptx), + first_cb->tnode_, /*prev tnode*/ + NULL, /*next tnode*/ + lmemtable, + ptx->mvcc_acc_ctx_.tx_id_, + INT64_MAX, /*trans_version*/ + wtx_seq_no2, + 1, /*modify_count*/ + ObMvccTransNode::F_INIT, + ObDmlFlag::DF_INSERT, + 1, /*key*/ + 3, /*value*/ + 1300 /*log_ts*/); + verify_tnode(first_cb->tnode_, + NULL, /*prev tnode*/ + get_tx_last_tnode(ptx), /*next tnode*/ + lmemtable, + ptx->mvcc_acc_ctx_.tx_id_, + INT64_MAX, /*trans_version*/ + wtx_seq_no1, + 0, /*modify_count*/ + ObMvccTransNode::F_INIT, + ObDmlFlag::DF_INSERT, + 1, /*key*/ + 2, /*value*/ + 1300 /*log_ts*/); + verify_mvcc_row(get_tx_last_mvcc_row(ptx), + ObDmlFlag::DF_NOT_EXIST, + ObDmlFlag::DF_NOT_EXIST, + get_tx_last_tnode(ptx), + 0, /*max_trans_version*/ + 2 /*total_trans_node_cnt*/); + commit_txn(ptx, + 2500,/*commit_version*/ + false/*need_write_back*/); + read_row(fmemtable, + rowkey, + 3000, /*snapshot version*/ + 1, /*key*/ + 3 /*value*/); + + //release resources + delete[] redo_log_buffer; + encrypt_meta_ = NULL; + lmemtable->destroy(); + fmemtable->destroy(); +} + TEST_F(TestMemtableV2, test_compact) { ObMemtable *lmemtable = create_memtable(); @@ -3004,14 +3305,16 @@ TEST_F(TestMemtableV2, test_seq_set_violation) tablet_id_.id(), read_info_, columns_, - write_row))); + write_row, + encrypt_meta_))); start_pdml_stmt(wtx, scn_3000, read_seq_no, 1000000000/*expire_time*/); EXPECT_EQ(OB_ERR_PRIMARY_KEY_DUPLICATE, (ret = memtable->set(*wtx, tablet_id_.id(), read_info_, columns_, - write_row))); + write_row, + encrypt_meta_))); memtable->destroy(); } diff --git a/mittest/mtlenv/tablelock/table_lock_tx_common_env.h b/mittest/mtlenv/tablelock/table_lock_tx_common_env.h index d0b409790..d8bd7ceea 100644 --- a/mittest/mtlenv/tablelock/table_lock_tx_common_env.h +++ b/mittest/mtlenv/tablelock/table_lock_tx_common_env.h @@ -73,7 +73,6 @@ void MockObTxCtx::init(const uint64_t tenant_id, ASSERT_EQ(OB_SUCCESS, lock_.init(this)); ASSERT_EQ(OB_SUCCESS, init_log_cbs_(ls_id, trans_id)); - ASSERT_EQ(OB_SUCCESS, clog_encrypt_info_.init()); init_memtable_ctx_(lock_memtable_handle); ctx_tx_data_.test_init(*tx_data, nullptr); diff --git a/mittest/mtlenv/test_multi_tenant.cpp b/mittest/mtlenv/test_multi_tenant.cpp index 80285ad68..4375ad84c 100644 --- a/mittest/mtlenv/test_multi_tenant.cpp +++ b/mittest/mtlenv/test_multi_tenant.cpp @@ -193,15 +193,6 @@ TEST_F(TestMultiTenant, create_and_remove_tenant) ASSERT_EQ(OB_SUCCESS, ret); } -TEST_F(TestMultiTenant, RS_TENANT) -{ - uint64_t tenant_id = OB_RS_TENANT_ID; - int ret = add_tenant(tenant_id); - ASSERT_EQ(OB_SUCCESS, ret); - lib::ObMallocAllocator *ma = lib::ObMallocAllocator::get_instance(); - ASSERT_EQ(OB_RS_MEMORY, ma->get_tenant_limit(OB_RS_TENANT_ID)); -} - //int get_tenant_with_tenant_lock(const uint64_t tenant_id, ObTenant *&tenant) const; TEST_F(TestMultiTenant, tenant_lock) { diff --git a/src/logservice/applyservice/ob_log_apply_service.cpp b/src/logservice/applyservice/ob_log_apply_service.cpp index 50c17ff37..a402bb46a 100644 --- a/src/logservice/applyservice/ob_log_apply_service.cpp +++ b/src/logservice/applyservice/ob_log_apply_service.cpp @@ -15,6 +15,7 @@ #include "logservice/ob_ls_adapter.h" #include "logservice/palf/palf_env.h" #include "share/scn.h" +#include "common/ob_clock_generator.h" #include "share/rc/ob_tenant_base.h" #include "share/ob_thread_mgr.h" #include "storage/tx_storage/ob_ls_service.h" @@ -904,8 +905,8 @@ void ObApplyStatus::statistics_cb_cost_(const LSN &lsn, const int64_t idx) { // no need to print debug log when config [default value is true] is false; - if (GCONF.enable_record_trace_log) { - const int64_t cb_finish_time = ObTimeUtility::fast_current_time(); + if (REACH_TIME_INTERVAL(10 * 1000)) { + const int64_t cb_finish_time = common::ObClockGenerator::getClock(); int64_t total_cost_time = cb_finish_time - append_start_time; int64_t append_cost_time = append_finish_time - append_start_time; int64_t cb_wait_thread_time = cb_first_handle_time - append_finish_time; diff --git a/src/logservice/archiveservice/ob_archive_service.h b/src/logservice/archiveservice/ob_archive_service.h index ff8f61587..b1764fc3f 100644 --- a/src/logservice/archiveservice/ob_archive_service.h +++ b/src/logservice/archiveservice/ob_archive_service.h @@ -166,7 +166,7 @@ private: void print_archive_status_(); private: - const int64_t THREAD_RUN_INTERVAL = 1000 * 1000L; + const int64_t THREAD_RUN_INTERVAL = 5 * 1000 * 1000L; private: bool inited_; uint64_t tenant_id_; diff --git a/src/logservice/leader_coordinator/ob_leader_coordinator.cpp b/src/logservice/leader_coordinator/ob_leader_coordinator.cpp index ac51b6f1a..0ee08ac07 100644 --- a/src/logservice/leader_coordinator/ob_leader_coordinator.cpp +++ b/src/logservice/leader_coordinator/ob_leader_coordinator.cpp @@ -78,9 +78,9 @@ int ObLeaderCoordinator::mtl_init(ObLeaderCoordinator *&p_coordinator)// init ti if(p_coordinator->is_running_) { ret = OB_INIT_TWICE; COORDINATOR_LOG(ERROR, "has been inited alread]y", KR(ret)); - } else if (CLICK_FAIL(p_coordinator->recovery_detect_timer_.init_and_start(1, 1_ms, "CoordTR"))) { + } else if (CLICK_FAIL(p_coordinator->recovery_detect_timer_.init_and_start(1, 100_ms, "CoordTR"))) { COORDINATOR_LOG(ERROR, "fail to init and start recovery_detect_timer", KR(ret)); - } else if (CLICK_FAIL(p_coordinator->failure_detect_timer_.init_and_start(1, 1_ms, "CoordTF"))) { + } else if (CLICK_FAIL(p_coordinator->failure_detect_timer_.init_and_start(1, 100_ms, "CoordTF"))) { COORDINATOR_LOG(ERROR, "fail to init and start failure_detect_timer", KR(ret)); } else { COORDINATOR_LOG(INFO, "ObLeaderCoordinator mtl init success", KR(ret)); @@ -99,7 +99,7 @@ int ObLeaderCoordinator::mtl_start(ObLeaderCoordinator *&p_coordinator)// start new(p_coordinator->all_ls_election_reference_info_) ObArray(); if (CLICK_FAIL(p_coordinator->recovery_detect_timer_.schedule_task_repeat( p_coordinator->refresh_priority_task_handle_, - 500_ms, + 5000_ms, [p_coordinator](){ p_coordinator->refresh(); return false; }))) { COORDINATOR_LOG(ERROR, "schedule repeat task failed", KR(ret)); } else { @@ -177,6 +177,12 @@ int ObLeaderCoordinator::get_ls_election_reference_info(const share::ObLSID &ls_ return ret; } +int ObLeaderCoordinator::schedule_refresh_priority_task() +{ + // the parameter is set to 1us to pass the checking + return refresh_priority_task_handle_.reschedule_after(1); +} + } } } diff --git a/src/logservice/leader_coordinator/ob_leader_coordinator.h b/src/logservice/leader_coordinator/ob_leader_coordinator.h index c572d8834..70f657082 100644 --- a/src/logservice/leader_coordinator/ob_leader_coordinator.h +++ b/src/logservice/leader_coordinator/ob_leader_coordinator.h @@ -66,6 +66,7 @@ public: */ void refresh(); int get_ls_election_reference_info(const share::ObLSID &ls_id, LsElectionReferenceInfo &reference_info) const; + int schedule_refresh_priority_task(); private: common::ObArray *all_ls_election_reference_info_; // refresh priority and detect recovery from failure diff --git a/src/logservice/leader_coordinator/table_accessor.cpp b/src/logservice/leader_coordinator/table_accessor.cpp index dc1ced107..5d6130b85 100644 --- a/src/logservice/leader_coordinator/table_accessor.cpp +++ b/src/logservice/leader_coordinator/table_accessor.cpp @@ -107,6 +107,9 @@ int LsElectionReferenceInfoRow::change_zone_priority(const ObArrayschedule_refresh_priority_task())) { + COORDINATOR_LOG_(WARN, "failed to schedule refresh priority task", KR(ret)); + } + return ret; + #undef PRINT_WRAPPER +} + int LsElectionReferenceInfoRow::convert_user_info_to_table_info_() { LC_TIME_GUARD(1_s); diff --git a/src/logservice/leader_coordinator/table_accessor.h b/src/logservice/leader_coordinator/table_accessor.h index 8aa4dad30..9a22d2876 100644 --- a/src/logservice/leader_coordinator/table_accessor.h +++ b/src/logservice/leader_coordinator/table_accessor.h @@ -115,6 +115,7 @@ private: int get_row_from_table_(); int start_and_read_(); int write_and_commit_(); + int schedule_refresh_priority_task_(); private: const uint64_t tenant_id_; const share::ObLSID ls_id_; diff --git a/src/logservice/palf/block_gc_timer_task.h b/src/logservice/palf/block_gc_timer_task.h index 3260f367b..b168c3eb9 100644 --- a/src/logservice/palf/block_gc_timer_task.h +++ b/src/logservice/palf/block_gc_timer_task.h @@ -30,7 +30,7 @@ public: void destroy(); virtual void runTimerTask(); private: - static const int64_t BLOCK_GC_TIMER_INTERVAL_MS = 10 * 1000; + static const int64_t BLOCK_GC_TIMER_INTERVAL_MS = 30 * 1000; PalfEnvImpl *palf_env_impl_; int64_t warn_time_; int tg_id_; diff --git a/src/logservice/palf/log_define.h b/src/logservice/palf/log_define.h index 8d3a6cc5f..f7125d6bc 100644 --- a/src/logservice/palf/log_define.h +++ b/src/logservice/palf/log_define.h @@ -102,8 +102,8 @@ const int64_t PALF_LEADER_RECONFIRM_SYNC_TIMEOUT_US = 10 * 1000 * 1000L; // const int64_t PREPARE_LOG_BUFFER_SIZE = 2048; const int64_t PALF_LEADER_ACTIVE_SYNC_TIMEOUT_US = 10 * 1000 * 1000L; // 10s const int32_t PALF_MAX_REPLAY_TIMEOUT = 500 * 1000; -const int32_t DEFAULT_PALF_LOG_LOOP_INTERVAL_US = 1 * 1000; // 1ms -const int32_t PALF_LOG_LOOP_INTERVAL_US_UPPER_BOUND = 100 * 1000; // 100ms +const int32_t DEFAULT_LOG_LOOP_INTERVAL_US = 100 * 1000; // 100ms +const int32_t LOG_LOOP_INTERVAL_FOR_PERIOD_FREEZE_US = 1 * 1000; // 1ms const int64_t PALF_SLIDING_WINDOW_SIZE = 1 << 11; // must be 2^n(n>0), default 2^11 = 2048 const int64_t PALF_MAX_LEADER_SUBMIT_LOG_COUNT = PALF_SLIDING_WINDOW_SIZE / 2; // max number of concurrent submitting group log in leader const int64_t PALF_RESEND_MSLOG_INTERVAL_US = 500 * 1000L; // 500 ms diff --git a/src/logservice/palf/log_loop_thread.cpp b/src/logservice/palf/log_loop_thread.cpp index 0f5b25c60..228bb8915 100644 --- a/src/logservice/palf/log_loop_thread.cpp +++ b/src/logservice/palf/log_loop_thread.cpp @@ -22,7 +22,7 @@ namespace palf { LogLoopThread::LogLoopThread() : palf_env_impl_(NULL), - run_interval_(DEFAULT_PALF_LOG_LOOP_INTERVAL_US), + run_interval_(DEFAULT_LOG_LOOP_INTERVAL_US), is_inited_(false) { } @@ -32,7 +32,7 @@ LogLoopThread::~LogLoopThread() destroy(); } -int LogLoopThread::init(const bool is_normal_mode, IPalfEnvImpl *palf_env_impl) +int LogLoopThread::init(IPalfEnvImpl *palf_env_impl) { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_inited_)) { @@ -44,9 +44,7 @@ int LogLoopThread::init(const bool is_normal_mode, IPalfEnvImpl *palf_env_impl) } else { palf_env_impl_ = palf_env_impl; share::ObThreadPool::set_run_wrapper(MTL_CTX()); - if (false == is_normal_mode) { - run_interval_ = PALF_LOG_LOOP_INTERVAL_US_UPPER_BOUND; - } + run_interval_ = DEFAULT_LOG_LOOP_INTERVAL_US; is_inited_ = true; } @@ -94,14 +92,47 @@ void LogLoopThread::log_loop_() last_switch_state_time = start_ts; } - auto switch_freeze_mode_func = [](IPalfHandleImpl *ipalf_handle_impl) { - return ipalf_handle_impl->check_and_switch_freeze_mode(); - }; if (start_ts - last_check_freeze_mode_time >= 1 * 1000 * 1000) { + auto switch_freeze_mode_func = [](IPalfHandleImpl *ipalf_handle_impl) { + return ipalf_handle_impl->check_and_switch_freeze_mode(); + }; if (OB_SUCCESS != (tmp_ret = palf_env_impl_->for_each(switch_freeze_mode_func))) { PALF_LOG_RET(WARN, tmp_ret, "for_each switch_freeze_mode_func failed", K(tmp_ret)); } + // Check whether some palf is in period_freeze_mode. + bool any_in_period_freeze_mode = false; + auto check_freeze_mode_func = [&any_in_period_freeze_mode](IPalfHandleImpl *ipalf_handle_impl) { + any_in_period_freeze_mode = (true == ipalf_handle_impl->is_in_period_freeze_mode()) \ + ? true : any_in_period_freeze_mode; + int ret = OB_SUCCESS; + if (any_in_period_freeze_mode) { + // If any one returns true, break iteration. + ret = OB_ITER_END; + } + return ret; + }; + if (OB_SUCCESS != (tmp_ret = palf_env_impl_->for_each(check_freeze_mode_func))) { + PALF_LOG_RET(WARN, tmp_ret, "for_each check_freeze_mode_func failed", K(tmp_ret)); + } + // update ts for each round last_check_freeze_mode_time = start_ts; + + // Try switch run_interval_ according to whether some palf is in period_freeze_mode. + if (any_in_period_freeze_mode) { + if (run_interval_ > LOG_LOOP_INTERVAL_FOR_PERIOD_FREEZE_US) { + // Some palf_handle is in period_freeze mode, the run_interval_ + // need be adjusted to DEFAULT_LOG_LOOP_INTERVAL_US here. + run_interval_ = LOG_LOOP_INTERVAL_FOR_PERIOD_FREEZE_US; + PALF_LOG(INFO, "LogLoopThread switch run_interval(us)", K_(run_interval), K(any_in_period_freeze_mode)); + } + } else { + // There is not any ls in period_freeze mode, + // try set run_interval_ to 100ms. + if (run_interval_ < DEFAULT_LOG_LOOP_INTERVAL_US) { + run_interval_ = DEFAULT_LOG_LOOP_INTERVAL_US; + PALF_LOG(INFO, "LogLoopThread switch run_interval(us)", K_(run_interval), K(any_in_period_freeze_mode)); + } + } } auto try_freeze_log_func = [](IPalfHandleImpl *ipalf_handle_impl) { @@ -119,7 +150,7 @@ void LogLoopThread::log_loop_() ob_usleep(sleep_ts); if (REACH_TENANT_TIME_INTERVAL(5 * 1000 * 1000)) { - PALF_LOG(INFO, "LogLoopThread round_cost_time", K(round_cost_time)); + PALF_LOG(INFO, "LogLoopThread round_cost_time(us)", K(round_cost_time)); } } } diff --git a/src/logservice/palf/log_loop_thread.h b/src/logservice/palf/log_loop_thread.h index 8a382ba70..860bc1588 100644 --- a/src/logservice/palf/log_loop_thread.h +++ b/src/logservice/palf/log_loop_thread.h @@ -27,7 +27,7 @@ public: LogLoopThread(); virtual ~LogLoopThread(); public: - int init(const bool is_normal_mode, IPalfEnvImpl *palf_env_impl); + int init(IPalfEnvImpl *palf_env_impl); void destroy(); void run1(); private: diff --git a/src/logservice/palf/log_sliding_window.cpp b/src/logservice/palf/log_sliding_window.cpp index 73c70662d..85238f33a 100644 --- a/src/logservice/palf/log_sliding_window.cpp +++ b/src/logservice/palf/log_sliding_window.cpp @@ -1210,6 +1210,11 @@ int LogSlidingWindow::feedback_freeze_last_log_() return ret; } +bool LogSlidingWindow::is_in_period_freeze_mode() const +{ + return (PERIOD_FREEZE_MODE == freeze_mode_); +} + int LogSlidingWindow::check_and_switch_freeze_mode() { int ret = OB_SUCCESS; diff --git a/src/logservice/palf/log_sliding_window.h b/src/logservice/palf/log_sliding_window.h index bc4472296..bb2ff12f8 100644 --- a/src/logservice/palf/log_sliding_window.h +++ b/src/logservice/palf/log_sliding_window.h @@ -243,6 +243,7 @@ public: virtual int get_last_slide_end_lsn(LSN &out_end_lsn) const; virtual const share::SCN get_last_slide_scn() const; virtual int check_and_switch_freeze_mode(); + virtual bool is_in_period_freeze_mode() const; virtual int period_freeze_last_log(); virtual int inc_update_scn_base(const share::SCN &scn); virtual int get_server_ack_info(const common::ObAddr &server, LsnTsInfo &ack_info) const; diff --git a/src/logservice/palf/palf_env_impl.cpp b/src/logservice/palf/palf_env_impl.cpp index c027ee4c3..3bcad407a 100644 --- a/src/logservice/palf/palf_env_impl.cpp +++ b/src/logservice/palf/palf_env_impl.cpp @@ -209,10 +209,10 @@ int PalfEnvImpl::init( PALF_LOG(ERROR, "construct log path failed", K(ret), K(pret)); } else if (OB_FAIL(palf_handle_impl_map_.init("LOG_HASH_MAP", tenant_id))) { PALF_LOG(ERROR, "palf_handle_impl_map_ init failed", K(ret)); - } else if (OB_FAIL(log_loop_thread_.init(true, this))) { + } else if (OB_FAIL(log_loop_thread_.init(this))) { PALF_LOG(ERROR, "log_loop_thread_ init failed", K(ret)); } else if (OB_FAIL( - election_timer_.init_and_start(1, 1_ms, "ElectTimer"))) { // just one worker thread + election_timer_.init_and_start(1, 10_ms, "ElectTimer"))) { // just one worker thread PALF_LOG(ERROR, "election_timer_ init failed", K(ret)); } else if (OB_FAIL(election::GLOBAL_INIT_ELECTION_MODULE())) { PALF_LOG(ERROR, "global init election module failed", K(ret)); @@ -827,7 +827,6 @@ int PalfEnvImpl::for_each(const common::ObFunction &fun int ret = OB_SUCCESS; if (OB_FAIL(func(ipalf_handle_impl))) { PALF_LOG(WARN, "execute func failed", K(ret), K(ls_key)); - } else { } if (OB_FAIL(ret)) { bool_ret = false; diff --git a/src/logservice/palf/palf_handle_impl.cpp b/src/logservice/palf/palf_handle_impl.cpp index 90975fd0b..57523e479 100644 --- a/src/logservice/palf/palf_handle_impl.cpp +++ b/src/logservice/palf/palf_handle_impl.cpp @@ -2120,6 +2120,11 @@ int PalfHandleImpl::check_and_switch_freeze_mode() return ret; } +bool PalfHandleImpl::is_in_period_freeze_mode() const +{ + return sw_.is_in_period_freeze_mode(); +} + int PalfHandleImpl::leader_sync_mode_meta_to_arb_member_() { // caller need hold rdlock diff --git a/src/logservice/palf/palf_handle_impl.h b/src/logservice/palf/palf_handle_impl.h index e6db41476..28d08de88 100644 --- a/src/logservice/palf/palf_handle_impl.h +++ b/src/logservice/palf/palf_handle_impl.h @@ -515,6 +515,7 @@ public: virtual int inner_flashback(const share::SCN &flashback_scn) = 0; virtual int check_and_switch_state() = 0; virtual int check_and_switch_freeze_mode() = 0; + virtual bool is_in_period_freeze_mode() const = 0; virtual int period_freeze_last_log() = 0; virtual int handle_prepare_request(const common::ObAddr &server, const int64_t &proposal_id) = 0; @@ -845,6 +846,7 @@ public: // ================================================================== int check_and_switch_state() override final; int check_and_switch_freeze_mode() override final; + bool is_in_period_freeze_mode() const override final; int period_freeze_last_log() override final; int handle_prepare_request(const common::ObAddr &server, const int64_t &proposal_id) override final; diff --git a/src/observer/dbms_scheduler/ob_dbms_sched_job_master.h b/src/observer/dbms_scheduler/ob_dbms_sched_job_master.h index fbb7fcc06..1cc5ad95e 100644 --- a/src/observer/dbms_scheduler/ob_dbms_sched_job_master.h +++ b/src/observer/dbms_scheduler/ob_dbms_sched_job_master.h @@ -215,7 +215,7 @@ public: private: const static int MAX_READY_JOBS_CAPACITY = 1024 * 1024; - const static int MIN_SCHEDULER_INTERVAL = 5 * 1000 * 1000; + const static int MIN_SCHEDULER_INTERVAL = 20 * 1000 * 1000; bool inited_; bool stoped_; diff --git a/src/observer/mysql/ob_mysql_request_manager.cpp b/src/observer/mysql/ob_mysql_request_manager.cpp index a57a136c5..6fdfff494 100644 --- a/src/observer/mysql/ob_mysql_request_manager.cpp +++ b/src/observer/mysql/ob_mysql_request_manager.cpp @@ -22,6 +22,7 @@ #include "lib/alloc/alloc_func.h" #include "lib/thread/thread_mgr.h" #include "lib/rc/ob_rc.h" +#include "common/ob_clock_generator.h" #include "share/rc/ob_context.h" #include "observer/mysql/ob_mysql_request_manager.h" #include "observer/ob_server.h" @@ -151,7 +152,9 @@ void ObMySQLRequestManager::destroy() *11.tenant_name varchar */ -int ObMySQLRequestManager::record_request(const ObAuditRecordData &audit_record, bool is_sensitive) +int ObMySQLRequestManager::record_request(const ObAuditRecordData &audit_record, + const bool enable_query_response_time_stats, + bool is_sensitive) { int ret = OB_SUCCESS; if (!inited_) { @@ -217,19 +220,20 @@ int ObMySQLRequestManager::record_request(const ObAuditRecordData &audit_record, record->data_.db_name_ = buf + pos; pos += db_len; } - int64_t timestamp = common::ObTimeUtility::current_time(); //for find bug // only print this log if enable_perf_event is enable, // for `receive_ts_` might be invalid if `enable_perf_event` is false if (lib::is_diagnose_info_enabled() - && OB_UNLIKELY(timestamp - audit_record.exec_timestamp_.receive_ts_ > US_PER_HOUR)) { + && OB_UNLIKELY(ObClockGenerator::getClock() - audit_record.exec_timestamp_.receive_ts_ > US_PER_HOUR)) { SERVER_LOG(WARN, "record: query too slow ", - "elapsed", timestamp - audit_record.exec_timestamp_.receive_ts_, + "elapsed", ObClockGenerator::getClock() - audit_record.exec_timestamp_.receive_ts_, "receive_ts", audit_record.exec_timestamp_.receive_ts_); } // query response time - observer::ObRSTCollector::get_instance().collect_query_response_time(audit_record.tenant_id_,audit_record.get_elapsed_time()); + if (enable_query_response_time_stats) { + observer::ObRSTCollector::get_instance().collect_query_response_time(audit_record.tenant_id_,audit_record.get_elapsed_time()); + } //push into queue if (OB_SUCC(ret)) { diff --git a/src/observer/mysql/ob_mysql_request_manager.h b/src/observer/mysql/ob_mysql_request_manager.h index 678893a49..41a6807c4 100644 --- a/src/observer/mysql/ob_mysql_request_manager.h +++ b/src/observer/mysql/ob_mysql_request_manager.h @@ -106,7 +106,9 @@ public: common::ObConcurrentFIFOAllocator *get_allocator() { return &allocator_; } int64_t get_request_id() { ATOMIC_INC(&request_id_); return request_id_; } - int record_request(const ObAuditRecordData &audit_record, bool is_sensitive = false); + int record_request(const ObAuditRecordData &audit_record, + const bool enable_query_response_time_stats, + bool is_sensitive = false); int64_t get_start_idx() const { return (int64_t)queue_.get_pop_idx(); } int64_t get_end_idx() const { return (int64_t)queue_.get_push_idx(); } diff --git a/src/observer/mysql/ob_query_driver.cpp b/src/observer/mysql/ob_query_driver.cpp index 56b25810b..91a705236 100644 --- a/src/observer/mysql/ob_query_driver.cpp +++ b/src/observer/mysql/ob_query_driver.cpp @@ -279,13 +279,24 @@ int ObQueryDriver::convert_string_value_charset(ObObj& value, ObResultSet &resul ObCharsetType charset_type = CHARSET_INVALID; const ObSQLSessionInfo &my_session = result.get_session(); ObArenaAllocator *allocator = NULL; - if (OB_FAIL(result.get_exec_context().get_convert_charset_allocator(allocator))) { + ObCollationType from_collation_type = value.get_collation_type(); + ObCollationType to_collation_type = CS_TYPE_INVALID; + if (OB_FAIL(my_session.get_character_set_results(charset_type))) { + LOG_WARN("fail to get result charset", K(ret)); + } else if (from_collation_type == (to_collation_type = ObCharset::get_default_collation(charset_type))) { + const ObCharsetInfo *charset_info = ObCharset::get_charset(from_collation_type); + if (OB_ISNULL(charset_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("charsetinfo is null", K(ret), K(from_collation_type), K(to_collation_type), K(value)); + } else if (CS_TYPE_INVALID == from_collation_type) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid collation", K(ret), K(from_collation_type), K(to_collation_type), K(value)); + } + } else if (OB_FAIL(result.get_exec_context().get_convert_charset_allocator(allocator))) { LOG_WARN("fail to get lob fake allocator", K(ret)); } else if (OB_ISNULL(allocator)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("lob fake allocator is null.", K(ret), K(value)); - } else if (OB_FAIL(my_session.get_character_set_results(charset_type))) { - LOG_WARN("fail to get result charset", K(ret)); } else { OZ (value.convert_string_value_charset(charset_type, *allocator)); } diff --git a/src/observer/mysql/obmp_query.cpp b/src/observer/mysql/obmp_query.cpp index f491749e5..f9b2ace99 100644 --- a/src/observer/mysql/obmp_query.cpp +++ b/src/observer/mysql/obmp_query.cpp @@ -520,7 +520,6 @@ int ObMPQuery::process_single_stmt(const ObMultiStmtItem &multi_stmt_item, session.set_curr_trans_last_stmt_end_time(0); //============================ 注意这些变量的生命周期 ================================ - ObSessionStatEstGuard stat_est_guard(get_conn()->tenant_->id(), session.get_sessid()); if (OB_FAIL(init_process_var(ctx_, multi_stmt_item, session))) { LOG_WARN("init process var failed.", K(ret), K(multi_stmt_item)); } else { @@ -598,7 +597,7 @@ int ObMPQuery::process_single_stmt(const ObMultiStmtItem &multi_stmt_item, // 也不需要在这里设置结束时间,因为这已经相当于事务的最后一条语句了。 // 最后,需要判断ret错误码,只有成功执行的sql才记录结束时间 if (session.get_in_transaction() && !async_resp_used && OB_SUCC(ret)) { - session.set_curr_trans_last_stmt_end_time(ObTimeUtility::current_time()); + session.set_curr_trans_last_stmt_end_time(ObClockGenerator::getClock()); } // need_response_error这个变量保证仅在 diff --git a/src/observer/ob_rpc_processor_simple.cpp b/src/observer/ob_rpc_processor_simple.cpp index d2f00dd60..2415ab4fc 100644 --- a/src/observer/ob_rpc_processor_simple.cpp +++ b/src/observer/ob_rpc_processor_simple.cpp @@ -68,6 +68,7 @@ using namespace share; using namespace sql; using namespace obmysql; using namespace omt; +using namespace logservice::coordinator; namespace rpc { @@ -412,6 +413,56 @@ int ObRpcNotifyTenantServerUnitResourceP::process() return ret; } +int ObRpcNotifySwitchLeaderP::process() +{ + int ret = OB_SUCCESS; + uint64_t tenant_id = arg_.get_tenant_id(); + if (OB_ISNULL(GCTX.omt_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null of GCTX.omt_", KR(ret)); + } else if (OB_INVALID_TENANT_ID != tenant_id) { + // only refresh the status of specified tenant + if (GCTX.omt_->is_available_tenant(tenant_id)) { + MTL_SWITCH(tenant_id) { + ObLeaderCoordinator* coordinator = MTL(ObLeaderCoordinator*); + if (OB_ISNULL(coordinator)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("unexpected null of leader coordinator", KR(ret), K_(arg)); + } else if (OB_FAIL(coordinator->schedule_refresh_priority_task())) { + LOG_WARN("failed to schedule refresh priority task", KR(ret), K_(arg)); + } + } + } + } else { + // refresh the status of all tenants + common::ObArray tenant_ids; + if (OB_FAIL(GCTX.omt_->get_mtl_tenant_ids(tenant_ids))) { + LOG_WARN("fail to get_mtl_tenant_ids", KR(ret)); + } else { + int tmp_ret = OB_SUCCESS; + for (int64_t i = 0; i < tenant_ids.size(); i++) { + if (GCTX.omt_->is_available_tenant(tenant_ids.at(i))) { + MTL_SWITCH(tenant_ids.at(i)) { + ObLeaderCoordinator* coordinator = MTL(ObLeaderCoordinator*); + if (OB_ISNULL(coordinator)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("unexpected null of leader coordinator", KR(ret), K_(arg)); + } else if (OB_FAIL(coordinator->schedule_refresh_priority_task())) { + LOG_WARN("failed to schedule refresh priority task", KR(ret), K_(arg)); + } + } + } + if (OB_FAIL(ret)) { + tmp_ret = ret; + } + } + ret = tmp_ret; + } + } + LOG_INFO("receive notify switch leader", KR(ret), K_(arg)); + return ret; +} + int ObCheckFrozenVersionP::process() { int ret = OB_SUCCESS; @@ -2251,6 +2302,18 @@ int ObRefreshTenantInfoP::process() return ret; } +int ObUpdateTenantInfoCacheP::process() +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(gctx_.ob_service_)) { + ret = OB_ERR_UNEXPECTED; + COMMON_LOG(WARN, "ob_service is null", KR(ret)); + } else if (OB_FAIL(gctx_.ob_service_->update_tenant_info_cache(arg_, result_))) { + COMMON_LOG(WARN, "failed to update_tenant_info_cache", KR(ret), K(arg_)); + } + return ret; +} + int ObSyncRewriteRulesP::process() { int ret = OB_SUCCESS; diff --git a/src/observer/ob_rpc_processor_simple.h b/src/observer/ob_rpc_processor_simple.h index a6f3d563d..5d6a3c14d 100644 --- a/src/observer/ob_rpc_processor_simple.h +++ b/src/observer/ob_rpc_processor_simple.h @@ -111,6 +111,7 @@ OB_DEFINE_PROCESSOR_S(Srv, OB_CHECK_FROZEN_SCN, ObCheckFrozenVersionP); OB_DEFINE_PROCESSOR_S(Srv, OB_GET_MIN_SSTABLE_SCHEMA_VERSION, ObGetMinSSTableSchemaVersionP); OB_DEFINE_PROCESSOR_S(Srv, OB_INIT_TENANT_CONFIG, ObInitTenantConfigP); OB_DEFINE_PROCESSOR_S(Srv, OB_GET_LEADER_LOCATIONS, ObGetLeaderLocationsP); +OB_DEFINE_PROCESSOR_S(Srv, OB_NOTIFY_SWITCH_LEADER, ObRpcNotifySwitchLeaderP); // oceanbase service provied OB_DEFINE_PROCESSOR_S(Srv, OB_FETCH_SYS_LS, ObRpcFetchSysLSP); @@ -216,6 +217,7 @@ OB_DEFINE_PROCESSOR_S(Srv, OB_ESTIMATE_TABLET_BLOCK_COUNT, ObEstimateTabletBlock OB_DEFINE_PROCESSOR_S(Srv, OB_DDL_CHECK_TABLET_MERGE_STATUS, ObRpcDDLCheckTabletMergeStatusP); OB_DEFINE_PROCESSOR_S(Srv, OB_REFRESH_TENANT_INFO, ObRefreshTenantInfoP); OB_DEFINE_PROCESSOR_S(Srv, OB_SYNC_REWRITE_RULES, ObSyncRewriteRulesP); +OB_DEFINE_PROCESSOR_S(Srv, OB_UPDATE_TENANT_INFO_CACHE, ObUpdateTenantInfoCacheP); } // end of namespace observer } // end of namespace oceanbase diff --git a/src/observer/ob_server.cpp b/src/observer/ob_server.cpp index a658d97e9..4b0bb40ff 100644 --- a/src/observer/ob_server.cpp +++ b/src/observer/ob_server.cpp @@ -1619,15 +1619,17 @@ int ObServer::set_running_mode() { int ret = OB_SUCCESS; const int64_t memory_limit = GMEMCONF.get_server_memory_limit(); + const int64_t cnt = GCONF.cpu_count; + const int64_t cpu_cnt = cnt > 0 ? cnt : common::get_cpu_num(); if (memory_limit < lib::ObRunningModeConfig::MINI_MEM_LOWER) { ret = OB_MACHINE_RESOURCE_NOT_ENOUGH; LOG_ERROR("memory limit too small", KR(ret), K(memory_limit)); } else if (memory_limit < lib::ObRunningModeConfig::MINI_MEM_UPPER) { ObTaskController::get().allow_next_syslog(); LOG_INFO("observer start with mini_mode", K(memory_limit)); - lib::update_mini_mode(memory_limit); + lib::update_mini_mode(memory_limit, cpu_cnt); } else { - lib::update_mini_mode(memory_limit); + lib::update_mini_mode(memory_limit, cpu_cnt); } _OB_LOG(INFO, "mini mode: %s", lib::is_mini_mode() ? "true" : "false"); return ret; diff --git a/src/observer/ob_server.h b/src/observer/ob_server.h index 92786b0b8..6c2e5c28c 100644 --- a/src/observer/ob_server.h +++ b/src/observer/ob_server.h @@ -98,7 +98,7 @@ class ObServer { public: static const int64_t DEFAULT_ETHERNET_SPEED = 1000 / 8 * 1024 * 1024; // default 125m/s 1000Mbit - static const int64_t DISK_USAGE_REPORT_INTERVAL = 1000L * 1000L * 10L; // 60s + static const int64_t DISK_USAGE_REPORT_INTERVAL = 1000L * 1000L * 300L; // 5min static const uint64_t DEFAULT_CPU_FREQUENCY = 2500 * 1000; // 2500 * 1000 khz static ObServer &get_instance(); diff --git a/src/observer/ob_service.cpp b/src/observer/ob_service.cpp index fb510f879..af122e286 100644 --- a/src/observer/ob_service.cpp +++ b/src/observer/ob_service.cpp @@ -2680,5 +2680,39 @@ int ObService::init_tenant_config( return OB_SUCCESS; } +int ObService::update_tenant_info_cache( + const ObUpdateTenantInfoCacheArg &arg, + ObUpdateTenantInfoCacheRes &result) +{ + int ret = OB_SUCCESS; + MAKE_TENANT_SWITCH_SCOPE_GUARD(guard); + + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (!arg.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("arg is invaild", KR(ret), K(arg)); + } else if (arg.get_tenant_id() != MTL_ID() && OB_FAIL(guard.switch_to(arg.get_tenant_id()))) { + LOG_WARN("switch tenant failed", KR(ret), K(arg)); + } + + if (OB_SUCC(ret)) { + rootserver::ObTenantInfoLoader *tenant_info_loader = MTL(rootserver::ObTenantInfoLoader*); + + if (OB_ISNULL(tenant_info_loader)) { + ret = OB_ERR_UNEXPECTED; + COMMON_LOG(ERROR, "tenant_info_loader should not be null", KR(ret)); + } else if (OB_FAIL(tenant_info_loader->update_tenant_info_cache(arg.get_ora_rowscn(), arg.get_tenant_info()))) { + COMMON_LOG(WARN, "update_tenant_info_cache failed", KR(ret), K(arg)); + } else if (OB_FAIL(result.init(arg.get_tenant_id()))) { + LOG_WARN("failed to init res", KR(ret), K(arg.get_tenant_id())); + } else { + LOG_TRACE("finish update_tenant_info_cache", KR(ret), K(arg), K(result)); + } + } + return ret; +} + }// end namespace observer }// end namespace oceanbase diff --git a/src/observer/ob_service.h b/src/observer/ob_service.h index e20d0ee2e..63fdd0ad8 100644 --- a/src/observer/ob_service.h +++ b/src/observer/ob_service.h @@ -153,7 +153,8 @@ public: obrpc::ObEstPartRes &res) const; int estimate_tablet_block_count(const obrpc::ObEstBlockArg &arg, obrpc::ObEstBlockRes &res) const; - + int update_tenant_info_cache(const obrpc::ObUpdateTenantInfoCacheArg &arg, + obrpc::ObUpdateTenantInfoCacheRes &result); //////////////////////////////////////////////////////////////// // ObRpcMinorFreezeP @RS minor freeze int minor_freeze(const obrpc::ObMinorFreezeArg &arg, diff --git a/src/observer/ob_srv_xlator_partition.cpp b/src/observer/ob_srv_xlator_partition.cpp index 51c6009ea..0cfe4baa3 100644 --- a/src/observer/ob_srv_xlator_partition.cpp +++ b/src/observer/ob_srv_xlator_partition.cpp @@ -276,6 +276,7 @@ void oceanbase::observer::init_srv_xlator_for_others(ObSrvRpcXlator *xlator) { //standby switchover/failover RPC_PROCESSOR(ObRpcGetLSSyncScnP, gctx_); RPC_PROCESSOR(ObRefreshTenantInfoP, gctx_); + RPC_PROCESSOR(ObUpdateTenantInfoCacheP, gctx_); RPC_PROCESSOR(ObSyncRewriteRulesP, gctx_); } diff --git a/src/observer/ob_srv_xlator_primary.cpp b/src/observer/ob_srv_xlator_primary.cpp index 6e08186bd..d2b93f227 100644 --- a/src/observer/ob_srv_xlator_primary.cpp +++ b/src/observer/ob_srv_xlator_primary.cpp @@ -72,6 +72,7 @@ void oceanbase::observer::init_srv_xlator_for_sys(ObSrvRpcXlator *xlator) { RPC_PROCESSOR(ObInitTenantConfigP, gctx_); RPC_PROCESSOR(ObGetLeaderLocationsP, gctx_); RPC_PROCESSOR(ObBatchBroadcastSchemaP, gctx_); + RPC_PROCESSOR(ObRpcNotifySwitchLeaderP, gctx_); // interrupt RPC_PROCESSOR(obrpc::ObInterruptProcessor); diff --git a/src/observer/omt/ob_multi_tenant.cpp b/src/observer/omt/ob_multi_tenant.cpp index 38cf696b8..0ccd79fc6 100644 --- a/src/observer/omt/ob_multi_tenant.cpp +++ b/src/observer/omt/ob_multi_tenant.cpp @@ -620,12 +620,6 @@ int ObMultiTenant::create_virtual_tenants() int ret = OB_SUCCESS; if (OB_FAIL(create_tenant_without_unit( - OB_EXT_LOG_TENANT_ID, - EXT_LOG_TENANT_CPU, - EXT_LOG_TENANT_CPU))) { - LOG_ERROR("add ext_log tenant fail", K(ret)); - - } else if (OB_FAIL(create_tenant_without_unit( OB_DATA_TENANT_ID, OB_DATA_CPU, OB_DATA_CPU))) { @@ -637,17 +631,6 @@ int ObMultiTenant::create_virtual_tenants() OB_DTL_CPU))) { LOG_ERROR("add DTL tenant fail", K(ret)); - } else if (OB_FAIL(create_tenant_without_unit( - OB_RS_TENANT_ID, - OB_RS_CPU, - OB_RS_CPU))) { - LOG_ERROR("add RS tenant fail", K(ret)); - - } else if (OB_FAIL(create_tenant_without_unit( - OB_SVR_BLACKLIST_TENANT_ID, - OB_SVR_BLACKLIST_CPU, - OB_SVR_BLACKLIST_CPU))) { - LOG_ERROR("add server balcklist tenant fail", K(ret)); } else { // init allocator for OB_SERVER_TENANT_ID ObMallocAllocator *allocator = ObMallocAllocator::get_instance(); @@ -678,10 +661,6 @@ int ObMultiTenant::create_tenant_without_unit(const uint64_t tenant_id, if (OB_SERVER_TENANT_ID == tenant_id) { mem_limit = INT64_MAX; - } else if (OB_EXT_LOG_TENANT_ID == tenant_id) { - mem_limit = EXT_LOG_TENANT_MEMORY_LIMIT; - } else if (OB_RS_TENANT_ID == tenant_id) { - mem_limit = GCONF.rootservice_memory_limit; } else { static const int64_t VIRTUAL_TENANT_MEMORY_LIMTI = 1L << 30; mem_limit = VIRTUAL_TENANT_MEMORY_LIMTI; diff --git a/src/observer/omt/ob_multi_tenant.h b/src/observer/omt/ob_multi_tenant.h index 09e9cecd0..7cc87cc68 100644 --- a/src/observer/omt/ob_multi_tenant.h +++ b/src/observer/omt/ob_multi_tenant.h @@ -40,8 +40,7 @@ class ObTenantConfig; #define VIRTUAL_TENANTS_CPU_RESERVED_QUOTA \ (GCONF.user_location_cpu_quota() + GCONF.sys_location_cpu_quota() \ + GCONF.root_location_cpu_quota() + GCONF.core_location_cpu_quota() \ - + EXT_LOG_TENANT_CPU + OB_MONITOR_CPU \ - + OB_SVR_BLACKLIST_CPU + OB_DATA_CPU + OB_DTL_CPU + OB_RS_CPU + OB_DIAG_CPU) + + OB_SVR_BLACKLIST_CPU + OB_DATA_CPU + OB_DTL_CPU + OB_DIAG_CPU) struct ObCtxMemConfig { diff --git a/src/observer/omt/ob_tenant.cpp b/src/observer/omt/ob_tenant.cpp index 7289c3db7..b7519065f 100644 --- a/src/observer/omt/ob_tenant.cpp +++ b/src/observer/omt/ob_tenant.cpp @@ -669,11 +669,13 @@ int ObTenant::init(const ObTenantMeta &meta) // there must be 2 workers. static_cast(workers_.get_first()->get_data())->set_priority_limit(QQ_HIGH); static_cast(workers_.get_last()->get_data())->set_priority_limit(QQ_NORMAL); - for (int level = MULTI_LEVEL_THRESHOLD; level < MAX_REQUEST_LEVEL; level++) { - if (OB_FAIL(acquire_level_worker(1, succ_cnt, level))) { - break; + if (!is_virtual_tenant_id(id_) && !is_meta_tenant(id_)) { + for (int level = MULTI_LEVEL_THRESHOLD; level < MAX_REQUEST_LEVEL; level++) { + if (OB_FAIL(acquire_level_worker(1, succ_cnt, level))) { + break; + } + succ_cnt = 0L; } - succ_cnt = 0L; } timeup(); } diff --git a/src/observer/omt/ob_tenant.h b/src/observer/omt/ob_tenant.h index 74b02c071..9a287c995 100644 --- a/src/observer/omt/ob_tenant.h +++ b/src/observer/omt/ob_tenant.h @@ -397,6 +397,7 @@ public: int init_ctx(); + int init_multi_level_queue(); int init(const ObTenantMeta &meta); void stop() { ATOMIC_STORE(&stopped_, true); } void start() { ATOMIC_STORE(&stopped_, false); } diff --git a/src/observer/omt/ob_th_worker.cpp b/src/observer/omt/ob_th_worker.cpp index 1a1038fbd..8f719add3 100644 --- a/src/observer/omt/ob_th_worker.cpp +++ b/src/observer/omt/ob_th_worker.cpp @@ -178,7 +178,7 @@ ObThWorker::Status ObThWorker::check_throttle() Status st = WS_NOWAIT; if (!OB_ISNULL(tenant_) && !OB_ISNULL(session_) && !static_cast(session_)->is_inner()) { - const int64_t curr_time = ObTimeUtility::current_time(); + const int64_t curr_time = common::ObClockGenerator::getClock(); auto &st_metrics = tenant_->get_sql_throttle_metrics(); if (st_current_priority_ != -1 && st_current_priority_ <= st_metrics.priority_) { if ((st_metrics.rt_ >= .0) && @@ -213,7 +213,7 @@ ObThWorker::Status ObThWorker::check_rate_limiter() ObThWorker::Status ObThWorker::check_wait() { const int64_t threshold = GCONF.large_query_threshold; - const int64_t curr_time = ObTimeUtility::current_time(); + const int64_t curr_time = common::ObClockGenerator::getClock(); Status st = WS_NOWAIT; if (OB_UNLIKELY(tenant_->has_stopped())) { st = WS_INVALID; @@ -266,7 +266,7 @@ inline void ObThWorker::process_request(rpc::ObRequest &req) if (retry_times == 1) { LOG_WARN("tenant push retry request to wait queue", "tenant", tenant_->id(), K(req)); } - uint64_t curr_timestamp = ObTimeUtility::current_time(); + uint64_t curr_timestamp = common::ObClockGenerator::getClock(); uint64_t delta_us = curr_timestamp - req.get_receive_timestamp(); uint64_t timestamp = curr_timestamp + min(delta_us, 100 * 1000UL); if (OB_FAIL(tenant_->push_retry_queue(req, timestamp))) { @@ -389,7 +389,7 @@ void ObThWorker::worker(int64_t &tenant_id, int64_t &req_recv_timestamp, int32_t /// get request from tenant { ObWaitEventGuard wait_guard(ObWaitEventIds::OMT_IDLE, 0, wait_start_time, 0, 0); - ret = tenant_->get_new_request(*this, REQUEST_WAIT_TIME, req); + ret = tenant_->get_new_request(*this, is_level_worker() ? NESTING_REQUEST_WAIT_TIME : REQUEST_WAIT_TIME, req); wait_end_time = ObTimeUtility::current_time(); } diff --git a/src/observer/omt/ob_th_worker.h b/src/observer/omt/ob_th_worker.h index a211d1c4f..982423b57 100644 --- a/src/observer/omt/ob_th_worker.h +++ b/src/observer/omt/ob_th_worker.h @@ -35,6 +35,7 @@ class ObResourceGroup; static const int64_t WORKER_CHECK_PERIOD = 500L; static const int64_t REQUEST_WAIT_TIME = 10 * 1000L; +static const int64_t NESTING_REQUEST_WAIT_TIME = 1 * 1000 * 1000L; // Quick Queue Priorities enum { QQ_HIGH = 0, QQ_NORMAL, QQ_LOW, QQ_MAX_PRIO }; diff --git a/src/observer/table/ob_table_rpc_processor.cpp b/src/observer/table/ob_table_rpc_processor.cpp index fba1b72d4..05d6d4850 100644 --- a/src/observer/table/ob_table_rpc_processor.cpp +++ b/src/observer/table/ob_table_rpc_processor.cpp @@ -707,7 +707,7 @@ void ObTableApiProcessorBase::end_audit() if (nullptr == req_manager) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to get request manager for current tenant", K(ret)); - } else if (OB_FAIL(req_manager->record_request(audit_record_))) { + } else if (OB_FAIL(req_manager->record_request(audit_record_, true))) { if (OB_SIZE_OVERFLOW == ret || OB_ALLOCATE_MEMORY_FAILED == ret) { LOG_DEBUG("cannot allocate mem for record", K(ret)); ret = OB_SUCCESS; diff --git a/src/observer/table_load/ob_table_load_coordinator_ctx.cpp b/src/observer/table_load/ob_table_load_coordinator_ctx.cpp index 1810e09a0..0e7792b4e 100644 --- a/src/observer/table_load/ob_table_load_coordinator_ctx.cpp +++ b/src/observer/table_load/ob_table_load_coordinator_ctx.cpp @@ -366,6 +366,7 @@ int ObTableLoadCoordinatorCtx::generate_autoinc_params(AutoincParam &autoinc_par autoinc_param.total_value_count_ = 1; autoinc_param.autoinc_desired_count_ = 0; autoinc_param.autoinc_mode_is_order_ = table_schema->is_order_auto_increment_mode(); + autoinc_param.autoinc_auto_increment_ = table_schema->get_auto_increment(); autoinc_param.autoinc_increment_ = 1; autoinc_param.autoinc_offset_ = 1; autoinc_param.part_value_no_order_ = true; diff --git a/src/observer/table_load/ob_table_load_store_ctx.cpp b/src/observer/table_load/ob_table_load_store_ctx.cpp index 84266c28e..33990f8eb 100644 --- a/src/observer/table_load/ob_table_load_store_ctx.cpp +++ b/src/observer/table_load/ob_table_load_store_ctx.cpp @@ -450,6 +450,7 @@ int ObTableLoadStoreCtx::generate_autoinc_params(AutoincParam &autoinc_param) autoinc_param.total_value_count_ = 1; autoinc_param.autoinc_desired_count_ = 0; autoinc_param.autoinc_mode_is_order_ = table_schema->is_order_auto_increment_mode(); + autoinc_param.autoinc_auto_increment_ = table_schema->get_auto_increment(); autoinc_param.autoinc_increment_ = 1; autoinc_param.autoinc_offset_ = 1; autoinc_param.part_value_no_order_ = true; diff --git a/src/observer/virtual_table/ob_gv_sql_audit.cpp b/src/observer/virtual_table/ob_gv_sql_audit.cpp index b02ef3e5b..50be466de 100644 --- a/src/observer/virtual_table/ob_gv_sql_audit.cpp +++ b/src/observer/virtual_table/ob_gv_sql_audit.cpp @@ -847,7 +847,7 @@ int ObGvSqlAudit::fill_cells(obmysql::ObMySQLRequestRecord &record) cells[cell_idx].set_uint64(record.data_.exec_record_.user_io_time_); } break; case SCHEDULE_TIME: { - cells[cell_idx].set_uint64(record.data_.exec_record_.sched_time_); + cells[cell_idx].set_uint64(0); } break; case ROW_CACHE_HIT: { cells[cell_idx].set_int(record.data_.exec_record_.row_cache_hit_); diff --git a/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp b/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp index f53660e8e..d4f60f8d3 100644 --- a/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp +++ b/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp @@ -1023,6 +1023,7 @@ int ObDDLRedefinitionTask::sync_auto_increment_position() param.autoinc_offset_ = 1; param.auto_increment_cache_size_ = 1; // TODO(shuangcan): should we use the sysvar on session? param.autoinc_mode_is_order_ = dest_table_schema->is_order_auto_increment_mode(); + param.autoinc_auto_increment_ = dest_table_schema->get_auto_increment(); if (OB_FAIL(auto_inc_service.get_sequence_value(tenant_id_, object_id_, cur_column_id, param.autoinc_mode_is_order_, sequence_value))) { LOG_WARN("get sequence value failed", K(ret), K(tenant_id_), K(object_id_), K(cur_column_id)); } else if (FALSE_IT(param.global_value_to_sync_ = sequence_value - 1)) { diff --git a/src/rootserver/ob_primary_ls_service.cpp b/src/rootserver/ob_primary_ls_service.cpp index a77f8fdc3..619bfa2be 100755 --- a/src/rootserver/ob_primary_ls_service.cpp +++ b/src/rootserver/ob_primary_ls_service.cpp @@ -1705,6 +1705,7 @@ int ObTenantLSInfo::try_update_ls_primary_zone_( const common::ObSqlString &zone_priority) { int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; if (OB_UNLIKELY(!primary_zone_info.is_valid() || new_primary_zone.is_empty() || zone_priority.empty())) { ret = OB_INVALID_ARGUMENT; @@ -1716,11 +1717,22 @@ int ObTenantLSInfo::try_update_ls_primary_zone_( } else if (new_primary_zone != primary_zone_info.get_primary_zone() || zone_priority.string() != primary_zone_info.get_zone_priority_str()) { ObLSLifeAgentManager ls_life_agent(*GCTX.sql_proxy_); - if (OB_FAIL(ls_life_agent.update_ls_primary_zone(primary_zone_info.get_tenant_id(), - primary_zone_info.get_ls_id(), + ObLSInfo ls_info; + const uint64_t tenant_id = primary_zone_info.get_tenant_id(); + const ObLSID ls_id = primary_zone_info.get_ls_id(); + if (OB_FAIL(ls_life_agent.update_ls_primary_zone(tenant_id, ls_id, new_primary_zone, zone_priority.string()))) { LOG_WARN("failed to update ls primary zone", KR(ret), K(primary_zone_info), K(new_primary_zone), K(zone_priority)); + } else if (OB_ISNULL(GCTX.lst_operator_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("GCTX.lst_operator_ is NULL", KR(ret)); + } else if (OB_FAIL(GCTX.lst_operator_->get(GCONF.cluster_id, + tenant_id, ls_id, share::ObLSTable::DEFAULT_MODE, ls_info))) { + LOG_WARN("get ls info from GCTX.lst_operator_ failed", KR(ret), K(tenant_id), K(ls_id)); + } else if (OB_TMP_FAIL(ObRootUtils::try_notify_switch_ls_leader(GCTX.srv_rpc_proxy_, ls_info, + obrpc::ObNotifySwitchLeaderArg::MODIFY_PRIMARY_ZONE))) { + LOG_WARN("failed to switch ls leader", KR(ret), KR(tmp_ret), K(ls_id), K(ls_info)); } LOG_INFO("[LS_MGR] update ls primary zone", KR(ret), K(new_primary_zone), K(zone_priority), K(primary_zone_info)); diff --git a/src/rootserver/ob_root_service.cpp b/src/rootserver/ob_root_service.cpp index affe18031..c59e9aa48 100644 --- a/src/rootserver/ob_root_service.cpp +++ b/src/rootserver/ob_root_service.cpp @@ -7212,6 +7212,10 @@ int ObRootService::stop_server(const obrpc::ObAdminServerArg &arg) LOG_WARN("stop server failed", "server", arg.servers_, "zone", arg.zone_, K(ret)); } else { LOG_INFO("stop server ok", K(arg)); + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(try_notify_switch_leader(obrpc::ObNotifySwitchLeaderArg::STOP_SERVER))) { + LOG_WARN("failed to notify switch leader", KR(ret), KR(tmp_ret)); + } } } } @@ -7462,6 +7466,10 @@ int ObRootService::stop_zone(const obrpc::ObAdminZoneArg &arg) LOG_WARN("stop zone failed", K(arg), K(ret)); } else { LOG_INFO("stop zone ok", K(arg)); + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(try_notify_switch_leader(obrpc::ObNotifySwitchLeaderArg::STOP_ZONE))) { + LOG_WARN("failed to notify switch leader", KR(ret), KR(tmp_ret)); + } } } else { //set other error code to 4179 @@ -7516,6 +7524,24 @@ int ObRootService::generate_stop_server_log_in_sync_dest_server_array( return ret; } +int ObRootService::try_notify_switch_leader(const obrpc::ObNotifySwitchLeaderArg::SwitchLeaderComment &comment) +{ + int ret = OB_SUCCESS; + ObServerManager::ObServerArray server_list; + ObZone zone; + obrpc::ObNotifySwitchLeaderArg arg; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_FAIL(server_manager_.get_alive_servers(zone, server_list))) { + LOG_WARN("failed to get server list", KR(ret), K(zone)); + } else if (OB_FAIL(arg.init(OB_INVALID_TENANT_ID, ObLSID(), ObAddr(), comment))) { + LOG_WARN("failed to init switch leader arg", KR(ret), K(comment)); + } else if (OB_FAIL(ObRootUtils::notify_switch_leader(&rpc_proxy_, OB_SYS_TENANT_ID, arg, server_list))) { + LOG_WARN("failed to notify switch leader", KR(ret), K(arg), K(server_list)); + } + return ret; +} //////////////////////////////////////////////////////////////// // system admin command (alter system ...) //////////////////////////////////////////////////////////////// diff --git a/src/rootserver/ob_root_service.h b/src/rootserver/ob_root_service.h index 4c00cd630..0038e7100 100644 --- a/src/rootserver/ob_root_service.h +++ b/src/rootserver/ob_root_service.h @@ -872,6 +872,7 @@ private: int check_all_ls_has_leader_(const char *print_str); void update_cpu_quota_concurrency_in_memory_(); int set_cpu_quota_concurrency_config_(); + int try_notify_switch_leader(const obrpc::ObNotifySwitchLeaderArg::SwitchLeaderComment &comment); private: int construct_rs_list_arg(obrpc::ObRsListArg &rs_list_arg); int precheck_interval_part(const obrpc::ObAlterTableArg &arg); diff --git a/src/rootserver/ob_root_utils.cpp b/src/rootserver/ob_root_utils.cpp index 386d5b4be..12be6e55a 100644 --- a/src/rootserver/ob_root_utils.cpp +++ b/src/rootserver/ob_root_utils.cpp @@ -1961,6 +1961,69 @@ int ObRootUtils::check_left_f_in_primary_zone(ObZoneManager &zone_mgr, return ret; } +int ObRootUtils::try_notify_switch_ls_leader( + obrpc::ObSrvRpcProxy *rpc_proxy, + const share::ObLSInfo &ls_info, + const obrpc::ObNotifySwitchLeaderArg::SwitchLeaderComment &comment) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!ls_info.is_valid()) || OB_ISNULL(rpc_proxy)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(ls_info), K(rpc_proxy)); + } else { + ObArray server_list; + obrpc::ObNotifySwitchLeaderArg arg; + const uint64_t tenant_id = ls_info.get_tenant_id(); + for (int64_t i = 0; OB_SUCC(ret) && i < ls_info.get_replicas_cnt(); ++i) { + if (OB_FAIL(server_list.push_back(ls_info.get_replicas().at(i).get_server()))) { + LOG_WARN("failed to push back server", KR(ret), K(i), K(ls_info)); + } + } + if (FAILEDx(arg.init(tenant_id, ls_info.get_ls_id(), ObAddr(), comment))) { + LOG_WARN("failed to init switch leader arg", KR(ret), K(tenant_id), K(ls_info), K(comment)); + } else if (OB_FAIL(notify_switch_leader(rpc_proxy, tenant_id, arg, server_list))) { + LOG_WARN("failed to notify switch leader", KR(ret), K(arg), K(tenant_id), K(server_list)); + } + } + return ret; + +} + + +int ObRootUtils::notify_switch_leader( + obrpc::ObSrvRpcProxy *rpc_proxy, + const uint64_t tenant_id, + const obrpc::ObNotifySwitchLeaderArg &arg, + const ObIArray &addr_list) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) + || !arg.is_valid() || 0 == addr_list.count()) || OB_ISNULL(rpc_proxy)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(arg), K(addr_list), KP(rpc_proxy)); + } else { + ObTimeoutCtx ctx; + int tmp_ret = OB_SUCCESS; + if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, GCONF.rpc_timeout))) { + LOG_WARN("fail to set timeout ctx", KR(ret)); + } else { + ObNotifySwitchLeaderProxy proxy(*rpc_proxy, &obrpc::ObSrvRpcProxy::notify_switch_leader); + for (int64_t i = 0; i < addr_list.count(); ++i) { + const int64_t timeout = ctx.get_timeout(); + if (OB_TMP_FAIL(proxy.call(addr_list.at(i), timeout, GCONF.cluster_id, tenant_id, arg))) { + ret = OB_SUCC(ret) ? tmp_ret : ret; + LOG_WARN("failed to send rpc", KR(ret), K(i), K(tenant_id), K(arg), K(addr_list)); + } + }//end for + if (OB_TMP_FAIL(proxy.wait())) { + ret = OB_SUCC(ret) ? tmp_ret : ret; + LOG_WARN("failed to wait all result", KR(ret), KR(tmp_ret)); + } + } + } + return ret; +} + /////////////////////////////// ObClusterRole ObClusterInfoGetter::get_cluster_role_v2() diff --git a/src/rootserver/ob_root_utils.h b/src/rootserver/ob_root_utils.h index 11fdf9d3e..88c138853 100644 --- a/src/rootserver/ob_root_utils.h +++ b/src/rootserver/ob_root_utils.h @@ -25,6 +25,7 @@ #include "rootserver/ob_locality_util.h" #include "rootserver/ob_zone_manager.h" #include "share/ob_cluster_role.h" +#include "share/ob_rpc_struct.h" namespace oceanbase { namespace share @@ -643,6 +644,16 @@ public: common::ObIArray &other_server_list, common::ObIArray &tenant_ids); + static int notify_switch_leader( + obrpc::ObSrvRpcProxy *rpc_proxy, + const uint64_t tenant_id, + const obrpc::ObNotifySwitchLeaderArg &arg, + const ObIArray &addr_list); + static int try_notify_switch_ls_leader( + obrpc::ObSrvRpcProxy *rpc_proxy, + const share::ObLSInfo &ls_info, + const obrpc::ObNotifySwitchLeaderArg::SwitchLeaderComment &comment); + }; class ObClusterInfoGetter diff --git a/src/rootserver/ob_rs_async_rpc_proxy.h b/src/rootserver/ob_rs_async_rpc_proxy.h index 01e635a13..ce4ba320f 100644 --- a/src/rootserver/ob_rs_async_rpc_proxy.h +++ b/src/rootserver/ob_rs_async_rpc_proxy.h @@ -69,6 +69,10 @@ RPC_F(obrpc::OB_GET_LEADER_LOCATIONS, obrpc::ObGetLeaderLocationsArg, RPC_F(obrpc::OB_DDL_CHECK_TABLET_MERGE_STATUS, obrpc::ObDDLCheckTabletMergeStatusArg, obrpc::ObDDLCheckTabletMergeStatusResult, ObCheckTabletMergeStatusProxy); RPC_F(obrpc::OB_REFRESH_TENANT_INFO, obrpc::ObRefreshTenantInfoArg, obrpc::ObRefreshTenantInfoRes, ObRefreshTenantInfoProxy); +RPC_F(obrpc::OB_NOTIFY_SWITCH_LEADER, obrpc::ObNotifySwitchLeaderArg, + obrpc::ObSrvRpcProxy::ObRpc::Response, ObNotifySwitchLeaderProxy); +RPC_F(obrpc::OB_UPDATE_TENANT_INFO_CACHE, obrpc::ObUpdateTenantInfoCacheArg, obrpc::ObUpdateTenantInfoCacheRes, ObUpdateTenantInfoCacheProxy); + }//end namespace rootserver }//end namespace oceanbase diff --git a/src/rootserver/ob_rs_reentrant_thread.h b/src/rootserver/ob_rs_reentrant_thread.h index a68515689..2d5542562 100644 --- a/src/rootserver/ob_rs_reentrant_thread.h +++ b/src/rootserver/ob_rs_reentrant_thread.h @@ -33,9 +33,7 @@ public: virtual void run2() override { int ret = common::OB_SUCCESS; - CREATE_WITH_TEMP_ENTITY(RESOURCE_OWNER, common::OB_RS_TENANT_ID) { - run3(); - } + run3(); } virtual void run3() = 0; diff --git a/src/rootserver/ob_system_admin_util.cpp b/src/rootserver/ob_system_admin_util.cpp index b9a318de3..fd36c882d 100644 --- a/src/rootserver/ob_system_admin_util.cpp +++ b/src/rootserver/ob_system_admin_util.cpp @@ -197,6 +197,12 @@ int ObAdminSwitchReplicaRole::execute(const ObAdminSwitchReplicaRoleArg &arg) LOG_WARN("get ls info from GCTX.lst_operator_ failed", K(arg), KR(ret), K(tenant_id)); } else if (OB_FAIL(update_ls_election_reference_info_table(arg, tenant_id, ls_info))) { LOG_WARN("fail to update ls election reference info", K(arg), KR(ret), K(tenant_id)); + } else { + int tmp_ret = OB_SUCCESS;//ignore ret + if (OB_TMP_FAIL(ObRootUtils::try_notify_switch_ls_leader(ctx_.rpc_proxy_, ls_info, + obrpc::ObNotifySwitchLeaderArg::SwitchLeaderComment::MANUAL_SWITCH))) { + LOG_WARN("failed to notify switch ls leader", KR(ret), K(ls_info)); + } } LOG_INFO("switch leader done", KR(ret), K(arg), K(tenant_id), K(ls_info)); return ret; diff --git a/src/rootserver/ob_tenant_info_loader.cpp b/src/rootserver/ob_tenant_info_loader.cpp index f2207049a..f06dc3027 100644 --- a/src/rootserver/ob_tenant_info_loader.cpp +++ b/src/rootserver/ob_tenant_info_loader.cpp @@ -12,11 +12,14 @@ #define USING_LOG_PREFIX RS +#include "lib/trace/ob_trace_event.h" #include "share/rc/ob_tenant_base.h" // MTL_ID #include "share/scn.h"//SCN +#include "share/ob_all_server_tracer.h" // ObAllServerTracer #include "observer/ob_server_struct.h" // GCTX #include "rootserver/ob_tenant_info_loader.h" -#include "rootserver/ob_tenant_role_transition_service.h"//ObTenantRoleTransitionConstants +#include "rootserver/ob_rs_async_rpc_proxy.h" +#include "logservice/ob_log_service.h" // ObLogService namespace oceanbase { @@ -51,6 +54,11 @@ int ObTenantInfoLoader::init() sql_proxy_ = GCTX.sql_proxy_; tenant_id_ = MTL_ID(); tenant_info_cache_.reset(); + ATOMIC_STORE(&broadcast_times_, 0); + ATOMIC_STORE(&rpc_update_times_, 0); + ATOMIC_STORE(&sql_update_times_, 0); + ATOMIC_STORE(&last_rpc_update_time_us_, OB_INVALID_TIMESTAMP); + if (!is_user_tenant(tenant_id_)) { } else if (OB_ISNULL(GCTX.sql_proxy_)) { ret = OB_ERR_UNEXPECTED; @@ -74,6 +82,10 @@ void ObTenantInfoLoader::destroy() tenant_id_ = OB_INVALID_TENANT_ID; tenant_info_cache_.reset(); sql_proxy_ = NULL; + ATOMIC_STORE(&broadcast_times_, 0); + ATOMIC_STORE(&rpc_update_times_, 0); + ATOMIC_STORE(&sql_update_times_, 0); + ATOMIC_STORE(&last_rpc_update_time_us_, OB_INVALID_TIMESTAMP); } int ObTenantInfoLoader::start() @@ -87,8 +99,6 @@ int ObTenantInfoLoader::start() } else if (OB_FAIL(logical_start())) { LOG_WARN("failed to start", KR(ret)); } else { - ObThreadCondGuard guard(get_cond()); - get_cond().broadcast(); LOG_INFO("tenant info loader start", KPC(this)); } return ret; @@ -108,7 +118,7 @@ void ObTenantInfoLoader::wakeup() int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; - LOG_WARN("not init", KR(ret)); + LOG_TRACE("not init", KR(ret)); } else { ObThreadCondGuard guard(get_cond()); get_cond().broadcast(); @@ -124,12 +134,38 @@ void ObTenantInfoLoader::run2() LOG_WARN("not init", KR(ret)); } else { ObThreadCondGuard guard(get_cond()); - const int64_t idle_time = ObTenantRoleTransitionConstants::TENANT_INFO_REFRESH_TIME_US; + int64_t last_dump_time_us = ObTimeUtility::current_time(); + while (!stop_) { - if (OB_FAIL(tenant_info_cache_.refresh_tenant_info(tenant_id_, sql_proxy_))) { + const int64_t start_time_us = ObTimeUtility::current_time(); + share::ObAllTenantInfo tenant_info; + bool refreshed = false; + bool is_sys_ls_leader = is_sys_ls_leader_(); + const int64_t refresh_time_interval_us = act_as_standby_() && is_sys_ls_leader ? + ObTenantRoleTransitionConstants::STS_TENANT_INFO_REFRESH_TIME_US : + ObTenantRoleTransitionConstants::DEFAULT_TENANT_INFO_REFRESH_TIME_US; + + if (need_refresh(refresh_time_interval_us) + && OB_FAIL(tenant_info_cache_.refresh_tenant_info(tenant_id_, sql_proxy_, refreshed))) { LOG_WARN("failed to update tenant info", KR(ret), K_(tenant_id), KP(sql_proxy_)); } + const int64_t now_us = ObTimeUtility::current_time(); + const int64_t sql_update_cost_time = now_us - start_time_us; + if (OB_FAIL(ret)) { + } else if (refreshed && is_sys_ls_leader) { + (void)ATOMIC_AAF(&sql_update_times_, 1); + (void)broadcast_tenant_info_content_(); + } + const int64_t broadcast_cost_time = ObTimeUtility::current_time() - now_us; + + const int64_t end_time_us = ObTimeUtility::current_time(); + const int64_t cost_time_us = end_time_us - start_time_us; + if (refreshed) { + (void)dump_tenant_info_(sql_update_cost_time, is_sys_ls_leader, broadcast_cost_time, end_time_us, last_dump_time_us); + } + const int64_t idle_time = max(10 * 1000, refresh_time_interval_us - cost_time_us); + //At least sleep 10ms, allowing the thread to release the lock if (!stop_) { get_cond().wait_us(idle_time); } @@ -137,25 +173,215 @@ void ObTenantInfoLoader::run2() } } +void ObTenantInfoLoader::dump_tenant_info_( + const int64_t sql_update_cost_time, + const bool is_sys_ls_leader, + const int64_t broadcast_cost_time, + const int64_t end_time_us, + int64_t &last_dump_time_us) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (0 >= last_dump_time_us || end_time_us < last_dump_time_us) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(last_dump_time_us), K(end_time_us)); + } else { + const int64_t dump_interval_s = (end_time_us - last_dump_time_us) / 1000000; // 1s unit; + if (dump_tenant_info_cache_update_action_interval_.reach() && 1 <= dump_interval_s) { + const uint64_t broadcast_times = ATOMIC_LOAD(&broadcast_times_); + const uint64_t rpc_update_times = ATOMIC_LOAD(&rpc_update_times_); + const uint64_t sql_update_times = ATOMIC_LOAD(&sql_update_times_); + const int64_t broadcast_per_sec = broadcast_times / dump_interval_s; // per second + const int64_t rpc_update_per_sec = rpc_update_times / dump_interval_s; // per second + const int64_t sql_update_per_sec = sql_update_times / dump_interval_s; // per second + if (OB_NOT_NULL(THE_TRACE)) { + THE_TRACE->reset(); + } + NG_TRACE_EXT(ob_tenant_info_loader, OB_ID(tenant_id), tenant_id_, + OB_ID(is_sys_ls_leader), is_sys_ls_leader, + OB_ID(broadcast_cost_time), broadcast_cost_time, + OB_ID(broadcast_times), broadcast_times_, + OB_ID(broadcast_per_sec), broadcast_per_sec, + OB_ID(rpc_update_times), rpc_update_times, + OB_ID(rpc_update_per_sec), rpc_update_per_sec, + OB_ID(last_rpc_update_time_us), last_rpc_update_time_us_, + OB_ID(sql_update_cost_time), sql_update_cost_time, + OB_ID(sql_update_times), sql_update_times, + OB_ID(tenant_info_cache), tenant_info_cache_, OB_ID(is_inited), is_inited_); + FORCE_PRINT_TRACE(THE_TRACE, "[dump tenant_info_loader]"); + ATOMIC_STORE(&broadcast_times_, 0); + ATOMIC_STORE(&rpc_update_times_, 0); + ATOMIC_STORE(&sql_update_times_, 0); + last_dump_time_us = ObTimeUtility::current_time(); + } + } +} + +bool ObTenantInfoLoader::need_refresh(const int64_t refresh_time_interval_us) +{ + int ret = OB_SUCCESS; + ObAllTenantInfo tenant_info; + bool need_refresh = true; + const int64_t now = ObTimeUtility::current_time(); + int64_t last_sql_update_time = OB_INVALID_TIMESTAMP; + int64_t ora_rowscn = 0; + + if (ObTenantRoleTransitionConstants::DEFAULT_TENANT_INFO_REFRESH_TIME_US < refresh_time_interval_us) { + LOG_WARN("unexpected refresh_time_interval_us"); + need_refresh = true; + } else if (OB_FAIL(tenant_info_cache_.get_tenant_info(tenant_info, last_sql_update_time, ora_rowscn))) { + LOG_WARN("failed to get tenant info", KR(ret)); + } else if (now - last_sql_update_time < refresh_time_interval_us) { + need_refresh = false; + } + + return need_refresh; +} + +bool ObTenantInfoLoader::is_sys_ls_leader_() +{ + int ret = OB_SUCCESS; + bool is_sys_ls_leader = false; + logservice::ObLogService *log_svr = MTL(logservice::ObLogService*); + int64_t proposal_id = 0; + common::ObRole role; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_ISNULL(log_svr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mtl pointer is null", KR(ret), KP(log_svr)); + } else if (OB_FAIL(log_svr->get_palf_role(share::SYS_LS, role, proposal_id))) { + LOG_WARN("failed to get palf role", KR(ret)); + } else if (is_strong_leader(role)) { + is_sys_ls_leader = true; + } + + return is_sys_ls_leader; +} + +bool ObTenantInfoLoader::act_as_standby_() +{ + int ret = OB_SUCCESS; + bool act_as_standby = true; + ObAllTenantInfo tenant_info; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_FAIL(get_tenant_info(tenant_info))) { + LOG_WARN("failed to get tenant_info", KR(ret)); + } else if (tenant_info.is_primary() && tenant_info.is_normal_status()) { + act_as_standby = false; + } + + return act_as_standby; +} + +void ObTenantInfoLoader::broadcast_tenant_info_content_() +{ + int ret = OB_SUCCESS; + const int64_t DEFAULT_TIMEOUT_US = 200 * 1000; // 200ms + const int64_t begin_time = ObTimeUtility::current_time(); + ObArray return_code_array; + share::ObAllTenantInfo tenant_info; + int64_t last_sql_update_time = OB_INVALID_TIMESTAMP; + int64_t ora_rowscn = 0; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_ISNULL(GCTX.srv_rpc_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("pointer is null", KR(ret), KP(GCTX.srv_rpc_proxy_)); + } else { + ObUpdateTenantInfoCacheProxy proxy( + *GCTX.srv_rpc_proxy_, &obrpc::ObSrvRpcProxy::update_tenant_info_cache); + int64_t rpc_count = 0; + + if (OB_FAIL(tenant_info_cache_.get_tenant_info(tenant_info, last_sql_update_time, ora_rowscn))) { + LOG_WARN("failed to get tenant info", KR(ret)); + } else if (OB_FAIL(share::ObAllServerTracer::get_instance().for_each_server_status( + [&rpc_count, &tenant_info, &proxy, ora_rowscn](const share::ObServerStatus &status) -> int { + int ret = OB_SUCCESS; + obrpc::ObUpdateTenantInfoCacheArg arg; + if (!status.is_valid()) { + LOG_WARN("skip invalid status", KR(ret), K(status)); + } else if (!status.is_alive()) { + //not send to alive + } else if (OB_FAIL(arg.init(tenant_info.get_tenant_id(), tenant_info, ora_rowscn))) { + LOG_WARN("failed to init arg", KR(ret), K(tenant_info), K(ora_rowscn)); + // use meta rpc process thread + } else if (OB_FAIL(proxy.call(status.server_, DEFAULT_TIMEOUT_US, gen_meta_tenant_id(tenant_info.get_tenant_id()), arg))) { + LOG_WARN("failed to send rpc", KR(ret), K(status), K(tenant_info), K(arg)); + } else { + rpc_count++; + } + + return ret; + }))) { + LOG_WARN("for each server status failed", KR(ret)); + } + + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = proxy.wait_all(return_code_array))) { + LOG_WARN("wait all batch result failed", KR(ret), KR(tmp_ret)); + ret = OB_SUCCESS == ret ? tmp_ret : ret; + } else if (proxy.get_results().count() != return_code_array.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result count not match", KR(ret), + K(rpc_count), K(return_code_array), "arg count", + proxy.get_args().count(), K(proxy.get_results().count())); + } else if (OB_FAIL(ret)) { + } else { + (void)ATOMIC_AAF(&broadcast_times_, 1); + ObUpdateTenantInfoCacheRes res; + for (int64_t i = 0; OB_SUCC(ret) && i < return_code_array.count(); ++i) { + ret = return_code_array.at(i); + const ObAddr &dest = proxy.get_dests().at(i); + if (OB_FAIL(ret)) { + LOG_WARN("send rpc is failed", KR(ret), K(i), K(dest)); + } else { + LOG_INFO("refresh tenant info content success", KR(ret), K(i), K(dest)); + } + } + } + } + const int64_t cost_time_us = ObTimeUtility::current_time() - begin_time; + const int64_t PRINT_INTERVAL = 3 * 1000 * 1000L; + if (REACH_TIME_INTERVAL(PRINT_INTERVAL)) { + LOG_INFO("broadcast_tenant_info_content_ finished", KR(ret), K_(tenant_id), K(cost_time_us), + K(return_code_array), K_(broadcast_times)); + } + return ; +} + int ObTenantInfoLoader::get_valid_sts_after(const int64_t specified_time_us, share::SCN &standby_scn) { int ret = OB_SUCCESS; standby_scn.set_min(); share::ObAllTenantInfo tenant_info; - int64_t refresh_time_us = OB_INVALID_TIMESTAMP; + int64_t last_sql_update_time = OB_INVALID_TIMESTAMP; + int64_t ora_rowscn = 0; - if (OB_INVALID_TIMESTAMP == specified_time_us) { + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_INVALID_TIMESTAMP == specified_time_us) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(specified_time_us)); - } else if (OB_FAIL(tenant_info_cache_.get_tenant_info(tenant_info, refresh_time_us))) { + } else if (OB_FAIL(tenant_info_cache_.get_tenant_info(tenant_info, last_sql_update_time, ora_rowscn))) { if (OB_NEED_WAIT == ret) { LOG_TRACE("tenant info cache is not refreshed, need wait", KR(ret)); } else { LOG_WARN("failed to get tenant info", KR(ret)); } - } else if (refresh_time_us <= specified_time_us) { + } else if (last_sql_update_time <= specified_time_us) { ret = OB_NEED_WAIT; - LOG_TRACE("tenant info cache is old, need wait", KR(ret), K(refresh_time_us), K(specified_time_us), K(tenant_info)); + LOG_TRACE("tenant info cache is old, need wait", KR(ret), K(last_sql_update_time), K(specified_time_us), K(tenant_info)); wakeup(); } else if (!tenant_info.is_sts_ready()) { ret = OB_NEED_WAIT; @@ -166,7 +392,7 @@ int ObTenantInfoLoader::get_valid_sts_after(const int64_t specified_time_us, sha const int64_t PRINT_INTERVAL = 3 * 1000 * 1000L; if (REACH_TIME_INTERVAL(PRINT_INTERVAL)) { - LOG_INFO("get_valid_sts_after", KR(ret), K(specified_time_us), K(refresh_time_us), K(tenant_info)); + LOG_INFO("get_valid_sts_after", KR(ret), K(specified_time_us), K(last_sql_update_time), K(tenant_info)); } return ret; @@ -186,32 +412,59 @@ int ObTenantInfoLoader::get_tenant_info(share::ObAllTenantInfo &tenant_info) int ObTenantInfoLoader::refresh_tenant_info() { int ret = OB_SUCCESS; + bool refreshed = false; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret)); - } else if (OB_FAIL(tenant_info_cache_.refresh_tenant_info(tenant_id_, sql_proxy_))) { + } else if (OB_FAIL(tenant_info_cache_.refresh_tenant_info(tenant_id_, sql_proxy_, refreshed))) { LOG_WARN("failed to refresh_tenant_info", KR(ret), K_(tenant_id), KP(sql_proxy_)); } return ret; } +int ObTenantInfoLoader::update_tenant_info_cache(const int64_t new_ora_rowscn, const ObAllTenantInfo &new_tenant_info) +{ + int ret = OB_SUCCESS; + bool refreshed = false; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_FAIL(tenant_info_cache_.update_tenant_info_cache(new_ora_rowscn, new_tenant_info, refreshed))) { + LOG_WARN("failed to update_tenant_info_cache", KR(ret), K(new_ora_rowscn), K(new_tenant_info)); + } else if (refreshed) { + (void)ATOMIC_AAF(&rpc_update_times_, 1); + (void)ATOMIC_STORE(&last_rpc_update_time_us_, ObTimeUtility::current_time()); + } + return ret; +} + +DEFINE_TO_YSON_KV(ObAllTenantInfoCache, + OB_ID(tenant_info), tenant_info_, + OB_ID(last_sql_update_time), last_sql_update_time_, + OB_ID(ora_rowscn), ora_rowscn_); + void ObAllTenantInfoCache::reset() { SpinWLockGuard guard(lock_); tenant_info_.reset(); - refresh_time_us_ = OB_INVALID_TIMESTAMP; + last_sql_update_time_ = OB_INVALID_TIMESTAMP; + ora_rowscn_ = 0; } -int ObAllTenantInfoCache::refresh_tenant_info(const uint64_t tenant_id, common::ObMySQLProxy *sql_proxy) +int ObAllTenantInfoCache::refresh_tenant_info(const uint64_t tenant_id, + common::ObMySQLProxy *sql_proxy, + bool &refreshed) { int ret = OB_SUCCESS; ObAllTenantInfo new_tenant_info; + int64_t ora_rowscn = 0; const int64_t new_refresh_time_us = ObClockGenerator::getCurrentTime(); + refreshed = false; if (OB_ISNULL(sql_proxy) || !is_user_tenant(tenant_id)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(tenant_id), KP(sql_proxy)); } else if (OB_FAIL(ObAllTenantInfoProxy::load_tenant_info(tenant_id, - sql_proxy, false /* for_update */, new_tenant_info))) { + sql_proxy, false /* for_update */, ora_rowscn, new_tenant_info))) { LOG_WARN("failed to load tenant info", KR(ret), K(tenant_id)); } else { /** @@ -224,33 +477,67 @@ int ObAllTenantInfoCache::refresh_tenant_info(const uint64_t tenant_id, common:: MTL_SET_TENANT_ROLE(new_tenant_info.get_tenant_role().value()); SpinWLockGuard guard(lock_); (void)tenant_info_.assign(new_tenant_info); - refresh_time_us_ = new_refresh_time_us; + last_sql_update_time_ = new_refresh_time_us; + ora_rowscn_ = ora_rowscn; + refreshed = true; } if (dump_tenant_info_interval_.reach()) { LOG_INFO("refresh tenant info", KR(ret), K(new_tenant_info), K(new_refresh_time_us), - K(tenant_id), K(tenant_info_), K(refresh_time_us_)); + K(tenant_id), K(tenant_info_), K(last_sql_update_time_), K(ora_rowscn_)); } return ret; } -int ObAllTenantInfoCache::get_tenant_info(share::ObAllTenantInfo &tenant_info, int64_t &refresh_time_us) +int ObAllTenantInfoCache::update_tenant_info_cache( + const int64_t new_ora_rowscn, + const ObAllTenantInfo &new_tenant_info, + bool &refreshed) +{ + int ret = OB_SUCCESS; + refreshed = false; + if (!new_tenant_info.is_valid() || 0 == new_ora_rowscn) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(new_tenant_info), K(new_ora_rowscn)); + } else { + SpinWLockGuard guard(lock_); + if (!tenant_info_.is_valid() || 0 == ora_rowscn_) { + ret = OB_EAGAIN; + LOG_WARN("my tenant_info is invalid, don't refresh", KR(ret), K_(tenant_info), K_(ora_rowscn)); + } else if (new_ora_rowscn > ora_rowscn_) { + MTL_SET_TENANT_ROLE(new_tenant_info.get_tenant_role().value()); + (void)tenant_info_.assign(new_tenant_info); + ora_rowscn_ = new_ora_rowscn; + refreshed = true; + LOG_TRACE("refresh_tenant_info_content", K(new_tenant_info), K(new_ora_rowscn), K(tenant_info_), K(ora_rowscn_)); + } + } + + return ret; +} + +int ObAllTenantInfoCache::get_tenant_info( + share::ObAllTenantInfo &tenant_info, + int64_t &last_sql_update_time, + int64_t &ora_rowscn) { int ret = OB_SUCCESS; tenant_info.reset(); - refresh_time_us = OB_INVALID_TIMESTAMP; + last_sql_update_time = OB_INVALID_TIMESTAMP; + ora_rowscn = 0; SpinRLockGuard guard(lock_); - if (!tenant_info_.is_valid() || OB_INVALID_TIMESTAMP == refresh_time_us_) { + if (!tenant_info_.is_valid() || OB_INVALID_TIMESTAMP == last_sql_update_time_ || 0 == ora_rowscn_) { ret = OB_NEED_WAIT; const int64_t PRINT_INTERVAL = 1 * 1000 * 1000L; if (REACH_TIME_INTERVAL(PRINT_INTERVAL)) { - LOG_WARN("tenant info is invalid, need wait", KR(ret), K(refresh_time_us_), K(tenant_info_)); + LOG_WARN("tenant info is invalid, need wait", KR(ret), K(last_sql_update_time_), K(tenant_info_), K(ora_rowscn_)); } } else { (void)tenant_info.assign(tenant_info_); - refresh_time_us = refresh_time_us_; + last_sql_update_time = last_sql_update_time_; + ora_rowscn = ora_rowscn_; } return ret; } @@ -259,9 +546,10 @@ int ObAllTenantInfoCache::get_tenant_info(share::ObAllTenantInfo &tenant_info) { int ret = OB_SUCCESS; tenant_info.reset(); - int64_t refresh_time_us = OB_INVALID_TIMESTAMP; + int64_t last_sql_update_time = OB_INVALID_TIMESTAMP; + int64_t ora_rowscn = 0; - if (OB_FAIL(get_tenant_info(tenant_info, refresh_time_us))) { + if (OB_FAIL(get_tenant_info(tenant_info, last_sql_update_time, ora_rowscn))) { LOG_WARN("failed to get tenant info", KR(ret)); } return ret; diff --git a/src/rootserver/ob_tenant_info_loader.h b/src/rootserver/ob_tenant_info_loader.h index 1d1149703..b23cc0fae 100644 --- a/src/rootserver/ob_tenant_info_loader.h +++ b/src/rootserver/ob_tenant_info_loader.h @@ -17,6 +17,7 @@ #include "lib/utility/ob_print_utils.h" //TO_STRING_KV #include "share/ob_tenant_info_proxy.h"//ObAllTenantInfo #include "lib/lock/ob_spin_rwlock.h" //lock +#include "rootserver/ob_tenant_role_transition_service.h"//ObTenantRoleTransitionConstants namespace oceanbase { namespace common @@ -36,22 +37,29 @@ public: ObAllTenantInfoCache() : lock_(), tenant_info_(), - refresh_time_us_(OB_INVALID_TIMESTAMP), - dump_tenant_info_interval_(3 * 1000 * 1000) {} // 3s + last_sql_update_time_(OB_INVALID_TIMESTAMP), + dump_tenant_info_interval_(DUMP_TENANT_INFO_INTERVAL), + ora_rowscn_(0) {} ~ObAllTenantInfoCache() {} int get_tenant_info(share::ObAllTenantInfo &tenant_info); - int get_tenant_info(share::ObAllTenantInfo &tenant_info, int64_t &refresh_time_us); - int refresh_tenant_info(const uint64_t tenant_id, common::ObMySQLProxy *sql_proxy); + int get_tenant_info(share::ObAllTenantInfo &tenant_info, int64_t &last_sql_update_time, int64_t &ora_rowscn); + int refresh_tenant_info(const uint64_t tenant_id, common::ObMySQLProxy *sql_proxy, bool &refreshed); void reset(); + void set_refresh_interval_for_sts(); + int update_tenant_info_cache(const int64_t new_ora_rowscn, const ObAllTenantInfo &new_tenant_info, bool &refreshed); +private: + const static int64_t DUMP_TENANT_INFO_INTERVAL = 3 * 1000 * 1000; // 3s public: - TO_STRING_KV(K_(refresh_time_us), K_(tenant_info)); + TO_STRING_KV(K_(last_sql_update_time), K_(tenant_info), K_(ora_rowscn)); + DECLARE_TO_YSON_KV; private: common::SpinRWLock lock_; share::ObAllTenantInfo tenant_info_; - int64_t refresh_time_us_; + int64_t last_sql_update_time_; common::ObTimeInterval dump_tenant_info_interval_; + int64_t ora_rowscn_; DISALLOW_COPY_AND_ASSIGN(ObAllTenantInfoCache); }; @@ -64,7 +72,12 @@ public: : is_inited_(false), tenant_id_(common::OB_INVALID_TENANT_ID), tenant_info_cache_(), - sql_proxy_(nullptr) {} + sql_proxy_(nullptr), + broadcast_times_(0), + rpc_update_times_(0), + sql_update_times_(0), + last_rpc_update_time_us_(OB_INVALID_TIMESTAMP), + dump_tenant_info_cache_update_action_interval_(DUMP_TENANT_INFO_CACHE_UPDATE_ACTION_INTERVAL) {} ~ObTenantInfoLoader() {} static int mtl_init(ObTenantInfoLoader *&ka); int init(); @@ -91,15 +104,35 @@ public: */ int get_valid_sts_after(const int64_t specified_time_us, share::SCN &standby_scn); int refresh_tenant_info(); + int update_tenant_info_cache(const int64_t new_ora_rowscn, const ObAllTenantInfo &new_tenant_info); + bool need_refresh(const int64_t refresh_time_interval_us); private: + bool is_sys_ls_leader_(); + void broadcast_tenant_info_content_(); + void dump_tenant_info_( + const int64_t sql_update_cost_time, + const bool is_sys_ls_leader, + const int64_t broadcast_cost_time, + const int64_t end_time_us, + int64_t &last_dump_time_us); + bool act_as_standby_(); public: - TO_STRING_KV(K_(is_inited), K_(tenant_id), K_(tenant_info_cache)); + TO_STRING_KV(K_(is_inited), K_(tenant_id), K_(tenant_info_cache), K_(broadcast_times), K_(rpc_update_times), K_(sql_update_times), K_(last_rpc_update_time_us)); + +private: + const static int64_t DUMP_TENANT_INFO_CACHE_UPDATE_ACTION_INTERVAL = 1 * 1000 * 1000; // 1s + private: bool is_inited_; uint64_t tenant_id_; ObAllTenantInfoCache tenant_info_cache_; common::ObMySQLProxy *sql_proxy_; + uint64_t broadcast_times_; + uint64_t rpc_update_times_; + uint64_t sql_update_times_; + int64_t last_rpc_update_time_us_; + common::ObTimeInterval dump_tenant_info_cache_update_action_interval_; private: DISALLOW_COPY_AND_ASSIGN(ObTenantInfoLoader); }; diff --git a/src/rootserver/ob_tenant_recovery_reportor.cpp b/src/rootserver/ob_tenant_recovery_reportor.cpp index 9f0d26b14..5632f1380 100644 --- a/src/rootserver/ob_tenant_recovery_reportor.cpp +++ b/src/rootserver/ob_tenant_recovery_reportor.cpp @@ -87,8 +87,6 @@ int ObTenantRecoveryReportor::start() } else if (OB_FAIL(logical_start())) { LOG_WARN("failed to start", KR(ret)); } else { - ObThreadCondGuard guard(get_cond()); - get_cond().broadcast(); LOG_INFO("tenant recovery service start", KPC(this)); } return ret; @@ -122,7 +120,6 @@ void ObTenantRecoveryReportor::run2() LOG_WARN("not init", KR(ret)); } else { ObThreadCondGuard guard(get_cond()); - const int64_t idle_time = ObTenantRoleTransitionConstants::TENANT_INFO_REFRESH_TIME_US; const uint64_t meta_tenant_id = gen_meta_tenant_id(tenant_id_); while (!stop_) { if (OB_ISNULL(GCTX.schema_service_)) { @@ -145,6 +142,9 @@ void ObTenantRecoveryReportor::run2() if (OB_SUCCESS != (tmp_ret = update_replayable_point_())) { LOG_WARN("failed to update_replayable_point", KR(tmp_ret)); } + + const int64_t idle_time = get_idle_time_(); + if (!stop_) { get_cond().wait_us(idle_time); } @@ -152,6 +152,72 @@ void ObTenantRecoveryReportor::run2() } } +int64_t ObTenantRecoveryReportor::get_idle_time_() +{ + int ret = OB_SUCCESS; + rootserver::ObTenantInfoLoader *tenant_info_loader = MTL(rootserver::ObTenantInfoLoader*); + ObAllTenantInfo tenant_info; + int64_t idle_time = ObTenantRoleTransitionConstants::STANDBY_UPDATE_LS_RECOVERY_STAT_TIME_US; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_ISNULL(tenant_info_loader)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("pointer is null", KR(ret), KP(tenant_info_loader)); + } else if (OB_FAIL(tenant_info_loader->get_tenant_info(tenant_info))) { + LOG_WARN("fail to get tenant info", KR(ret), K_(tenant_id)); + } else if (tenant_info.is_primary() && tenant_info.is_normal_status()) { + idle_time = ObTenantRoleTransitionConstants::PRIMARY_UPDATE_LS_RECOVERY_STAT_TIME_US; + } + return idle_time; +} + +int ObTenantRecoveryReportor::submit_tenant_refresh_schema_task_() +{ + int ret = OB_SUCCESS; + ObAllTenantInfo tenant_info; + rootserver::ObTenantInfoLoader *tenant_info_loader = MTL(rootserver::ObTenantInfoLoader*); + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_ISNULL(GCTX.ob_service_) || OB_ISNULL(GCTX.schema_service_) || OB_ISNULL(sql_proxy_) || OB_ISNULL(tenant_info_loader)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("pointer is null", KR(ret), KP(GCTX.ob_service_), KP(GCTX.schema_service_), KP(sql_proxy_), KP(tenant_info_loader)); + } else if (OB_FAIL(tenant_info_loader->get_tenant_info(tenant_info))) { + LOG_WARN("fail to get tenant info", KR(ret), K_(tenant_id)); + } else if (tenant_info.is_standby() && tenant_info.is_normal_status()) { + ObRefreshSchemaStatus schema_status; + ObSchemaStatusProxy *schema_status_proxy = GCTX.schema_status_proxy_; + if (OB_ISNULL(schema_status_proxy)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("schema_status_proxy is null", KR(ret)); + } else if (OB_FAIL(schema_status_proxy->get_refresh_schema_status(tenant_id_, schema_status))) { + LOG_WARN("fail to get schema status", KR(ret), K(tenant_id_)); + } else if (common::OB_INVALID_TIMESTAMP == schema_status.snapshot_timestamp_) { + int64_t version_in_inner_table = OB_INVALID_VERSION; + int64_t local_schema_version = OB_INVALID_VERSION; + if (OB_FAIL(GCTX.schema_service_->get_tenant_refreshed_schema_version( + tenant_id_, local_schema_version))) { + LOG_WARN("fail to get tenant refreshed schema version", KR(ret), K_(tenant_id)); + } else if (OB_FAIL(GCTX.schema_service_->get_schema_version_in_inner_table( + *sql_proxy_, schema_status, version_in_inner_table))) { + LOG_WARN("fail to get_schema_version_in_inner_table", KR(ret), K(schema_status)); + } else if (local_schema_version > version_in_inner_table) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("local_schema_version > version_in_inner_table", KR(ret), K_(tenant_id), + K(local_schema_version), K(version_in_inner_table)); + } else if (local_schema_version == version_in_inner_table) { + // do nothing + } else if (OB_FAIL(GCTX.ob_service_->submit_async_refresh_schema_task(tenant_id_, version_in_inner_table))) { + LOG_WARN("failed to submit_async_refresh_schema_task", KR(ret), K_(tenant_id)); + } + } + } + return ret; +} + int ObTenantRecoveryReportor::update_ls_recovery_stat_() { int ret = OB_SUCCESS; diff --git a/src/rootserver/ob_tenant_recovery_reportor.h b/src/rootserver/ob_tenant_recovery_reportor.h index 31aceb18c..f76c13d0b 100644 --- a/src/rootserver/ob_tenant_recovery_reportor.h +++ b/src/rootserver/ob_tenant_recovery_reportor.h @@ -67,6 +67,7 @@ public: private: static int get_sync_point_(const share::ObLSID &id, share::SCN &scn, share::SCN &read_scn); int update_ls_recovery_stat_(); + int64_t get_idle_time_(); public: TO_STRING_KV(K_(is_inited), K_(tenant_id)); diff --git a/src/rootserver/ob_tenant_role_transition_service.cpp b/src/rootserver/ob_tenant_role_transition_service.cpp index 87a177e16..b39e34498 100644 --- a/src/rootserver/ob_tenant_role_transition_service.cpp +++ b/src/rootserver/ob_tenant_role_transition_service.cpp @@ -36,6 +36,7 @@ namespace rootserver const char* const ObTenantRoleTransitionConstants::SWITCH_TO_PRIMARY_LOG_MOD_STR = "SWITCH_TO_PRIMARY"; const char* const ObTenantRoleTransitionConstants::SWITCH_TO_STANDBY_LOG_MOD_STR = "SWITCH_TO_STANDBY"; +const char* const ObTenantRoleTransitionConstants::RESTORE_TO_STANDBY_LOG_MOD_STR = "RESTORE_TO_STANDBY"; ////////////LSAccessModeInfo///////////////// int ObTenantRoleTransitionService::LSAccessModeInfo::init( diff --git a/src/rootserver/ob_tenant_role_transition_service.h b/src/rootserver/ob_tenant_role_transition_service.h index 8529c7d68..32c8ae885 100644 --- a/src/rootserver/ob_tenant_role_transition_service.h +++ b/src/rootserver/ob_tenant_role_transition_service.h @@ -45,11 +45,14 @@ using namespace share; class ObTenantRoleTransitionConstants { public: - static constexpr int64_t TENANT_INFO_REFRESH_TIME_US = 100 * 1000; // 100ms - static constexpr int64_t TENANT_INFO_LEASE_TIME_US = 2 * TENANT_INFO_REFRESH_TIME_US; // 200ms - + static constexpr int64_t PRIMARY_UPDATE_LS_RECOVERY_STAT_TIME_US = 1000 * 1000; // 1s + static constexpr int64_t STANDBY_UPDATE_LS_RECOVERY_STAT_TIME_US = 100 * 1000; // 100ms + static constexpr int64_t STS_TENANT_INFO_REFRESH_TIME_US = 100 * 1000; // 100ms + static constexpr int64_t DEFAULT_TENANT_INFO_REFRESH_TIME_US = 1000 * 1000; // 1s + static constexpr int64_t TENANT_INFO_LEASE_TIME_US = 2 * DEFAULT_TENANT_INFO_REFRESH_TIME_US; // 2s static const char* const SWITCH_TO_PRIMARY_LOG_MOD_STR; static const char* const SWITCH_TO_STANDBY_LOG_MOD_STR; + static const char* const RESTORE_TO_STANDBY_LOG_MOD_STR; }; /*description: diff --git a/src/rootserver/restore/ob_restore_scheduler.cpp b/src/rootserver/restore/ob_restore_scheduler.cpp index 175745349..b25c796fe 100644 --- a/src/rootserver/restore/ob_restore_scheduler.cpp +++ b/src/rootserver/restore/ob_restore_scheduler.cpp @@ -582,6 +582,11 @@ int ObRestoreService::post_check(const ObPhysicalRestoreJob &job_info) share::STANDBY_TENANT_ROLE, all_tenant_info.get_switchover_status(), share::NORMAL_SWITCHOVER_STATUS, new_switch_ts))) { LOG_WARN("failed to update tenant role", KR(ret), K(tenant_id_), K(all_tenant_info)); + } else { + ObTenantRoleTransitionService role_transition_service(tenant_id_, sql_proxy_, + GCTX.srv_rpc_proxy_, ObSwitchTenantArg::OpType::INVALID); + (void)role_transition_service.broadcast_tenant_info( + ObTenantRoleTransitionConstants::RESTORE_TO_STANDBY_LOG_MOD_STR); } } diff --git a/src/share/cache/ob_kv_storecache.cpp b/src/share/cache/ob_kv_storecache.cpp index 364126ec0..52bb013b5 100644 --- a/src/share/cache/ob_kv_storecache.cpp +++ b/src/share/cache/ob_kv_storecache.cpp @@ -150,6 +150,7 @@ ObKVGlobalCache::ObKVGlobalCache() map_once_clean_num_(0), map_replace_pos_(0), map_once_replace_num_(0), + map_replace_skip_count_(0), start_destory_(false), cache_wash_interval_(0) { @@ -714,7 +715,18 @@ void ObKVGlobalCache::wash() void ObKVGlobalCache::replace_map() { if (inited_ && !start_destory_) { - map_.replace_fragment_node(map_replace_pos_, map_once_replace_num_); + int ret = OB_SUCCESS; + int64_t replace_node_count = 0; + if (map_replace_skip_count_ <= 0) { + if (OB_FAIL(map_.replace_fragment_node(map_replace_pos_, replace_node_count, map_once_replace_num_))) { + } else if (0 == replace_node_count) { + map_replace_skip_count_ = MAP_REPLACE_ONCE_SKIP_COUNT; + } + } else { + --map_replace_skip_count_; + } + COMMON_LOG(INFO, "replace map num details", K(ret), K(replace_node_count), K(map_once_replace_num_), + K(map_replace_skip_count_)); } } diff --git a/src/share/cache/ob_kv_storecache.h b/src/share/cache/ob_kv_storecache.h index 84af275cf..ffa8c8750 100644 --- a/src/share/cache/ob_kv_storecache.h +++ b/src/share/cache/ob_kv_storecache.h @@ -246,6 +246,7 @@ private: static const int64_t bucket_num_array_[MAX_BUCKET_NUM_LEVEL]; static const int64_t PRINT_INTERVAL = 30 * 1000L * 1000L; static const int64_t MAP_WASH_CLEAN_INTERNAL = 10; + static const int64_t MAP_REPLACE_ONCE_SKIP_COUNT = 10; private: class KVStoreWashTask: public ObTimerTask { @@ -300,6 +301,7 @@ private: KVStoreWashTask wash_task_; int64_t map_replace_pos_; int64_t map_once_replace_num_; + int64_t map_replace_skip_count_; KVMapReplaceTask replace_task_; bool start_destory_; int64_t cache_wash_interval_; diff --git a/src/share/cache/ob_kvcache_map.cpp b/src/share/cache/ob_kvcache_map.cpp index 4cf6eb3e8..d1b83209c 100644 --- a/src/share/cache/ob_kvcache_map.cpp +++ b/src/share/cache/ob_kvcache_map.cpp @@ -650,9 +650,10 @@ int ObKVCacheMap::clean_garbage_node(int64_t &start_pos, const int64_t clean_num return ret; } -int ObKVCacheMap::replace_fragment_node(int64_t &start_pos, const int64_t replace_num) +int ObKVCacheMap::replace_fragment_node(int64_t &start_pos, int64_t &replace_node_count, const int64_t replace_num) { int ret = OB_SUCCESS; + replace_node_count = 0; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; COMMON_LOG(WARN, "The ObKVCacheMap has not been inited, ", K(ret)); @@ -662,7 +663,6 @@ int ObKVCacheMap::replace_fragment_node(int64_t &start_pos, const int64_t replac ret = OB_INVALID_ARGUMENT; COMMON_LOG(WARN, "Invalid argument, ", K(start_pos), K_(bucket_num), K(replace_num), K(ret)); } else { - int64_t replace_node_count = 0; ObTimeGuard tg("replace_fragement_node", 100000); // The variable 'replace_start_pos' do not need atomic operation because it is only used by replace thread int64_t replace_start_pos = start_pos % bucket_num_; diff --git a/src/share/cache/ob_kvcache_map.h b/src/share/cache/ob_kvcache_map.h index f2bf2e693..849726a4d 100644 --- a/src/share/cache/ob_kvcache_map.h +++ b/src/share/cache/ob_kvcache_map.h @@ -40,7 +40,7 @@ public: int erase_tenant(const uint64_t tenant_id, const bool force_erase = false); int erase_tenant_cache(const uint64_t tenant_id, const int64_t cache_id); int clean_garbage_node(int64_t &start_pos, const int64_t clean_num); - int replace_fragment_node(int64_t &start_pos, const int64_t replace_num); + int replace_fragment_node(int64_t &start_pos, int64_t &replace_node_count, const int64_t replace_num); int put( ObKVCacheInst &inst, const ObIKVCacheKey &key, diff --git a/src/share/location_cache/ob_ls_location_service.h b/src/share/location_cache/ob_ls_location_service.h index 7699b229b..dcadc0bbb 100644 --- a/src/share/location_cache/ob_ls_location_service.h +++ b/src/share/location_cache/ob_ls_location_service.h @@ -210,7 +210,7 @@ private: common::ObArray &leaders); private: static const int64_t OB_LOCATION_CACHE_BUCKET_NUM = 512; - static const int64_t RENEW_LS_LOCATION_INTERVAL_US = 1000 * 1000L; // 1s + static const int64_t RENEW_LS_LOCATION_INTERVAL_US = 5 * 1000 * 1000L; // 5s static const int64_t RENEW_LS_LOCATION_BY_RPC_INTERVAL_US = 1000 * 1000L; // 1s static const int64_t DUMP_CACHE_INTERVAL_US = 10 * 1000 * 1000L; // 10s diff --git a/src/share/ob_autoincrement_param.cpp b/src/share/ob_autoincrement_param.cpp index ee67e1ffb..5ba8d524a 100644 --- a/src/share/ob_autoincrement_param.cpp +++ b/src/share/ob_autoincrement_param.cpp @@ -35,7 +35,8 @@ OB_SERIALIZE_MEMBER(AutoincParam, auto_increment_cache_size_, part_value_no_order_, autoinc_mode_is_order_, - autoinc_version_); + autoinc_version_, + autoinc_auto_increment_); }//end namespace share }//end namespace oceanbase diff --git a/src/share/ob_autoincrement_param.h b/src/share/ob_autoincrement_param.h index e93b11d60..07ce05947 100644 --- a/src/share/ob_autoincrement_param.h +++ b/src/share/ob_autoincrement_param.h @@ -99,7 +99,8 @@ struct AutoincParam part_level_(schema::PARTITION_LEVEL_ZERO), auto_increment_cache_size_(DEFAULT_INCREMENT_CACHE_SIZE), autoinc_mode_is_order_(true), - autoinc_version_(0) + autoinc_version_(0), + autoinc_auto_increment_(1) {} TO_STRING_KV("tenant_id" , tenant_id_, @@ -123,7 +124,8 @@ struct AutoincParam "auto_increment_cache_size" , auto_increment_cache_size_, "part_value_no_order" , part_value_no_order_, "autoinc_mode_is_order" , autoinc_mode_is_order_, - "autoinc_version" , autoinc_version_); + "autoinc_version" , autoinc_version_, + "autoinc_auto_increment" , autoinc_auto_increment_); inline bool with_order() const { return !part_value_no_order_; } // pay attention to schema changes @@ -161,6 +163,7 @@ struct AutoincParam int64_t auto_increment_cache_size_; bool autoinc_mode_is_order_; int64_t autoinc_version_; + uint64_t autoinc_auto_increment_; // auto increment value of table schema OB_UNIS_VERSION(1); }; diff --git a/src/share/ob_autoincrement_service.cpp b/src/share/ob_autoincrement_service.cpp index a301d62e9..73cbc5be4 100644 --- a/src/share/ob_autoincrement_service.cpp +++ b/src/share/ob_autoincrement_service.cpp @@ -304,6 +304,119 @@ int ObAutoincrementService::get_handle(AutoincParam ¶m, CacheHandle *&handle) { ObActiveSessionGuard::get_stat().in_sequence_load_ = true; + int ret = param.autoinc_mode_is_order_ ? + get_handle_order(param, handle) : get_handle_noorder(param, handle); + ObActiveSessionGuard::get_stat().in_sequence_load_ = false; + return ret; +} + +int ObAutoincrementService::get_handle_order(AutoincParam ¶m, CacheHandle *&handle) +{ + int ret = OB_SUCCESS; + const uint64_t tenant_id = param.tenant_id_; + const uint64_t table_id = param.autoinc_table_id_; + const uint64_t column_id = param.autoinc_col_id_; + const ObObjType column_type = param.autoinc_col_type_; + const uint64_t offset = param.autoinc_offset_; + const uint64_t increment = param.autoinc_increment_; + const uint64_t max_value = get_max_value(column_type); + const int64_t auto_increment_cache_size = param.auto_increment_cache_size_; + + uint64_t desired_count = 0; + // calc nb_desired_values in MySQL + if (0 == param.autoinc_intervals_count_) { + desired_count = param.total_value_count_; + } else if (param.autoinc_intervals_count_ <= AUTO_INC_DEFAULT_NB_MAX_BITS) { + desired_count = AUTO_INC_DEFAULT_NB_ROWS * (1 << param.autoinc_intervals_count_); + if (desired_count > AUTO_INC_DEFAULT_NB_MAX) { + desired_count = AUTO_INC_DEFAULT_NB_MAX; + } + } else { + desired_count = AUTO_INC_DEFAULT_NB_MAX; + } + + // allocate auto-increment value first time + if (0 == param.autoinc_desired_count_) { + param.autoinc_desired_count_ = desired_count; + } + + desired_count = param.autoinc_desired_count_; + const uint64_t batch_count = increment * desired_count; + + AutoincKey key(tenant_id, table_id, column_id); + uint64_t table_auto_increment = param.autoinc_auto_increment_; + uint64_t start_inclusive = 0; + uint64_t end_inclusive = 0; + uint64_t sync_value = 0; + if (OB_UNLIKELY(table_auto_increment > max_value)) { + ret = OB_ERR_REACH_AUTOINC_MAX; + LOG_WARN("reach max autoinc", K(ret), K(table_auto_increment)); + } else if (OB_FAIL(global_autoinc_service_.get_value(key, offset, increment, max_value, + table_auto_increment, batch_count, + auto_increment_cache_size, sync_value, + start_inclusive, end_inclusive))) { + LOG_WARN("fail get value", K(ret)); + } else if (OB_UNLIKELY(sync_value > max_value || start_inclusive > max_value)) { + ret = OB_ERR_REACH_AUTOINC_MAX; + LOG_WARN("reach max autoinc", K(start_inclusive), K(max_value), K(ret)); + } else { + uint64_t new_next_value = 0; + uint64_t needed_interval = 0; + if (start_inclusive >= max_value) { + new_next_value = max_value; + needed_interval = max_value; + } else if (OB_FAIL(calc_next_value(start_inclusive, offset, increment, new_next_value))) { + LOG_WARN("fail to calc next value", K(ret)); + } else if (OB_UNLIKELY(new_next_value > end_inclusive)) { + if (max_value == end_inclusive) { + new_next_value = max_value; + needed_interval = max_value; + } else { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("fail to alloc handle; cache is not enough", + K(start_inclusive), K(end_inclusive), K(offset), K(increment), K(max_value), + K(new_next_value), K(ret)); + } + } else { + needed_interval = new_next_value + increment * (desired_count - 1); + // check overflow + if (needed_interval < new_next_value) { + needed_interval = UINT64_MAX; + } + if (needed_interval > end_inclusive) { + if (max_value == end_inclusive) { + needed_interval = max_value; + } else { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("fail to alloc handle; cache is not enough", K(ret)); + } + } + } + if (OB_SUCC(ret)) { + if (OB_UNLIKELY(UINT64_MAX == needed_interval)) { + // compatible with MySQL; return error when reach UINT64_MAX + ret = OB_ERR_REACH_AUTOINC_MAX; + LOG_WARN("reach UINT64_MAX", K(ret)); + } else if (OB_ISNULL(handle = static_cast(handle_allocator_.alloc()))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("failed to alloc cache handle", K(ret)); + } else { + handle = new (handle) CacheHandle; + handle->offset_ = offset; + handle->increment_ = increment; + handle->next_value_ = new_next_value; + handle->prefetch_start_ = new_next_value; + handle->prefetch_end_ = needed_interval; + handle->max_value_ = max_value; + LOG_TRACE("succ to allocate cache handle", K(*handle), K(ret)); + } + } + } + return ret; +} + +int ObAutoincrementService::get_handle_noorder(AutoincParam ¶m, CacheHandle *&handle) +{ int ret = OB_SUCCESS; const uint64_t tenant_id = param.tenant_id_; const uint64_t table_id = param.autoinc_table_id_; @@ -343,7 +456,6 @@ int ObAutoincrementService::get_handle(AutoincParam ¶m, int ignore_ret = try_periodic_refresh_global_sync_value(tenant_id, table_id, column_id, - param.autoinc_mode_is_order_, *table_node); if (OB_SUCCESS != ignore_ret) { LOG_INFO("fail refresh global sync value. ignore this failure.", K(ignore_ret)); @@ -431,7 +543,6 @@ int ObAutoincrementService::get_handle(AutoincParam ¶m, if (OB_UNLIKELY(NULL != table_node)) { node_map_.revert(table_node); } - ObActiveSessionGuard::get_stat().in_sequence_load_ = false; return ret; } @@ -439,7 +550,6 @@ int ObAutoincrementService::try_periodic_refresh_global_sync_value( uint64_t tenant_id, uint64_t table_id, uint64_t column_id, - const bool is_order, TableNode &table_node) { int ret = OB_SUCCESS; @@ -457,7 +567,7 @@ int ObAutoincrementService::try_periodic_refresh_global_sync_value( delta = cur_time - ATOMIC_LOAD(&table_node.last_refresh_ts_); if (delta * PRE_OP_THRESHOLD < cache_refresh_interval) { // do nothing; refreshed already - } else if (OB_FAIL(fetch_global_sync(tenant_id, table_id, column_id, is_order, table_node, true))) { + } else if (OB_FAIL(fetch_global_sync(tenant_id, table_id, column_id, table_node, true))) { LOG_WARN("failed to get global sync", K(table_node), K(ret)); } table_node.sync_mutex_.unlock(); @@ -470,7 +580,7 @@ int ObAutoincrementService::try_periodic_refresh_global_sync_value( delta = cur_time - ATOMIC_LOAD(&table_node.last_refresh_ts_); if (delta * PRE_OP_THRESHOLD < cache_refresh_interval) { // do nothing; refreshed already - } else if (OB_FAIL(fetch_global_sync(tenant_id, table_id, column_id, is_order, table_node))) { + } else if (OB_FAIL(fetch_global_sync(tenant_id, table_id, column_id, table_node))) { LOG_WARN("failed to get global sync", K(table_node), K(ret)); } table_node.sync_mutex_.unlock(); @@ -786,36 +896,6 @@ int ObAutoincrementService::get_table_node(const AutoincParam ¶m, TableNode return ret; } -template -int ObAutoincrementService::get_schema(share::schema::ObSchemaGetterGuard &schema_guard, - const uint64_t tenant_id, - const uint64_t schema_id, - const std::function get_schema_func, - const SchemaType *&schema) -{ - int ret = OB_SUCCESS; - schema = NULL; - if (OB_ISNULL(schema_service_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("NULL ptr", K(schema_service_), K(ret)); - } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(tenant_id, schema_guard))) { - LOG_WARN("get schema guard failed", K(ret), K(tenant_id)); - } else if (OB_FAIL(get_schema_func(tenant_id, schema_id, schema))) { - LOG_WARN("get table schema failed", K(tenant_id), K(schema_id), K(ret)); - } else if (OB_ISNULL(schema)) { - // - // Why returns OB_ERR_WAIT_REMOTE_SCHEMA_REFRESH/OB_ERR_REMOTE_SCHEMA_NOT_FULL: - // 1. since we can reach here, it means that this table can be seen in our executor - // 2. schema is null imply schema can't be seen in this server. we may see it later. retry! - ret = schema_service_->is_schema_error_need_retry(&schema_guard, tenant_id) ? - OB_ERR_REMOTE_SCHEMA_NOT_FULL : OB_ERR_WAIT_REMOTE_SCHEMA_REFRESH; - LOG_WARN("NULL ptr, maybe schema not refreshed in this server", - K(tenant_id), K(schema_id), K(schema), K(ret)); - } - - return ret; -} - int ObAutoincrementService::alloc_autoinc_try_lock(lib::ObMutex &alloc_mutex) { int ret = OB_SUCCESS; @@ -847,7 +927,6 @@ int ObAutoincrementService::fetch_table_node(const AutoincParam ¶m, const uint64_t desired_count = param.autoinc_desired_count_; const uint64_t offset = param.autoinc_offset_; const uint64_t increment = param.autoinc_increment_; - const bool is_order = param.autoinc_mode_is_order_; if (part_num <= 0 || ObNullType == column_type) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(part_num), K(column_type), K(ret)); @@ -864,7 +943,7 @@ int ObAutoincrementService::fetch_table_node(const AutoincParam ¶m, } // For ORDER mode, the local cache_size is always 1, and the central(remote) cache_size // is the configuration value. - const uint64_t local_cache_size = is_order ? 1 : auto_increment_cache_size; + const uint64_t local_cache_size = auto_increment_cache_size; uint64_t prefetch_count = std::min(max_value / 100 / part_num, local_cache_size); uint64_t batch_count = 0; if (prefetch_count > 1) { @@ -873,19 +952,14 @@ int ObAutoincrementService::fetch_table_node(const AutoincParam ¶m, batch_count = increment * desired_count; } AutoincKey key(tenant_id, table_id, column_id); - uint64_t table_auto_increment = 0; - ObIGlobalAutoIncrementService *service = get_global_autoinc_service(is_order); - if (OB_ISNULL(service)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("auto inc service is null", K(ret)); - } else if (OB_FAIL(get_auto_increment(tenant_id, table_id, table_auto_increment))) { - LOG_WARN("fail get auto increment", K(ret)); - } else if (OB_UNLIKELY(table_auto_increment > max_value)) { + uint64_t table_auto_increment = param.autoinc_auto_increment_; + if (OB_UNLIKELY(table_auto_increment > max_value)) { ret = OB_ERR_REACH_AUTOINC_MAX; LOG_WARN("reach max autoinc", K(ret), K(table_auto_increment)); - } else if (OB_FAIL(service->get_value(key, offset, increment, max_value, table_auto_increment, - batch_count, auto_increment_cache_size, sync_value, - start_inclusive, end_inclusive))) { + } else if (OB_FAIL(distributed_autoinc_service_.get_value( + key, offset, increment, max_value, table_auto_increment, + batch_count, auto_increment_cache_size, sync_value, + start_inclusive, end_inclusive))) { LOG_WARN("fail get value", K(ret)); } else if (sync_value > max_value || start_inclusive > max_value) { ret = OB_ERR_REACH_AUTOINC_MAX; @@ -1037,15 +1111,60 @@ int ObAutoincrementService::get_server_set(const uint64_t tenant_id, return ret; } +int ObAutoincrementService::sync_insert_value_order(AutoincParam ¶m, + CacheHandle *&cache_handle, + const uint64_t insert_value) +{ + int ret = OB_SUCCESS; + const uint64_t tenant_id = param.tenant_id_; + const uint64_t table_id = param.autoinc_table_id_; + const uint64_t column_id = param.autoinc_col_id_; + const ObObjType column_type = param.autoinc_col_type_; + const uint64_t max_value = get_max_value(column_type); + const uint64_t part_num = param.autoinc_table_part_num_; + + uint64_t global_sync_value = 0; + AutoincKey key(tenant_id, table_id, column_id); + uint64_t value_to_sync = insert_value; + if (value_to_sync > max_value) { + value_to_sync = max_value; + } + if (OB_FAIL(global_autoinc_service_.local_push_to_global_value( + key, max_value, value_to_sync, global_sync_value))) { + LOG_WARN("fail sync value to global", K(key), K(insert_value), K(ret)); + } else if (NULL != cache_handle) { + LOG_DEBUG("insert value, generate next val", + K(insert_value), K(cache_handle->prefetch_end_), K(cache_handle->next_value_)); + if (insert_value < cache_handle->prefetch_end_) { + if (insert_value >= cache_handle->next_value_) { + if (OB_FAIL(calc_next_value(insert_value + 1, + param.autoinc_offset_, + param.autoinc_increment_, + cache_handle->next_value_))) { + LOG_WARN("failed to calc next value", K(cache_handle), K(param), K(ret)); + } + LOG_DEBUG("generate next value when sync_insert_value", K(insert_value), K(cache_handle->next_value_)); + } + } else { + // release handle No. + handle_allocator_.free(cache_handle); + cache_handle = NULL; + // invalid cache handle; record count + param.autoinc_intervals_count_++; + } + } + return ret; +} + /* core logic: * 1. write insert_value to global storage * - only if insert_value > local_sync_ * 2. notify other servers that a larger value written to global storage * 3. update local table node */ -int ObAutoincrementService::sync_insert_value(AutoincParam ¶m, - CacheHandle *&cache_handle, - const uint64_t insert_value) +int ObAutoincrementService::sync_insert_value_noorder(AutoincParam ¶m, + CacheHandle *&cache_handle, + const uint64_t insert_value) { int ret = OB_SUCCESS; const uint64_t tenant_id = param.tenant_id_; @@ -1082,11 +1201,8 @@ int ObAutoincrementService::sync_insert_value(AutoincParam ¶m, // is the configuration value. const uint64_t local_cache_size = is_order ? 1 : param.auto_increment_cache_size_; uint64_t value_to_sync = calc_next_cache_boundary(insert_value, local_cache_size, max_value); - ObIGlobalAutoIncrementService *service = get_global_autoinc_service(is_order); - if (OB_ISNULL(service)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("auto inc service is null", K(ret)); - } else if (OB_FAIL(service->local_push_to_global_value(key, max_value, value_to_sync, global_sync_value))) { + if (OB_FAIL(distributed_autoinc_service_.local_push_to_global_value(key, max_value, + value_to_sync, global_sync_value))) { LOG_WARN("fail sync value to global", K(key), K(insert_value), K(ret)); } else { if (OB_FAIL(alloc_autoinc_try_lock(table_node->alloc_mutex_))) { @@ -1243,14 +1359,22 @@ int ObAutoincrementService::sync_insert_value_global(AutoincParam ¶m) { int ret = OB_SUCCESS; if (0 != param.global_value_to_sync_) { - if (OB_FAIL(sync_insert_value(param, - param.cache_handle_, - param.global_value_to_sync_))) { - SQL_ENG_LOG(WARN, "failed to sync insert value", - "insert_value", param.global_value_to_sync_, K(ret)); + if (param.autoinc_mode_is_order_) { + if (OB_FAIL(sync_insert_value_order(param, + param.cache_handle_, + param.global_value_to_sync_))) { + SQL_ENG_LOG(WARN, "failed to sync insert value", + "insert_value", param.global_value_to_sync_, K(ret)); + } } else { - param.global_value_to_sync_ = 0; + if (OB_FAIL(sync_insert_value_noorder(param, + param.cache_handle_, + param.global_value_to_sync_))) { + SQL_ENG_LOG(WARN, "failed to sync insert value", + "insert_value", param.global_value_to_sync_, K(ret)); + } } + param.global_value_to_sync_ = 0; } return ret; } @@ -1330,7 +1454,6 @@ int ObAutoincrementService::sync_auto_increment_all(const uint64_t tenant_id, int ObAutoincrementService::fetch_global_sync(const uint64_t tenant_id, const uint64_t table_id, const uint64_t column_id, - const bool is_order, TableNode &table_node, const bool sync_presync) { @@ -1347,11 +1470,7 @@ int ObAutoincrementService::fetch_global_sync(const uint64_t tenant_id, if (OB_SUCC(ret)) { AutoincKey key(tenant_id, table_id, column_id); - ObIGlobalAutoIncrementService *service = get_global_autoinc_service(is_order); - if (OB_ISNULL(service)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("auto inc service is null", K(ret)); - } else if (OB_FAIL(service->local_sync_with_global_value(key, sync_value))) { + if (OB_FAIL(distributed_autoinc_service_.local_sync_with_global_value(key, sync_value))) { LOG_WARN("fail refresh global value", K(ret)); } else { atomic_update(table_node.local_sync_, sync_value); @@ -1471,37 +1590,11 @@ int ObAutoincrementService::get_sequence_value(const uint64_t tenant_id, key.tenant_id_ = tenant_id; key.table_id_ = table_id; key.column_id_ = column_id; - ObIGlobalAutoIncrementService *service = get_global_autoinc_service(is_order); - if (OB_ISNULL(service)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("auto inc service is null", K(ret)); - } else if (OB_FAIL(service->get_sequence_value(key, seq_value))) { - LOG_WARN("service get sequence value failed", K(ret)); - } - return ret; -} - -// TODO: don't want to depend on schema. -// we'd better save table_auto_increment to AutoincParam -int ObAutoincrementService::get_auto_increment(const uint64_t tenant_id, - const uint64_t table_id, - uint64_t &auto_increment) -{ - int ret = OB_SUCCESS; - if (OB_SUCC(ret)) { - ObSchemaGetterGuard schema_guard; - const ObTableSchema *table = NULL; - std::function get_schema_func = - std::bind((int(ObSchemaGetterGuard::*)(const uint64_t, const uint64_t, const ObTableSchema *&)) - &ObSchemaGetterGuard::get_table_schema, std::ref(schema_guard), - std::placeholders::_1, std::placeholders::_2, std::placeholders::_3); - if (OB_FAIL(get_schema(schema_guard, tenant_id, table_id, - get_schema_func, - table))) { - LOG_WARN("fail get table schema", K(table_id), K(ret)); - } else { - auto_increment = table->get_auto_increment(); - } + if (is_order && OB_FAIL(global_autoinc_service_.get_sequence_value(key, seq_value))) { + LOG_WARN("global autoinc service get sequence value failed", K(ret)); + } else if (!is_order && + OB_FAIL(distributed_autoinc_service_.get_sequence_value(key, seq_value))) { + LOG_WARN("distributed autoinc service get sequence value failed", K(ret)); } return ret; } @@ -1763,6 +1856,9 @@ int ObAutoIncInnerTableProxy::next_autoinc_value(const AutoincKey &key, if (max_value < desired_count || curr_new_value >= max_value - desired_count) { end_inclusive = max_value; next_sequence_value = max_value; + if (OB_UNLIKELY(curr_new_value > max_value)) { + curr_new_value = max_value; + } } else { end_inclusive = curr_new_value + desired_count - 1; next_sequence_value = curr_new_value + desired_count; diff --git a/src/share/ob_autoincrement_service.h b/src/share/ob_autoincrement_service.h index cf04b085e..0617d41af 100644 --- a/src/share/ob_autoincrement_service.h +++ b/src/share/ob_autoincrement_service.h @@ -367,14 +367,6 @@ public: const common::ObArray* alive_server_list = nullptr); int clear_autoinc_cache(const obrpc::ObAutoincSyncArg &arg); - static int calc_next_value(const uint64_t last_next_value, - const uint64_t offset, - const uint64_t increment, - uint64_t &new_next_value); - static int calc_prev_value(const uint64_t last_next_value, - const uint64_t offset, - const uint64_t increment, - uint64_t &prev_value); int get_sequence_value(const uint64_t tenant_id, const uint64_t table_id, const uint64_t column_id, @@ -397,6 +389,21 @@ public: const uint64_t &table_id, const uint64_t &column_id, common::ObMySQLTransaction &trans); + static int calc_next_value(const uint64_t last_next_value, + const uint64_t offset, + const uint64_t increment, + uint64_t &new_next_value); + static int calc_prev_value(const uint64_t last_next_value, + const uint64_t offset, + const uint64_t increment, + uint64_t &prev_value); +private: + int get_handle_order(AutoincParam ¶m, CacheHandle *&handle); + int get_handle_noorder(AutoincParam ¶m, CacheHandle *&handle); + int sync_insert_value_order(AutoincParam ¶m, CacheHandle *&cache_handle, + const uint64_t value_to_sync); + int sync_insert_value_noorder(AutoincParam ¶m, CacheHandle *&cache_handle, + const uint64_t value_to_sync); private: uint64_t get_max_value(const common::ObObjType type); @@ -407,17 +414,12 @@ private: int fetch_global_sync(const uint64_t tenant_id, const uint64_t table_id, const uint64_t column_id, - const bool is_order, TableNode &table_node, const bool sync_presync = false); int get_server_set(const uint64_t tenant_id, const uint64_t table_id, common::hash::ObHashSet &server_set, const bool get_follower = false); - - int get_auto_increment(const uint64_t tenant_id, - const uint64_t table_id, - uint64_t &auto_increment); int sync_value_to_other_servers( AutoincParam ¶m, uint64_t insert_value); @@ -426,7 +428,6 @@ private: uint64_t tenant_id, uint64_t table_id, uint64_t column_id, - const bool is_order, TableNode &table_node); // align insert value to next cache boundary (end) @@ -434,30 +435,8 @@ private: uint64_t insert_value, uint64_t cache_size, uint64_t max_value); - - int sync_insert_value(AutoincParam ¶m, - CacheHandle *&cache_handle, - const uint64_t value_to_sync); // for prefetch or presync int set_pre_op_timeout(common::ObTimeoutCtx &ctx); - template - int get_schema(share::schema::ObSchemaGetterGuard &schema_guard, - const uint64_t tenant_id, - const uint64_t schema_id, - const std::function get_schema_func, - const SchemaType *&schema); - - inline ObIGlobalAutoIncrementService *get_global_autoinc_service(const bool is_order) - { - ObIGlobalAutoIncrementService *service = nullptr; - if (is_order) { - service = &global_autoinc_service_; - } else { - service = &distributed_autoinc_service_; - } - return service; - } - int alloc_autoinc_try_lock(lib::ObMutex &alloc_mutex); private: diff --git a/src/share/ob_encryption_struct.h b/src/share/ob_encryption_struct.h index 4adbba088..e6999ead2 100644 --- a/src/share/ob_encryption_struct.h +++ b/src/share/ob_encryption_struct.h @@ -132,16 +132,14 @@ struct ObEncryptMeta int replace_tenant_id(const uint64_t real_tenant_id); bool is_valid_before_decrypt_table_key() const { - return OB_INVALID_ID != tenant_id_ - && encrypt_algorithm_ > 0 + return encrypt_algorithm_ > 0 && master_key_version_ > 0 && random_.size() > 0 && encrypted_table_key_.size() > 0; } bool is_valid() const { - return OB_INVALID_ID != tenant_id_ - && encrypt_algorithm_ > 0 + return encrypt_algorithm_ > 0 && master_key_version_ > 0 && random_.size() > 0 && master_key_.size() > 0 diff --git a/src/share/ob_global_autoinc_service.cpp b/src/share/ob_global_autoinc_service.cpp index cf3c69d88..01563e08a 100644 --- a/src/share/ob_global_autoinc_service.cpp +++ b/src/share/ob_global_autoinc_service.cpp @@ -313,14 +313,10 @@ int ObGlobalAutoIncService::handle_push_autoinc_request( ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to get seq value", K(ret), K(key)); } else if (OB_UNLIKELY(OB_HASH_NOT_EXIST == err || cache_node.need_sync(request.base_value_))) { - const bool is_valid_before_sync = cache_node.is_valid(); if (OB_FAIL(sync_value_to_inner_table_(request, cache_node, sync_value))) { LOG_WARN("sync to inner table failed", K(ret)); - } else { - bool need_update_node_map = cache_node.is_valid() || is_valid_before_sync; - if (need_update_node_map && OB_FAIL(autoinc_map_.set_refactored(key, cache_node, 1))) { - LOG_WARN("set autoinc_map_ failed", K(ret)); - } + } else if (OB_FAIL(autoinc_map_.set_refactored(key, cache_node, 1))) { + LOG_WARN("set autoinc_map_ failed", K(ret)); } } else { sync_value = cache_node.sync_value_; @@ -444,6 +440,7 @@ int ObGlobalAutoIncService::sync_value_to_inner_table_( if (seq_value > node.last_available_value_) { // the node is expired. node.reset(); + node.sync_value_ = sync_value; // update sync value for next sync } else if (sync_value == request.max_value_) { if (node.last_available_value_ != request.max_value_) { ret = OB_ERR_UNEXPECTED; @@ -457,7 +454,8 @@ int ObGlobalAutoIncService::sync_value_to_inner_table_( LOG_WARN("fail to update sync value", K(ret), K(sync_value)); } } else { - // do nothing for non-valid node. + node.reset(); + node.sync_value_ = sync_value; // update sync value for next sync } return ret; } diff --git a/src/share/ob_global_autoinc_service.h b/src/share/ob_global_autoinc_service.h index d05bce3d4..d2c0e55bb 100644 --- a/src/share/ob_global_autoinc_service.h +++ b/src/share/ob_global_autoinc_service.h @@ -61,7 +61,7 @@ struct ObAutoIncCacheNode const uint64_t max_value) const; inline bool need_sync(const uint64_t new_sync_value) const { - return !is_valid() || new_sync_value > sync_value_; + return new_sync_value > sync_value_; } int update_sequence_value(const uint64_t sequence_value); int update_available_value(const uint64_t available_value); diff --git a/src/share/ob_rpc_struct.cpp b/src/share/ob_rpc_struct.cpp index 5edb5a01f..a1933b9af 100644 --- a/src/share/ob_rpc_struct.cpp +++ b/src/share/ob_rpc_struct.cpp @@ -5530,6 +5530,73 @@ int ObRefreshTenantInfoRes::assign(const ObRefreshTenantInfoRes &other) return ret; } +OB_SERIALIZE_MEMBER(ObUpdateTenantInfoCacheArg, tenant_id_, tenant_info_, ora_rowscn_); + +bool ObUpdateTenantInfoCacheArg::is_valid() const +{ + return OB_INVALID_TENANT_ID != tenant_id_ + && tenant_info_.is_valid() + && 0 != ora_rowscn_; +} + +int ObUpdateTenantInfoCacheArg::init( + const uint64_t tenant_id, + const ObAllTenantInfo &tenant_info, + const int64_t ora_rowscn) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id + || !tenant_info.is_valid() + || 0 == ora_rowscn)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(tenant_info), K(ora_rowscn)); + } else { + tenant_id_ = tenant_id; + tenant_info_ = tenant_info; + ora_rowscn_ = ora_rowscn; + } + return ret; +} + +int ObUpdateTenantInfoCacheArg::assign(const ObUpdateTenantInfoCacheArg &other) +{ + int ret = OB_SUCCESS; + if (this != &other) { + tenant_id_ = other.tenant_id_; + tenant_info_ = other.tenant_info_; + ora_rowscn_ = other.ora_rowscn_; + } + return ret; +} + +OB_SERIALIZE_MEMBER(ObUpdateTenantInfoCacheRes, tenant_id_); + +bool ObUpdateTenantInfoCacheRes::is_valid() const +{ + return OB_INVALID_TENANT_ID != tenant_id_; +} + +int ObUpdateTenantInfoCacheRes::init(const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id)); + } else { + tenant_id_ = tenant_id; + } + return ret; +} + +int ObUpdateTenantInfoCacheRes::assign(const ObUpdateTenantInfoCacheRes &other) +{ + int ret = OB_SUCCESS; + if (this != &other) { + tenant_id_ = other.tenant_id_; + } + return ret; +} + int ObSwitchTenantArg::init( const uint64_t exec_tenant_id, const OpType op_type, @@ -6757,6 +6824,49 @@ int ObChangeLSAccessModeRes::assign(const ObChangeLSAccessModeRes &other) OB_SERIALIZE_MEMBER(ObChangeLSAccessModeRes, tenant_id_, ls_id_, ret_); +int ObNotifySwitchLeaderArg::init(const uint64_t tenant_id, const share::ObLSID &ls_id, + const common::ObAddr &leader, const SwitchLeaderComment &comment) +{ + int ret = OB_SUCCESS; + if (ObNotifySwitchLeaderArg::INVALID_COMMENT == comment) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(comment)); + } else { + tenant_id_ = tenant_id; + ls_id_ = ls_id; + advise_leader_ = leader; + comment_ = comment; + } + return ret; +} + +int ObNotifySwitchLeaderArg::assign(const ObNotifySwitchLeaderArg &other) +{ + int ret = OB_SUCCESS; + tenant_id_ = other.tenant_id_; + ls_id_ = other.ls_id_; + advise_leader_ = other.advise_leader_; + comment_ = other.comment_; + return ret; +} + +const char* SwitchLeaderCommentStr[] = +{ + "STOP SERVER", "STOP ZONE", "CHANGE MEMBER", "MODIFY PRIMARY_ZONE", "MANUAL SWITCH" +}; +const char* ObNotifySwitchLeaderArg::comment_to_str() const +{ + const char* str = "UNKNOWN"; + if (ObNotifySwitchLeaderArg::INVALID_COMMENT == comment_) { + LOG_WARN_RET(OB_NOT_INIT, "comment is invalid", K(comment_)); + } else { + str = SwitchLeaderCommentStr[comment_]; + } + return str; +} + +OB_SERIALIZE_MEMBER(ObNotifySwitchLeaderArg, tenant_id_, ls_id_, advise_leader_, comment_); + bool ObBatchRemoveTabletArg::is_valid() const { bool is_valid = id_.is_valid(); diff --git a/src/share/ob_rpc_struct.h b/src/share/ob_rpc_struct.h index 1f8d57629..ad5f4c2b2 100644 --- a/src/share/ob_rpc_struct.h +++ b/src/share/ob_rpc_struct.h @@ -3023,6 +3023,48 @@ private: int ret_; }; +struct ObNotifySwitchLeaderArg +{ + OB_UNIS_VERSION(1); +public: + enum SwitchLeaderComment + { + INVALID_COMMENT = -1, + STOP_SERVER, + STOP_ZONE, + CHANGE_MEMBER,//remove member, migrate replica, type transfer + MODIFY_PRIMARY_ZONE, + MANUAL_SWITCH, + }; + const char* comment_to_str() const; + ObNotifySwitchLeaderArg() : tenant_id_(OB_INVALID_TENANT_ID), ls_id_(), advise_leader_(), + comment_(INVALID_COMMENT) {} + ~ObNotifySwitchLeaderArg() {} + TO_STRING_KV(K_(tenant_id), K_(ls_id), K_(advise_leader), K_(comment), "commet_str", comment_to_str()); + int init(const uint64_t tenant_id, const share::ObLSID &ls_id, const common::ObAddr &leader, const SwitchLeaderComment &comment); + bool is_valid() const + { + return INVALID_COMMENT != comment_; + } + int assign(const ObNotifySwitchLeaderArg &other); + uint64_t get_tenant_id() const + { + return tenant_id_; + } +private: + DISALLOW_COPY_AND_ASSIGN(ObNotifySwitchLeaderArg); +private: + //tenant_id is invalid weakup all tenant + //teenant_id is valid weakup target tenant + uint64_t tenant_id_; + //ls_id is invalid iterator all ls + //ls_id is valid, only check target ls + share::ObLSID ls_id_; + //advise_leader maybe invalid + common::ObAddr advise_leader_; + SwitchLeaderComment comment_; +}; + struct ObCreateTabletInfo { OB_UNIS_VERSION(1); @@ -6261,6 +6303,60 @@ private: uint64_t tenant_id_; }; +struct ObUpdateTenantInfoCacheArg +{ + OB_UNIS_VERSION(1); +public: + ObUpdateTenantInfoCacheArg(): tenant_id_(OB_INVALID_TENANT_ID), tenant_info_(), ora_rowscn_(0) {} + ~ObUpdateTenantInfoCacheArg() {} + bool is_valid() const; + int init(const uint64_t tenant_id, const share::ObAllTenantInfo &tenant_info, const int64_t ora_rowscn); + int assign(const ObUpdateTenantInfoCacheArg &other); + TO_STRING_KV(K_(tenant_id), K_(tenant_info), K_(ora_rowscn)); + + uint64_t get_tenant_id() const + { + return tenant_id_; + } + + const share::ObAllTenantInfo &get_tenant_info() const + { + return tenant_info_; + } + + int64_t get_ora_rowscn() const + { + return ora_rowscn_; + } + +private: + DISALLOW_COPY_AND_ASSIGN(ObUpdateTenantInfoCacheArg); +private: + uint64_t tenant_id_; + share::ObAllTenantInfo tenant_info_; + int64_t ora_rowscn_; +}; + +struct ObUpdateTenantInfoCacheRes +{ + OB_UNIS_VERSION(1); +public: + ObUpdateTenantInfoCacheRes(): tenant_id_(OB_INVALID_TENANT_ID) {} + ~ObUpdateTenantInfoCacheRes() {} + bool is_valid() const; + int init(const uint64_t tenant_id); + int assign(const ObUpdateTenantInfoCacheRes &other); + TO_STRING_KV(K_(tenant_id)); + uint64_t get_tenant_id() const + { + return tenant_id_; + } +private: + DISALLOW_COPY_AND_ASSIGN(ObUpdateTenantInfoCacheRes); +private: + uint64_t tenant_id_; +}; + struct ObCheckpoint { OB_UNIS_VERSION(1); diff --git a/src/share/ob_server_blacklist.h b/src/share/ob_server_blacklist.h index 79be799bf..c31124bf5 100644 --- a/src/share/ob_server_blacklist.h +++ b/src/share/ob_server_blacklist.h @@ -85,6 +85,7 @@ public: ObDstServerInfo dst_info_; }; +// TODO shanyan.g to be removed class ObServerBlacklist : public lib::TGRunnable { public: diff --git a/src/share/ob_srv_rpc_proxy.h b/src/share/ob_srv_rpc_proxy.h index a3c48e74d..dc7b51911 100644 --- a/src/share/ob_srv_rpc_proxy.h +++ b/src/share/ob_srv_rpc_proxy.h @@ -185,6 +185,8 @@ public: RPC_AP(PR5 refresh_tenant_info, OB_REFRESH_TENANT_INFO, (obrpc::ObRefreshTenantInfoArg), obrpc::ObRefreshTenantInfoRes); RPC_S(PR5 sync_rewrite_rules, OB_SYNC_REWRITE_RULES, (ObSyncRewriteRuleArg)); RPC_S(PR5 force_set_ls_as_single_replica, OB_LOG_FORCE_SET_LS_AS_SINGLE_REPLICA, (obrpc::ObForceSetLSAsSingleReplicaArg)); + RPC_AP(PR5 notify_switch_leader, OB_NOTIFY_SWITCH_LEADER, (obrpc::ObNotifySwitchLeaderArg)); + RPC_AP(PR5 update_tenant_info_cache, OB_UPDATE_TENANT_INFO_CACHE, (obrpc::ObUpdateTenantInfoCacheArg), obrpc::ObUpdateTenantInfoCacheRes); }; // end of class ObSrvRpcProxy } // end of namespace rpc diff --git a/src/share/ob_tenant_info_proxy.cpp b/src/share/ob_tenant_info_proxy.cpp index ee3b94ac4..af2557073 100644 --- a/src/share/ob_tenant_info_proxy.cpp +++ b/src/share/ob_tenant_info_proxy.cpp @@ -58,6 +58,16 @@ SCN gen_new_standby_scn(const SCN &cur_standby_scn, const SCN &desired_standby_s } ////////////ObAllTenantInfo +DEFINE_TO_YSON_KV(ObAllTenantInfo, + OB_ID(tenant_id), tenant_id_, + OB_ID(switchover_epoch), switchover_epoch_, + OB_ID(sync_scn), sync_scn_, + OB_ID(replayable_scn), replayable_scn_, + OB_ID(standby_scn), standby_scn_, + OB_ID(recovery_until_scn), recovery_until_scn_, + OB_ID(tenant_role), tenant_role_, + OB_ID(switchover_status), switchover_status_); + bool ObAllTenantInfo::is_valid() const { return OB_INVALID_TENANT_ID != tenant_id_ @@ -309,6 +319,24 @@ int ObAllTenantInfoProxy::load_tenant_info(const uint64_t tenant_id, { int ret = OB_SUCCESS; tenant_info.reset(); + int64_t ora_rowscn = 0; + + if (OB_FAIL(load_tenant_info(tenant_id, proxy, for_update, ora_rowscn, tenant_info))) { + LOG_WARN("failed to get tenant info", KR(ret), K(tenant_id), KP(proxy), K(for_update)); + } + return ret; +} + + +int ObAllTenantInfoProxy::load_tenant_info(const uint64_t tenant_id, + ObISQLClient *proxy, + const bool for_update, + int64_t &ora_rowscn, + ObAllTenantInfo &tenant_info) +{ + int ret = OB_SUCCESS; + tenant_info.reset(); + ora_rowscn = 0; ObTimeoutCtx ctx; if (OB_ISNULL(proxy)) { ret = OB_ERR_UNEXPECTED; @@ -326,7 +354,7 @@ int ObAllTenantInfoProxy::load_tenant_info(const uint64_t tenant_id, uint64_t exec_tenant_id = gen_meta_tenant_id(tenant_id); if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) { LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx)); - } else if (OB_FAIL(sql.assign_fmt("select * from %s where tenant_id = %lu ", + } else if (OB_FAIL(sql.assign_fmt("select ORA_ROWSCN, * from %s where tenant_id = %lu ", OB_ALL_TENANT_INFO_TNAME, tenant_id))) { LOG_WARN("failed to assign sql", KR(ret), K(sql)); } else if(for_update && OB_FAIL(sql.append(" for update"))) { @@ -341,7 +369,7 @@ int ObAllTenantInfoProxy::load_tenant_info(const uint64_t tenant_id, LOG_WARN("failed to get sql result", KR(ret)); } else if (OB_FAIL(result->next())) { LOG_WARN("failed to get tenant info", KR(ret), K(sql)); - } else if (OB_FAIL(fill_cell(result, tenant_info))) { + } else if (OB_FAIL(fill_cell(result, tenant_info, ora_rowscn))) { LOG_WARN("failed to fill cell", KR(ret), K(sql)); } } @@ -419,7 +447,7 @@ int ObAllTenantInfoProxy::update_tenant_recovery_status( return ret; } -int ObAllTenantInfoProxy::fill_cell(common::sqlclient::ObMySQLResult *result, ObAllTenantInfo &tenant_info) +int ObAllTenantInfoProxy::fill_cell(common::sqlclient::ObMySQLResult *result, ObAllTenantInfo &tenant_info, int64_t &ora_rowscn) { int ret = OB_SUCCESS; tenant_info.reset(); @@ -435,6 +463,7 @@ int ObAllTenantInfoProxy::fill_cell(common::sqlclient::ObMySQLResult *result, Ob uint64_t replay_scn_val = OB_INVALID_SCN_VAL; uint64_t sts_scn_val = OB_INVALID_SCN_VAL; uint64_t recovery_until_scn_val = OB_INVALID_SCN_VAL; + ora_rowscn = 0; ObString log_mode_str; ObString log_mode_default_value("NOARCHIVELOG"); SCN sync_scn; @@ -451,6 +480,7 @@ int ObAllTenantInfoProxy::fill_cell(common::sqlclient::ObMySQLResult *result, Ob EXTRACT_UINT_FIELD_MYSQL_WITH_DEFAULT_VALUE(*result, "recovery_until_scn", recovery_until_scn_val, uint64_t, false /* skip_null_error */, true /* skip_column_error */, OB_MAX_SCN_TS_NS); EXTRACT_VARCHAR_FIELD_MYSQL_WITH_DEFAULT_VALUE(*result, "log_mode", log_mode_str, false /* skip_null_error */, true /* skip_column_error */, log_mode_default_value); + EXTRACT_INT_FIELD_MYSQL(*result, "ORA_ROWSCN", ora_rowscn, int64_t); ObTenantRole tmp_tenant_role(tenant_role_str); ObTenantSwitchoverStatus tmp_tenant_sw_status(status_str); ObArchiveMode tmp_log_mode(log_mode_str); diff --git a/src/share/ob_tenant_info_proxy.h b/src/share/ob_tenant_info_proxy.h index 802e2296f..a40ed62a0 100644 --- a/src/share/ob_tenant_info_proxy.h +++ b/src/share/ob_tenant_info_proxy.h @@ -113,6 +113,7 @@ IS_TENANT_STATUS(prepare_flashback_for_switch_to_primary) TO_STRING_KV(K_(tenant_id), K_(tenant_role), K_(switchover_status), K_(switchover_epoch), K_(sync_scn), K_(replayable_scn), K_(standby_scn), K_(recovery_until_scn), K_(log_mode)); + DECLARE_TO_YSON_KV; // Getter&Setter const ObTenantRole &get_tenant_role() const { return tenant_role_; } @@ -169,6 +170,19 @@ public: static int load_tenant_info(const uint64_t tenant_id, ObISQLClient *proxy, const bool for_update, ObAllTenantInfo &tenant_info); + + /** + * @description: get target tenant's tenant_info from inner table + * @param[in] tenant_id + * @param[in] proxy + * @param[in] for_update select for_update + * @param[out] ora_rowscn + * @param[out] tenant_info + */ + static int load_tenant_info(const uint64_t tenant_id, ObISQLClient *proxy, + const bool for_update, + int64_t &ora_rowscn, + ObAllTenantInfo &tenant_info); /** * @description: update tenant recovery status * @param[in] tenant_id @@ -244,7 +258,7 @@ public: const ObTenantSwitchoverStatus &old_status, const ObTenantSwitchoverStatus &new_status, int64_t &new_switchover_epoch); - static int fill_cell(common::sqlclient::ObMySQLResult *result, ObAllTenantInfo &tenant_info); + static int fill_cell(common::sqlclient::ObMySQLResult *result, ObAllTenantInfo &tenant_info, int64_t &ora_rowscn); /** * @description: update tenant recovery_until_scn in trans diff --git a/src/share/ob_tenant_role.cpp b/src/share/ob_tenant_role.cpp index 528a1ab25..ee0c6b7fb 100644 --- a/src/share/ob_tenant_role.cpp +++ b/src/share/ob_tenant_role.cpp @@ -12,7 +12,10 @@ #define USING_LOG_PREFIX SHARE +#include "lib/ob_name_id_def.h" //OB_ID #include "share/ob_tenant_role.h" +#include "lib/utility/ob_print_utils.h" //TO_STRING_KV +#include "lib/trace/ob_trace_event.h" using namespace oceanbase; using namespace oceanbase::common; @@ -29,6 +32,8 @@ static const char* TENANT_ROLE_ARRAY[] = }; OB_SERIALIZE_MEMBER(ObTenantRole, value_); +DEFINE_TO_YSON_KV(ObTenantRole, + OB_ID(value), value_); const char* ObTenantRole::to_str() const { diff --git a/src/share/ob_tenant_role.h b/src/share/ob_tenant_role.h index c66f09b88..6dcc38f79 100644 --- a/src/share/ob_tenant_role.h +++ b/src/share/ob_tenant_role.h @@ -59,6 +59,7 @@ public: bool is_restore() const { return RESTORE_TENANT == value_; } TO_STRING_KV(K_(value)); + DECLARE_TO_YSON_KV; private: Role value_; }; diff --git a/src/share/ob_tenant_switchover_status.cpp b/src/share/ob_tenant_switchover_status.cpp index ccbe39469..57a455194 100644 --- a/src/share/ob_tenant_switchover_status.cpp +++ b/src/share/ob_tenant_switchover_status.cpp @@ -12,7 +12,10 @@ #define USING_LOG_PREFIX SHARE +#include "lib/ob_name_id_def.h" //OB_ID #include "share/ob_tenant_switchover_status.h" +#include "lib/utility/ob_print_utils.h" //TO_STRING_KV +#include "lib/trace/ob_trace_event.h" using namespace oceanbase; using namespace oceanbase::common; @@ -33,6 +36,8 @@ static const char* TENANT_SWITCHOVER_ARRAY[] = }; OB_SERIALIZE_MEMBER(ObTenantSwitchoverStatus, value_); +DEFINE_TO_YSON_KV(ObTenantSwitchoverStatus, + OB_ID(value), value_); const char* ObTenantSwitchoverStatus::to_str() const { diff --git a/src/share/ob_tenant_switchover_status.h b/src/share/ob_tenant_switchover_status.h index ba3f08a5b..cd6f6620c 100644 --- a/src/share/ob_tenant_switchover_status.h +++ b/src/share/ob_tenant_switchover_status.h @@ -75,6 +75,7 @@ IS_TENANT_STATUS(PREPARE_FLASHBACK_FOR_SWITCH_TO_PRIMARY_STATUS, prepare_flashba #undef IS_TENANT_STATUS TO_STRING_KV(K_(value)); + DECLARE_TO_YSON_KV; private: ObTenantSwitchoverStatus::Status value_; }; diff --git a/src/share/parameter/ob_parameter_seed.ipp b/src/share/parameter/ob_parameter_seed.ipp index 159a0d6fe..4462c664b 100644 --- a/src/share/parameter/ob_parameter_seed.ipp +++ b/src/share/parameter/ob_parameter_seed.ipp @@ -116,7 +116,7 @@ DEF_BOOL(enable_sql_audit, OB_CLUSTER_PARAMETER, "true", "specifies whether SQL audit is turned on. " "The default value is TRUE. Value: TRUE: turned on FALSE: turned off", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); -DEF_BOOL(enable_record_trace_id, OB_CLUSTER_PARAMETER, "true", +DEF_BOOL(enable_record_trace_id, OB_CLUSTER_PARAMETER, "False", "specifies whether record app trace id is turned on.", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); DEF_BOOL(enable_rich_error_msg, OB_CLUSTER_PARAMETER, "false", @@ -162,7 +162,7 @@ DEF_STR_WITH_CHECKER(default_compress, OB_CLUSTER_PARAMETER, "archive", "default compress strategy for create new table within oracle mode", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); -DEF_TIME(weak_read_version_refresh_interval, OB_CLUSTER_PARAMETER, "100ms", "[50ms,)", +DEF_TIME(weak_read_version_refresh_interval, OB_CLUSTER_PARAMETER, "500ms", "[50ms,)", "the time interval to refresh cluster weak read version " "Range: [50ms, +∞)", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); @@ -372,7 +372,7 @@ DEF_CAP(plan_cache_low_watermark, OB_CLUSTER_PARAMETER, "1500M", "(don't use now) memory usage at which plan cache eviction will be stopped. " "Range: [0, plan_cache_high_watermark)", ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); -DEF_TIME(plan_cache_evict_interval, OB_CLUSTER_PARAMETER, "1s", "[0s,)", +DEF_TIME(plan_cache_evict_interval, OB_CLUSTER_PARAMETER, "5s", "[0s,)", "time interval for periodic plan cache eviction. Range: [0s, +∞)", ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); DEF_INT(max_px_worker_count, OB_CLUSTER_PARAMETER, "64", "[0,65535]", @@ -712,7 +712,7 @@ DEF_TIME(virtual_table_location_cache_expire_time, OB_CLUSTER_PARAMETER, "8s", " "expiration time for virtual table location info in partiton location cache. " "Range: [1s, +∞)", ObParameterAttr(Section::LOCATION_CACHE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); -DEF_INT(location_refresh_thread_count, OB_CLUSTER_PARAMETER, "4", "(1,64]", +DEF_INT(location_refresh_thread_count, OB_CLUSTER_PARAMETER, "2", "(1,64]", "the number of threads for fetching location cache in the background. Range: (1, 64]", ObParameterAttr(Section::LOCATION_CACHE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); @@ -885,7 +885,7 @@ DEF_INT(_migrate_block_verify_level, OB_CLUSTER_PARAMETER, "1", "[0,2]", "2 : logical verification", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); -DEF_TIME(_cache_wash_interval, OB_CLUSTER_PARAMETER, "200ms", "[1ms, 3s]", +DEF_TIME(_cache_wash_interval, OB_CLUSTER_PARAMETER, "1s", "[1ms, 3s]", "specify interval of cache background wash", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); diff --git a/src/share/rpc/ob_batch_proxy.h b/src/share/rpc/ob_batch_proxy.h index fb448a332..7dfb4208f 100644 --- a/src/share/rpc/ob_batch_proxy.h +++ b/src/share/rpc/ob_batch_proxy.h @@ -23,32 +23,32 @@ namespace oceanbase namespace obrpc { enum { - ELECTION_BATCH_REQ = 0, - ELECTION_GROUP_BATCH_REQ = 1, - CLOG_BATCH_REQ = 2, - CLOG_BATCH_REQ_NODELAY = 3, - TRX_BATCH_REQ_NODELAY = 4, - SQL_BATCH_REQ_NODELAY1 = 5, - SQL_BATCH_REQ_NODELAY2 = 6, - CLOG_BATCH_REQ_NODELAY2 = 7, - BATCH_REQ_TYPE_COUNT = 8 + TRX_BATCH_REQ_NODELAY = 0, + BATCH_REQ_TYPE_COUNT = 1, + ELECTION_BATCH_REQ, + ELECTION_GROUP_BATCH_REQ, + CLOG_BATCH_REQ, + CLOG_BATCH_REQ_NODELAY, + SQL_BATCH_REQ_NODELAY1, + SQL_BATCH_REQ_NODELAY2, + CLOG_BATCH_REQ_NODELAY2 }; inline int64_t get_batch_delay_us(const int batch_type) { - int64_t delay[BATCH_REQ_TYPE_COUNT] = {2 * 1000, 2 * 1000, 1 * 1000, 0, 0, 0, 0, 0}; + int64_t delay[BATCH_REQ_TYPE_COUNT] = {0}; return (batch_type >= 0 && batch_type < BATCH_REQ_TYPE_COUNT) ? delay[batch_type]: 0; } inline int64_t get_batch_buffer_size(const int batch_type) { - int64_t batch_buffer_size_k[BATCH_REQ_TYPE_COUNT] = {256, 256, 2048, 2048, 256, 256, 256, 2048}; + int64_t batch_buffer_size_k[BATCH_REQ_TYPE_COUNT] = {256}; return batch_buffer_size_k[batch_type] * 1024; } inline bool is_hp_rpc(const int batch_type) { - static const bool hp_rpc_map[BATCH_REQ_TYPE_COUNT] = {true, true, false, false, false, false, false, false}; + static const bool hp_rpc_map[BATCH_REQ_TYPE_COUNT] = {false}; return (batch_type >= 0 && batch_type < BATCH_REQ_TYPE_COUNT) ? hp_rpc_map[batch_type] : false; } diff --git a/src/share/rpc/ob_blacklist_proxy.cpp b/src/share/rpc/ob_blacklist_proxy.cpp index 2991b6c9f..2746e46ac 100644 --- a/src/share/rpc/ob_blacklist_proxy.cpp +++ b/src/share/rpc/ob_blacklist_proxy.cpp @@ -98,7 +98,7 @@ int ObBlacklistRpcProxy::send_req(const common::ObAddr &dst, const int64_t dst_c ret = OB_INVALID_ARGUMENT; } else { static BlacklistReqCallBack cb; - ret = this->to(dst).dst_cluster_id(dst_cluster_id).by(OB_SVR_BLACKLIST_TENANT_ID).as(OB_SERVER_TENANT_ID).timeout(BLACK_LIST_MSG_TIMEOUT).post_request(req, &cb); + ret = this->to(dst).dst_cluster_id(dst_cluster_id).by(OB_SERVER_TENANT_ID).timeout(BLACK_LIST_MSG_TIMEOUT).post_request(req, &cb); } return ret; } @@ -110,7 +110,7 @@ int ObBlacklistRpcProxy::send_resp(const common::ObAddr &dst, const int64_t dst_ ret = OB_INVALID_ARGUMENT; } else { static BlacklistRespCallBack cb; - ret = this->to(dst).dst_cluster_id(dst_cluster_id).by(OB_SVR_BLACKLIST_TENANT_ID).as(OB_SERVER_TENANT_ID).timeout(BLACK_LIST_MSG_TIMEOUT).post_response(resp, &cb); + ret = this->to(dst).dst_cluster_id(dst_cluster_id).by(OB_SERVER_TENANT_ID).timeout(BLACK_LIST_MSG_TIMEOUT).post_response(resp, &cb); } return ret; } diff --git a/src/sql/das/ob_data_access_service.cpp b/src/sql/das/ob_data_access_service.cpp index e33a56208..0ccdfee6a 100644 --- a/src/sql/das/ob_data_access_service.cpp +++ b/src/sql/das/ob_data_access_service.cpp @@ -232,6 +232,7 @@ int ObDataAccessService::retry_das_task(ObDASRef &das_ref, ObIDASTaskOp &task_op is_partition_change_error(ret) || OB_REPLICA_NOT_READABLE == ret) && !is_virtual_table(task_op.get_ref_table_id())) { + int tmp_ret = ret; if (!can_fast_fail(task_op)) { task_op.in_part_retry_ = true; das_ref.get_exec_ctx().get_my_session()->set_session_in_retry(true, ret); @@ -240,7 +241,7 @@ int ObDataAccessService::retry_das_task(ObDASRef &das_ref, ObIDASTaskOp &task_op } else if (OB_FAIL(das_ref.get_exec_ctx().check_status())) { LOG_WARN("query is timeout, terminate retry", K(ret)); } else if (OB_FAIL(refresh_partition_location(das_ref, task_op))) { - LOG_WARN("refresh partition location failed", K(ret)); + LOG_WARN("refresh partition location failed", K(ret), "ori_err_code", tmp_ret, K(lbt())); } else if (FALSE_IT(das_task_wrapper.reuse())) { } else if (FALSE_IT(task_op.set_task_status(ObDasTaskStatus::UNSTART))) { } else if (OB_FAIL(das_task_wrapper.push_back_task(&task_op))) { @@ -370,7 +371,7 @@ int ObDataAccessService::do_async_remote_das_task( ObSQLSessionInfo *session = das_ref.get_exec_ctx().get_my_session(); ObPhysicalPlanCtx *plan_ctx = das_ref.get_exec_ctx().get_physical_plan_ctx(); int64_t timeout_ts = plan_ctx->get_timeout_timestamp(); - int64_t current_ts = ObTimeUtility::current_time(); + int64_t current_ts = ObClockGenerator::getClock(); int64_t timeout = timeout_ts - current_ts; int64_t simulate_timeout = - EVENT_CALL(EventTable::EN_DAS_SIMULATE_ASYNC_RPC_TIMEOUT); if (OB_UNLIKELY(simulate_timeout > 0)) { @@ -465,7 +466,7 @@ int ObDataAccessService::do_sync_remote_das_task( FLTSpanGuard(do_sync_remote_das_task); ObSQLSessionInfo *session = das_ref.get_exec_ctx().get_my_session(); ObPhysicalPlanCtx *plan_ctx = das_ref.get_exec_ctx().get_physical_plan_ctx(); - int64_t timeout = plan_ctx->get_timeout_timestamp() - ObTimeUtility::current_time(); + int64_t timeout = plan_ctx->get_timeout_timestamp() - ObClockGenerator::getClock(); uint64_t tenant_id = session->get_rpc_tenant_id(); common::ObSEArray &task_ops = task_arg.get_task_ops(); ObIDASTaskResult *op_result = nullptr; diff --git a/src/sql/engine/aggregate/ob_aggregate_function.cpp b/src/sql/engine/aggregate/ob_aggregate_function.cpp index e887e66d2..f446d9487 100644 --- a/src/sql/engine/aggregate/ob_aggregate_function.cpp +++ b/src/sql/engine/aggregate/ob_aggregate_function.cpp @@ -164,7 +164,7 @@ int ObGroupConcatRowStore::init(const uint64_t tenant_id, iter_idx_ = 0; if (sort_columns.empty()) { int64_t sort_area_size = 0; - if (OB_FAIL(ObSqlWorkareaUtil::get_workarea_size(SORT_WORK_AREA, tenant_id, sort_area_size))) { + if (OB_FAIL(ObSqlWorkareaUtil::get_workarea_size(SORT_WORK_AREA, tenant_id, NULL, sort_area_size))) { LOG_WARN("failed to get workarea size", K(ret), K(tenant_id)); } else if (OB_FAIL(rs_.init(sort_area_size, tenant_id, ObCtxIds::WORK_AREA, ObModIds::OB_SQL_AGGR_FUN_GROUP_CONCAT, diff --git a/src/sql/engine/aggregate/ob_aggregate_processor.cpp b/src/sql/engine/aggregate/ob_aggregate_processor.cpp index ea9919c34..ba1859213 100644 --- a/src/sql/engine/aggregate/ob_aggregate_processor.cpp +++ b/src/sql/engine/aggregate/ob_aggregate_processor.cpp @@ -353,7 +353,7 @@ int ObAggregateProcessor::GroupConcatExtraResult::init(const uint64_t tenant_id, } else { int64_t sort_area_size = 0; if (OB_FAIL(ObSqlWorkareaUtil::get_workarea_size( - SORT_WORK_AREA, tenant_id, sort_area_size))) { + SORT_WORK_AREA, tenant_id, &eval_ctx.exec_ctx_, sort_area_size))) { LOG_WARN("failed to get workarea size", K(ret), K(tenant_id)); } else if (OB_FAIL(row_store_.init(sort_area_size, tenant_id, diff --git a/src/sql/engine/join/ob_hash_join_op.cpp b/src/sql/engine/join/ob_hash_join_op.cpp index f796b75f8..ea5faa123 100644 --- a/src/sql/engine/join/ob_hash_join_op.cpp +++ b/src/sql/engine/join/ob_hash_join_op.cpp @@ -1417,7 +1417,7 @@ int ObHashJoinOp::get_max_memory_size(int64_t input_size) int64_t extra_memory_size = get_extra_memory_size(); int64_t memory_size = (extra_memory_size + input_size) < 0 ? input_size : (extra_memory_size + input_size); if (OB_FAIL(ObSqlWorkareaUtil::get_workarea_size( - ObSqlWorkAreaType::HASH_WORK_AREA, tenant_id, hash_area_size))) { + ObSqlWorkAreaType::HASH_WORK_AREA, tenant_id, &ctx_, hash_area_size))) { LOG_WARN("failed to get workarea size", K(ret), K(tenant_id)); } else if (FALSE_IT(remain_data_memory_size_ = hash_area_size * 80 / 100)) { // default data memory size: 80% @@ -1670,7 +1670,7 @@ int ObHashJoinOp::get_processor_type() int64_t hash_area_size = 0; if (OB_FAIL(ObSqlWorkareaUtil::get_workarea_size( ObSqlWorkAreaType::HASH_WORK_AREA, - ctx_.get_my_session()->get_effective_tenant_id(), hash_area_size))) { + ctx_.get_my_session()->get_effective_tenant_id(), &ctx_, hash_area_size))) { LOG_WARN("failed to get workarea size", K(ret)); } remain_data_memory_size_ = hash_area_size * 10; diff --git a/src/sql/engine/ob_physical_plan.cpp b/src/sql/engine/ob_physical_plan.cpp index bf7e0b29d..93b25080a 100644 --- a/src/sql/engine/ob_physical_plan.cpp +++ b/src/sql/engine/ob_physical_plan.cpp @@ -456,7 +456,7 @@ void ObPhysicalPlan::update_plan_stat(const ObAuditRecordData &record, const bool is_evolution, const ObIArray *table_row_count_list) { - const int64_t current_time = ObTimeUtility::current_time(); + const int64_t current_time = ObClockGenerator::getClock(); int64_t execute_count = 0; if (record.is_timeout()) { ATOMIC_INC(&(stat_.timeout_count_)); diff --git a/src/sql/engine/ob_sql_mem_mgr_processor.cpp b/src/sql/engine/ob_sql_mem_mgr_processor.cpp index ec69bd565..deff24fe2 100644 --- a/src/sql/engine/ob_sql_mem_mgr_processor.cpp +++ b/src/sql/engine/ob_sql_mem_mgr_processor.cpp @@ -97,7 +97,7 @@ int ObSqlMemMgrProcessor::init( int64_t max_mem_size = MAX_SQL_MEM_SIZE; if (OB_FAIL(ret)) { } else if (OB_FAIL(ObSqlWorkareaUtil::get_workarea_size( - profile_.get_work_area_type(), tenant_id_, max_mem_size))) { + profile_.get_work_area_type(), tenant_id_, exec_ctx, max_mem_size))) { LOG_WARN("failed to get workarea size", K(ret), K(tenant_id_), K(max_mem_size)); } if (!profile_.get_auto_policy()) { @@ -310,24 +310,47 @@ int ObSqlMemMgrProcessor::alloc_dir_id(int64_t &dir_id) return ret; } -int ObSqlWorkareaUtil::get_workarea_size(const ObSqlWorkAreaType wa_type, const int64_t tenant_id, int64_t &value) +int ObSqlWorkareaUtil::get_workarea_size(const ObSqlWorkAreaType wa_type, + const int64_t tenant_id, + ObExecContext *exec_ctx, + int64_t &value) { int ret = OB_SUCCESS; - ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id)); - if (tenant_config.is_valid()) { - if (HASH_WORK_AREA == wa_type) { - value = tenant_config->_hash_area_size; - } else if (SORT_WORK_AREA == wa_type) { - value = tenant_config->_sort_area_size; + + if (NULL != exec_ctx) { + if (OB_ISNULL(exec_ctx->get_my_session())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected exec_ctx or session", K(ret), K(wa_type), K(tenant_id), KP(exec_ctx)); + } else { + if (HASH_WORK_AREA == wa_type) { + value = exec_ctx->get_my_session()->get_tenant_hash_area_size(); + } else if (SORT_WORK_AREA == wa_type) { + value = exec_ctx->get_my_session()->get_tenant_sort_area_size(); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected status: workarea type", K(wa_type), K(tenant_id)); + } + } + } else { + ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id)); + if (tenant_config.is_valid()) { + if (HASH_WORK_AREA == wa_type) { + value = tenant_config->_hash_area_size; + } else if (SORT_WORK_AREA == wa_type) { + value = tenant_config->_sort_area_size; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected status: workarea type", K(wa_type), K(tenant_id)); + } + LOG_DEBUG("debug workarea size", K(value), K(tenant_id), K(lbt())); } else { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected status: workarea type", K(wa_type), K(tenant_id)); + LOG_WARN("failed to init tenant config", K(tenant_id), K(ret)); } - LOG_DEBUG("debug workarea size", K(value), K(tenant_id), K(lbt())); - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("failed to init tenant config", K(tenant_id), K(ret)); } + + LOG_DEBUG("debug workarea size", K(value), K(tenant_id), K(lbt())); + return ret; } diff --git a/src/sql/engine/ob_sql_mem_mgr_processor.h b/src/sql/engine/ob_sql_mem_mgr_processor.h index 9cce5ffc4..be4a1783d 100644 --- a/src/sql/engine/ob_sql_mem_mgr_processor.h +++ b/src/sql/engine/ob_sql_mem_mgr_processor.h @@ -183,6 +183,7 @@ public: static int get_workarea_size( const ObSqlWorkAreaType wa_type, const int64_t tenant_id, + ObExecContext *exec_ctx, int64_t &value ); }; diff --git a/src/sql/engine/px/ob_px_target_mgr.h b/src/sql/engine/px/ob_px_target_mgr.h index 1a532f44b..5b869cd2d 100644 --- a/src/sql/engine/px/ob_px_target_mgr.h +++ b/src/sql/engine/px/ob_px_target_mgr.h @@ -91,7 +91,7 @@ class ObPxTargetMgr : public share::ObThreadPool { -#define PX_REFRESH_TARGET_INTERVEL_US (100 * 1000) +#define PX_REFRESH_TARGET_INTERVEL_US (500 * 1000) #define PX_MAX_ALIVE_SERVER_NUM (2000) public: diff --git a/src/sql/monitor/ob_exec_stat.h b/src/sql/monitor/ob_exec_stat.h index 3b0c7ad9a..8bfa6b31c 100644 --- a/src/sql/monitor/ob_exec_stat.h +++ b/src/sql/monitor/ob_exec_stat.h @@ -13,33 +13,15 @@ #ifdef EVENT_INFO EVENT_INFO(WAIT_TIME, wait_time) EVENT_INFO(WAIT_COUNT, wait_count) -EVENT_INFO(SCHED_TIME, sched_time) +EVENT_INFO(IO_READ_COUNT, io_read_count) EVENT_INFO(USER_IO_TIME, user_io_time) EVENT_INFO(APPLICATION_TIME, application_time) EVENT_INFO(CONCURRENCY_TIME, concurrency_time) -EVENT_INFO(IO_READ_COUNT, io_read_count) EVENT_INFO(IO_WRITES_COUNT, io_write_count) -EVENT_INFO(IO_READ_BYTES, io_read_bytes) -EVENT_INFO(IO_WRITE_BYTES, io_write_bytes) -EVENT_INFO(RPC_PACKET_IN, rpc_packet_in) -EVENT_INFO(RPC_PACKET_IN_BYTES, rpc_packet_in_bytes) EVENT_INFO(RPC_PACKET_OUT, rpc_packet_out) -EVENT_INFO(RPC_PACKET_OUT_BYTES, rpc_packet_out_bytes) EVENT_INFO(ROW_CACHE_HIT, row_cache_hit) -EVENT_INFO(ROW_CACHE_MISS, row_cache_miss) EVENT_INFO(BLOCK_CACHE_HIT, block_cache_hit) -EVENT_INFO(BLOCK_CACHE_MISS, block_cache_miss) EVENT_INFO(BLOOM_FILTER_FILTS, bloom_filter_filts) -EVENT_INFO(LOCATION_CACHE_HIT, location_cache_hit) -EVENT_INFO(LOCATION_CACHE_MISS, location_cache_miss) -EVENT_INFO(MEMSTORE_READ_LOCK_SUCC_COUNT, memstore_read_lock_succ_count) -EVENT_INFO(MEMSTORE_WRITE_LOCK_SUCC_COUNT, memstore_write_lock_succ_count) -EVENT_INFO(MEMSTORE_WAIT_READ_LOCK_TIME, memstore_wait_read_lock_time) -EVENT_INFO(MEMSTORE_WAIT_WRITE_LOCK_TIME, memstore_wait_write_lock_time) -EVENT_INFO(TRANS_COMMIT_LOG_SYNC_TIME, trans_commit_log_sync_time) -EVENT_INFO(TRANS_COMMIT_LOG_SYNC_COUNT, trans_commit_log_sync_count) -EVENT_INFO(TRANS_COMMIT_LOG_SUBMIT_COUNT, trans_commit_log_submit_count) -EVENT_INFO(TRANS_COMMIT_TIME, trans_commit_time) EVENT_INFO(MEMSTORE_READ_ROW_COUNT, memstore_read_row_count) EVENT_INFO(SSSTORE_READ_ROW_COUNT, ssstore_read_row_count) EVENT_INFO(DATA_BLOCK_READ_CNT, data_block_read_cnt) @@ -50,7 +32,6 @@ EVENT_INFO(BLOCKSCAN_BLOCK_CNT, blockscan_block_cnt) EVENT_INFO(BLOCKSCAN_ROW_CNT, blockscan_row_cnt) EVENT_INFO(PUSHDOWN_STORAGE_FILTER_ROW_CNT, pushdown_storage_filter_row_cnt) EVENT_INFO(FUSE_ROW_CACHE_HIT, fuse_row_cache_hit) -EVENT_INFO(FUSE_ROW_CACHE_MISS, fuse_row_cache_miss) #endif #ifndef OCEANBASE_SQL_OB_EXEC_STAT_H @@ -88,42 +69,37 @@ struct ObExecRecord #undef EVENT_INFO +#define EVENT_STAT_GET(event_stats_array, stat_no) \ + ({ \ + int64_t ret = 0; \ + oceanbase::common::ObStatEventAddStat *stat = NULL; \ + if (NULL != (stat = event_stats_array.get(::oceanbase::common::stat_no))) { \ + ret = stat->get_stat_value(); \ + } \ + ret; \ + }) + #define RECORD(se, di) \ do { \ - oceanbase::common::ObDiagnoseSessionInfo *diag_session_info = (NULL != di) ? di : oceanbase::common::ObDiagnoseSessionInfo::get_local_diagnose_info(); \ + oceanbase::common::ObDiagnoseSessionInfo *diag_session_info = \ + (NULL != di) ? di : oceanbase::common::ObDiagnoseSessionInfo::get_local_diagnose_info(); \ if (NULL != diag_session_info) { \ - io_read_count_##se##_= EVENT_GET(ObStatEventIds::IO_READ_COUNT, diag_session_info); \ - io_write_count_##se##_ = EVENT_GET(ObStatEventIds::IO_WRITE_COUNT, diag_session_info); \ - block_cache_hit_##se##_= EVENT_GET(ObStatEventIds::BLOCK_CACHE_HIT, diag_session_info); \ - io_read_bytes_##se##_= EVENT_GET(ObStatEventIds::IO_READ_BYTES, diag_session_info); \ - io_write_bytes_##se##_= EVENT_GET(ObStatEventIds::IO_WRITE_BYTES, diag_session_info); \ - rpc_packet_in_##se##_= EVENT_GET(ObStatEventIds::RPC_PACKET_IN, diag_session_info); \ - rpc_packet_in_bytes_##se##_= EVENT_GET(ObStatEventIds::RPC_PACKET_IN_BYTES, diag_session_info); \ - rpc_packet_out_##se##_= EVENT_GET(ObStatEventIds::RPC_PACKET_OUT, diag_session_info); \ - rpc_packet_out_bytes_##se##_= EVENT_GET(ObStatEventIds::RPC_PACKET_OUT_BYTES, diag_session_info); \ - trans_commit_log_sync_time_##se##_= EVENT_GET(ObStatEventIds::TRANS_COMMIT_LOG_SYNC_TIME, diag_session_info); \ - row_cache_hit_##se##_= EVENT_GET(ObStatEventIds::ROW_CACHE_HIT, diag_session_info); \ - row_cache_miss_##se##_= EVENT_GET(ObStatEventIds::ROW_CACHE_MISS, diag_session_info); \ - block_cache_hit_##se##_= EVENT_GET(ObStatEventIds::BLOCK_CACHE_HIT, diag_session_info); \ - block_cache_miss_##se##_= EVENT_GET(ObStatEventIds::BLOCK_CACHE_MISS, diag_session_info); \ - bloom_filter_filts_##se##_ = EVENT_GET(ObStatEventIds::BLOOM_FILTER_FILTS, diag_session_info); \ - location_cache_hit_##se##_= EVENT_GET(ObStatEventIds::LOCATION_CACHE_HIT, diag_session_info); \ - location_cache_miss_##se##_= EVENT_GET(ObStatEventIds::LOCATION_CACHE_MISS, diag_session_info); \ - memstore_read_lock_succ_count_##se##_= EVENT_GET(ObStatEventIds::MEMSTORE_READ_LOCK_SUCC_COUNT, diag_session_info); \ - memstore_write_lock_succ_count_##se##_= EVENT_GET(ObStatEventIds::MEMSTORE_WRITE_LOCK_SUCC_COUNT, diag_session_info); \ - memstore_wait_read_lock_time_##se##_= EVENT_GET(ObStatEventIds::MEMSTORE_WAIT_READ_LOCK_TIME, diag_session_info); \ - memstore_wait_write_lock_time_##se##_= EVENT_GET(ObStatEventIds::MEMSTORE_WAIT_WRITE_LOCK_TIME, diag_session_info); \ - memstore_read_row_count_##se##_ = EVENT_GET(ObStatEventIds::MEMSTORE_READ_ROW_COUNT, diag_session_info); \ - ssstore_read_row_count_##se##_ = EVENT_GET(ObStatEventIds::SSSTORE_READ_ROW_COUNT, diag_session_info); \ - data_block_read_cnt_##se##_ = EVENT_GET(ObStatEventIds::DATA_BLOCK_READ_CNT, diag_session_info); \ - data_block_cache_hit_##se##_ = EVENT_GET(ObStatEventIds::DATA_BLOCK_CACHE_HIT, diag_session_info); \ - index_block_read_cnt_##se##_ = EVENT_GET(ObStatEventIds::INDEX_BLOCK_READ_CNT, diag_session_info); \ - index_block_cache_hit_##se##_ = EVENT_GET(ObStatEventIds::INDEX_BLOCK_CACHE_HIT, diag_session_info); \ - blockscan_block_cnt_##se##_ = EVENT_GET(ObStatEventIds::BLOCKSCAN_BLOCK_CNT, diag_session_info); \ - blockscan_row_cnt_##se##_ = EVENT_GET(ObStatEventIds::BLOCKSCAN_ROW_CNT, diag_session_info); \ - pushdown_storage_filter_row_cnt_##se##_ = EVENT_GET(ObStatEventIds::PUSHDOWN_STORAGE_FILTER_ROW_CNT, diag_session_info); \ - fuse_row_cache_hit_##se##_= EVENT_GET(ObStatEventIds::FUSE_ROW_CACHE_HIT, diag_session_info); \ - fuse_row_cache_miss_##se##_= EVENT_GET(ObStatEventIds::FUSE_ROW_CACHE_MISS, diag_session_info); \ + oceanbase::common::ObStatEventAddStatArray &arr = diag_session_info->get_add_stat_stats(); \ + io_read_count_##se##_= EVENT_STAT_GET(arr, ObStatEventIds::IO_READ_COUNT); \ + block_cache_hit_##se##_= EVENT_STAT_GET(arr, ObStatEventIds::BLOCK_CACHE_HIT); \ + rpc_packet_out_##se##_= EVENT_STAT_GET(arr, ObStatEventIds::RPC_PACKET_OUT); \ + row_cache_hit_##se##_= EVENT_STAT_GET(arr, ObStatEventIds::ROW_CACHE_HIT); \ + bloom_filter_filts_##se##_ = EVENT_STAT_GET(arr, ObStatEventIds::BLOOM_FILTER_FILTS); \ + memstore_read_row_count_##se##_ = EVENT_STAT_GET(arr, ObStatEventIds::MEMSTORE_READ_ROW_COUNT); \ + ssstore_read_row_count_##se##_ = EVENT_STAT_GET(arr, ObStatEventIds::SSSTORE_READ_ROW_COUNT); \ + data_block_read_cnt_##se##_ = EVENT_STAT_GET(arr, ObStatEventIds::DATA_BLOCK_READ_CNT); \ + data_block_cache_hit_##se##_ = EVENT_STAT_GET(arr, ObStatEventIds::DATA_BLOCK_CACHE_HIT); \ + index_block_read_cnt_##se##_ = EVENT_STAT_GET(arr, ObStatEventIds::INDEX_BLOCK_READ_CNT); \ + index_block_cache_hit_##se##_ = EVENT_STAT_GET(arr, ObStatEventIds::INDEX_BLOCK_CACHE_HIT); \ + blockscan_block_cnt_##se##_ = EVENT_STAT_GET(arr, ObStatEventIds::BLOCKSCAN_BLOCK_CNT); \ + blockscan_row_cnt_##se##_ = EVENT_STAT_GET(arr, ObStatEventIds::BLOCKSCAN_ROW_CNT); \ + pushdown_storage_filter_row_cnt_##se##_ = EVENT_STAT_GET(arr, ObStatEventIds::PUSHDOWN_STORAGE_FILTER_ROW_CNT); \ + fuse_row_cache_hit_##se##_= EVENT_STAT_GET(arr, ObStatEventIds::FUSE_ROW_CACHE_HIT); \ } \ } while(0); @@ -147,24 +123,10 @@ struct ObExecRecord UPDATE_EVENT(io_read_count); UPDATE_EVENT(io_write_count); UPDATE_EVENT(block_cache_hit); - UPDATE_EVENT(io_read_bytes); - UPDATE_EVENT(io_write_bytes); - UPDATE_EVENT(rpc_packet_in); - UPDATE_EVENT(rpc_packet_in_bytes); UPDATE_EVENT(rpc_packet_out); - UPDATE_EVENT(rpc_packet_out_bytes); - UPDATE_EVENT(trans_commit_log_sync_time); UPDATE_EVENT(row_cache_hit); - UPDATE_EVENT(row_cache_miss); UPDATE_EVENT(bloom_filter_filts); - UPDATE_EVENT(location_cache_hit); - UPDATE_EVENT(location_cache_miss); - UPDATE_EVENT(memstore_read_lock_succ_count); - UPDATE_EVENT(memstore_write_lock_succ_count); - UPDATE_EVENT(memstore_wait_read_lock_time); - UPDATE_EVENT(memstore_wait_write_lock_time); UPDATE_EVENT(user_io_time); - UPDATE_EVENT(sched_time); UPDATE_EVENT(concurrency_time); UPDATE_EVENT(application_time); UPDATE_EVENT(memstore_read_row_count); diff --git a/src/sql/ob_sql.cpp b/src/sql/ob_sql.cpp index 87b9e959c..67873fac6 100644 --- a/src/sql/ob_sql.cpp +++ b/src/sql/ob_sql.cpp @@ -2424,9 +2424,8 @@ OB_NOINLINE int ObSql::handle_large_query(int tmp_ret, || !ObStmt::is_dml_stmt(result.get_stmt_type())) { ret = (tmp_ret == OB_PC_LOCK_CONFLICT) ? OB_SUCCESS : tmp_ret; } else { - const int64_t curr_time = ObTimeUtility::current_time(); const int64_t lqt = GCONF.large_query_threshold; - int64_t elapsed_time = curr_time - THIS_THWORKER.get_query_start_time(); + int64_t elapsed_time = ObClockGenerator::getClock() - THIS_THWORKER.get_query_start_time(); bool is_large_query = false; bool lq_from_plan = true; int64_t total_process_time = 0; diff --git a/src/sql/ob_sql_utils.cpp b/src/sql/ob_sql_utils.cpp index 13bdd7063..4b8137b10 100644 --- a/src/sql/ob_sql_utils.cpp +++ b/src/sql/ob_sql_utils.cpp @@ -3979,14 +3979,12 @@ int ObSQLUtils::handle_audit_record(bool need_retry, // failed to get request manager, maybe tenant has been dropped, NOT NEED TO record; } else { const ObAuditRecordData &audit_record = session.get_final_audit_record(exec_mode); - if (OB_FAIL(req_manager->record_request(audit_record, is_sensitive))) { + if (OB_FAIL(req_manager->record_request(audit_record, + session.enable_query_response_time_stats(), + is_sensitive))) { if (OB_SIZE_OVERFLOW == ret || OB_ALLOCATE_MEMORY_FAILED == ret) { LOG_DEBUG("cannot allocate mem for record", K(ret)); ret = OB_SUCCESS; - } else { - if (REACH_TIME_INTERVAL(100 * 1000)) { // in case logging is too frequent - LOG_WARN("failed to record request info in request manager", K(ret)); - } } } } diff --git a/src/sql/optimizer/ob_table_location.cpp b/src/sql/optimizer/ob_table_location.cpp index a3db1c262..afd260ae5 100644 --- a/src/sql/optimizer/ob_table_location.cpp +++ b/src/sql/optimizer/ob_table_location.cpp @@ -1318,7 +1318,8 @@ int ObTableLocation::get_is_weak_read(const ObDMLStmt &dml_stmt, ret = OB_ERR_UNEXPECTED; LOG_ERROR("unexpected null", K(ret), K(session), K(sql_ctx)); } else if (dml_stmt.get_query_ctx()->has_dml_write_stmt_ || - dml_stmt.get_query_ctx()->is_contain_select_for_update_) { + dml_stmt.get_query_ctx()->is_contain_select_for_update_ || + dml_stmt.get_query_ctx()->is_contain_inner_table_) { is_weak_read = false; } else { ObConsistencyLevel consistency_level = INVALID_CONSISTENCY; diff --git a/src/sql/plan_cache/ob_i_lib_cache_node.cpp b/src/sql/plan_cache/ob_i_lib_cache_node.cpp index 0871112b5..e174fa27e 100644 --- a/src/sql/plan_cache/ob_i_lib_cache_node.cpp +++ b/src/sql/plan_cache/ob_i_lib_cache_node.cpp @@ -153,7 +153,7 @@ int ObILibCacheNode::lock(bool is_rdlock) int ObILibCacheNode::update_node_stat(ObILibCacheCtx &ctx) { int ret = OB_SUCCESS; - ATOMIC_STORE(&(node_stat_.last_active_timestamp_), ObTimeUtility::current_time()); + ATOMIC_STORE(&(node_stat_.last_active_timestamp_), ObClockGenerator::getClock()); ATOMIC_INC(&(node_stat_.execute_count_)); return ret; } diff --git a/src/sql/plan_cache/ob_plan_cache_util.h b/src/sql/plan_cache/ob_plan_cache_util.h index ebf8d4218..ab10c2beb 100644 --- a/src/sql/plan_cache/ob_plan_cache_util.h +++ b/src/sql/plan_cache/ob_plan_cache_util.h @@ -772,8 +772,7 @@ struct ObPlanStat inline void update_cache_stat(const ObTableScanStat &stat) { - const int64_t current_time = common::ObTimeUtility::current_time(); - if (current_time > gen_time_ + CACHE_POLICY_UPDATE_INTERVAL) { + if (ObClockGenerator::getClock() > gen_time_ + CACHE_POLICY_UPDATE_INTERVAL) { const int64_t update_times = ATOMIC_AAF(&cache_stat_update_times_, 1); ATOMIC_AAF(&bf_filter_cnt_, stat.bf_filter_cnt_); ATOMIC_AAF(&bf_access_cnt_, stat.bf_access_cnt_); diff --git a/src/sql/resolver/dml/ob_del_upd_resolver.cpp b/src/sql/resolver/dml/ob_del_upd_resolver.cpp index 8c8a65e6d..7557b09e6 100644 --- a/src/sql/resolver/dml/ob_del_upd_resolver.cpp +++ b/src/sql/resolver/dml/ob_del_upd_resolver.cpp @@ -2800,6 +2800,8 @@ int ObDelUpdResolver::generate_autoinc_params(ObInsertTableInfo &table_info) param.autoinc_desired_count_ = 0; param.autoinc_mode_is_order_ = table_schema->is_order_auto_increment_mode(); param.autoinc_version_ = table_schema->get_truncate_version(); + param.autoinc_auto_increment_ = table_schema->get_auto_increment(); + // hidden pk auto-increment variables' default value is 1 // auto-increment variables for other columns are set in ob_sql.cpp // because physical plan may come from plan cache; it need be reset every time diff --git a/src/sql/session/ob_basic_session_info.cpp b/src/sql/session/ob_basic_session_info.cpp index b4396e491..06a7b43bb 100644 --- a/src/sql/session/ob_basic_session_info.cpp +++ b/src/sql/session/ob_basic_session_info.cpp @@ -66,7 +66,6 @@ ObBasicSessionInfo::ObBasicSessionInfo() sessid_(0), master_sessid_(INVALID_SESSID), proxy_sessid_(VALID_PROXY_SESSID), - variables_last_modify_time_(0), global_vars_version_(0), sys_var_base_version_(OB_INVALID_VERSION), tx_desc_(NULL), @@ -321,7 +320,6 @@ void ObBasicSessionInfo::reset(bool skip_sys_var) sessid_ = 0; master_sessid_ = INVALID_SESSID; proxy_sessid_ = VALID_PROXY_SESSID; - variables_last_modify_time_ = 0; global_vars_version_ = 0; tx_result_.reset(); @@ -1056,8 +1054,6 @@ int ObBasicSessionInfo::load_default_sys_variable(const bool print_info_log, con LOG_WARN("fail create all sys variables", K(ret)); } else if (OB_FAIL(init_system_variables(print_info_log, is_sys_tenant))) { LOG_WARN("Init system variables failed !", K(ret)); - } else { - variables_last_modify_time_ = ObTimeUtility::current_time(); } return ret; } @@ -1276,8 +1272,6 @@ int ObBasicSessionInfo::load_sys_variable(ObIAllocator &calc_buf, false /*check_timezone_valid*/, false /*is_update_sys_var*/))) { LOG_WARN("process system variable error", K(name), K(type), K(real_val), K(value), K(ret)); - } else { - variables_last_modify_time_ = ObTimeUtility::current_time(); } return ret; } @@ -1600,7 +1594,6 @@ int ObBasicSessionInfo::update_sys_variable(const ObSysVarClassType sys_var_id, if (OB_FAIL(track_sys_var(sys_var_id, sys_var->get_value()))) { LOG_WARN("failed to track sys var", K(ret), K(sys_var_id), K(val)); } else { - variables_last_modify_time_ = ObTimeUtility::current_time(); LOG_DEBUG("succ to track system variable", K(ret), K(sys_var_id), K(val), K(sys_var->get_value())); } @@ -3430,9 +3423,6 @@ int ObBasicSessionInfo::replace_user_variable(const ObString &var, const ObSessi LOG_WARN("fail to track user var", K(var), K(ret)); } } - if (OB_SUCC(ret)) { - variables_last_modify_time_ = ObTimeUtility::current_time(); - } } return ret; } @@ -3446,8 +3436,6 @@ int ObBasicSessionInfo::remove_user_variable(const ObString &var) } else if (OB_SUCCESS != user_var_val_map_.erase_refactored(var)) { ret = OB_ERR_USER_VARIABLE_UNKNOWN; LOG_WARN("unknown variable", K(var), K(ret)); - } else { - variables_last_modify_time_ = ObTimeUtility::current_time(); } return ret; } diff --git a/src/sql/session/ob_basic_session_info.h b/src/sql/session/ob_basic_session_info.h index feb49ee61..f71da0fd1 100644 --- a/src/sql/session/ob_basic_session_info.h +++ b/src/sql/session/ob_basic_session_info.h @@ -930,7 +930,6 @@ public: int is_use_transmission_checksum(bool &use_transmission_checksum) const; int is_select_index_enabled(bool &select_index_enabled) const; int get_name_case_mode(common::ObNameCaseMode &case_mode) const; - int64_t get_variables_last_modify_time() const { return variables_last_modify_time_;} int get_init_connect(common::ObString &str) const; /// @} @@ -1897,8 +1896,6 @@ private: uint32_t sessid_; uint32_t master_sessid_; uint64_t proxy_sessid_; - //TODO(qianfu):this is not used,may be removed - int64_t variables_last_modify_time_; int64_t global_vars_version_; // used for obproxy synchronize variables int64_t sys_var_base_version_; /******************************************* diff --git a/src/sql/session/ob_sql_session_info.cpp b/src/sql/session/ob_sql_session_info.cpp index 670167fa9..b9a17a2de 100644 --- a/src/sql/session/ob_sql_session_info.cpp +++ b/src/sql/session/ob_sql_session_info.cpp @@ -1629,8 +1629,7 @@ void ObSQLSessionInfo::update_stat_from_audit_record() + audit_record_.exec_record_.ssstore_read_row_count_); // session_stat_.total_logical_write_ += 0; // session_stat_.total_physical_read_ += 0; - session_stat_.total_lock_count_ += (audit_record_.exec_record_.memstore_read_lock_succ_count_ - + audit_record_.exec_record_.memstore_write_lock_succ_count_); + session_stat_.total_lock_count_ += 0; session_stat_.total_cpu_time_us_ += audit_record_.exec_timestamp_.executor_t_; session_stat_.total_exec_time_us_ += audit_record_.exec_timestamp_.elapsed_t_; } @@ -2276,6 +2275,8 @@ void ObSQLSessionInfo::ObCachedTenantConfigInfo::refresh() } // 4.sort area size ATOMIC_STORE(&sort_area_size_, tenant_config->_sort_area_size); + ATOMIC_STORE(&hash_area_size_, tenant_config->_hash_area_size); + ATOMIC_STORE(&enable_query_response_time_stats_, tenant_config->query_response_time_stats); // 5.allow security audit if (OB_SUCCESS != (tmp_ret = ObSecurityAuditUtils::check_allow_audit(*session_, at_type_))) { LOG_WARN_RET(tmp_ret, "fail get tenant_config", "ret", tmp_ret, diff --git a/src/sql/session/ob_sql_session_info.h b/src/sql/session/ob_sql_session_info.h index 45fa8a8a9..d005c469e 100644 --- a/src/sql/session/ob_sql_session_info.h +++ b/src/sql/session/ob_sql_session_info.h @@ -490,6 +490,8 @@ public: px_join_skew_minfreq_(30), at_type_(ObAuditTrailType::NONE), sort_area_size_(128*1024*1024), + hash_area_size_(128*1024*1024), + enable_query_response_time_stats_(false), print_sample_ppm_(0), last_check_ec_ts_(0), session_(session) @@ -503,6 +505,8 @@ public: bool get_enable_sql_extension() const { return enable_sql_extension_; } ObAuditTrailType get_at_type() const { return at_type_; } int64_t get_sort_area_size() const { return ATOMIC_LOAD(&sort_area_size_); } + int64_t get_hash_area_size() const { return ATOMIC_LOAD(&hash_area_size_); } + bool enable_query_response_time_stats() const { return enable_query_response_time_stats_; } int64_t get_print_sample_ppm() const { return ATOMIC_LOAD(&print_sample_ppm_); } bool get_px_join_skew_handling() const { return px_join_skew_handling_; } int64_t get_px_join_skew_minfreq() const { return px_join_skew_minfreq_; } @@ -517,6 +521,8 @@ public: int64_t px_join_skew_minfreq_; ObAuditTrailType at_type_; int64_t sort_area_size_; + int64_t hash_area_size_; + bool enable_query_response_time_stats_; // for record sys config print_sample_ppm int64_t print_sample_ppm_; int64_t last_check_ec_ts_; @@ -967,11 +973,21 @@ public: at_type = cached_tenant_config_info_.get_at_type(); return common::OB_SUCCESS; } + int64_t get_tenant_hash_area_size() + { + cached_tenant_config_info_.refresh(); + return cached_tenant_config_info_.get_hash_area_size(); + } int64_t get_tenant_sort_area_size() { cached_tenant_config_info_.refresh(); return cached_tenant_config_info_.get_sort_area_size(); } + bool enable_query_response_time_stats() + { + cached_tenant_config_info_.refresh(); + return cached_tenant_config_info_.enable_query_response_time_stats(); + } int64_t get_tenant_print_sample_ppm() { cached_tenant_config_info_.refresh(); diff --git a/src/storage/compaction/ob_tenant_freeze_info_mgr.h b/src/storage/compaction/ob_tenant_freeze_info_mgr.h index 816fe6b07..7a6d9246b 100644 --- a/src/storage/compaction/ob_tenant_freeze_info_mgr.h +++ b/src/storage/compaction/ob_tenant_freeze_info_mgr.h @@ -133,7 +133,7 @@ private: typedef common::RWLock::RLockGuard RLockGuard; typedef common::RWLock::WLockGuard WLockGuard; - static const int64_t RELOAD_INTERVAL = 1L * 1000L * 1000L; + static const int64_t RELOAD_INTERVAL = 3L * 1000L * 1000L; static const int64_t UPDATE_LS_RESERVED_SNAPSHOT_INTERVAL = 10L * 1000L * 1000L; static const int64_t MAX_GC_SNAPSHOT_TS_REFRESH_TS = 10L * 60L * 1000L * 1000L; static const int64_t FLUSH_GC_SNAPSHOT_TS_REFRESH_TS = diff --git a/src/storage/compaction/ob_tenant_tablet_scheduler.cpp b/src/storage/compaction/ob_tenant_tablet_scheduler.cpp index 0ed650eca..b90f99264 100755 --- a/src/storage/compaction/ob_tenant_tablet_scheduler.cpp +++ b/src/storage/compaction/ob_tenant_tablet_scheduler.cpp @@ -168,6 +168,8 @@ ObTenantTabletScheduler::ObTenantTabletScheduler() sstable_gc_task_(), fast_freeze_checker_(), enable_adaptive_compaction_(false), + minor_ls_tablet_iter_(false/*is_major*/), + medium_ls_tablet_iter_(true/*is_major*/), error_tablet_cnt_(0), ls_locality_cache_() { @@ -195,6 +197,8 @@ void ObTenantTabletScheduler::destroy() medium_loop_tg_id_ = 0; sstable_gc_tg_id_ = 0; schedule_interval_ = 0; + minor_ls_tablet_iter_.reset(); + medium_ls_tablet_iter_.reset(); ls_locality_cache_.reset(); is_inited_ = false; LOG_INFO("The ObTenantTabletScheduler destroy"); @@ -404,25 +408,28 @@ int ObTenantTabletScheduler::schedule_all_tablets_minor() { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - ObSharedGuard ls_iter_guard; + ObLSHandle ls_handle; ObLS *ls = nullptr; + int64_t schedule_tablet_cnt = 0; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("The ObTenantTabletScheduler has not been inited", K(ret)); - } else if (OB_FAIL(MTL(ObLSService *)->get_ls_iter(ls_iter_guard, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("failed to get ls iterator", K(ret)); + } else if (!minor_ls_tablet_iter_.is_valid() && OB_FAIL(minor_ls_tablet_iter_.build_iter())) { + LOG_WARN("failed to init iterator", K(ret)); + } else { + LOG_INFO("start schedule all tablet minor merge", K(minor_ls_tablet_iter_)); } - while (OB_SUCC(ret)) { - if (OB_FAIL(ls_iter_guard.get_ptr()->get_next(ls))) { + while (OB_SUCC(ret) && schedule_tablet_cnt < SCHEDULE_TABLET_BATCH_CNT) { + if (OB_FAIL(minor_ls_tablet_iter_.get_next_ls(ls_handle))) { if (OB_ITER_END == ret) { ret = OB_SUCCESS; break; } else { - LOG_WARN("failed to get ls", K(ret), KP(ls_iter_guard.get_ptr())); + LOG_WARN("failed to get ls", K(ret), K(minor_ls_tablet_iter_)); } - } else if (OB_ISNULL(ls)) { + } else if (OB_ISNULL(ls = ls_handle.get_ls())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ls is null", K(ret), K(ls)); } else { @@ -432,7 +439,7 @@ int ObTenantTabletScheduler::schedule_all_tablets_minor() LOG_WARN("failed to check ls state", K(ret), K(ls_id)); } else if (!need_merge) { // no need to merge, do nothing - } else if (OB_TMP_FAIL(schedule_ls_minor_merge(*ls))) { + } else if (OB_TMP_FAIL(schedule_ls_minor_merge(ls_handle, schedule_tablet_cnt))) { LOG_WARN("failed to schedule ls minor merge", K(tmp_ret), K(ls_id)); } } @@ -831,32 +838,29 @@ int ObTenantTabletScheduler::schedule_merge_execute_dag( } int ObTenantTabletScheduler::schedule_ls_minor_merge( - ObLS &ls) + ObLSHandle &ls_handle, + int64_t &schedule_tablet_cnt) { int ret = OB_SUCCESS; ObLSTabletIterator tablet_iter(ObTabletCommon::DIRECT_GET_COMMITTED_TABLET_TIMEOUT_US); bool need_merge = false; bool need_fast_freeze = false; + ObLS &ls = *ls_handle.get_ls(); const ObLSID &ls_id = ls.get_ls_id(); - ObLSHandle ls_handle; if (OB_FAIL(check_ls_state(ls, need_merge))) { LOG_WARN("failed to check ls state", K(ret), K(ls)); } else if (!need_merge) { // no need to merge, do nothing - } else if (OB_FAIL(MTL(ObLSService *)->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id)); - } else if (OB_FAIL(ls.build_tablet_iter(tablet_iter))) { - LOG_WARN("failed to build ls tablet iter", K(ret), K(ls)); } else { ObTabletID tablet_id; ObTabletHandle tablet_handle; ObTablet *tablet = nullptr; int tmp_ret = OB_SUCCESS; bool schedule_minor_flag = true; - while (OB_SUCC(ret) && schedule_minor_flag) { // loop all tablet in ls + while (OB_SUCC(ret) && schedule_minor_flag && schedule_tablet_cnt < SCHEDULE_TABLET_BATCH_CNT) { // loop all tablet in ls bool tablet_merge_finish = false; - if (OB_FAIL(tablet_iter.get_next_tablet(tablet_handle))) { - if (OB_ITER_END == ret) { + if (OB_FAIL(minor_ls_tablet_iter_.get_next_tablet(ls_handle, tablet_handle))) { + if (OB_ITER_END == ret || OB_LS_NOT_EXIST == ret) { ret = OB_SUCCESS; break; } else { @@ -880,6 +884,7 @@ int ObTenantTabletScheduler::schedule_ls_minor_merge( } } } else { // data tablet + schedule_tablet_cnt++; if (OB_TMP_FAIL(schedule_tablet_minor_merge(ls_handle, tablet_handle))) { if (OB_SIZE_OVERFLOW == tmp_ret) { schedule_minor_flag = false; @@ -913,24 +918,20 @@ int ObTenantTabletScheduler::schedule_ls_minor_merge( int ObTenantTabletScheduler::schedule_ls_medium_merge( int64_t &merge_version, - ObLS &ls, + ObLSHandle &ls_handle, bool &ls_merge_finish, - bool &all_ls_weak_read_ts_ready) + bool &all_ls_weak_read_ts_ready, + int64_t &schedule_tablet_cnt) { int ret = OB_SUCCESS; - ObLSTabletIterator tablet_iter(ObTabletCommon::DIRECT_GET_COMMITTED_TABLET_TIMEOUT_US); bool need_merge = false; + ObLS &ls = *ls_handle.get_ls(); const ObLSID &ls_id = ls.get_ls_id(); - ObLSHandle ls_handle; ObLSRestoreStatus restore_status; if (OB_FAIL(check_ls_state(ls, need_merge))) { LOG_WARN("failed to check ls state", K(ret), K(ls)); } else if (!need_merge) { // no need to merge, do nothing - } else if (OB_FAIL(MTL(ObLSService *)->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("failed to get ls", K(ret), K(ls_id)); - } else if (OB_FAIL(ls.build_tablet_iter(tablet_iter))) { - LOG_WARN("failed to build ls tablet iter", K(ret), K(ls)); } else if (OB_FAIL(ls.get_ls_meta().get_restore_status(restore_status))) { LOG_WARN("failed to get restore status", K(ret), K(ls)); } else if (OB_UNLIKELY(!restore_status.is_restore_none())) { @@ -970,10 +971,10 @@ int ObTenantTabletScheduler::schedule_ls_medium_merge( } } - while (OB_SUCC(ret)) { // loop all tablet in ls + while (OB_SUCC(ret) && schedule_tablet_cnt < SCHEDULE_TABLET_BATCH_CNT) { // loop all tablet in ls bool tablet_merge_finish = false; - if (OB_FAIL(tablet_iter.get_next_tablet(tablet_handle))) { - if (OB_ITER_END == ret) { + if (OB_FAIL(medium_ls_tablet_iter_.get_next_tablet(ls_handle, tablet_handle))) { + if (OB_ITER_END == ret || OB_LS_NOT_EXIST == ret) { ret = OB_SUCCESS; break; } else { @@ -987,6 +988,8 @@ int ObTenantTabletScheduler::schedule_ls_medium_merge( } else if (tablet_id.is_special_merge_tablet()) { // data tablet // do nothing } else { + ++schedule_tablet_cnt; // inc tablet cnt + ObMediumCompactionScheduleFunc func(ls, tablet_handle); ObITable *latest_major = tablet->get_table_store().get_major_sstables().get_boundary_table(true/*last*/); if (OB_NOT_NULL(latest_major) && latest_major->get_snapshot_version() >= merge_version) { @@ -1052,7 +1055,6 @@ int ObTenantTabletScheduler::schedule_all_tablets_medium() { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; - ObSharedGuard ls_iter_guard; uint64_t compat_version = 0; if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -1067,15 +1069,17 @@ int ObTenantTabletScheduler::schedule_all_tablets_medium() "invalid data version to schedule all tablets medium", K(compat_version), "DATA_VERSION_4_1_0_0", DATA_VERSION_4_1_0_0); } - } else if (OB_FAIL(MTL(ObLSService *)->get_ls_iter(ls_iter_guard, ObLSGetMod::STORAGE_MOD))) { - LOG_WARN("failed to get ls iterator", K(ret)); + } else if (!medium_ls_tablet_iter_.is_valid() && OB_FAIL(medium_ls_tablet_iter_.build_iter())) { + LOG_WARN("failed to init iterator", K(ret)); } else { bool tenant_merge_finish = true; bool all_ls_weak_read_ts_ready = true; bool check_report_scn_flag = false; int64_t merge_version = get_frozen_version(); + ObLSHandle ls_handle; ObLS *ls = nullptr; - LOG_INFO("start schedule all tablet merge", K(merge_version)); + int64_t schedule_tablet_cnt = 0; + LOG_INFO("start schedule all tablet merge", K(merge_version), K(medium_ls_tablet_iter_)); if (INIT_COMPACTION_SCN == merge_version) { merge_version = 0; @@ -1100,19 +1104,21 @@ int ObTenantTabletScheduler::schedule_all_tablets_medium() check_report_scn_flag = true; #endif - while (OB_SUCC(ret)) { + while (OB_SUCC(ret) && schedule_tablet_cnt < SCHEDULE_TABLET_BATCH_CNT) { bool ls_merge_finish = true; - if (OB_FAIL(ls_iter_guard.get_ptr()->get_next(ls))) { + if (OB_FAIL(medium_ls_tablet_iter_.get_next_ls(ls_handle))) { if (OB_ITER_END == ret) { ret = OB_SUCCESS; break; } else { - LOG_WARN("failed to get ls", K(ret), KP(ls_iter_guard.get_ptr())); + LOG_WARN("failed to get ls", K(ret), K(ls_handle)); } - } else if (OB_ISNULL(ls)) { + } else if (OB_ISNULL(ls = ls_handle.get_ls())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ls is null", K(ret), K(ls)); - } else if (OB_TMP_FAIL(schedule_ls_medium_merge(merge_version, *ls, ls_merge_finish, all_ls_weak_read_ts_ready))) { + } else if (OB_TMP_FAIL(schedule_ls_medium_merge( + merge_version, ls_handle, ls_merge_finish, + all_ls_weak_read_ts_ready, schedule_tablet_cnt))) { tenant_merge_finish = false; if (OB_SIZE_OVERFLOW == tmp_ret) { break; @@ -1184,7 +1190,7 @@ int ObTenantTabletScheduler::schedule_all_tablets_medium() } LOG_INFO("finish schedule all tablet merge", K(merge_version), K(schedule_stats_), K(tenant_merge_finish), - K(merged_version_)); + K(merged_version_), K(schedule_tablet_cnt)); schedule_stats_.clear_tablet_cnt(); } return ret; @@ -1252,5 +1258,119 @@ int ObTenantTabletScheduler::update_report_scn_as_ls_leader(ObLS &ls) return ret; } + +// ------------------- ObCompactionScheduleIterator -------------------- // +int ObCompactionScheduleIterator::build_iter() +{ + int ret = OB_SUCCESS; + ls_ids_.reuse(); + if (OB_FAIL(MTL(ObLSService *)->get_ls_ids(ls_ids_))) { + LOG_WARN("failed to get all ls id", K(ret)); + } else { + ls_idx_ = -1; + tablet_idx_ = 0; + tablet_ids_.reuse(); + LOG_TRACE("build iter", K(ret), K(ls_ids_)); + } + return ret; +} + +int ObCompactionScheduleIterator::get_next_ls(ObLSHandle &ls_handle) +{ + int ret = OB_SUCCESS; + if (-1 == ls_idx_ + || tablet_idx_ >= tablet_ids_.count()) { // tablet iter end, need get next ls + ++ls_idx_; + tablet_ids_.reuse(); + } + do { + if (ls_idx_ >= ls_ids_.count()) { + ret = OB_ITER_END; + } else if (OB_FAIL((MTL(storage::ObLSService *)->get_ls(ls_ids_[ls_idx_], ls_handle, mod_)))) { + if (OB_LS_NOT_EXIST == ret) { + LOG_TRACE("ls not exist", K(ret), K(ls_idx_), K(ls_ids_[ls_idx_])); + ++ls_idx_; + tablet_ids_.reuse(); + } else { + LOG_WARN("failed to get ls", K(ret), K(ls_idx_), K(ls_ids_[ls_idx_])); + } + } + } while (OB_LS_NOT_EXIST == ret); + return ret; +} + +void ObCompactionScheduleIterator::reset() +{ + mod_ = ObLSGetMod::INVALID_MOD; + timeout_us_ = 0; + ls_idx_ = 0; + tablet_idx_ = 0; + ls_ids_.reuse(); + tablet_ids_.reuse(); +} + +bool ObCompactionScheduleIterator::is_valid() const +{ + return ls_ids_.count() > 0 && ls_idx_ >= 0 + && (ls_idx_ < ls_ids_.count() - 1 + || (ls_idx_ == ls_ids_.count() - 1 && tablet_idx_ < tablet_ids_.count())); + // have remain ls or have remain tablet +} + +int ObCompactionScheduleIterator::get_next_tablet( + ObLSHandle &ls_handle, + ObTabletHandle &tablet_handle) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!ls_handle.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ls_handle is invalid", K(ret), K(ls_idx_), K(ls_ids_[ls_idx_])); + } else if (tablet_ids_.empty()) { + if (OB_FAIL(ls_handle.get_ls()->get_tablet_svr()->get_all_tablet_ids( + is_major_/*except_ls_inner_tablet*/, tablet_ids_))) { + LOG_WARN("failed to get tablet ids", K(ret)); + } else { + tablet_idx_ = 0; // for new ls, set tablet_idx_ = 0 + LOG_TRACE("build iter", K(ret), K_(ls_idx), "ls_id", ls_ids_[ls_idx_], K(tablet_ids_)); + } + } + if (OB_SUCC(ret)) { + do { + if (tablet_idx_ >= tablet_ids_.count()) { + ret = OB_ITER_END; + } else { + const common::ObTabletID &tablet_id = tablet_ids_.at(tablet_idx_); + if (OB_FAIL(ls_handle.get_ls()->get_tablet_svr()->get_tablet(tablet_id, tablet_handle, timeout_us_))) { + if (OB_TABLET_NOT_EXIST == ret) { + tablet_idx_++; + } else { + LOG_WARN("fail to get tablet", K(ret), K(tablet_idx_), K(tablet_id), K_(timeout_us)); + } + } else { + tablet_handle.set_wash_priority(WashTabletPriority::WTP_LOW); + tablet_idx_++; + } + } + } while (OB_TABLET_NOT_EXIST == ret); + } + return ret; +} +int64_t ObCompactionScheduleIterator::to_string(char *buf, const int64_t buf_len) const +{ + int64_t pos = 0; + J_OBJ_START(); + J_KV(K_(ls_idx), K_(ls_ids), K_(tablet_idx), K(tablet_ids_.count())); + if (is_valid()) { + J_COMMA(); + J_KV("cur_ls", ls_ids_.at(ls_idx_)); + if (!tablet_ids_.empty() && tablet_idx_ < tablet_ids_.count()) { + J_COMMA(); + J_KV("next_tablet", tablet_ids_.at(tablet_idx_)); + } + } + J_OBJ_END(); + return pos; +} + } // namespace storage } // namespace oceanbase diff --git a/src/storage/compaction/ob_tenant_tablet_scheduler.h b/src/storage/compaction/ob_tenant_tablet_scheduler.h index d827fa4e4..986a826ad 100644 --- a/src/storage/compaction/ob_tenant_tablet_scheduler.h +++ b/src/storage/compaction/ob_tenant_tablet_scheduler.h @@ -55,6 +55,43 @@ private: bool enable_fast_freeze_; }; + +// record ls_id/tablet_id +class ObCompactionScheduleIterator +{ +public: + ObCompactionScheduleIterator( + const bool is_major, + ObLSGetMod mod = ObLSGetMod::STORAGE_MOD, + const int64_t timeout_us = ObTabletCommon::DIRECT_GET_COMMITTED_TABLET_TIMEOUT_US) + : mod_(mod), + is_major_(is_major), + timeout_us_(timeout_us), + ls_idx_(0), + tablet_idx_(0), + ls_ids_(), + tablet_ids_() + {} + ~ObCompactionScheduleIterator() { reset(); } + int build_iter(); + int get_next_ls(ObLSHandle &ls_handle); + int get_next_tablet(ObLSHandle &ls_handle, ObTabletHandle &tablet_handle); + void reset(); + bool is_valid() const; + OB_INLINE int64_t to_string(char *buf, const int64_t buf_len) const; +private: + static const int64_t LS_ID_ARRAY_CNT = 10; + static const int64_t TABLET_ID_ARRAY_CNT = 2000; + ObLSGetMod mod_; + bool is_major_; + int64_t timeout_us_; + int64_t ls_idx_; + uint64_t tablet_idx_; + common::ObSEArray ls_ids_; + common::ObSEArray tablet_ids_; +}; + + class ObTenantTabletScheduler { public: @@ -171,11 +208,13 @@ private: int schedule_all_tablets_medium(); int schedule_ls_medium_merge( int64_t &merge_version, - ObLS &ls, + ObLSHandle &ls_handle, bool &ls_merge_finish, - bool &all_ls_weak_read_ts_ready); + bool &all_ls_weak_read_ts_ready, + int64_t &schedule_tablet_cnt); int schedule_ls_minor_merge( - ObLS &ls); + ObLSHandle &ls_handle, + int64_t &schedule_tablet_cnt); int try_remove_old_table(ObLS &ls); int restart_schedule_timer_task( const int64_t interval, @@ -226,10 +265,11 @@ private: static const int64_t DEFAULT_HASH_MAP_BUCKET_CNT = 1009; static const int64_t DEFAULT_COMPACTION_SCHEDULE_INTERVAL = 30 * 1000 * 1000L; // 30s static const int64_t CHECK_WEAK_READ_TS_SCHEDULE_INTERVAL = 10 * 1000 * 1000L; // 10s - static const int64_t CHECK_REPORT_SCN_INTERVAL = 5 * 60 * 1000 * 1000L; // 5m + static const int64_t CHECK_REPORT_SCN_INTERVAL = 5 * 60 * 1000 * 1000L; // 600s static const int64_t ADD_LOOP_EVENT_INTERVAL = 120 * 1000 * 1000L; // 120s static const int64_t WAIT_MEDIUM_CHECK_THRESHOLD = 10 * 60 * 1000 * 1000L; // 10m static const int64_t PRINT_LOG_INVERVAL = 2 * 60 * 1000 * 1000L; // 2m + static const int64_t SCHEDULE_TABLET_BATCH_CNT = 50 * 1000L; // 5w static const int64_t CHECK_LS_LOCALITY_INTERVAL = 5 * 60 * 1000 * 1000L; // 5m private: bool is_inited_; @@ -251,6 +291,8 @@ private: SSTableGCTask sstable_gc_task_; ObFastFreezeChecker fast_freeze_checker_; bool enable_adaptive_compaction_; + ObCompactionScheduleIterator minor_ls_tablet_iter_; + ObCompactionScheduleIterator medium_ls_tablet_iter_; int64_t error_tablet_cnt_; // for diagnose compaction::ObStorageLocalityCache ls_locality_cache_; }; diff --git a/src/storage/ls/ob_ls_tablet_service.cpp b/src/storage/ls/ob_ls_tablet_service.cpp index a43a821fe..077ea8f0f 100644 --- a/src/storage/ls/ob_ls_tablet_service.cpp +++ b/src/storage/ls/ob_ls_tablet_service.cpp @@ -3821,7 +3821,7 @@ int ObLSTabletService::insert_tablet_rows( && OB_FAIL(check_new_row_legitimacy(run_ctx, tbl_row.row_val_))) { LOG_WARN("check new row legitimacy failed", K(ret), K(tbl_row.row_val_)); } else if (OB_FAIL(tablet_handle.get_obj()->insert_row_without_rowkey_check(table, run_ctx.store_ctx_, - *run_ctx.col_descs_, tbl_row))) { + *run_ctx.col_descs_, tbl_row, run_ctx.dml_param_.encrypt_meta_))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret) { LOG_WARN("fail to insert row to data tablet", K(ret), K(tbl_row)); } @@ -4504,7 +4504,7 @@ int ObLSTabletService::process_old_row( del_row.flag_.set_flag(ObDmlFlag::DF_DELETE); if (!is_delete_total_quantity_log) { if (OB_FAIL(tablet_handle.get_obj()->insert_row_without_rowkey_check(relative_table, - run_ctx.store_ctx_, col_descs, del_row))) { + run_ctx.store_ctx_, col_descs, del_row, run_ctx.dml_param_.encrypt_meta_))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { LOG_WARN("failed to write data tablet row", K(ret), K(del_row)); } @@ -4516,7 +4516,7 @@ int ObLSTabletService::process_old_row( del_row.flag_.set_flag(ObDmlFlag::DF_UPDATE); ObSEArray update_idx; if (OB_FAIL(tablet_handle.get_obj()->update_row(relative_table, - run_ctx.store_ctx_, col_descs, update_idx, del_row, new_tbl_row))) { + run_ctx.store_ctx_, col_descs, update_idx, del_row, new_tbl_row, run_ctx.dml_param_.encrypt_meta_))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { LOG_WARN("failed to write data tablet row", K(ret), K(del_row), K(new_tbl_row)); } @@ -4629,14 +4629,14 @@ int ObLSTabletService::process_data_table_row( old_row.flag_.set_flag(ObDmlFlag::DF_UPDATE); old_row.row_val_ = old_tbl_row.row_val_; if (OB_FAIL(data_tablet.get_obj()->update_row(relative_table, - ctx, col_descs, update_idx, old_row, new_row))) { + ctx, col_descs, update_idx, old_row, new_row, run_ctx.dml_param_.encrypt_meta_))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { LOG_WARN("failed to update to row", K(ret), K(old_row), K(new_row)); } } } else { if (OB_FAIL(data_tablet.get_obj()->insert_row_without_rowkey_check(relative_table, - ctx, col_descs, new_row))) { + ctx, col_descs, new_row, run_ctx.dml_param_.encrypt_meta_))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { LOG_WARN("failed to update to row", K(ret), K(new_row)); } @@ -5055,7 +5055,8 @@ int ObLSTabletService::insert_row_to_tablet( relative_table, store_ctx, col_descs, - tbl_row))) { + tbl_row, + run_ctx.dml_param_.encrypt_meta_))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret) { LOG_WARN("failed to write table row", K(ret), "table id", relative_table.get_table_id(), @@ -5220,7 +5221,7 @@ int ObLSTabletService::delete_row_in_tablet( LOG_WARN("failed to process old row lob col", K(ret), K(tbl_row)); } else if (!dml_param.is_total_quantity_log_) { if (OB_FAIL(tablet_handle.get_obj()->insert_row_without_rowkey_check(relative_table, - ctx, *run_ctx.col_descs_, tbl_row))) { + ctx, *run_ctx.col_descs_, tbl_row, dml_param.encrypt_meta_))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { LOG_WARN("failed to set row", K(ret), K(*run_ctx.col_descs_), K(tbl_row)); } @@ -5234,7 +5235,7 @@ int ObLSTabletService::delete_row_in_tablet( new_tbl_row.row_val_ = tbl_row.row_val_; tbl_row.flag_.set_flag(ObDmlFlag::DF_UPDATE); if (OB_FAIL(tablet_handle.get_obj()->update_row(relative_table, ctx, - *run_ctx.col_descs_, update_idx, tbl_row, new_tbl_row))) { + *run_ctx.col_descs_, update_idx, tbl_row, new_tbl_row, dml_param.encrypt_meta_))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) { LOG_WARN("failed to set row", K(ret), K(*run_ctx.col_descs_), K(tbl_row), K(new_tbl_row)); } diff --git a/src/storage/memtable/mvcc/ob_mvcc.h b/src/storage/memtable/mvcc/ob_mvcc.h index 7649beac1..ee33570e0 100644 --- a/src/storage/memtable/mvcc/ob_mvcc.h +++ b/src/storage/memtable/mvcc/ob_mvcc.h @@ -156,6 +156,8 @@ protected: bool need_submit_log_ : 1; // Identifies whether log has been submitted }; share::SCN scn_; +public: + int64_t owner_; private: ObITransCallback *prev_; ObITransCallback *next_; diff --git a/src/storage/memtable/mvcc/ob_mvcc_iterator.cpp b/src/storage/memtable/mvcc/ob_mvcc_iterator.cpp index cfeb7a70d..1f13195ba 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_iterator.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_iterator.cpp @@ -37,7 +37,6 @@ int ObMvccValueIterator::init(ObMvccAccessCtx &ctx, { int ret = OB_SUCCESS; skip_compact_ = skip_compact; - bool enable_sql_audit = GCONF.enable_sql_audit;; reset(); int64_t lock_for_read_start = ObClockGenerator::getClock(); ctx_ = &ctx; @@ -59,11 +58,6 @@ int ObMvccValueIterator::init(ObMvccAccessCtx &ctx, } } - if (OB_SUCC(ret) && enable_sql_audit) { - // stat lock for read time : FIXME: (yunxing.cyx) feedback via .. - // const_cast(ctx).add_lock_for_read_elapse( - // ObClockGenerator::getClock() - lock_for_read_start); - } TRANS_LOG(TRACE, "value_iter.init", K(ret), KPC(value), KPC_(version_iter), @@ -78,12 +72,10 @@ int ObMvccValueIterator::init(ObMvccAccessCtx &ctx, int ObMvccValueIterator::lock_for_read_(const ObQueryFlag &flag) { int ret = OB_SUCCESS; - int64_t lock_start_time = OB_INVALID_TIMESTAMP; // the head of the read position ObMvccTransNode *iter = value_->get_list_head(); // the resolved mvcc read position version_iter_ = NULL; - lock_begin(lock_start_time); while (OB_SUCC(ret) && NULL != iter && NULL == version_iter_) { if (OB_FAIL(lock_for_read_inner_(flag, iter))) { @@ -103,31 +95,9 @@ int ObMvccValueIterator::lock_for_read_(const ObQueryFlag &flag) } } - lock_for_read_end(lock_start_time, ret); return ret; } -void ObMvccValueIterator::lock_begin(int64_t &lock_start_time) const -{ - if (GCONF.enable_sql_audit) { - lock_start_time = OB_TSC_TIMESTAMP.current_time(); - } -} - -void ObMvccValueIterator::lock_for_read_end(const int64_t lock_start_time, int64_t ret) const -{ - // TODO: Add ELR check back - if (GCONF.enable_sql_audit && OB_INVALID_TIMESTAMP != lock_start_time) { - const int64_t lock_use_time = OB_TSC_TIMESTAMP.current_time() - lock_start_time; - EVENT_ADD(MEMSTORE_WAIT_READ_LOCK_TIME, lock_use_time); - if (OB_FAIL(ret)) { - EVENT_INC(MEMSTORE_READ_LOCK_FAIL_COUNT); - } else { - EVENT_INC(MEMSTORE_READ_LOCK_SUCC_COUNT); - } - } -} - int ObMvccValueIterator::lock_for_read_inner_(const ObQueryFlag &flag, ObMvccTransNode *&iter) { diff --git a/src/storage/memtable/mvcc/ob_mvcc_iterator.h b/src/storage/memtable/mvcc/ob_mvcc_iterator.h index b620dfb7a..64ef48b15 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_iterator.h +++ b/src/storage/memtable/mvcc/ob_mvcc_iterator.h @@ -144,8 +144,6 @@ private: int lock_for_read_inner_(const ObQueryFlag &flag, ObMvccTransNode *&iter); int try_cleanout_tx_node_(ObMvccTransNode *tnode); void move_to_next_node_(); - void lock_begin(int64_t &lock_start_time) const; - void lock_for_read_end(const int64_t lock_start_time, int64_t ret) const; private: static const int64_t WAIT_COMMIT_US = 20 * 1000; private: diff --git a/src/storage/memtable/mvcc/ob_mvcc_row.cpp b/src/storage/memtable/mvcc/ob_mvcc_row.cpp index 088e3ec6d..78e58cc81 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_row.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_row.cpp @@ -689,30 +689,6 @@ int ObMvccRow::elr(const ObTransID &tx_id, return ret; } -void ObMvccRow::lock_begin(ObIMemtableCtx &ctx) const -{ - if (GCONF.enable_sql_audit) { - ctx.set_lock_start_time(OB_TSC_TIMESTAMP.current_time()); - } -} - -void ObMvccRow::mvcc_write_end(ObIMemtableCtx &ctx, int64_t ret) const -{ - if (GCONF.enable_sql_audit) { - const int64_t lock_use_time = OB_TSC_TIMESTAMP.current_time() - ctx.get_lock_start_time(); - EVENT_ADD(MEMSTORE_WAIT_WRITE_LOCK_TIME, lock_use_time); - if (OB_FAIL(ret)) { - EVENT_INC(MEMSTORE_WRITE_LOCK_FAIL_COUNT); - } else { - EVENT_INC(MEMSTORE_WRITE_LOCK_SUCC_COUNT); - } - if (lock_use_time >= WARN_TIME_US && TC_REACH_TIME_INTERVAL(LOG_INTERVAL)) { - TRANS_LOG_RET(WARN, OB_ERR_TOO_MUCH_TIME, "wait mvcc write use too much time", - K(ctx), K(ret), K(lock_use_time)); - } - } -} - SCN ObMvccRow::get_max_trans_version() const { const SCN max_elr_commit_version = max_elr_trans_version_.atomic_get(); @@ -1020,7 +996,6 @@ int ObMvccRow::mvcc_write(ObIMemtableCtx &ctx, { int ret = OB_SUCCESS; const SCN snapshot_version = snapshot.version_; - lock_begin(ctx); if (max_trans_version_.atomic_load() > snapshot_version || max_elr_trans_version_.atomic_load() > snapshot_version) { @@ -1052,7 +1027,6 @@ int ObMvccRow::mvcc_write(ObIMemtableCtx &ctx, } } - mvcc_write_end(ctx, ret); return ret; } diff --git a/src/storage/memtable/mvcc/ob_mvcc_row.h b/src/storage/memtable/mvcc/ob_mvcc_row.h index 3b94e4078..a2484b209 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_row.h +++ b/src/storage/memtable/mvcc/ob_mvcc_row.h @@ -370,9 +370,6 @@ struct ObMvccRow const transaction::ObTransID &tx_id); int64_t get_total_trans_node_cnt() const { return total_trans_node_cnt_; } int64_t get_last_compact_cnt() const { return last_compact_cnt_; } - // ===================== ObMvccRow Event Statistic ===================== - void lock_begin(ObIMemtableCtx &ctx) const; - void mvcc_write_end(ObIMemtableCtx &ctx, int64_t ret) const; // ===================== ObMvccRow Flag Interface ===================== OB_INLINE bool is_btree_indexed() const { diff --git a/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp b/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp index 82e245020..c4946e97b 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp @@ -212,10 +212,19 @@ void ObTransCallbackMgr::reset() } } } + if (NULL != cb_allocators_) { + for (int i = 0; i < MAX_CB_ALLOCATOR_COUNT; ++i) { + cb_allocators_[i].reset(); + } + } if (OB_NOT_NULL(callback_lists_)) { cb_allocator_.free(callback_lists_); callback_lists_ = NULL; } + if (OB_NOT_NULL(cb_allocators_)) { + cb_allocator_.free(cb_allocators_); + cb_allocators_ = NULL; + } parallel_stat_ = 0; callback_main_list_append_count_ = 0; callback_slave_list_append_count_ = 0; @@ -228,6 +237,77 @@ void ObTransCallbackMgr::reset() flushed_log_size_ = 0; } +void ObTransCallbackMgr::callback_free(ObITransCallback *cb) +{ + int64_t owner = cb->owner_; + if (-1 == owner) { + TRANS_LOG_RET(WARN, OB_ERR_UNEXPECTED, "callback free failed", KPC(cb)); + } else if (0 == owner) { + cb_allocator_.free(cb); + } else if (0 < owner) { + cb_allocators_[owner - 1].free(cb); + } else { + TRANS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "unexpected cb", KPC(cb)); + ob_abort(); + } +} + +void *ObTransCallbackMgr::callback_alloc(const int64_t size) +{ + int ret = OB_SUCCESS; + ObITransCallback *callback = nullptr; + const int64_t tid = get_itid() + 1; + const int64_t slot = tid % MAX_CB_ALLOCATOR_COUNT; + int64_t stat = ATOMIC_LOAD(¶llel_stat_); + + if (PARALLEL_STMT == stat) { + if (NULL == cb_allocators_) { + WRLockGuard guard(rwlock_); + if (NULL == cb_allocators_) { + ObMemtableCtxCbAllocator *tmp_cb_allocators = nullptr; + if (NULL == (tmp_cb_allocators = (ObMemtableCtxCbAllocator *)cb_allocator_.alloc( + sizeof(ObMemtableCtxCbAllocator) * MAX_CB_ALLOCATOR_COUNT))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + TRANS_LOG(WARN, "alloc cb allocator fail", K(ret)); + } else { + for (int i = 0; OB_SUCC(ret) && i < MAX_CB_ALLOCATOR_COUNT; ++i) { + UNUSED(new(tmp_cb_allocators + i) ObMemtableCtxCbAllocator()); + if (OB_FAIL(tmp_cb_allocators[i].init(MTL_ID()))) { + TRANS_LOG(ERROR, "cb_allocator_ init error", K(ret)); + } + } + if (OB_SUCC(ret)) { + cb_allocators_ = tmp_cb_allocators; + } + } + } + } + + if (OB_SUCC(ret)) { + if (NULL == cb_allocators_) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "cb allocators is not inited", K(ret)); + } else { + callback = (ObITransCallback *)(cb_allocators_[slot].alloc(size)); + if (nullptr != callback) { + callback->owner_ = slot + 1; + } + } + } + } else { + callback = (ObITransCallback *)(cb_allocator_.alloc(size)); + if (nullptr != callback) { + callback->owner_ = 0; + } + } + + if (OB_FAIL(ret)) { + callback = nullptr; + } + + return callback; +} + int ObTransCallbackMgr::append(ObITransCallback *node) { int ret = OB_SUCCESS; diff --git a/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.h b/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.h index 3fda9791a..c8648d9e4 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.h +++ b/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.h @@ -172,6 +172,7 @@ public: friend class ObITransCallbackIterator; enum { MAX_CALLBACK_LIST_COUNT = OB_MAX_CPU_NUM }; + enum { MAX_CB_ALLOCATOR_COUNT = OB_MAX_CPU_NUM }; enum { PARALLEL_STMT = -1 }; @@ -192,12 +193,15 @@ public: callback_remove_for_rollback_to_count_(0), pending_log_size_(0), flushed_log_size_(0), - cb_allocator_(cb_allocator) + cb_allocator_(cb_allocator), + cb_allocators_(NULL) { } ~ObTransCallbackMgr() {} void reset(); ObIMvccCtx &get_ctx() { return host_; } + void *callback_alloc(const int64_t size); + void callback_free(ObITransCallback *cb); int append(ObITransCallback *node); void before_append(ObITransCallback *node); void after_append(ObITransCallback *node, const int ret_code); @@ -323,6 +327,7 @@ private: // current flushed log size in leader participant int64_t flushed_log_size_; ObMemtableCtxCbAllocator &cb_allocator_; + ObMemtableCtxCbAllocator *cb_allocators_; }; //class ObIMvccCtx; diff --git a/src/storage/memtable/ob_lock_wait_mgr.cpp b/src/storage/memtable/ob_lock_wait_mgr.cpp index 526e86234..8388e9e95 100644 --- a/src/storage/memtable/ob_lock_wait_mgr.cpp +++ b/src/storage/memtable/ob_lock_wait_mgr.cpp @@ -192,7 +192,7 @@ void ObLockWaitMgr::run1() row_holder_mapper_.clear(); } } - ob_usleep(10000); + ob_usleep(500000); } } diff --git a/src/storage/memtable/ob_memtable.cpp b/src/storage/memtable/ob_memtable.cpp index c6dd65794..3e64e903b 100644 --- a/src/storage/memtable/ob_memtable.cpp +++ b/src/storage/memtable/ob_memtable.cpp @@ -127,7 +127,9 @@ ObMemtable::ObMemtable() minor_merged_time_(0), contain_hotspot_row_(false), multi_source_data_(local_allocator_), - multi_source_data_lock_() + multi_source_data_lock_(), + encrypt_meta_(nullptr), + encrypt_meta_lock_(ObLatchIds::DEFAULT_SPIN_RWLOCK) { mt_stat_.reset(); migration_clog_checkpoint_scn_.set_min(); @@ -273,6 +275,7 @@ void ObMemtable::destroy() is_inited_ = false; contain_hotspot_row_ = false; snapshot_version_.set_max(); + encrypt_meta_ = nullptr; } int ObMemtable::safe_to_destroy(bool &is_safe) @@ -318,7 +321,8 @@ int ObMemtable::set( const uint64_t table_id, const storage::ObTableReadInfo &read_info, const common::ObIArray &columns, - const storage::ObStoreRow &row) + const storage::ObStoreRow &row, + const share::ObEncryptMeta *encrypt_meta) { int ret = OB_SUCCESS; ObMvccWriteGuard guard; @@ -331,6 +335,9 @@ int ObMemtable::set( TRANS_LOG(WARN, "invalid param", K(ctx), K(read_info), K(columns.count()), K(row.row_val_.count_)); ret = OB_INVALID_ARGUMENT; + } + + if (OB_FAIL(ret)) { } else if (OB_FAIL(guard.write_auth(ctx))) { TRANS_LOG(WARN, "not allow to write", K(ctx)); } else { @@ -355,7 +362,8 @@ int ObMemtable::set( const ObIArray &columns, const ObIArray &update_idx, const storage::ObStoreRow &old_row, - const storage::ObStoreRow &new_row) + const storage::ObStoreRow &new_row, + const share::ObEncryptMeta *encrypt_meta) { int ret = OB_SUCCESS; ObMvccWriteGuard guard; @@ -366,6 +374,9 @@ int ObMemtable::set( || read_info.get_schema_rowkey_count() > columns.count()) { ret = OB_INVALID_ARGUMENT; TRANS_LOG(ERROR, "invalid param", K(ret), K(ctx), K(read_info)); + } + + if (OB_FAIL(ret)){ } else if (OB_FAIL(guard.write_auth(ctx))) { TRANS_LOG(WARN, "not allow to write", K(ctx)); } else { @@ -1097,8 +1108,7 @@ int ObMemtable::replay_schema_version_change_log(const int64_t schema_version) } int ObMemtable::replay_row(ObStoreCtx &ctx, - ObMemtableMutatorIterator *mmi, - ObEncryptRowBuf &decrypt_buf) + ObMemtableMutatorIterator *mmi) { int ret = OB_SUCCESS; @@ -2383,6 +2393,7 @@ bool ObMemtable::has_multi_source_data_unit(const MultiSourceDataUnitType type) return multi_source_data_.is_valid() && multi_source_data_.has_multi_source_data_unit(type); } + ObMemtableStat::ObMemtableStat() : lock_(common::ObLatchIds::MEMTABLE_STAT_LOCK), memtables_() diff --git a/src/storage/memtable/ob_memtable.h b/src/storage/memtable/ob_memtable.h index ca0fc573a..f45680084 100644 --- a/src/storage/memtable/ob_memtable.h +++ b/src/storage/memtable/ob_memtable.h @@ -193,7 +193,8 @@ public: const uint64_t table_id, const storage::ObTableReadInfo &read_info, const common::ObIArray &columns, // TODO: remove columns - const storage::ObStoreRow &row); + const storage::ObStoreRow &row, + const share::ObEncryptMeta *encrypt_meta); virtual int set( storage::ObStoreCtx &ctx, const uint64_t table_id, @@ -201,7 +202,8 @@ public: const common::ObIArray &columns, // TODO: remove columns const ObIArray &update_idx, const storage::ObStoreRow &old_row, - const storage::ObStoreRow &new_row); + const storage::ObStoreRow &new_row, + const share::ObEncryptMeta *encrypt_meta); // lock is used to lock the row(s) // ctx is the locker tx's context, we need the tx_id, version and scn to do the concurrent control(mvcc_write) @@ -291,8 +293,7 @@ public: // decrypt_buf is used for decryption virtual int replay_row( storage::ObStoreCtx &ctx, - ObMemtableMutatorIterator *mmi, - ObEncryptRowBuf &decrypt_buf); + ObMemtableMutatorIterator *mmi); virtual int replay_schema_version_change_log( const int64_t schema_version); @@ -466,6 +467,8 @@ public: const bool for_replay, const MemtableRefOp ref_op = MemtableRefOp::NONE, const bool is_callback = false); + + // Print stat data in log. // For memtable debug. int print_stat() const; @@ -581,6 +584,8 @@ private: bool contain_hotspot_row_; ObMultiSourceData multi_source_data_; mutable common::TCRWLock multi_source_data_lock_; + transaction::ObTxEncryptMeta *encrypt_meta_; + common::SpinRWLock encrypt_meta_lock_; }; template diff --git a/src/storage/memtable/ob_memtable_context.cpp b/src/storage/memtable/ob_memtable_context.cpp index a7150ea0a..bd0556e23 100644 --- a/src/storage/memtable/ob_memtable_context.cpp +++ b/src/storage/memtable/ob_memtable_context.cpp @@ -139,7 +139,6 @@ void ObMemtableCtx::reset() truncate_cnt_ = 0; unsynced_cnt_ = 0; unsubmitted_cnt_ = 0; - partition_audit_info_cache_.reset(); lock_mem_ctx_.reset(); //FIXME: ctx_ is not reset log_conflict_interval_.reset(); @@ -344,7 +343,7 @@ void ObMemtableCtx::old_row_free(void *row) void *ObMemtableCtx::callback_alloc(const int64_t size) { void* ret = NULL; - if (OB_ISNULL(ret = ctx_cb_allocator_.alloc(size))) { + if (OB_ISNULL(ret = trans_mgr_.callback_alloc(size))) { TRANS_LOG_RET(ERROR, OB_ALLOCATE_MEMORY_FAILED, "callback alloc error, no memory", K(size), K(*this)); } else { ATOMIC_FAA(&callback_mem_used_, size); @@ -363,7 +362,7 @@ void ObMemtableCtx::callback_free(ObITransCallback *cb) } else { ATOMIC_INC(&callback_free_count_); TRANS_LOG(DEBUG, "callback release succ", KP(cb), K(*this), K(lbt())); - ctx_cb_allocator_.free(cb); + trans_mgr_.callback_free(cb); cb = NULL; } } @@ -507,12 +506,6 @@ int ObMemtableCtx::do_trans_end( } else if (OB_FAIL(clear_table_lock_(commit, trans_version, final_scn))) { TRANS_LOG(ERROR, "clear table lock failed.", K(ret), K(*this)); } - (void)partition_audit_info_cache_.stmt_end_update_audit_info(commit); - // flush partition audit statistics cached in ctx to partition - if (NULL != ATOMIC_LOAD(&ctx_) && - OB_UNLIKELY(OB_SUCCESS != (tmp_ret = flush_audit_partition_cache_(commit)))) { - TRANS_LOG(WARN, "flush audit partition cache error", K(tmp_ret), K(commit), K(*ctx_)); - } } return ret; } @@ -782,50 +775,6 @@ int ObMemtableCtx::add_conflict_trans_id(const ObTransID conflict_trans_id) return ret; } -int ObMemtableCtx::audit_partition(const enum ObPartitionAuditOperator op, - const int64_t count) -{ - return audit_partition_cache_(op, (int32_t)count); -} - -// cache partition audit info in current context -int ObMemtableCtx::audit_partition_cache_(const enum ObPartitionAuditOperator op, - const int32_t count) -{ - int ret = OB_SUCCESS; - - if (!GCONF.enable_sql_audit) { - // do nothing - } else if (OB_FAIL(partition_audit_info_cache_.update_audit_info(op, count))) { - TRANS_LOG(WARN, "update audit info", K(ret), K(*ctx_)); - } - - return ret; -} - -// flush partition audit info into partition -int ObMemtableCtx::flush_audit_partition_cache_(bool commit) -{ - int ret = OB_SUCCESS; - //ObLSTxCtxMgr *partition_mgr = NULL; - - //if (!GCONF.enable_sql_audit) { - // // do nothing - //} else if (OB_ISNULL(ctx_)) { - // ret = OB_ERR_UNEXPECTED; - // TRANS_LOG(WARN, "memtable ctx is NULL", K(ret), KP(ctx_)); - //} else if (OB_ISNULL(partition_mgr = ctx_->get_ls_tx_ctx_mgr())) { - // // ret = OB_ERR_UNEXPECTED; - // // TRANS_LOG(WARN, "partition mgr is NULL", K(ret), K(*ctx_)); - //} else if (OB_FAIL(partition_mgr->audit_partition(partition_audit_info_cache_, commit))) { - // TRANS_LOG(WARN, "partition mgr audit partition error", K(ret), K(*ctx_)); - //} else { - // //do nothing - //} - - return ret; -} - void ObMemtableCtx::inc_lock_for_read_retry_count() { lock_for_read_retry_count_++; diff --git a/src/storage/memtable/ob_memtable_context.h b/src/storage/memtable/ob_memtable_context.h index 9816e05d2..810ea2c5c 100644 --- a/src/storage/memtable/ob_memtable_context.h +++ b/src/storage/memtable/ob_memtable_context.h @@ -390,8 +390,6 @@ public: virtual void set_trans_ctx(transaction::ObPartTransCtx *ctx); virtual transaction::ObPartTransCtx *get_trans_ctx() const { return ctx_; } virtual void inc_truncate_cnt() override { truncate_cnt_++; } - virtual int audit_partition(const enum transaction::ObPartitionAuditOperator op, - const int64_t count); int get_memtable_key_arr(transaction::ObMemtableKeyArray &memtable_key_arr); uint64_t get_lock_for_read_retry_count() const { return lock_for_read_retry_count_; } virtual void add_trans_mem_total_size(const int64_t size); @@ -491,8 +489,6 @@ private: const transaction::tablelock::ObTableLockOp &lock_op, const bool is_replay); static int64_t get_us() { return ::oceanbase::common::ObTimeUtility::current_time(); } - int audit_partition_cache_(const enum transaction::ObPartitionAuditOperator op, const int32_t count); - int flush_audit_partition_cache_(bool commit); void set_read_elr_data(const bool read_elr_data) { read_elr_data_ = read_elr_data; } int reset_log_generator_(); int reuse_log_generator_(); @@ -520,7 +516,6 @@ private: MemtableCtxStat mtstat_; ObTimeInterval log_conflict_interval_; transaction::ObPartTransCtx *ctx_; - transaction::ObPartitionAuditInfoCache partition_audit_info_cache_; int64_t truncate_cnt_; // the retry count of lock for read uint64_t lock_for_read_retry_count_; diff --git a/src/storage/memtable/ob_memtable_interface.h b/src/storage/memtable/ob_memtable_interface.h index 88ff58869..af506d824 100644 --- a/src/storage/memtable/ob_memtable_interface.h +++ b/src/storage/memtable/ob_memtable_interface.h @@ -89,8 +89,6 @@ public: int64_t &buf_pos, ObRedoLogSubmitHelper &helper, const bool log_for_lock_node = true) = 0; - virtual int audit_partition(const enum transaction::ObPartitionAuditOperator op, - const int64_t count) = 0; common::ActiveResource resource_link_; }; @@ -167,7 +165,8 @@ public: const uint64_t table_id, const storage::ObTableReadInfo &read_info, const common::ObIArray &columns, // TODO: remove columns - const storage::ObStoreRow &row) = 0; + const storage::ObStoreRow &row, + const share::ObEncryptMeta *encrypt_meta) = 0; // // Lock rows // diff --git a/src/storage/memtable/ob_memtable_mutator.cpp b/src/storage/memtable/ob_memtable_mutator.cpp index c66d65118..20ca5c628 100644 --- a/src/storage/memtable/ob_memtable_mutator.cpp +++ b/src/storage/memtable/ob_memtable_mutator.cpp @@ -394,41 +394,48 @@ int ObMemtableMutatorRow::copy(uint64_t &table_id, } -int ObMemtableMutatorRow::serialize(char *buf, const int64_t buf_len, int64_t &pos, - const transaction::ObCLogEncryptInfo &encrypt_info, +int ObMemtableMutatorRow::serialize(char *buf, int64_t &buf_len, int64_t &pos, + const transaction::ObTxEncryptMeta *encrypt_meta, + transaction::ObCLogEncryptInfo &encrypt_info, const bool is_big_row) { int ret = OB_SUCCESS; - int64_t new_pos = pos + encoded_length_i32(0); - int64_t data_pos = new_pos + encoded_length_vi64(table_id_); - share::ObEncryptMeta meta; + transaction::ObTxEncryptMeta *old_meta = NULL; bool need_encrypt = false; + bool use_old = false; UNUSED(is_big_row); + if (OB_ISNULL(buf) || pos < 0 || pos > buf_len) { ret = OB_INVALID_ARGUMENT; - } else if (OB_FAIL(encode_vi64(buf, buf_len, new_pos, table_id_))) { - TRANS_LOG(WARN, "serialize table id failed", K(ret), KP(buf), K(buf_len), K(pos)); - } else if (OB_FAIL(rowkey_.serialize(buf, buf_len, new_pos)) - || OB_FAIL(encode_vi64(buf, buf_len, new_pos, table_version_)) - || OB_FAIL(encode_i8(buf, buf_len, new_pos, dml_flag_)) - || OB_FAIL(encode_vi32(buf, buf_len, new_pos, update_seq_)) - || OB_FAIL(new_row_.serialize(buf, buf_len, new_pos)) - || OB_FAIL(old_row_.serialize(buf, buf_len, new_pos)) - || OB_FAIL(encode_vi32(buf, buf_len, new_pos, acc_checksum_)) - || OB_FAIL(encode_vi64(buf, buf_len, new_pos, version_)) - || OB_FAIL(encode_vi32(buf, buf_len, new_pos, flag_)) - || OB_FAIL(encode_vi64(buf, buf_len, new_pos, seq_no_))) { - if (OB_BUF_NOT_ENOUGH != ret || buf_len > common::OB_MAX_LOG_ALLOWED_SIZE) { - TRANS_LOG(INFO, "serialize row fail", K(ret), KP(buf), K(buf_len), K(pos)); - } } if (OB_SUCC(ret)) { - row_size_ = (uint32_t)(new_pos - pos); - if (OB_FAIL(encode_i32(buf, buf_len, pos, row_size_))) { - TRANS_LOG(WARN, "serialize row fail", K(ret), K(buf_len), K(pos), K(table_id_)); - } else { - pos = new_pos; + int64_t data_pos; + int64_t new_pos = pos + encoded_length_i32(0); + if (OB_FAIL(encode_vi64(buf, buf_len, new_pos, table_id_))) { + TRANS_LOG(WARN, "serialize table id failed", K(ret), KP(buf), K(buf_len), K(pos)); + } else if (FALSE_IT(data_pos = new_pos)) { + } else if (OB_FAIL(rowkey_.serialize(buf, buf_len, new_pos)) + || OB_FAIL(encode_vi64(buf, buf_len, new_pos, table_version_)) + || OB_FAIL(encode_i8(buf, buf_len, new_pos, dml_flag_)) + || OB_FAIL(encode_vi32(buf, buf_len, new_pos, update_seq_)) + || OB_FAIL(new_row_.serialize(buf, buf_len, new_pos)) + || OB_FAIL(old_row_.serialize(buf, buf_len, new_pos)) + || OB_FAIL(encode_vi32(buf, buf_len, new_pos, acc_checksum_)) + || OB_FAIL(encode_vi64(buf, buf_len, new_pos, version_)) + || OB_FAIL(encode_vi32(buf, buf_len, new_pos, flag_)) + || OB_FAIL(encode_vi64(buf, buf_len, new_pos, seq_no_))) { + if (OB_BUF_NOT_ENOUGH != ret || buf_len > common::OB_MAX_LOG_ALLOWED_SIZE) { + TRANS_LOG(INFO, "serialize row fail", K(ret), KP(buf), K(buf_len), K(pos)); + } + } + if (OB_SUCC(ret)) { + row_size_ = (uint32_t)(new_pos - pos); + if (OB_FAIL(encode_i32(buf, buf_len, pos, row_size_))) { + TRANS_LOG(WARN, "serialize row size fail", K(ret), K(buf_len), K(pos), K(table_id_)); + } else { + pos = new_pos; + } } } return ret; @@ -460,22 +467,10 @@ int ObMemtableMutatorRow::deserialize(const char *buf, const int64_t buf_len, in TRANS_LOG(WARN, "deserialize table id failed", K(ret), K(buf_len), K(new_pos)); } else { int64_t data_pos = new_pos; - share::ObEncryptMeta meta; - bool need_decrypt = false; - if (OB_FAIL(encrypt_info.get_encrypt_info(table_id_, need_decrypt, meta))) { - TRANS_LOG(ERROR, "failed to get encrypt info", K(ret), K(table_id_)); - } else if (need_decrypt) { - ret = OB_NOT_IMPLEMENT; - } else { - //no encryption - decrypted_buf = buf + data_pos; - decrypted_len = encrypted_len - (data_pos - pos); - row_size_ = encrypted_len; - new_pos = 0; - if (need_extract_encrypt_meta) { - encrypt_stat_map.set_map(CLOG_CONTAIN_NON_ENCRYPTED_ROW); - } - } + decrypted_buf = buf + data_pos; + decrypted_len = encrypted_len - (data_pos - pos); + row_size_ = encrypted_len; + new_pos = 0; if (OB_SUCC(ret)) { if (NULL == decrypted_buf) { ret = OB_ERR_UNEXPECTED; @@ -830,6 +825,7 @@ int ObMutatorWriter::set_buffer(char *buf, const int64_t buf_len) TRANS_LOG(WARN, "set_data fail", KP(buf), K(buf_len)); } else { buf_.get_position() = meta_size; + row_capacity_ = buf_len; } return ret; } @@ -837,7 +833,8 @@ int ObMutatorWriter::set_buffer(char *buf, const int64_t buf_len) int ObMutatorWriter::append_row_kv( const int64_t table_version, const RedoDataNode &redo, - const transaction::ObCLogEncryptInfo &encrypt_info, + const transaction::ObTxEncryptMeta *encrypt_meta, + transaction::ObCLogEncryptInfo &encrypt_info, const bool is_big_row) { int ret = OB_SUCCESS; @@ -866,6 +863,8 @@ int ObMutatorWriter::append_row_kv( ObMutatorRowHeader row_header; row_header.mutator_type_ = MutatorType::MUTATOR_ROW; row_header.tablet_id_ = redo.tablet_id_; + //TODO: table_id is just used as encrypt_index, + // if table_id is no longer used, we may rename it in the future ObMemtableMutatorRow row(table_id, rowkey, table_version, @@ -878,19 +877,20 @@ int ObMutatorWriter::append_row_kv( redo.flag_, redo.seq_no_); int64_t tmp_pos = buf_.get_position(); + int64_t row_capacity = row_capacity_; if (OB_ISNULL(buf_.get_data())) { ret = OB_NOT_INIT; TRANS_LOG(WARN, "not init", K(ret)); } else if (is_with_head && - OB_FAIL(row_header.serialize(buf_.get_data(), buf_.get_capacity(), tmp_pos))) { + OB_FAIL(row_header.serialize(buf_.get_data(), row_capacity, tmp_pos))) { if (ret == OB_ALLOCATE_MEMORY_FAILED) { //do nothing } else { ret = OB_BUF_NOT_ENOUGH; } - } else if (OB_FAIL(row.serialize(buf_.get_data(), buf_.get_capacity(), - tmp_pos, encrypt_info, is_big_row))) { + } else if (OB_FAIL(row.serialize(buf_.get_data(), row_capacity, tmp_pos, + encrypt_meta, encrypt_info, is_big_row))) { if (ret == OB_ALLOCATE_MEMORY_FAILED) { //do nothing } else { @@ -900,6 +900,7 @@ int ObMutatorWriter::append_row_kv( TRANS_LOG(WARN, "meta inc_row_count failed", K(ret)); } else { buf_.get_position() = tmp_pos; + row_capacity_ = row_capacity; } } if (OB_SUCCESS != ret && OB_BUF_NOT_ENOUGH != ret) { @@ -910,13 +911,15 @@ int ObMutatorWriter::append_row_kv( int ObMutatorWriter::append_row( ObMemtableMutatorRow &row, - const transaction::ObCLogEncryptInfo &encrypt_info, + transaction::ObCLogEncryptInfo &encrypt_info, const bool is_big_row, const bool is_with_head) { int ret = OB_SUCCESS; ObMutatorRowHeader row_header; row_header.mutator_type_ = MutatorType::MUTATOR_ROW; + const transaction::ObTxEncryptMeta *encrypt_meta = NULL; + int64_t buf_len; //TODO replace pkey with tablet_id for clog_encrypt_info //row_header.pkey_ = pkey; if (OB_ISNULL(buf_.get_data())) { @@ -931,8 +934,9 @@ int ObMutatorWriter::append_row( } else { ret = OB_BUF_NOT_ENOUGH; } - } else if (OB_FAIL(row.serialize(buf_.get_data(), buf_.get_capacity(), - tmp_pos, encrypt_info, is_big_row))) { + } else if (FALSE_IT(buf_len = buf_.get_capacity())) { + } else if (OB_FAIL(row.serialize(buf_.get_data(), buf_len, + tmp_pos, encrypt_meta, encrypt_info, is_big_row))) { if (ret == OB_ALLOCATE_MEMORY_FAILED) { //do nothing } else { @@ -1033,11 +1037,13 @@ int ObMutatorWriter::append_table_lock_kv( return ret; } -int ObMutatorWriter::serialize(const uint8_t row_flag, int64_t &res_len) +int ObMutatorWriter::serialize(const uint8_t row_flag, int64_t &res_len, + transaction::ObCLogEncryptInfo &encrypt_info) { int ret = OB_SUCCESS; const int64_t meta_size = meta_.get_serialize_size(); int64_t meta_pos = 0; + int64_t end_pos = buf_.get_position(); if (OB_ISNULL(buf_.get_data())) { ret = OB_NOT_INIT; TRANS_LOG(WARN, "not init", K(ret)); @@ -1053,6 +1059,7 @@ int ObMutatorWriter::serialize(const uint8_t row_flag, int64_t &res_len) buf_.get_position() - meta_size))) { } else if (OB_FAIL(meta_.serialize(buf_.get_data(), meta_size, meta_pos))) { } else { + buf_.get_position() = end_pos; res_len = buf_.get_position(); } if (OB_FAIL(ret) && OB_ENTRY_NOT_EXIST != ret) { @@ -1094,12 +1101,11 @@ void ObMemtableMutatorIterator::reset() } int ObMemtableMutatorIterator::deserialize(const char *buf, const int64_t data_len, int64_t &pos, - const transaction::ObCLogEncryptInfo &encrypt_info) + transaction::ObCLogEncryptInfo &encrypt_info) { int ret = OB_SUCCESS; int64_t data_pos = pos; - - UNUSED(encrypt_info); + int64_t end_pos = pos; if (OB_ISNULL(buf) || (data_len - pos) <= 0 || data_len < 0) { TRANS_LOG(WARN, "invalid argument", KP(buf), K(data_len), K(pos)); @@ -1109,8 +1115,9 @@ int ObMemtableMutatorIterator::deserialize(const char *buf, const int64_t data_l ret = (OB_SUCCESS == ret) ? OB_INVALID_DATA : ret; } else if (!buf_.set_data(const_cast(buf + pos), meta_.get_total_size())) { TRANS_LOG(WARN, "set_data fail", KP(buf), K(pos), K(meta_.get_total_size())); + } else if (FALSE_IT(end_pos += meta_.get_total_size())) { } else { - pos += meta_.get_total_size(); + pos = end_pos; buf_.get_limit() = meta_.get_total_size(); buf_.get_position() = meta_.get_meta_size(); } @@ -1120,7 +1127,8 @@ int ObMemtableMutatorIterator::deserialize(const char *buf, const int64_t data_l return ret; } -int ObMemtableMutatorIterator::iterate_next_row() +int ObMemtableMutatorIterator::iterate_next_row(ObEncryptRowBuf &decrypt_buf, + const transaction::ObCLogEncryptInfo &encrypt_info) { int ret = OB_SUCCESS; @@ -1137,15 +1145,11 @@ int ObMemtableMutatorIterator::iterate_next_row() case MutatorType::MUTATOR_ROW: { row_.reset(); const bool unused_need_extract_encrypt_meta = false; - ObEncryptMeta unused_encrypt_meta; ObCLogEncryptStatMap unused_encrypt_stat_map; - ObEncryptRowBuf unused_row_buf; - transaction::ObCLogEncryptInfo unused_encrypt_info; - unused_encrypt_info.init(); - + ObEncryptMeta encrypt_meta; if (OB_FAIL(row_.deserialize( - buf_.get_data(), buf_.get_limit(), buf_.get_position(), unused_row_buf, - unused_encrypt_info, unused_need_extract_encrypt_meta, unused_encrypt_meta, + buf_.get_data(), buf_.get_limit(), buf_.get_position(), decrypt_buf, + encrypt_info, unused_need_extract_encrypt_meta, encrypt_meta, unused_encrypt_stat_map, ObTransRowFlag::is_big_row(meta_.get_flags())))) { TRANS_LOG(WARN, "deserialize mutator row fail", K(ret)); } diff --git a/src/storage/memtable/ob_memtable_mutator.h b/src/storage/memtable/ob_memtable_mutator.h index a86abac22..eca2f48b4 100644 --- a/src/storage/memtable/ob_memtable_mutator.h +++ b/src/storage/memtable/ob_memtable_mutator.h @@ -176,8 +176,9 @@ public: int32_t &flag, int64_t &seq_no) const; - int serialize(char *buf, const int64_t buf_len, int64_t &pos, - const transaction::ObCLogEncryptInfo &encrypt_meta, + int serialize(char *buf, int64_t &buf_len, int64_t &pos, + const transaction::ObTxEncryptMeta *encrypt_meta, + transaction::ObCLogEncryptInfo &new_encrypt_info, const bool is_big_row = false); // the deserialize function need to be virtual function so as for // the extended classes to implement their own deserializaation logic @@ -309,20 +310,23 @@ public: int append_row_kv( const int64_t table_version, const RedoDataNode &redo, - const transaction::ObCLogEncryptInfo &clog_encrypt_info, + const transaction::ObTxEncryptMeta *encrypt_meta, + transaction::ObCLogEncryptInfo &encrypt_info, const bool is_big_row = false); int append_row( ObMemtableMutatorRow &row, - const transaction::ObCLogEncryptInfo &encrypt_info, + transaction::ObCLogEncryptInfo &encrypt_info, const bool is_big_row = false, const bool is_with_head = false); int append_row_buf(const char *buf, const int64_t buf_len); - int serialize(const uint8_t row_flag, int64_t &res_len); + int serialize(const uint8_t row_flag, int64_t &res_len, + transaction::ObCLogEncryptInfo &encrypt_info); ObMemtableMutatorMeta& get_meta() { return meta_; } int64_t get_serialize_size() const; private: ObMemtableMutatorMeta meta_; common::ObDataBuffer buf_; + int64_t row_capacity_; DISALLOW_COPY_AND_ASSIGN(ObMutatorWriter); }; @@ -335,12 +339,13 @@ public: void reset(); public: int deserialize(const char *buf, const int64_t data_len, int64_t &pos, - const transaction::ObCLogEncryptInfo &encrypt_info); + transaction::ObCLogEncryptInfo &encrypt_info); bool is_iter_end() const { return buf_.get_remain() <= 0; } const ObMemtableMutatorMeta &get_meta() const { return meta_; } //4.0 new interface for replay - int iterate_next_row(); + int iterate_next_row(ObEncryptRowBuf &decrypt_buf, + const transaction::ObCLogEncryptInfo &encrypt_info); const ObMutatorRowHeader &get_row_head(); const ObMemtableMutatorRow &get_mutator_row(); const ObMutatorTableLock &get_table_lock_row(); diff --git a/src/storage/memtable/ob_redo_log_generator.cpp b/src/storage/memtable/ob_redo_log_generator.cpp index 9810fb6f3..854be7ad5 100644 --- a/src/storage/memtable/ob_redo_log_generator.cpp +++ b/src/storage/memtable/ob_redo_log_generator.cpp @@ -35,6 +35,10 @@ void ObRedoLogGenerator::reset() generate_cursor_.reset(); callback_mgr_ = nullptr; mem_ctx_ = NULL; + if (clog_encrypt_meta_ != NULL) { + op_free(clog_encrypt_meta_); + clog_encrypt_meta_ = NULL; + } } void ObRedoLogGenerator::reuse() @@ -86,6 +90,9 @@ int ObRedoLogGenerator::fill_redo_log(char *buf, ObCallbackScope callbacks; int64_t data_size = 0; ObITransCallbackIterator cursor; + // for encrypt + transaction::ObCLogEncryptInfo encrypt_info; + encrypt_info.init(); for (cursor = generate_cursor_ + 1; OB_SUCC(ret) && callback_mgr_->end() != cursor; ++cursor) { ObITransCallback *iter = (ObITransCallback *)*cursor; @@ -102,7 +109,7 @@ int ObRedoLogGenerator::fill_redo_log(char *buf, } else { bool fake_fill = false; if (MutatorType::MUTATOR_ROW == iter->get_mutator_type()) { - ret = fill_row_redo(cursor, mmw, redo, log_for_lock_node, fake_fill); + ret = fill_row_redo(cursor, mmw, redo, log_for_lock_node, fake_fill, encrypt_info); } else if (MutatorType::MUTATOR_TABLE_LOCK == iter->get_mutator_type()) { ret = fill_table_lock_redo(cursor, mmw, table_lock_redo, log_for_lock_node, fake_fill); } else { @@ -151,7 +158,8 @@ int ObRedoLogGenerator::fill_redo_log(char *buf, if (OB_LIKELY(OB_ERR_TOO_BIG_ROWSIZE != ret)) { int64_t res_len = 0; - if (OB_SUCCESS != (tmp_ret = mmw.serialize(ObTransRowFlag::NORMAL_ROW, res_len))) { + uint8_t row_flag = ObTransRowFlag::NORMAL_ROW; + if (OB_SUCCESS != (tmp_ret = mmw.serialize(row_flag, res_len, encrypt_info))) { if (OB_ENTRY_NOT_EXIST != tmp_ret) { TRANS_LOG(ERROR, "mmw.serialize fail", K(ret), K(tmp_ret)); ret = tmp_ret; @@ -268,7 +276,8 @@ int ObRedoLogGenerator::fill_row_redo(ObITransCallbackIterator &cursor, ObMutatorWriter &mmw, RedoDataNode &redo, const bool log_for_lock_node, - bool &fake_fill) + bool &fake_fill, + transaction::ObCLogEncryptInfo &encrypt_info) { int ret = OB_SUCCESS; @@ -290,13 +299,14 @@ int ObRedoLogGenerator::fill_row_redo(ObITransCallbackIterator &cursor, } else if (OB_ENTRY_NOT_EXIST == ret) { ret = OB_SUCCESS; } else { - transaction::ObPartTransCtx *part_ctx = static_cast(mem_ctx_->get_trans_ctx()); - if (OB_ISNULL(part_ctx)) { - TRANS_LOG(ERROR, "part ctx is null", K(mem_ctx_)); + ObMemtable *memtable = static_cast(riter->get_memtable()); + if (OB_ISNULL(memtable)) { + TRANS_LOG(ERROR, "memtable is null", K(riter)); ret = OB_ERR_UNEXPECTED; } else if (OB_FAIL(mmw.append_row_kv(mem_ctx_->get_max_table_version(), redo, - part_ctx->get_clog_encrypt_info(), + clog_encrypt_meta_, + encrypt_info, false))) { if (OB_BUF_NOT_ENOUGH != ret) { TRANS_LOG(WARN, "mutator writer append_kv fail", "ret", ret); diff --git a/src/storage/memtable/ob_redo_log_generator.h b/src/storage/memtable/ob_redo_log_generator.h index de96561f3..624fb3bad 100644 --- a/src/storage/memtable/ob_redo_log_generator.h +++ b/src/storage/memtable/ob_redo_log_generator.h @@ -61,9 +61,16 @@ public: redo_sync_fail_cnt_(0), generate_cursor_(), callback_mgr_(nullptr), - mem_ctx_(NULL) + mem_ctx_(NULL), + clog_encrypt_meta_(NULL) {} - ~ObRedoLogGenerator() {} + ~ObRedoLogGenerator() + { + if (clog_encrypt_meta_ != NULL) { + op_free(clog_encrypt_meta_); + clog_encrypt_meta_ = NULL; + } + } void reset(); void reuse(); int set(ObTransCallbackMgr *mgr, ObIMemtableCtx *mem_ctx); @@ -86,7 +93,8 @@ private: ObMutatorWriter &mmw, RedoDataNode &redo, const bool log_for_lock_node, - bool &fake_fill); + bool &fake_fill, + transaction::ObCLogEncryptInfo &encrypt_info); int fill_table_lock_redo(ObITransCallbackIterator &cursor, ObMutatorWriter &mmw, TableLockRedoDataNode &redo, @@ -102,6 +110,7 @@ private: ObITransCallbackIterator generate_cursor_; // the pos of callback which already generated log ObTransCallbackMgr *callback_mgr_; ObIMemtableCtx *mem_ctx_; + transaction::ObTxEncryptMeta *clog_encrypt_meta_; }; }; // end namespace memtable }; // end namespace oceanbase diff --git a/src/storage/meta_mem/ob_meta_obj_struct.h b/src/storage/meta_mem/ob_meta_obj_struct.h index ca80c0fe0..fb1c1f00e 100644 --- a/src/storage/meta_mem/ob_meta_obj_struct.h +++ b/src/storage/meta_mem/ob_meta_obj_struct.h @@ -14,6 +14,7 @@ #define OCEANBASE_STORAGE_OB_META_OBJ_STRUCT_H_ #include "common/log/ob_log_constants.h" +#include "common/ob_clock_generator.h" #include "share/ob_define.h" #include "storage/meta_mem/ob_tenant_meta_obj_pool.h" @@ -206,7 +207,7 @@ void ObMetaObjGuard::set_obj(ObMetaObj &obj) } else { obj_ = obj.ptr_; obj_->inc_ref(); - hold_start_time_ = ObTimeUtility::current_time(); + hold_start_time_ = ObClockGenerator::getClock(); } } } @@ -223,7 +224,7 @@ void ObMetaObjGuard::set_obj(T *obj, common::ObIAllocator *allocator) } else { obj_ = obj; obj_->inc_ref(); - hold_start_time_ = ObTimeUtility::current_time(); + hold_start_time_ = ObClockGenerator::getClock(); } } } @@ -262,7 +263,7 @@ ObMetaObjGuard &ObMetaObjGuard::operator = (const ObMetaObjGuard &other ob_abort(); } else { obj_ = other.obj_; - hold_start_time_ = ObTimeUtility::current_time(); + hold_start_time_ = ObClockGenerator::getClock(); other.obj_->inc_ref(); if (OB_UNLIKELY(other.obj_->get_ref() < 2)) { STORAGE_LOG_RET(ERROR, common::OB_ERR_UNEXPECTED, "obj guard may be accessed by multiple threads or ref cnt leak", KP(obj_), KP(obj_pool_)); @@ -301,7 +302,7 @@ void ObMetaObjGuard::reset_obj() ob_abort(); } else { const int64_t ref_cnt = obj_->dec_ref(); - const int64_t hold_time = ObTimeUtility::current_time() - hold_start_time_; + const int64_t hold_time = ObClockGenerator::getClock() - hold_start_time_; if (OB_UNLIKELY(hold_time > HOLD_OBJ_MAX_TIME && need_hold_time_check())) { int ret = OB_ERR_TOO_MUCH_TIME; STORAGE_LOG(WARN, "The meta obj reference count was held for more " diff --git a/src/storage/tablelock/ob_lock_memtable.cpp b/src/storage/tablelock/ob_lock_memtable.cpp index 7fba5aad9..da39290ac 100644 --- a/src/storage/tablelock/ob_lock_memtable.cpp +++ b/src/storage/tablelock/ob_lock_memtable.cpp @@ -647,13 +647,15 @@ int ObLockMemtable::set( const uint64_t table_id, const storage::ObTableReadInfo &read_info, const ObIArray &columns, - const storage::ObStoreRow &row) + const storage::ObStoreRow &row, + const share::ObEncryptMeta *encrypt_meta) { UNUSED(ctx); UNUSED(table_id); UNUSED(read_info); UNUSED(columns); UNUSED(row); + UNUSED(encrypt_meta); return OB_NOT_SUPPORTED; } diff --git a/src/storage/tablelock/ob_lock_memtable.h b/src/storage/tablelock/ob_lock_memtable.h index 71a4a0627..a0cc61ab3 100644 --- a/src/storage/tablelock/ob_lock_memtable.h +++ b/src/storage/tablelock/ob_lock_memtable.h @@ -145,7 +145,8 @@ public: const uint64_t table_id, const storage::ObTableReadInfo &read_info, const common::ObIArray &columns, - const storage::ObStoreRow &row) override; + const storage::ObStoreRow &row, + const share::ObEncryptMeta *encrypt_meta) override; virtual int lock(storage::ObStoreCtx &ctx, const uint64_t table_id, diff --git a/src/storage/tablet/ob_tablet.cpp b/src/storage/tablet/ob_tablet.cpp index c8d2445de..cf13d403b 100644 --- a/src/storage/tablet/ob_tablet.cpp +++ b/src/storage/tablet/ob_tablet.cpp @@ -1316,6 +1316,7 @@ int ObTablet::insert_row( { int ret = OB_SUCCESS; bool b_exist = false; + common::ObIArray *encrypt_meta_arr = NULL; if (OB_UNLIKELY(!store_ctx.is_valid() || col_descs.count() <= 0 || !row.is_valid() || !relative_table.is_valid())) { ret = OB_INVALID_ARGUMENT; @@ -1331,7 +1332,7 @@ int ObTablet::insert_row( } else if (OB_UNLIKELY(b_exist)) { ret = OB_ERR_PRIMARY_KEY_DUPLICATE; LOG_WARN("rowkey already exists", K(relative_table.get_table_id()), K(row), K(ret)); - } else if (OB_FAIL(insert_row_without_rowkey_check(relative_table, store_ctx, col_descs, row))) { + } else if (OB_FAIL(insert_row_without_rowkey_check(relative_table, store_ctx, col_descs, row, encrypt_meta_arr))) { if (OB_TRY_LOCK_ROW_CONFLICT != ret) { LOG_WARN("failed to set row", K(row), K(ret)); } @@ -1346,13 +1347,15 @@ int ObTablet::update_row( const common::ObIArray &col_descs, const ObIArray &update_idx, const storage::ObStoreRow &old_row, - const storage::ObStoreRow &new_row) + const storage::ObStoreRow &new_row, + const common::ObIArray *encrypt_meta_arr) { int ret = OB_SUCCESS; { ObStorageTableGuard guard(this, store_ctx, true); ObMemtable *write_memtable = nullptr; + const transaction::ObSerializeEncryptMeta *encrypt_meta = NULL; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; @@ -1380,7 +1383,7 @@ int ObTablet::update_row( } else if (OB_FAIL(prepare_memtable(relative_table, store_ctx, write_memtable))) { LOG_WARN("prepare write memtable fail", K(ret), K(relative_table)); } else if (OB_FAIL(write_memtable->set(store_ctx, relative_table.get_table_id(), - full_read_info_, col_descs, update_idx, old_row, new_row))) { + full_read_info_, col_descs, update_idx, old_row, new_row, encrypt_meta))) { LOG_WARN("failed to set write memtable", K(ret)); } } @@ -1392,12 +1395,14 @@ int ObTablet::insert_row_without_rowkey_check( ObRelativeTable &relative_table, ObStoreCtx &store_ctx, const common::ObIArray &col_descs, - const storage::ObStoreRow &row) + const storage::ObStoreRow &row, + const common::ObIArray *encrypt_meta_arr) { int ret = OB_SUCCESS; { ObStorageTableGuard guard(this, store_ctx, true); ObMemtable *write_memtable = nullptr; + const transaction::ObSerializeEncryptMeta *encrypt_meta = NULL; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; @@ -1423,7 +1428,7 @@ int ObTablet::insert_row_without_rowkey_check( } else if (OB_FAIL(prepare_memtable(relative_table, store_ctx, write_memtable))) { LOG_WARN("prepare write memtable fail", K(ret), K(relative_table)); } else if (OB_FAIL(write_memtable->set(store_ctx, relative_table.get_table_id(), - full_read_info_, col_descs, row))) { + full_read_info_, col_descs, row, encrypt_meta))) { LOG_WARN("failed to set memtable", K(ret)); } } diff --git a/src/storage/tablet/ob_tablet.h b/src/storage/tablet/ob_tablet.h index 967d00380..b4fc490bb 100644 --- a/src/storage/tablet/ob_tablet.h +++ b/src/storage/tablet/ob_tablet.h @@ -190,14 +190,16 @@ public: ObRelativeTable &relative_table, ObStoreCtx &store_ctx, const ObColDescIArray &col_descs, - const storage::ObStoreRow &row); + const storage::ObStoreRow &row, + const common::ObIArray *encrypt_meta_arr); int update_row( ObRelativeTable &relative_table, ObStoreCtx &store_ctx, const ObColDescIArray &col_descs, const ObIArray &update_idx, const storage::ObStoreRow &old_row, - const storage::ObStoreRow &new_row); + const storage::ObStoreRow &new_row, + const common::ObIArray *encrypt_meta_arr); int lock_row( ObRelativeTable &relative_table, ObStoreCtx &store_ctx, @@ -506,6 +508,7 @@ private: const memtable::MemtableRefOp ref_op); void print_memtables_for_table(); + private: static const int32_t TABLET_VERSION = 1; private: @@ -888,6 +891,7 @@ int ObTablet::save_multi_source_data_unit( return ret; } + } // namespace storage } // namespace oceanbase diff --git a/src/storage/tx/ob_clog_encrypt_info.h b/src/storage/tx/ob_clog_encrypt_info.h index e2cd00cb3..f0386033c 100644 --- a/src/storage/tx/ob_clog_encrypt_info.h +++ b/src/storage/tx/ob_clog_encrypt_info.h @@ -30,58 +30,12 @@ using namespace common; namespace transaction { -struct ObPGPartitionEncryptMeta -{ - share::ObEncryptMeta meta_; - TO_STRING_KV(K_(meta)); - bool is_valid() const - { - return meta_.is_valid(); - } -}; - -typedef hash::ObHashMap ObEncryptMap; - -class ObCLogEncryptInfo -{ -public: - ObCLogEncryptInfo() : cached_encrypt_meta_(), encrypt_meta_(nullptr), - cached_table_id_(OB_INVALID_ID), is_inited_(false) {} - ~ObCLogEncryptInfo() { destroy(); } - int init(); - bool is_inited() const { return is_inited_; } - virtual void destroy(); - bool is_valid() const; - void reset(); - int get_encrypt_info(const uint64_t table_id, bool &need_encrypt, share::ObEncryptMeta &meta) const; - int store(const ObPGPartitionEncryptMeta &meta); - int store(uint64_t table_id, const share::ObEncryptMeta &meta); - int add_clog_encrypt_info(const ObCLogEncryptInfo &rhs); - ObEncryptMap* get_info() { return encrypt_meta_; } - bool has_encrypt_meta() const; - int replace_tenant_id(const uint64_t real_tenant_id); - int decrypt_table_key(); - OB_UNIS_VERSION(1); -public: -private: - int init_encrypt_meta_(); - int get_(const uint64_t table_id, share::ObEncryptMeta &meta) const; -private: - DISALLOW_COPY_AND_ASSIGN(ObCLogEncryptInfo); -private: - mutable share::ObEncryptMeta cached_encrypt_meta_; - ObEncryptMap *encrypt_meta_; - mutable uint64_t cached_table_id_; - bool is_inited_; -}; - -typedef ObSEArray ObPGPartitionEncryptMetaArray; - struct ObSerializeEncryptMeta : public share::ObEncryptMeta { ObSerializeEncryptMeta() : share::ObEncryptMeta() {} OB_UNIS_VERSION(1); }; + struct ObEncryptMetaCache { int64_t table_id_; @@ -93,6 +47,357 @@ struct ObEncryptMetaCache OB_UNIS_VERSION(1); }; +struct ObTxEncryptMeta +{ + ObTxEncryptMeta() : table_id_(OB_INVALID_ID), prev_(NULL), next_(NULL) {} + int store_encrypt_meta(uint64_t table_id, const share::ObEncryptMeta &meta); + bool is_valid() const { return ((table_id_ != OB_INVALID_ID) && meta_.is_valid()); } + bool contain(const uint64_t &table_id) const { return table_id_ == table_id; } + //for memtable compare + bool is_memtable_equal(const share::ObEncryptMeta &other) + { + //now, we only take random_、table_key_、master_key_version_、encrypt_algorithm_ into consideration. + return (meta_.random_.get_content() == other.random_.get_content() && + meta_.table_key_.get_content() == other.table_key_.get_content() && + meta_.master_key_version_ == other.master_key_version_ && + meta_.encrypt_algorithm_ == other.encrypt_algorithm_); + } + int assign(const ObTxEncryptMeta &other); + void reset() + { + table_id_ = OB_INVALID_ID; + meta_.reset(); + prev_ = NULL; + next_ = NULL; + } + //table_id is used as encrypt_index + uint64_t table_id_; + share::ObEncryptMeta meta_; + //for node in hash + ObTxEncryptMeta *prev_; + ObTxEncryptMeta *next_; + + TO_STRING_KV(K_(table_id), K_(meta)); + OB_UNIS_VERSION(1); +}; + +class ObEncryptMetaAlloc +{ +public: + static ObTxEncryptMeta* alloc_value() + { + ObTxEncryptMeta *it = nullptr; + void *buf = ob_malloc(sizeof(ObTxEncryptMeta), "ClogEncrypt"); + if (nullptr != buf) { + it = new(buf) ObTxEncryptMeta(); + } + return it; + } + static void free_value(ObTxEncryptMeta *v) + { + if (NULL != v) { + ob_free(v); + v = NULL; + } + } +}; + +//this hashMap is just for clog encryption to save encryption data +//without lock protect +template, + int64_t BUCKETS_CNT = 10> +class ObSimpleHashMap +{ +public: + ObSimpleHashMap() : is_inited_(false), total_cnt_(0) + { + OB_ASSERT(BUCKETS_CNT > 0); + } + ~ObSimpleHashMap() { destroy(); } + int64_t count() const { return ATOMIC_LOAD(&total_cnt_); } + int init() + { + int ret = OB_SUCCESS; + if (is_inited_) { + TRANS_LOG(WARN, "ObSimpleHashMap init twice"); + ret = OB_INIT_TWICE; + } else { + is_inited_ = true; + } + return ret; + } + + void reset() + { + if (is_inited_) { + Value *curr = nullptr; + Value *next = nullptr; + for (int64_t i = 0; i < BUCKETS_CNT; ++i) { + curr = buckets_[i].next_; + while (OB_NOT_NULL(curr)) { + next = curr->next_; + del_from_bucket_(i, curr); + // dec ref and free curr value + curr = next; + } + // reset bucket + buckets_[i].reset(); + } + total_cnt_ = 0; + is_inited_ = false; + } + } + + void destroy() { reset(); } + + int insert_and_get(const Key &key, const Value &value, Value **old_value) + { return insert_(key, value, old_value); } + int insert(const Key &key, const Value &value) + { return insert_(key, value, NULL); } + int insert_(const Key &key, const Value &value, Value **old_value) + { + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + TRANS_LOG(WARN, "ObSimpleHashMap not init", K(ret)); + } else { + uint64_t pos = hashfunc_(key) % BUCKETS_CNT; + Value *curr = buckets_[pos].next_; + while (OB_NOT_NULL(curr)) { + if (curr->contain(key)) { + break; + } else { + curr = curr->next_; + } + } + if (OB_ISNULL(curr)) { + Value *new_value = NULL; + if (NULL == (new_value = alloc_handle_.alloc_value())) { + ret = OB_ALLOCATE_MEMORY_FAILED; + } else { + new_value->assign(value); + if (NULL != buckets_[pos].next_) { + buckets_[pos].next_->prev_ = new_value; + } + new_value->next_ = buckets_[pos].next_; + new_value->prev_ = NULL; + buckets_[pos].next_ = new_value; + ATOMIC_INC(&total_cnt_); + } + } else { + ret = OB_ENTRY_EXIST; + if (old_value) { + *old_value = curr; + } + } + } + return ret; + } + + int del(const Key &key) + { + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + TRANS_LOG(WARN, "ObTransHashMap not init", K(ret)); + } else { + int64_t pos = hashfunc_(key) % BUCKETS_CNT; + Value *curr = buckets_[pos].next_; + while (OB_NOT_NULL(curr)) { + if (curr->contain(key)) { + break; + } else { + curr = curr->next_; + } + } + if (OB_ISNULL(curr)) { + ret = OB_ENTRY_NOT_EXIST; + } else { + del_from_bucket_(pos, curr); + } + } + return ret; + } + + int get(const Key &key, Value *&value) + { + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + TRANS_LOG(WARN, "ObTransHashMap not init", K(ret), K(key)); + } else { + Value *tmp_value = NULL; + int64_t pos = hashfunc_(key) % BUCKETS_CNT; + + tmp_value = buckets_[pos].next_; + while (OB_NOT_NULL(tmp_value)) { + if (tmp_value->contain(key)) { + value = tmp_value; + break; + } else { + tmp_value = tmp_value->next_; + } + } + + if (OB_ISNULL(tmp_value)) { + ret = OB_ENTRY_NOT_EXIST; + } + } + return ret; + } + + class ObEncryptIterator + { + public: + ObEncryptIterator(const ObSimpleHashMap* hm, int64_t bucket_pos, Value *node) : + hm_(hm), bucket_pos_(bucket_pos), node_(node) {} + Value* operator ->() const + { + Value *p = NULL; + if (OB_ISNULL(node_)) { + HASH_WRITE_LOG_RET(HASH_FATAL, OB_ERR_UNEXPECTED, "node is null, backtrace=%s", lbt()); + } else { + p = node_; + } + return p; + } + ObEncryptIterator &operator ++() + { + if (OB_ISNULL(hm_)) { + HASH_WRITE_LOG_RET(HASH_FATAL, OB_ERR_UNEXPECTED, "hashmap is null, backtrace=%s", lbt()); + } else if (NULL != node_ && NULL != (node_ = node_->next_)) { + // do nothing + } else { + for (int64_t i = bucket_pos_ + 1; i < BUCKETS_CNT; i++) { + if (NULL != (node_ = hm_->buckets_[i].next_)) { + bucket_pos_ = i; + break; + } + } + if (NULL == node_) { + bucket_pos_ = BUCKETS_CNT; + } + } + return *this; + } + + bool operator !=(const ObEncryptIterator &iter) const + { + return node_ != iter.node_; + } + private: + //for iterator + const ObSimpleHashMap *hm_; + int64_t bucket_pos_; + Value *node_; + }; + + ObEncryptIterator begin() + { + int64_t bucket_pos = 0; + Value *node = NULL; + while (NULL == node && bucket_pos < BUCKETS_CNT) { + node = buckets_[bucket_pos].next_; + if (NULL == node) { + ++bucket_pos; + } + } + return ObEncryptIterator(this, bucket_pos, node); + } + + ObEncryptIterator end() + { + return ObEncryptIterator(this, BUCKETS_CNT, NULL); + } + +private: + void del_from_bucket_(const int64_t pos, Value *curr) + { + if (curr == buckets_[pos].next_) { + if (NULL == curr->next_) { + buckets_[pos].next_ = NULL; + } else { + buckets_[pos].next_ = curr->next_; + curr->next_->prev_ = curr->prev_; + } + } else { + curr->prev_->next_ = curr->next_; + if (NULL != curr->next_) { + curr->next_->prev_ = curr->prev_; + } + } + curr->prev_ = NULL; + curr->next_ = NULL; + alloc_handle_.free_value(curr); + ATOMIC_DEC(&total_cnt_); + } + +private: + struct ObSimpleHashHeader + { + Value *next_; + ObSimpleHashHeader() : next_(NULL) {} + ~ObSimpleHashHeader() { destroy(); } + //maybe we can add init() in the future + void reset() + { + next_ = NULL; + } + void destroy() + { + reset(); + } + }; +private: + bool is_inited_; + ObSimpleHashHeader buckets_[BUCKETS_CNT]; + int64_t total_cnt_; + AllocHandle alloc_handle_; + _hashfunc hashfunc_; +}; + +typedef ObSimpleHashMap ObTxEncryptMap; + +class ObCLogEncryptInfo +{ +public: + ObCLogEncryptInfo() : encrypt_map_(nullptr), + cached_encrypt_meta_(nullptr), + cached_table_id_(OB_INVALID_ID), + is_inited_(false) {} + ~ObCLogEncryptInfo() { destroy(); } + int init(); + bool is_inited() const { return is_inited_; } + virtual void destroy(); + bool is_valid() const; + void reset(); + int get_encrypt_info(const uint64_t table_id, ObTxEncryptMeta *&meta) const; + int store(uint64_t table_id, const share::ObEncryptMeta &meta); + int store_and_get_old(const uint64_t table_id, const ObTxEncryptMeta &new_meta, ObTxEncryptMeta **old_meta); + int remove(uint64_t table_id); + int add_clog_encrypt_info(const ObCLogEncryptInfo &rhs); + ObTxEncryptMap* get_info() { return encrypt_map_; } + bool has_encrypt_meta() const; + int replace_tenant_id(const uint64_t real_tenant_id); + int64_t to_string(char *buf, const int64_t buf_len) const; + int decrypt_table_key(); + OB_UNIS_VERSION(1); +private: + int init_encrypt_meta_(); +private: + DISALLOW_COPY_AND_ASSIGN(ObCLogEncryptInfo); +private: + ObTxEncryptMap *encrypt_map_; + mutable ObTxEncryptMeta *cached_encrypt_meta_; + mutable uint64_t cached_table_id_; + bool is_inited_; +}; + }//transaction }//oceanbase diff --git a/src/storage/tx/ob_clog_encrypt_info_os.cpp b/src/storage/tx/ob_clog_encrypt_info_os.cpp index e8abc88f5..e518da4c4 100644 --- a/src/storage/tx/ob_clog_encrypt_info_os.cpp +++ b/src/storage/tx/ob_clog_encrypt_info_os.cpp @@ -49,11 +49,9 @@ void ObCLogEncryptInfo::destroy() { } -int ObCLogEncryptInfo::get_encrypt_info(const uint64_t table_id, bool &need_encrypt, - ObEncryptMeta &meta) const +int ObCLogEncryptInfo::get_encrypt_info(const uint64_t table_id, ObTxEncryptMeta *&meta) const { int ret = OB_SUCCESS; - need_encrypt = false; return ret; } @@ -67,20 +65,20 @@ OB_DEF_SERIALIZE(ObCLogEncryptInfo) } else if (OB_UNLIKELY(!is_inited_)) { TRANS_LOG(ERROR, "serialize uninitialized clog encrypt info"); ret = OB_ERR_UNEXPECTED; - } else if (OB_ISNULL(encrypt_meta_)) { + } else if (OB_ISNULL(encrypt_map_)) { size = 0; if (OB_FAIL(serialization::encode_vi64(buf, buf_len, pos, size))) { TRANS_LOG(WARN, "failed to encode count of encrypt info map", K(ret)); } } else { - size = encrypt_meta_->size(); + size = encrypt_map_->count(); if (OB_FAIL(serialization::encode_vi64(buf, buf_len, pos, size))) { TRANS_LOG(WARN, "failed to encode count of encrypt info map", K(ret)); } - for (auto it = encrypt_meta_->begin(); OB_SUCC(ret) && it != encrypt_meta_->end(); ++it) { - if (OB_FAIL(serialization::encode_vi64(buf, buf_len, pos, it->first))) { + for (auto it = encrypt_map_->begin(); OB_SUCC(ret) && it != encrypt_map_->end(); ++it) { + if (OB_FAIL(serialization::encode_vi64(buf, buf_len, pos, it->table_id_))) { TRANS_LOG(WARN, "failed to encode encrypt info map table_id", K(ret)); - } else if (OB_FAIL(it->second.serialize(buf, buf_len, pos))) { + } else if (OB_FAIL(it->meta_.serialize(buf, buf_len, pos))) { TRANS_LOG(WARN, "failed to encode encrypt info map item", K(ret)); } } @@ -119,11 +117,11 @@ OB_DEF_SERIALIZE_SIZE(ObCLogEncryptInfo) { int64_t size = 0; int64_t count = 0; - if (OB_NOT_NULL(encrypt_meta_)) { - count = encrypt_meta_->size(); - for (auto it = encrypt_meta_->begin(); it != encrypt_meta_->end(); ++it) { - size += serialization::encoded_length_vi64(it->first); - size += it->second.get_serialize_size(); + if (OB_NOT_NULL(encrypt_map_)) { + count = encrypt_map_->count(); + for (auto it = encrypt_map_->begin(); it != encrypt_map_->end(); ++it) { + size += serialization::encoded_length_vi64(it->table_id_); + size += it->meta_.get_serialize_size(); } } size += serialization::encoded_length_vi64(count); diff --git a/src/storage/tx/ob_trans_ctx.h b/src/storage/tx/ob_trans_ctx.h index 8d861e926..2cb975f58 100644 --- a/src/storage/tx/ob_trans_ctx.h +++ b/src/storage/tx/ob_trans_ctx.h @@ -249,6 +249,7 @@ protected: int64_t ctx_create_time_; ObTransService *trans_service_; mutable CtxLock lock_; + ObTransTraceLog trace_log_; ObTransTraceLog *tlog_; uint64_t cluster_version_; ObLSTxCtxMgr *ls_tx_ctx_mgr_; diff --git a/src/storage/tx/ob_trans_define.h b/src/storage/tx/ob_trans_define.h index ecd6a3805..3772fb388 100644 --- a/src/storage/tx/ob_trans_define.h +++ b/src/storage/tx/ob_trans_define.h @@ -682,11 +682,11 @@ private: typedef common::ObReserveArenaAllocator<1024> ObTxReserveArenaAllocator; -class ObTransTraceLog : public common::ObTraceEventRecorderBase<100, 4000> +class ObTransTraceLog : public common::ObTraceEventRecorder { public: ObTransTraceLog() - : common::ObTraceEventRecorderBase<100, 4000>::ObTraceEventRecorderBase( + : common::ObTraceEventRecorder::ObTraceEventRecorderBase( true, common::ObLatchIds::TRANS_TRACE_RECORDER_LOCK) {} ~ObTransTraceLog() {} void destroy() {} diff --git a/src/storage/tx/ob_trans_event.cpp b/src/storage/tx/ob_trans_event.cpp index 9c2e572d8..62d172307 100644 --- a/src/storage/tx/ob_trans_event.cpp +++ b/src/storage/tx/ob_trans_event.cpp @@ -88,7 +88,7 @@ int64_t ObTransStatItem::to_string(char *buf, const int64_t buf_len) const void ObTransStatistic::add_sys_trans_count(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(TRANS_SYSTEM_TRANS_COUNT, value); + // EVENT_ADD(TRANS_SYSTEM_TRANS_COUNT, value); //sys_trans_count_stat_.add(value); } @@ -286,150 +286,150 @@ void ObTransStatistic::add_clear_log_replay_time(const uint64_t tenant_id, const void ObTransStatistic::add_sp_redo_log_cb_count(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(SP_REDO_LOG_CB_COUNT, value); + //EVENT_ADD(SP_REDO_LOG_CB_COUNT, value); } void ObTransStatistic::add_sp_redo_log_cb_time(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(SP_REDO_LOG_CB_TIME, value); + //EVENT_ADD(SP_REDO_LOG_CB_TIME, value); } void ObTransStatistic::add_sp_commit_log_cb_count(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(SP_COMMIT_LOG_CB_COUNT, value); + //EVENT_ADD(SP_COMMIT_LOG_CB_COUNT, value); } void ObTransStatistic::add_sp_commit_log_cb_time(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(SP_COMMIT_LOG_CB_TIME, value); + //EVENT_ADD(SP_COMMIT_LOG_CB_TIME, value); } void ObTransStatistic::add_redo_log_cb_count(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(REDO_LOG_CB_COUNT, value); + //EVENT_ADD(REDO_LOG_CB_COUNT, value); } void ObTransStatistic::add_redo_log_cb_time(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(REDO_LOG_CB_TIME, value); + //EVENT_ADD(REDO_LOG_CB_TIME, value); } void ObTransStatistic::add_prepare_log_cb_count(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(PREPARE_LOG_CB_COUNT, value); + //EVENT_ADD(PREPARE_LOG_CB_COUNT, value); } void ObTransStatistic::add_prepare_log_cb_time(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(PREPARE_LOG_CB_TIME, value); + //EVENT_ADD(PREPARE_LOG_CB_TIME, value); } void ObTransStatistic::add_commit_log_cb_count(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(COMMIT_LOG_CB_COUNT, value); + //EVENT_ADD(COMMIT_LOG_CB_COUNT, value); } void ObTransStatistic::add_commit_log_cb_time(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(COMMIT_LOG_CB_TIME, value); + //EVENT_ADD(COMMIT_LOG_CB_TIME, value); } void ObTransStatistic::add_abort_log_cb_count(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(ABORT_LOG_CB_COUNT, value); + //EVENT_ADD(ABORT_LOG_CB_COUNT, value); } void ObTransStatistic::add_abort_log_cb_time(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(ABORT_LOG_CB_TIME, value); + //EVENT_ADD(ABORT_LOG_CB_TIME, value); } void ObTransStatistic::add_clear_log_cb_count(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(CLEAR_LOG_CB_COUNT, value); + //EVENT_ADD(CLEAR_LOG_CB_COUNT, value); } void ObTransStatistic::add_clear_log_cb_time(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(CLEAR_LOG_CB_TIME, value); + //EVENT_ADD(CLEAR_LOG_CB_TIME, value); } void ObTransStatistic::add_trans_callback_sql_count(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(TRANS_CALLBACK_SQL_COUNT, value); + //EVENT_ADD(TRANS_CALLBACK_SQL_COUNT, value); } void ObTransStatistic::add_trans_callback_sql_time(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(TRANS_CALLBACK_SQL_TIME, value); + //EVENT_ADD(TRANS_CALLBACK_SQL_TIME, value); } void ObTransStatistic::add_trans_mt_end_count(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(TRANS_MT_END_COUNT, value); + //EVENT_ADD(TRANS_MT_END_COUNT, value); } void ObTransStatistic::add_trans_mt_end_time(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(TRANS_MT_END_TIME, value); + //EVENT_ADD(TRANS_MT_END_TIME, value); } void ObTransStatistic::add_memstore_mutator_size(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(CLOG_MEMSTORE_MUTATOR_TOTAL_SIZE, value); + //EVENT_ADD(CLOG_MEMSTORE_MUTATOR_TOTAL_SIZE, value); } void ObTransStatistic::add_stmt_timeout_count(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(TRANS_STRONG_CONSISTENCY_STMT_TIMEOUT_COUNT, value); + //EVENT_ADD(TRANS_STRONG_CONSISTENCY_STMT_TIMEOUT_COUNT, value); } void ObTransStatistic::add_slave_read_stmt_timeout_count(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(TRANS_SLAVE_READ_STMT_TIMEOUT_COUNT, value); + //EVENT_ADD(TRANS_SLAVE_READ_STMT_TIMEOUT_COUNT, value); } void ObTransStatistic::add_slave_read_stmt_retry_count(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(TRANS_SLAVE_READ_STMT_RETRY_COUNT, value); + //EVENT_ADD(TRANS_SLAVE_READ_STMT_RETRY_COUNT, value); } void ObTransStatistic::add_fill_redo_log_count(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(TRANS_FILL_REDO_LOG_COUNT, value); + //EVENT_ADD(TRANS_FILL_REDO_LOG_COUNT, value); } void ObTransStatistic::add_fill_redo_log_time(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(TRANS_FILL_REDO_LOG_TIME, value); + //EVENT_ADD(TRANS_FILL_REDO_LOG_TIME, value); } void ObTransStatistic::add_submit_trans_log_count(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(TRANS_SUBMIT_LOG_COUNT, value); + //EVENT_ADD(TRANS_SUBMIT_LOG_COUNT, value); } void ObTransStatistic::add_submit_trans_log_time(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(TRANS_SUBMIT_LOG_TIME, value); + //EVENT_ADD(TRANS_SUBMIT_LOG_TIME, value); } void ObTransStatistic::add_gts_request_total_count(const uint64_t tenant_id, const int64_t value) { @@ -444,12 +444,12 @@ void ObTransStatistic::add_gts_acquire_total_time(const uint64_t tenant_id, cons void ObTransStatistic::add_gts_acquire_total_count(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(GTS_ACQUIRE_TOTAL_COUNT, value); + //EVENT_ADD(GTS_ACQUIRE_TOTAL_COUNT, value); } void ObTransStatistic::add_gts_acquire_total_wait_count(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(GTS_ACQUIRE_TOTAL_WAIT_COUNT, value); + //EVENT_ADD(GTS_ACQUIRE_TOTAL_WAIT_COUNT, value); } void ObTransStatistic::add_gts_wait_elapse_total_time(const uint64_t tenant_id, const int64_t value) { @@ -487,25 +487,25 @@ void ObTransStatistic::add_gts_try_wait_elapse_total_count(const uint64_t tenant void ObTransStatistic::add_stmt_total_count(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(TRANS_STMT_TOTAL_COUNT, value); + //EVENT_ADD(TRANS_STMT_TOTAL_COUNT, value); } void ObTransStatistic::add_stmt_interval_time(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(TRANS_STMT_INTERVAL_TIME, value); + //EVENT_ADD(TRANS_STMT_INTERVAL_TIME, value); } void ObTransStatistic::add_trans_multi_partition_update_stmt_count(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(TRANS_MULTI_PARTITION_UPDATE_STMT_COUNT, value); + //EVENT_ADD(TRANS_MULTI_PARTITION_UPDATE_STMT_COUNT, value); } void ObTransStatistic::add_batch_commit_trans_count(const uint64_t tenant_id, const int64_t value) { common::ObTenantStatEstGuard guard(tenant_id); - EVENT_ADD(TRANS_BATCH_COMMIT_COUNT, value); + //EVENT_ADD(TRANS_BATCH_COMMIT_COUNT, value); } void ObTransStatistic::add_trans_log_total_size(const uint64_t tenant_id, const int64_t value) diff --git a/src/storage/tx/ob_trans_part_ctx.cpp b/src/storage/tx/ob_trans_part_ctx.cpp index 8f10fc0ef..7d55e49d2 100644 --- a/src/storage/tx/ob_trans_part_ctx.cpp +++ b/src/storage/tx/ob_trans_part_ctx.cpp @@ -109,8 +109,6 @@ int ObPartTransCtx::init(const uint64_t tenant_id, TRANS_LOG(WARN, "timeout task init error", KR(ret)); } else if (OB_FAIL(init_memtable_ctx_(tenant_id, ls_id))) { TRANS_LOG(WARN, "ObPartTransCtx init memtable context error", KR(ret), K(trans_id), K(ls_id)); - } else if (OB_FAIL(clog_encrypt_info_.init())) { - TRANS_LOG(WARN, "init clog encrypt info failed", K(ret), K(trans_id)); } else if (OB_FAIL(init_log_cbs_(ls_id, trans_id))) { TRANS_LOG(WARN, "init log cbs failed", KR(ret), K(trans_id), K(ls_id)); } else if (OB_FAIL(ctx_tx_data_.init(ls_ctx_mgr, trans_id))) { @@ -150,11 +148,8 @@ int ObPartTransCtx::init(const uint64_t tenant_id, if (!GCONF.enable_sql_audit) { tlog_ = NULL; - } else if (OB_ISNULL(tlog_ = ObTransTraceLogFactory::alloc())) { - ret = OB_ALLOCATE_MEMORY_FAILED; - TRANS_LOG(WARN, "alloc ObTransTraceLog error", KR(ret), KP_(tlog)); } else { - // do nothing + tlog_ = &trace_log_; } is_inited_ = true; @@ -239,15 +234,15 @@ void ObPartTransCtx::destroy() if (NULL != tlog_) { print_trace_log_if_necessary_(); - ObTransTraceLogFactory::release(tlog_); tlog_ = NULL; } mds_cache_.destroy(); exec_info_.destroy(); + reset_log_cbs_(); + timeout_task_.destroy(); - clog_encrypt_info_.destroy(); trace_info_.reset(); block_frozen_memtable_ = nullptr; is_inited_ = false; @@ -273,7 +268,6 @@ void ObPartTransCtx::default_init_() can_elr_ = false; // TODO ObPartTransCtx - clog_encrypt_info_.reset(); ObTxCycleTwoPhaseCommitter::reset(); is_inited_ = false; mt_ctx_.reset(); @@ -313,16 +307,17 @@ void ObPartTransCtx::default_init_() int ObPartTransCtx::init_log_cbs_(const ObLSID &ls_id, const ObTransID &tx_id) { int ret = OB_SUCCESS; - for (int64_t i = 0; OB_SUCC(ret) && i < OB_TX_MAX_LOG_CBS; ++i) { - if (OB_FAIL(log_cbs_[i].init(ls_id, tx_id, this))) { + for (int64_t i = 0; OB_SUCC(ret) && i < PREALLOC_LOG_CALLBACK_COUNT; ++i) { + if (OB_FAIL(log_cbs_[i].init(ls_id, tx_id, this, false))) { TRANS_LOG(WARN, "log cb init failed", KR(ret)); } else if (!free_cbs_.add_last(&log_cbs_[i])) { - ret = OB_INIT_FAIL; + log_cbs_[i].destroy(); + ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "add to free list failed", KR(ret)); } } if (OB_SUCC(ret)) { - if (OB_FAIL(final_log_cb_.init(ls_id, tx_id, this))) { + if (OB_FAIL(final_log_cb_.init(ls_id, tx_id, this, false))) { TRANS_LOG(WARN, "init commit log cb failed", K(ret)); } else { TRANS_LOG(DEBUG, "init commit log cb success", K(ret), KP(&final_log_cb_), K(*this)); @@ -331,24 +326,63 @@ int ObPartTransCtx::init_log_cbs_(const ObLSID &ls_id, const ObTransID &tx_id) return ret; } +int ObPartTransCtx::extend_log_cbs_() +{ + int ret = OB_SUCCESS; + void *ptr = NULL; + ObTxLogCb *cb = NULL; + if (busy_cbs_.get_size() >= OB_TX_MAX_LOG_CBS + 1) { + ret = OB_TX_NOLOGCB; + } else if (ATOMIC_LOAD(&is_submitting_redo_log_for_freeze_) == false && + busy_cbs_.get_size() >= OB_TX_MAX_LOG_CBS + 1 - RESERVE_LOG_CALLBACK_COUNT_FOR_FREEZING) { + ret = OB_TX_NOLOGCB; + } else if (OB_ISNULL(ptr = reserve_allocator_.alloc(sizeof(ObTxLogCb)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + } else { + if (OB_ISNULL(cb = new(ptr) ObTxLogCb)) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "log callback construct failed", K(ret)); + } else if (OB_FAIL(cb->init(ls_id_, trans_id_, this, true))) { + TRANS_LOG(WARN, "log callback init failed", K(ret)); + cb->~ObTxLogCb(); + } else if (!free_cbs_.add_last(cb)) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "add to free list failed", KR(ret)); + cb->~ObTxLogCb(); + } else { + // do nothing + } + if (OB_FAIL(ret)) { + reserve_allocator_.free(ptr); + } + } + return ret; +} + +void ObPartTransCtx::reset_log_cb_list_(common::ObDList &cb_list) +{ + ObTxLogCb *cb = NULL; + while (OB_NOT_NULL(cb = cb_list.remove_first())) { + const bool is_dynamic = cb->is_dynamic(); + if (OB_NOT_NULL(cb->get_tx_data())) { + ObTxData *tx_data = cb->get_tx_data(); + ctx_tx_data_.free_tmp_tx_data(tx_data); + cb->set_tx_data(nullptr); + } + if (!is_dynamic) { + cb->reset(); + } else { + cb->~ObTxLogCb(); + reserve_allocator_.free(cb); + } + } +} + void ObPartTransCtx::reset_log_cbs_() { - for (int64_t i = 0; i < OB_TX_MAX_LOG_CBS; ++i) { - if (OB_NOT_NULL(log_cbs_[i].get_tx_data())) { - ObTxData *tx_data = log_cbs_[i].get_tx_data(); - ctx_tx_data_.free_tmp_tx_data(tx_data); - log_cbs_[i].set_tx_data(nullptr); - } - log_cbs_[i].reset(); - } - if (OB_NOT_NULL(final_log_cb_.get_tx_data())) { - ObTxData *tx_data = final_log_cb_.get_tx_data(); - ctx_tx_data_.free_tmp_tx_data(tx_data); - final_log_cb_.set_tx_data(nullptr); - } + reset_log_cb_list_(free_cbs_); + reset_log_cb_list_(busy_cbs_); final_log_cb_.reset(); - free_cbs_.reset(); - busy_cbs_.reset(); } // thread-unsafe @@ -2415,7 +2449,7 @@ int ObPartTransCtx::submit_redo_log_(ObTxLogBlock &log_block, ObTxLogCb *log_cb = NULL; while (OB_SUCC(ret) && need_continue) { - ObTxRedoLog redo_log(clog_encrypt_info_, get_redo_log_no_(), cluster_version_); + ObTxRedoLog redo_log(get_redo_log_no_(), cluster_version_); mutator_size = 0; need_undo_log = false; need_submit_log = false; @@ -3677,23 +3711,31 @@ int ObPartTransCtx::get_log_cb_(const bool need_final_cb, ObTxLogCb *&log_cb) log_cb = &final_log_cb_; } } else { - if (free_cbs_.is_empty()) { - ret = OB_TX_NOLOGCB; - //TRANS_LOG(INFO, "all log cbs are busy now, try again later", K(ret), K(*this)); - } else if (free_cbs_.get_size() <= RESERVE_LOG_CALLBACK_COUNT_FOR_FREEZING && - ATOMIC_LOAD(&is_submitting_redo_log_for_freeze_) == false) { - ret = OB_TX_NOLOGCB; - //TRANS_LOG(INFO, "reserve log callback for freezing, try again later", K(ret), K(*this)); - } else if (OB_ISNULL(log_cb = free_cbs_.remove_first())) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "unexpected null log cb", KR(ret), K(*this)); - } else { - // do nothing + for (int64_t i = 0; OB_SUCC(ret) && i < 2; i++) { + if (!free_cbs_.is_empty()) { + if (OB_ISNULL(log_cb = free_cbs_.remove_first())) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "unexpected null log cb", KR(ret), K(*this)); + } else { + break; + } + } else { + if (OB_FAIL(extend_log_cbs_())) { + TRANS_LOG(WARN, "extend log callback failed", K(ret)); + // rewrite ret + ret = OB_TX_NOLOGCB; + } + } } } if (OB_SUCC(ret)) { - log_cb->reuse(); - busy_cbs_.add_last(log_cb); + if (OB_ISNULL(log_cb)) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "unexpected log callback", K(ret)); + } else { + log_cb->reuse(); + busy_cbs_.add_last(log_cb); + } } return ret; } diff --git a/src/storage/tx/ob_trans_part_ctx.h b/src/storage/tx/ob_trans_part_ctx.h index 1047b1f5e..acb009a6f 100644 --- a/src/storage/tx/ob_trans_part_ctx.h +++ b/src/storage/tx/ob_trans_part_ctx.h @@ -19,7 +19,6 @@ #include "ob_ctx_tx_data.h" #include "lib/container/ob_mask_set2.h" #include "lib/list/ob_dlist.h" -#include "ob_clog_encrypt_info.h" #include "share/ob_ls_id.h" #include "logservice/palf/lsn.h" #include "ob_one_phase_committer.h" @@ -81,7 +80,8 @@ enum namespace transaction { -const static int64_t OB_TX_MAX_LOG_CBS = 3; +const static int64_t OB_TX_MAX_LOG_CBS = 15; +const static int64_t PREALLOC_LOG_CALLBACK_COUNT = 3; const static int64_t RESERVE_LOG_CALLBACK_COUNT_FOR_FREEZING = 1; // participant transaction context @@ -173,7 +173,6 @@ public: uint64_t get_lock_for_read_retry_count() const { return mt_ctx_.get_lock_for_read_retry_count(); } int check_scheduler_status(); - void audit_partition(const bool is_rollback, const sql::stmt::StmtType stmt_type); int remove_callback_for_uncommited_txn(memtable::ObMemtable* mt); int64_t get_trans_mem_total_size() const { return mt_ctx_.get_trans_mem_total_size(); } @@ -188,7 +187,6 @@ public: int64_t get_applying_log_ts() const; int64_t get_pending_log_size() { return mt_ctx_.get_pending_log_size(); } int64_t get_flushed_log_size() { return mt_ctx_.get_flushed_log_size(); } - ObCLogEncryptInfo& get_clog_encrypt_info() { return clog_encrypt_info_; } void get_audit_info(int64_t &lock_for_read_elapse) const; virtual int64_t get_part_trans_action() const override; inline bool has_pending_write() const { return pending_write_; } @@ -539,6 +537,8 @@ protected: private: int init_log_cbs_(const share::ObLSID&ls_id, const ObTransID &tx_id); + int extend_log_cbs_(); + void reset_log_cb_list_(common::ObDList &cb_list); void reset_log_cbs_(); int prepare_log_cb_(const bool need_final_cb, ObTxLogCb *&log_cb); int get_log_cb_(const bool need_final_cb, ObTxLogCb *&log_cb); @@ -745,7 +745,6 @@ private: int64_t last_ask_scheduler_status_ts_; int64_t cur_query_start_time_; - ObCLogEncryptInfo clog_encrypt_info_; /* * used during txn protected data access @@ -774,7 +773,7 @@ private: ObTxMDSCache mds_cache_; // sub_state_ is volatile ObTxSubState sub_state_; - ObTxLogCb log_cbs_[OB_TX_MAX_LOG_CBS]; + ObTxLogCb log_cbs_[PREALLOC_LOG_CALLBACK_COUNT]; common::ObDList free_cbs_; common::ObDList busy_cbs_; ObTxLogCb final_log_cb_; @@ -837,7 +836,8 @@ private: }; // reserve log callback for freezing and other two log callbacks for normal -STATIC_ASSERT(OB_TX_MAX_LOG_CBS >= RESERVE_LOG_CALLBACK_COUNT_FOR_FREEZING + 2, "log callback is not enough"); +STATIC_ASSERT(OB_TX_MAX_LOG_CBS >= PREALLOC_LOG_CALLBACK_COUNT && + PREALLOC_LOG_CALLBACK_COUNT >= (RESERVE_LOG_CALLBACK_COUNT_FOR_FREEZING + 2), "log callback is not enough"); #if defined(__x86_64__) /* uncomment this block to error messaging real size diff --git a/src/storage/tx/ob_trans_submit_log_cb.cpp b/src/storage/tx/ob_trans_submit_log_cb.cpp index ae9e5e715..5d1189703 100644 --- a/src/storage/tx/ob_trans_submit_log_cb.cpp +++ b/src/storage/tx/ob_trans_submit_log_cb.cpp @@ -68,7 +68,7 @@ int ObTxBaseLogCb::set_lsn(const LSN &lsn) } int ObTxLogCb::init(const ObLSID &key, - const ObTransID &trans_id, ObTransCtx *ctx) + const ObTransID &trans_id, ObTransCtx *ctx, const bool is_dynamic) { int ret = OB_SUCCESS; @@ -85,6 +85,7 @@ int ObTxLogCb::init(const ObLSID &key, ctx_ = ctx; tx_data_guard_.reset(); is_callbacked_ = false; + is_dynamic_ = is_dynamic; } return ret; @@ -101,6 +102,7 @@ void ObTxLogCb::reset() tx_data_guard_.reset(); callbacks_.reset(); is_callbacked_ = false; + is_dynamic_ = false; cb_arg_array_.reset(); mds_range_.reset(); //is_callbacking_ = false; diff --git a/src/storage/tx/ob_trans_submit_log_cb.h b/src/storage/tx/ob_trans_submit_log_cb.h index cf2c9a280..a526ee74f 100644 --- a/src/storage/tx/ob_trans_submit_log_cb.h +++ b/src/storage/tx/ob_trans_submit_log_cb.h @@ -74,7 +74,8 @@ public: ~ObTxLogCb() { destroy(); } int init(const share::ObLSID &key, const ObTransID &trans_id, - ObTransCtx *ctx); + ObTransCtx *ctx, + const bool is_dynamic); void reset(); void reuse(); void destroy() { reset(); } @@ -93,6 +94,7 @@ public: memtable::ObCallbackScope& get_callbacks() { return callbacks_; } void set_callbacked() { is_callbacked_ = true; } bool is_callbacked() const { return is_callbacked_; } + bool is_dynamic() const { return is_dynamic_; } ObTxCbArgArray &get_cb_arg_array() { return cb_arg_array_; } const ObTxCbArgArray &get_cb_arg_array() const { return cb_arg_array_; } bool is_valid() const; @@ -116,6 +118,7 @@ public: KP_(ctx), K_(tx_data_guard), K(is_callbacked_), + K(is_dynamic_), K(mds_range_), K(cb_arg_array_), K(first_part_scn_)); @@ -131,6 +134,7 @@ private: ObTxDataGuard tx_data_guard_; memtable::ObCallbackScope callbacks_; bool is_callbacked_; + bool is_dynamic_; ObTxMDSRange mds_range_; ObTxCbArgArray cb_arg_array_; share::SCN first_part_scn_; diff --git a/src/storage/tx/ob_trans_timer.h b/src/storage/tx/ob_trans_timer.h index 8a9819017..ea8e82714 100644 --- a/src/storage/tx/ob_trans_timer.h +++ b/src/storage/tx/ob_trans_timer.h @@ -129,7 +129,7 @@ private: DISALLOW_COPY_AND_ASSIGN(ObTransTimer); protected: // schedule timeout task precision. us - static const int64_t TRANS_TIMEOUT_TASK_PRECISION_US = 5000; + static const int64_t TRANS_TIMEOUT_TASK_PRECISION_US = 100 * 1000L; bool is_inited_; bool is_running_; diff --git a/src/storage/tx/ob_ts_mgr.h b/src/storage/tx/ob_ts_mgr.h index 8e2c2fbb0..ecebc9025 100644 --- a/src/storage/tx/ob_ts_mgr.h +++ b/src/storage/tx/ob_ts_mgr.h @@ -29,7 +29,7 @@ #include "ob_ts_worker.h" #include "ob_location_adapter.h" -#define REFRESH_GTS_INTERVEL_US (100 * 1000) +#define REFRESH_GTS_INTERVEL_US (500 * 1000) namespace oceanbase { diff --git a/src/storage/tx/ob_tx_log.cpp b/src/storage/tx/ob_tx_log.cpp index 17d40ced4..a7907817c 100644 --- a/src/storage/tx/ob_tx_log.cpp +++ b/src/storage/tx/ob_tx_log.cpp @@ -542,6 +542,8 @@ void ObTxRedoLog::reset_mutator_buf() mutator_size_ = -1; } +//TODO: if ob_admin_dump is called by others and clog is encrypted, +// unused_encrypt_info can work. This may be perfected in the future. int ObTxRedoLog::ob_admin_dump(memtable::ObMemtableMutatorIterator *iter_ptr, ObAdminMutatorStringArg &arg, palf::block_id_t block_id, @@ -552,6 +554,8 @@ int ObTxRedoLog::ob_admin_dump(memtable::ObMemtableMutatorIterator *iter_ptr, { int ret = OB_SUCCESS; int64_t pos = 0; + transaction::ObCLogEncryptInfo unused_encrypt_info; + unused_encrypt_info.init(); arg.log_stat_->tx_redo_log_size_ += get_serialize_size(); if (OB_ISNULL(iter_ptr) || OB_ISNULL(arg.writer_ptr_) || OB_ISNULL(arg.buf_) @@ -560,7 +564,7 @@ int ObTxRedoLog::ob_admin_dump(memtable::ObMemtableMutatorIterator *iter_ptr, TRANS_LOG(WARN, "invalid argument", KP(iter_ptr), KP(arg.writer_ptr_), KP(arg.buf_), KP(mutator_buf_), KP(replay_mutator_buf_)); } else if (OB_FAIL(iter_ptr->deserialize(replay_mutator_buf_, mutator_size_, pos, - ctx_redo_info_.clog_encrypt_info_))) { + unused_encrypt_info))) { TRANS_LOG(WARN, "deserialize replay_mutator_buf_ failed", K(ret)); } else { bool has_output = false; @@ -587,7 +591,8 @@ int ObTxRedoLog::ob_admin_dump(memtable::ObMemtableMutatorIterator *iter_ptr, //fill info in buf } bool has_dumped_meta_info = false; - while (OB_SUCC(iter_ptr->iterate_next_row())) { + memtable::ObEncryptRowBuf unused_row_buf; + while (OB_SUCC(iter_ptr->iterate_next_row(unused_row_buf, unused_encrypt_info))) { // arg.writer_ptr_->start_object(); if (arg.filter_.is_tablet_id_valid()) { if (arg.filter_.get_tablet_id() != iter_ptr->get_row_head().tablet_id_) { diff --git a/src/storage/tx/ob_tx_log.h b/src/storage/tx/ob_tx_log.h index 669fd3872..31c6ec0af 100644 --- a/src/storage/tx/ob_tx_log.h +++ b/src/storage/tx/ob_tx_log.h @@ -20,7 +20,7 @@ #include "storage/tx/ob_tx_serialization.h" #include "storage/tx/ob_tx_big_segment_buf.h" #include "storage/tx/ob_trans_factory.h" -#include "storage/tx/ob_clog_encrypter.h" +#include "storage/tx/ob_clog_encrypt_info.h" #include "share/ob_admin_dump_helper.h" #include "logservice/ob_log_base_type.h" #include "logservice/ob_log_base_header.h" @@ -191,23 +191,25 @@ private: class ObTxRedoLogTempRef { public: - ObTxRedoLogTempRef() : encrypt_info_() { encrypt_info_.init(); } + ObTxRedoLogTempRef() {} public: - ObCLogEncryptInfo encrypt_info_; + // remove member variable ObCLogEncryptInfo encrypt_info_ + // there may be added other variables in the future }; struct ObCtxRedoInfo { - ObCtxRedoInfo(ObCLogEncryptInfo &encrypt_info, const int64_t cluster_version) - : clog_encrypt_info_(encrypt_info), cluster_version_(cluster_version) + ObCtxRedoInfo(const int64_t cluster_version) + : cluster_version_(cluster_version) { }; int before_serialize(); ObTxSerCompatByte compat_bytes_; // serialize before mutator_buf by fixed length - ObCLogEncryptInfo &clog_encrypt_info_; uint64_t cluster_version_; + // remove member variable ObCLogEncryptInfo encrypt_info_ + // there may be added other variables in the future OB_UNIS_VERSION(1); }; @@ -221,14 +223,14 @@ class ObTxRedoLog public: ObTxRedoLog(ObTxRedoLogTempRef &temp_ref) : mutator_buf_(nullptr), replay_mutator_buf_(nullptr), mutator_size_(-1), - ctx_redo_info_(temp_ref.encrypt_info_, 0) + ctx_redo_info_(0) // (ctx_redo_info_.clog_encrypt_info_)(temp_ref.encrypt_info_), (ctx_redo_info_.cluster_version_)(0) { before_serialize(); } - ObTxRedoLog(ObCLogEncryptInfo &encrypt_info, const int64_t &log_no, const uint64_t &cluster_version) + ObTxRedoLog(const int64_t &log_no, const uint64_t &cluster_version) : mutator_buf_(nullptr), replay_mutator_buf_(nullptr), mutator_size_(-1), - ctx_redo_info_(encrypt_info, cluster_version) + ctx_redo_info_(cluster_version) // (ctx_redo_info_.clog_encrypt_info_)(encrypt_info),(ctx_redo_info_.cluster_version_)(cluster_version) { before_serialize(); @@ -237,7 +239,6 @@ public: char *get_mutator_buf() { return mutator_buf_; } const char *get_replay_mutator_buf() const { return replay_mutator_buf_; } const int64_t &get_mutator_size() const { return mutator_size_; } - const ObCLogEncryptInfo &get_clog_encrypt_info() const { return ctx_redo_info_.clog_encrypt_info_; } const uint64_t &get_cluster_version() const { return ctx_redo_info_.cluster_version_; } //------------ Only invoke in ObTxLogBlock diff --git a/src/storage/tx/ob_tx_replay_executor.cpp b/src/storage/tx/ob_tx_replay_executor.cpp index b55818ab4..7605d1718 100644 --- a/src/storage/tx/ob_tx_replay_executor.cpp +++ b/src/storage/tx/ob_tx_replay_executor.cpp @@ -524,6 +524,9 @@ int ObTxReplayExecutor::replay_redo_in_memtable_(ObTxRedoLog &redo) ObMutatorRowHeader row_head; uint8_t meta_flag = 0; + ObCLogEncryptInfo encrypt_info; + encrypt_info.init(); + if (OB_ISNULL(mmi_ptr_)) { if (nullptr == (mmi_ptr_ = static_cast( @@ -540,7 +543,7 @@ int ObTxReplayExecutor::replay_redo_in_memtable_(ObTxRedoLog &redo) if (OB_FAIL(ret)) { } else if (OB_FAIL(mmi_ptr_->deserialize(redo.get_replay_mutator_buf(), redo.get_mutator_size(), - pos, redo.get_clog_encrypt_info())) + pos, encrypt_info)) || redo.get_mutator_size() != pos) { TRANS_LOG(WARN, "[Replay Tx] deserialize fail or pos does not match data_len", K(ret)); } else { @@ -548,13 +551,13 @@ int ObTxReplayExecutor::replay_redo_in_memtable_(ObTxRedoLog &redo) ObEncryptRowBuf row_buf; while (OB_SUCC(ret)) { row_head.reset(); - if (OB_FAIL(mmi_ptr_->iterate_next_row())) { + if (OB_FAIL(mmi_ptr_->iterate_next_row(row_buf, encrypt_info))) { if (OB_ITER_END != ret) { TRANS_LOG(WARN, "[Replay Tx] iterate_next_row failed", K(ret)); } } else if (FALSE_IT(row_head = mmi_ptr_->get_row_head())) { // do nothing - } else if (OB_FAIL(replay_one_row_in_memtable_(row_head, mmi_ptr_, row_buf))) { + } else if (OB_FAIL(replay_one_row_in_memtable_(row_head, mmi_ptr_))) { if (OB_MINOR_FREEZE_NOT_ALLOW == ret) { if (TC_REACH_TIME_INTERVAL(1000 * 1000)) { TRANS_LOG(WARN, "[Replay Tx] cannot create more memtable", K(ret), @@ -592,8 +595,7 @@ int ObTxReplayExecutor::replay_redo_in_memtable_(ObTxRedoLog &redo) } int ObTxReplayExecutor::replay_one_row_in_memtable_(ObMutatorRowHeader &row_head, - memtable::ObMemtableMutatorIterator *mmi_ptr, - memtable::ObEncryptRowBuf &row_buf) + memtable::ObMemtableMutatorIterator *mmi_ptr) { int ret = OB_SUCCESS; lib::Worker::CompatMode mode; @@ -632,7 +634,7 @@ int ObTxReplayExecutor::replay_one_row_in_memtable_(ObMutatorRowHeader &row_head lib::CompatModeGuard compat_guard(mode); switch (row_head.mutator_type_) { case MutatorType::MUTATOR_ROW: { - if (OB_FAIL(replay_row_(storeCtx, tablet, mmi_ptr_, row_buf)) && OB_ITER_END != ret) { + if (OB_FAIL(replay_row_(storeCtx, tablet, mmi_ptr_)) && OB_ITER_END != ret) { if (OB_NO_NEED_UPDATE != ret && OB_MINOR_FREEZE_NOT_ALLOW != ret) { TRANS_LOG(WARN, "[Replay Tx] replay row failed.", K(ret), K(mt_ctx_), K(row_head.tablet_id_)); @@ -649,7 +651,7 @@ int ObTxReplayExecutor::replay_one_row_in_memtable_(ObMutatorRowHeader &row_head break; } case MutatorType::MUTATOR_TABLE_LOCK: { - if (OB_FAIL(replay_lock_(storeCtx, tablet, mmi_ptr_, row_buf)) && OB_ITER_END != ret) { + if (OB_FAIL(replay_lock_(storeCtx, tablet, mmi_ptr_)) && OB_ITER_END != ret) { TRANS_LOG(WARN, "[Replay Tx] replay lock failed.", K(ret), K(mt_ctx_), K(row_head.tablet_id_)); } else { @@ -684,8 +686,7 @@ int ObTxReplayExecutor::prepare_memtable_replay_(ObStorageTableGuard &w_guard, int ObTxReplayExecutor::replay_row_(storage::ObStoreCtx &store_ctx, ObTablet *tablet, - memtable::ObMemtableMutatorIterator *mmi_ptr, - memtable::ObEncryptRowBuf &row_buf) + memtable::ObMemtableMutatorIterator *mmi_ptr) { int ret = OB_SUCCESS; common::ObTimeGuard timeguard("replay_row_in_memtable", 10 * 1000); @@ -710,7 +711,7 @@ int ObTxReplayExecutor::replay_row_(storage::ObStoreCtx &store_ctx, TRANS_LOG(WARN, "[Replay Tx] this is not a ObMemtable", K(ret), KP(mem_ptr), KPC(mem_ptr), KP(mmi_ptr)); } else if (FALSE_IT(timeguard.click("get_memtable"))) { - } else if (OB_FAIL(data_mem_ptr->replay_row(store_ctx, mmi_ptr, row_buf))) { + } else if (OB_FAIL(data_mem_ptr->replay_row(store_ctx, mmi_ptr))) { TRANS_LOG(WARN, "[Replay Tx] replay row error", K(ret)); } else if (OB_FAIL(data_mem_ptr->set_max_end_scn(log_ts_ns_))) { // for freeze log_ts , may be TRANS_LOG(WARN, "[Replay Tx] set memtable max end log ts failed", K(ret), KP(data_mem_ptr)); @@ -731,12 +732,10 @@ int ObTxReplayExecutor::replay_row_(storage::ObStoreCtx &store_ctx, int ObTxReplayExecutor::replay_lock_(storage::ObStoreCtx &store_ctx, ObTablet *tablet, - memtable::ObMemtableMutatorIterator *mmi_ptr, - memtable::ObEncryptRowBuf &row_buf) + memtable::ObMemtableMutatorIterator *mmi_ptr) { // TODO: yanyuan.cxf lock is not encrypted. common::ObTimeGuard timeguard("replay_row_in_lock_memtable", 10 * 1000); - UNUSED(row_buf); int ret = OB_SUCCESS; ObTableHandleV2 handle; ObLockMemtable *memtable = nullptr; diff --git a/src/storage/tx/ob_tx_replay_executor.h b/src/storage/tx/ob_tx_replay_executor.h index 033dc0423..701388044 100644 --- a/src/storage/tx/ob_tx_replay_executor.h +++ b/src/storage/tx/ob_tx_replay_executor.h @@ -108,18 +108,15 @@ private: int replay_redo_in_memtable_(ObTxRedoLog &redo); virtual int replay_one_row_in_memtable_(memtable::ObMutatorRowHeader& row_head, - memtable::ObMemtableMutatorIterator *mmi_ptr, - memtable::ObEncryptRowBuf &row_buf); + memtable::ObMemtableMutatorIterator *mmi_ptr); int prepare_memtable_replay_(storage::ObStorageTableGuard &w_guard, memtable::ObIMemtable *&mem_ptr); int replay_row_(storage::ObStoreCtx &store_ctx, storage::ObTablet *tablet, - memtable::ObMemtableMutatorIterator *mmi_ptr, - memtable::ObEncryptRowBuf &row_buf); + memtable::ObMemtableMutatorIterator *mmi_ptr); int replay_lock_(storage::ObStoreCtx &store_ctx, storage::ObTablet *tablet, - memtable::ObMemtableMutatorIterator *mmi_ptr, - memtable::ObEncryptRowBuf &row_buf); + memtable::ObMemtableMutatorIterator *mmi_ptr); int get_compat_mode_(const ObTabletID &tablet_id, lib::Worker::CompatMode &mode); bool can_replay() const; diff --git a/src/storage/tx_storage/ob_access_service.cpp b/src/storage/tx_storage/ob_access_service.cpp index f83b7d174..61c49e404 100644 --- a/src/storage/tx_storage/ob_access_service.cpp +++ b/src/storage/tx_storage/ob_access_service.cpp @@ -1174,10 +1174,10 @@ int ObAccessService::audit_tablet_opt_dml_stat( const int64_t affected_rows) { int ret = OB_SUCCESS; - static __thread int64_t last_access_ts = 0; - if (!GCONF.enable_defensive_check() && ObClockGenerator::getClock() - last_access_ts < 1000000) { + //static __thread int64_t last_access_ts = 0; + //if (!GCONF.enable_defensive_check() && ObClockGenerator::getClock() - last_access_ts < 1000000) { // do nothing - } else if (OB_ISNULL(dml_param.table_param_)) { + if (OB_ISNULL(dml_param.table_param_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret), K(dml_param.table_param_)); } else if (dml_stat_type == ObOptDmlStatType::TABLET_OPT_INSERT_STAT || @@ -1199,7 +1199,7 @@ int ObAccessService::audit_tablet_opt_dml_stat( } else { LOG_TRACE("succeed to update dml stat local cache", K(dml_stat)); } - last_access_ts = ObClockGenerator::getClock(); + //last_access_ts = ObClockGenerator::getClock(); } return ret; } diff --git a/src/storage/tx_storage/ob_checkpoint_service.cpp b/src/storage/tx_storage/ob_checkpoint_service.cpp index 56f215403..95c2afceb 100644 --- a/src/storage/tx_storage/ob_checkpoint_service.cpp +++ b/src/storage/tx_storage/ob_checkpoint_service.cpp @@ -357,7 +357,7 @@ void ObCheckPointService::ObCheckClogDiskUsageTask::runTimerTask() } } - if (OB_FAIL(checkpoint_service_.flush_if_need_(need_flush))) { + if (need_flush && OB_FAIL(checkpoint_service_.flush_if_need_(need_flush))) { STORAGE_LOG(ERROR, "flush if needed failed", K(ret), K(need_flush)); } } diff --git a/src/storage/tx_storage/ob_ls_map.cpp b/src/storage/tx_storage/ob_ls_map.cpp index 202c407be..7b157b9fd 100644 --- a/src/storage/tx_storage/ob_ls_map.cpp +++ b/src/storage/tx_storage/ob_ls_map.cpp @@ -17,6 +17,7 @@ #include "lib/ob_errno.h" #include "storage/ls/ob_ls.h" #include "storage/tx_storage/ob_ls_handle.h" +#include "storage/tx_storage/ob_ls_service.h" using namespace oceanbase::share; using namespace oceanbase::common; @@ -321,6 +322,30 @@ int ObLSMap::get_ls(const share::ObLSID &ls_id, return ret; } +int ObLSMap::get_all_ls_id(ObIArray &ls_id_array) +{ + int ret = OB_SUCCESS; + ObLS *ls = NULL; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObLSMap not init", K(ret)); + } else { + for (int64_t bucket_idx = 0 ; OB_SUCC(ret) && bucket_idx < BUCKETS_CNT; ++bucket_idx) { + ObQSyncLockReadGuard bucket_guard(buckets_lock_[bucket_idx]); + ls = ls_buckets_[bucket_idx]; + while (OB_SUCC(ret) && OB_NOT_NULL(ls)) { + if (OB_FAIL(ls_id_array.push_back(ls->get_ls_id()))) { + LOG_WARN("failed to push back ls id", K(ret), KP(ls)); + } else { + ls = static_cast(ls->next_); + } + } // end of while + } // end of for + } + return ret; +} + int ObLSMap::remove_duplicate_ls() { int ret = OB_SUCCESS; diff --git a/src/storage/tx_storage/ob_ls_map.h b/src/storage/tx_storage/ob_ls_map.h index d00489d80..bd238c757 100644 --- a/src/storage/tx_storage/ob_ls_map.h +++ b/src/storage/tx_storage/ob_ls_map.h @@ -48,6 +48,7 @@ public: // allow_multi_true is used during replay int add_ls(ObLS &ls); int del_ls(const share::ObLSID &ls_id); + int get_all_ls_id(ObIArray &ls_id_array); int get_ls(const share::ObLSID &ls_id, ObLSHandle &handle, ObLSGetMod mod) const; diff --git a/src/storage/tx_storage/ob_ls_service.cpp b/src/storage/tx_storage/ob_ls_service.cpp index 3a0aa4d23..df5c80a81 100644 --- a/src/storage/tx_storage/ob_ls_service.cpp +++ b/src/storage/tx_storage/ob_ls_service.cpp @@ -1183,6 +1183,19 @@ int ObLSService::get_ls_iter(common::ObSharedGuard &guard, ObLSGet return ret; } +int ObLSService::get_ls_ids(common::ObIArray &ls_id_array) +{ + int ret = OB_SUCCESS; + ls_id_array.reuse(); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_FAIL(ls_map_.get_all_ls_id(ls_id_array))) { + LOG_WARN("failed to get all ls id", K(ret)); + } + return ret; +} + int ObLSService::create_tablet(const obrpc::ObBatchCreateTabletArg &batch_arg, obrpc::ObCreateTabletBatchRes &result) { diff --git a/src/storage/tx_storage/ob_ls_service.h b/src/storage/tx_storage/ob_ls_service.h index 0052ae16f..486c4aac6 100644 --- a/src/storage/tx_storage/ob_ls_service.h +++ b/src/storage/tx_storage/ob_ls_service.h @@ -115,6 +115,9 @@ public: // use guard just like a pointer of ObLSIterator int get_ls_iter(common::ObSharedGuard &guard, ObLSGetMod mod); + // get all ls ids + int get_ls_ids(common::ObIArray &ls_id_array); + // tablet operation int create_tablet(const obrpc::ObBatchCreateTabletArg &batch_arg, obrpc::ObCreateTabletBatchRes &result); diff --git a/src/storage/tx_table/ob_tx_ctx_memtable.cpp b/src/storage/tx_table/ob_tx_ctx_memtable.cpp index c6e11a19a..f01c64630 100644 --- a/src/storage/tx_table/ob_tx_ctx_memtable.cpp +++ b/src/storage/tx_table/ob_tx_ctx_memtable.cpp @@ -123,13 +123,15 @@ int ObTxCtxMemtable::set(storage::ObStoreCtx &ctx, const uint64_t table_id, const storage::ObTableReadInfo &read_info, const common::ObIArray &columns, - const storage::ObStoreRow &row) + const storage::ObStoreRow &row, + const share::ObEncryptMeta *encrypt_meta) { UNUSED(ctx); UNUSED(table_id); UNUSED(read_info); UNUSED(columns); UNUSED(row); + UNUSED(encrypt_meta); return OB_NOT_SUPPORTED; } diff --git a/src/storage/tx_table/ob_tx_ctx_memtable.h b/src/storage/tx_table/ob_tx_ctx_memtable.h index fa6ad0c70..f790035d0 100644 --- a/src/storage/tx_table/ob_tx_ctx_memtable.h +++ b/src/storage/tx_table/ob_tx_ctx_memtable.h @@ -83,7 +83,8 @@ public: const uint64_t table_id, const storage::ObTableReadInfo &read_info, const common::ObIArray &columns, - const storage::ObStoreRow &row) override; + const storage::ObStoreRow &row, + const share::ObEncryptMeta *encrypt_meta) override; virtual int lock(storage::ObStoreCtx &ctx, const uint64_t table_id, diff --git a/src/storage/tx_table/ob_tx_data_memtable.cpp b/src/storage/tx_table/ob_tx_data_memtable.cpp index a63e12d3b..07066a27e 100644 --- a/src/storage/tx_table/ob_tx_data_memtable.cpp +++ b/src/storage/tx_table/ob_tx_data_memtable.cpp @@ -1225,7 +1225,8 @@ int ObTxDataMemtable::set(storage::ObStoreCtx &ctx, const uint64_t table_id, const storage::ObTableReadInfo &read_info, const common::ObIArray &columns, - const storage::ObStoreRow &row) + const storage::ObStoreRow &row, + const share::ObEncryptMeta *encrypt_meta) { int ret = OB_NOT_SUPPORTED; UNUSED(ctx); @@ -1233,6 +1234,7 @@ int ObTxDataMemtable::set(storage::ObStoreCtx &ctx, UNUSED(read_info); UNUSED(columns); UNUSED(row); + UNUSED(encrypt_meta); return ret; } diff --git a/src/storage/tx_table/ob_tx_data_memtable.h b/src/storage/tx_table/ob_tx_data_memtable.h index 93fd46b34..09a172686 100644 --- a/src/storage/tx_table/ob_tx_data_memtable.h +++ b/src/storage/tx_table/ob_tx_data_memtable.h @@ -291,7 +291,8 @@ public: /* derived from ObIMemtable */ const uint64_t table_id, const storage::ObTableReadInfo &read_info, const common::ObIArray &columns, - const storage::ObStoreRow &row) override; + const storage::ObStoreRow &row, + const share::ObEncryptMeta *encrypt_meta) override; // not supported virtual int lock(storage::ObStoreCtx &ctx, const uint64_t table_id, diff --git a/tools/deploy/mysql_test/test_suite/join/r/mysql/anti_semi_join.result b/tools/deploy/mysql_test/test_suite/join/r/mysql/anti_semi_join.result index c2d259dbc..fb4474f98 100644 --- a/tools/deploy/mysql_test/test_suite/join/r/mysql/anti_semi_join.result +++ b/tools/deploy/mysql_test/test_suite/join/r/mysql/anti_semi_join.result @@ -1,5 +1,6 @@ result_format: 4 explain_protocol: 2 +alter system set _enable_newsort = true; drop table if exists xy_t1, xy_t2; create table xy_t1(c1 int, c2 int); create table xy_t2(c1 int, c2 int); diff --git a/tools/deploy/mysql_test/test_suite/join/t/anti_semi_join.test b/tools/deploy/mysql_test/test_suite/join/t/anti_semi_join.test index ccaada1f7..b828d002f 100644 --- a/tools/deploy/mysql_test/test_suite/join/t/anti_semi_join.test +++ b/tools/deploy/mysql_test/test_suite/join/t/anti_semi_join.test @@ -12,6 +12,13 @@ set @@session.explicit_defaults_for_timestamp=off; --result_format 4 --explain_protocol 2 +connect (syscon, $OBMYSQL_MS0,admin,$OBMYSQL_PWD,test,$OBMYSQL_PORT); +connection syscon; +alter system set _enable_newsort = true; +--sleep 3 +connection default; +--sleep 3 + --disable_warnings drop table if exists xy_t1, xy_t2; --enable_warnings diff --git a/unittest/libobcdc/log_generator.h b/unittest/libobcdc/log_generator.h index 20baafd18..c96c5f5c7 100644 --- a/unittest/libobcdc/log_generator.h +++ b/unittest/libobcdc/log_generator.h @@ -228,7 +228,7 @@ int ObTxLogBlockBuilder::fill_redo(ObTxRedoLog &redo_log) if (OB_FAIL(mmw.append_row_buf(row_str.ptr(), row_str.length()))) { LOG_ERROR("append_row failed", KR(ret)); - } else if (OB_FAIL(mmw.serialize(tmp_flag, mutator_pos))) { + } else if (OB_FAIL(mmw.serialize(tmp_flag, mutator_pos, encrypt_info))) { LOG_ERROR("serialize memtable_mutator failed", KR(ret)); } else if (OB_FAIL(tx_log_block_.finish_mutator_buf(redo_log, mutator_pos))) { LOG_ERROR("finish_mutator_buf failed", KR(ret), K_(tx_log_block), K(redo_log)); diff --git a/unittest/storage/memtable/mvcc/test_mvcc_callback.cpp b/unittest/storage/memtable/mvcc/test_mvcc_callback.cpp index 5b5f288a2..7c6309151 100644 --- a/unittest/storage/memtable/mvcc/test_mvcc_callback.cpp +++ b/unittest/storage/memtable/mvcc/test_mvcc_callback.cpp @@ -1149,6 +1149,20 @@ TEST_F(TestTxCallbackList, checksum_all_and_tx_end_test) { namespace memtable { +void ObMemtableCtx::callback_free(ObITransCallback *cb) +{ + if (OB_ISNULL(cb)) { + TRANS_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "cb is null, unexpected error", KP(cb), K(*this)); + } else if (cb->is_table_lock_callback()) { + free_table_lock_callback(cb); + } else { + ATOMIC_INC(&callback_free_count_); + TRANS_LOG(DEBUG, "callback release succ", KP(cb), K(*this), K(lbt())); + ctx_cb_allocator_.free(cb); + cb = NULL; + } +} + int ObTxCallbackList::remove_callbacks_for_fast_commit(bool &has_remove) { int ret = OB_SUCCESS; diff --git a/unittest/storage/mockcontainer/mock_ob_server.cpp b/unittest/storage/mockcontainer/mock_ob_server.cpp index efcdda5d4..adf10381e 100644 --- a/unittest/storage/mockcontainer/mock_ob_server.cpp +++ b/unittest/storage/mockcontainer/mock_ob_server.cpp @@ -248,8 +248,6 @@ int MockObServer::init_multi_tenant() STORAGE_LOG(WARN, "init multi_tenant failed", K(ret)); } else if (OB_SUCCESS != (ret = multi_tenant_.create_tenant_without_unit(OB_SYS_TENANT_ID, 3, 3))) { STORAGE_LOG(WARN, "add sys tenant failed", K(ret)); - } else if (OB_SUCCESS != (ret = multi_tenant_.create_tenant_without_unit(OB_ELECT_TENANT_ID, 5, 5))) { - STORAGE_LOG(WARN, "add election tenant failed", K(ret)); } else if (OB_SUCCESS != (ret = multi_tenant_.create_tenant_without_unit(OB_SERVER_TENANT_ID, 5, 5))) { STORAGE_LOG(WARN, "add election tenant failed", K(ret)); } else { diff --git a/unittest/storage/tx/it/tx_node.cpp b/unittest/storage/tx/it/tx_node.cpp index 6a3651b2f..4d0c151a1 100644 --- a/unittest/storage/tx/it/tx_node.cpp +++ b/unittest/storage/tx/it/tx_node.cpp @@ -597,6 +597,7 @@ int ObTxNode::write(ObTxDesc &tx, write_store_ctx.mvcc_acc_ctx_.tx_table_guard_.init(&fake_tx_table_); ObArenaAllocator allocator; ObTableReadInfo read_info; + const transaction::ObSerializeEncryptMeta *encrypt_meta = NULL; const int64_t schema_version = 100; read_info.init(allocator, schema_version, 1, false, columns_); ObStoreRow row; @@ -606,7 +607,7 @@ int ObTxNode::write(ObTxDesc &tx, row.row_val_.count_ = 2; row.flag_ = blocksstable::ObDmlFlag::DF_UPDATE; row.trans_id_.reset(); - OZ(memtable_->set(write_store_ctx, 1, read_info, columns_, row)); + OZ(memtable_->set(write_store_ctx, 1, read_info, columns_, row, encrypt_meta)); OZ(txs_.revert_store_ctx(write_store_ctx)); delete iter; return ret; @@ -639,6 +640,7 @@ int ObTxNode::write_one_row(ObStoreCtx& write_store_ctx, const int64_t key, cons ObArenaAllocator allocator; ObTableReadInfo read_info; + const transaction::ObSerializeEncryptMeta *encrypt_meta = NULL; const int64_t schema_version = 100; read_info.init(allocator, schema_version, 1, false, columns_); ObStoreRow row; @@ -646,7 +648,7 @@ int ObTxNode::write_one_row(ObStoreCtx& write_store_ctx, const int64_t key, cons row.flag_ = blocksstable::ObDmlFlag::DF_INSERT; row.row_val_.cells_ = cols; row.row_val_.count_ = 2; - OZ(memtable_->set(write_store_ctx, 1, read_info, columns_, row)); + OZ(memtable_->set(write_store_ctx, 1, read_info, columns_, row, encrypt_meta)); return ret; } diff --git a/unittest/storage/tx/mock_utils/basic_fake_define.h b/unittest/storage/tx/mock_utils/basic_fake_define.h index 22c34326a..16631891c 100644 --- a/unittest/storage/tx/mock_utils/basic_fake_define.h +++ b/unittest/storage/tx/mock_utils/basic_fake_define.h @@ -635,8 +635,7 @@ private: } int replay_one_row_in_memtable_(memtable::ObMutatorRowHeader& row_head, - memtable::ObMemtableMutatorIterator *mmi_ptr, - memtable::ObEncryptRowBuf &row_buf) override + memtable::ObMemtableMutatorIterator *mmi_ptr) override { int ret = OB_SUCCESS; storage::ObStoreCtx storeCtx; @@ -649,7 +648,7 @@ private: switch (row_head.mutator_type_) { case memtable::MutatorType::MUTATOR_ROW: { - if (OB_FAIL(memtable_->replay_row(storeCtx, mmi_ptr_, row_buf))) { + if (OB_FAIL(memtable_->replay_row(storeCtx, mmi_ptr_))) { TRANS_LOG(WARN, "[Replay Tx] replay row error", K(ret)); } else { TRANS_LOG(INFO, "[Replay Tx] replay row in memtable success"); diff --git a/unittest/storage/tx/ob_mock_tx_ctx.cpp b/unittest/storage/tx/ob_mock_tx_ctx.cpp index 5d2b545bb..665051e88 100644 --- a/unittest/storage/tx/ob_mock_tx_ctx.cpp +++ b/unittest/storage/tx/ob_mock_tx_ctx.cpp @@ -60,8 +60,6 @@ int MockObTxCtx::init(const ObLSID &ls_id, // ======================= mock trans ctx end ======================== } else if (OB_FAIL(init_log_cbs_(ls_id, trans_id))) { TRANS_LOG(WARN, "init log cbs failed", KR(ret), K(trans_id), K(ls_id)); - } else if (OB_FAIL(clog_encrypt_info_.init())) { - TRANS_LOG(WARN, "init clog encrypt info failed", K(ret), KPC(this), K(trans_id)); } if (OB_SUCC(ret)) { diff --git a/unittest/storage/tx/test_ob_tx_log.cpp b/unittest/storage/tx/test_ob_tx_log.cpp index 688c377fc..e2c4cf4dc 100644 --- a/unittest/storage/tx/test_ob_tx_log.cpp +++ b/unittest/storage/tx/test_ob_tx_log.cpp @@ -325,9 +325,7 @@ TEST_F(TestObTxLog, tx_log_body_redo) ObString TEST_MUTATOR_BUF("FFF"); int64_t mutator_pos = 0; - ObCLogEncryptInfo TEST_CLOG_ENCRYPT_INFO; - TEST_CLOG_ENCRYPT_INFO.init(); - ObTxRedoLog fill_redo(TEST_CLOG_ENCRYPT_INFO, TEST_LOG_NO, TEST_CLUSTER_VERSION); + ObTxRedoLog fill_redo(TEST_LOG_NO, TEST_CLUSTER_VERSION); ASSERT_EQ(OB_SUCCESS, fill_block.prepare_mutator_buf(fill_redo)); ASSERT_EQ(OB_SUCCESS, serialization::encode(fill_redo.get_mutator_buf(),