diff --git a/deps/oblib/src/common/ob_range.h b/deps/oblib/src/common/ob_range.h index 2daf63e9b..a9654f0f8 100644 --- a/deps/oblib/src/common/ob_range.h +++ b/deps/oblib/src/common/ob_range.h @@ -286,7 +286,6 @@ struct ObVersionRange OB_UNIS_VERSION(1); public: static const int64_t MIN_VERSION = 0; - static const int64_t MAX_VERSION = INT64_MAX; ObVersionRange(); OB_INLINE void reset(); @@ -310,7 +309,6 @@ struct ObNewVersionRange OB_UNIS_VERSION(1); public: static const int64_t MIN_VERSION = 0; - static const int64_t MAX_VERSION = INT64_MAX; ObNewVersionRange(); OB_INLINE void reset(); diff --git a/deps/oblib/src/lib/alloc/memory_dump.cpp b/deps/oblib/src/lib/alloc/memory_dump.cpp index bbea42deb..2e1f34caf 100644 --- a/deps/oblib/src/lib/alloc/memory_dump.cpp +++ b/deps/oblib/src/lib/alloc/memory_dump.cpp @@ -352,10 +352,11 @@ int label_stat(AChunk *chunk, ABlock *block, AObject *object, } else { hold = align_up2(chunk->alloc_bytes_ + ACHUNK_HEADER_SIZE, get_page_size()); } - void *label = &object->label_[0]; - void *end = memchr(label, '\0', sizeof(object->label_)); - int len = end ? (char*)end - (char*)label : sizeof(object->label_); - ObString str(len, (char*)label); + char label[AOBJECT_LABEL_SIZE + 1]; + STRNCPY(label, object->label_, sizeof(label)); + label[sizeof(label) - 1] = '\0'; + int len = strlen(label); + ObString str(len, label); LabelItem *litem = nullptr; LabelInfoItem *linfoitem = lmap.get(str); if (NULL != linfoitem) { diff --git a/deps/oblib/src/lib/alloc/memory_dump.h b/deps/oblib/src/lib/alloc/memory_dump.h index e531a00d3..0f621f910 100644 --- a/deps/oblib/src/lib/alloc/memory_dump.h +++ b/deps/oblib/src/lib/alloc/memory_dump.h @@ -102,7 +102,7 @@ struct LabelInfoItem void *block_; }; -typedef common::hash::ObHashMap LabelMap; +typedef common::hash::ObHashMap LabelMap; using lib::AChunk; using lib::ABlock; diff --git a/deps/oblib/src/lib/alloc/memory_sanity.cpp b/deps/oblib/src/lib/alloc/memory_sanity.cpp index d1ddb6b1b..1e1110593 100644 --- a/deps/oblib/src/lib/alloc/memory_sanity.cpp +++ b/deps/oblib/src/lib/alloc/memory_sanity.cpp @@ -49,13 +49,16 @@ void sanity_set_whitelist(const char *str) void memory_sanity_abort() { + if ('\0' == whitelist[0]) { + abort(); + } void *addrs[128]; int n_addr = backtrace(addrs, sizeof(addrs)/sizeof(addrs[0])); void *vip_addr = NULL; for (int i = 0; NULL == vip_addr && i < n_addr; i++) { - for (int j = 0; NULL == vip_addr && j < 8; j++) { + for (int j = 0; NULL == vip_addr && j < sizeof(vips)/sizeof(vips[0]); j++) { t_vip *vip = &vips[j]; - if (0 == strlen(vip->func_)) { + if ('\0' == vip->func_[0]) { break; } else if (0 == vip->min_addr_ || 0 == vip->max_addr_) { continue; @@ -81,9 +84,9 @@ void memory_sanity_abort() if (real_len < buf_len - pos) { pos += real_len; } - for (int i = 0; i < 8; i++) { + for (int i = 0; i < sizeof(vips)/sizeof(vips[0]); i++) { t_vip *vip = &vips[i]; - if (0 == strlen(vip->func_)) { + if ('\0' == vip->func_[0]) { break; } else if (strstr(func_name, vip->func_) != NULL) { strncpy(vip_func, func_name, sizeof(vip_func)); diff --git a/deps/oblib/src/lib/alloc/memory_sanity.h b/deps/oblib/src/lib/alloc/memory_sanity.h index b8c134223..774f17c84 100644 --- a/deps/oblib/src/lib/alloc/memory_sanity.h +++ b/deps/oblib/src/lib/alloc/memory_sanity.h @@ -72,7 +72,7 @@ static constexpr int64_t sanity_max_canonical_addr = 0x4f210376cf1c; static inline bool sanity_addr_in_range(const void *ptr) { - return (int64_t)ptr >= sanity_min_canonical_addr && (int64_t)ptr < sanity_max_canonical_addr; + return (int64_t)ptr < sanity_max_canonical_addr && (int64_t)ptr >= sanity_min_canonical_addr; } static inline void* sanity_to_shadow(const void *ptr) @@ -139,12 +139,12 @@ static inline void sanity_check_range(const void *ptr, ssize_t len) char *start_align = (char*)sanity_align_up((uint64_t)start, 8); char *end_align = (char*)sanity_align_down((uint64_t)end, 8); if (start_align > start && - (*(uint8_t*)sanity_to_shadow(start_align - 8) != 0x0 && - *(uint8_t*)sanity_to_shadow(start_align - 8) < (len + start - (start_align - 8)))) { + (*(int8_t*)sanity_to_shadow(start_align - 8) != 0x0 && + *(int8_t*)sanity_to_shadow(start_align - 8) < (len + start - (start_align - 8)))) { memory_sanity_abort(); } if (end_align >= start_align + 8) { - if (*(uint8_t*)sanity_to_shadow(start_align) != 0x0) { + if (*(int8_t*)sanity_to_shadow(start_align) != 0x0) { memory_sanity_abort(); } if (end_align > start_align + 8) { @@ -157,8 +157,8 @@ static inline void sanity_check_range(const void *ptr, ssize_t len) } } if (end_align < end && - (*(uint8_t*)sanity_to_shadow(end_align) != 0x0 && - *(uint8_t*)sanity_to_shadow(end_align) < (end - end_align))) { + (*(int8_t*)sanity_to_shadow(end_align) != 0x0 && + *(int8_t*)sanity_to_shadow(end_align) < (end - end_align))) { memory_sanity_abort(); } } diff --git a/deps/oblib/src/lib/allocator/ob_slice_alloc.h b/deps/oblib/src/lib/allocator/ob_slice_alloc.h index ffb8ebdca..dee840ae9 100644 --- a/deps/oblib/src/lib/allocator/ob_slice_alloc.h +++ b/deps/oblib/src/lib/allocator/ob_slice_alloc.h @@ -131,9 +131,11 @@ public: } public: uint32_t total() { return total_; } + int32_t stock() { return stock_; } + int32_t remain() { return stock_ > K ? stock_ - K : (stock_ < 0 ? stock_ + K : stock_); } bool acquire() { return dec_if_gt(K, K) > K; } bool release() { return faa(-K) > 0; } - bool recyle() { + bool recycle() { int32_t total = total_; return inc_if_lt(2 * K, -K + total) == -K + total; } @@ -271,13 +273,37 @@ public: LIB_LOG(INFO, "ObSliceAlloc init finished", K(bsize_), K(isize_), K(slice_limit_), KP(tmallocator_)); } ~ObSliceAlloc() { - tmallocator_ = NULL; + destroy(); } int init(const int size, const int block_size, BlockAlloc& block_alloc, const ObMemAttr& attr) { int ret = common::OB_SUCCESS; new(this)ObSliceAlloc(size, attr, block_size, block_alloc, NULL); return ret; } + void destroy() { + for(int i = MAX_ARENA_NUM - 1; i >= 0; i--) { + Arena& arena = arena_[i]; + Block* old_blk = arena.clear(); + if (NULL != old_blk) { + blk_ref_[ObBlockSlicer::hash((uint64_t)old_blk) % MAX_REF_NUM].sync(); + if (old_blk->release()) { + blk_list_.add(&old_blk->dlink_); + if (old_blk->recycle()) { + destroy_block(old_blk); + } else { + _LIB_LOG(ERROR, "there was memory leak, stock=%d, remain=%d", old_blk->stock(), old_blk->remain()); + } + } + } + } + ObDLink* dlink = nullptr; + if (OB_NOT_NULL(dlink = blk_list_.top())) { + Block* blk = CONTAINER_OF(dlink, Block, dlink_); + _LIB_LOG(ERROR, "there was memory leak, stock=%d, remain=%d", blk->stock(), blk->remain()); + } + tmallocator_ = NULL; + bsize_ = 0; + } void set_nway(int nway) { if (nway <= 0) { nway = 1; @@ -351,6 +377,7 @@ public: Block* blk = item->host_; #ifndef NDEBUG abort_unless(blk->get_slice_alloc() == this); + abort_unless(bsize_ != 0); #else if (this != blk->get_slice_alloc()) { LIB_LOG(ERROR, "blk is freed or alloced by different slice_alloc", K(this), K(blk->get_slice_alloc())); @@ -391,7 +418,7 @@ private: } void add_to_blist(Block* blk) { blk_list_.add(&blk->dlink_); - if (blk->recyle()) { + if (blk->recycle()) { destroy_block(blk); } } diff --git a/deps/oblib/src/lib/allocator/ob_vslice_alloc.h b/deps/oblib/src/lib/allocator/ob_vslice_alloc.h index 2aeb81eb3..d3d4be95e 100644 --- a/deps/oblib/src/lib/allocator/ob_vslice_alloc.h +++ b/deps/oblib/src/lib/allocator/ob_vslice_alloc.h @@ -25,6 +25,7 @@ namespace common extern ObBlockAllocMgr default_blk_alloc; class ObBlockVSlicer { + friend class ObVSliceAlloc; public: static const uint32_t ITEM_MAGIC_CODE = 0XCCEEDDF1; static const uint32_t ITEM_MAGIC_CODE_MASK = 0XFFFFFFF0; @@ -72,6 +73,7 @@ private: class ObVSliceAlloc : public common::ObIAllocator { + friend class ObBlockVSlicer; public: enum { MAX_ARENA_NUM = 32, DEFAULT_BLOCK_SIZE = OB_MALLOC_NORMAL_BLOCK_SIZE }; typedef ObBlockAllocMgr BlockAlloc; @@ -91,12 +93,31 @@ public: ObVSliceAlloc(): nway_(0), bsize_(0), blk_alloc_(default_blk_alloc) {} ObVSliceAlloc(const ObMemAttr &attr, int block_size = DEFAULT_BLOCK_SIZE, BlockAlloc &blk_alloc = default_blk_alloc) : nway_(1), bsize_(block_size), mattr_(attr), blk_alloc_(blk_alloc) {} - virtual ~ObVSliceAlloc() override {} + virtual ~ObVSliceAlloc() override { destroy(); } int init(int block_size, BlockAlloc& block_alloc, const ObMemAttr& attr) { int ret = OB_SUCCESS; new(this)ObVSliceAlloc(attr, block_size, block_alloc); return ret; } + void destroy() { + for(int i = MAX_ARENA_NUM - 1; i >= 0; i--) { + Arena& arena = arena_[i]; + Block* old_blk = arena.clear(); + if (NULL != old_blk) { + int64_t old_pos = INT64_MAX; + if (old_blk->freeze(old_pos)) { + arena.sync(); + if (old_blk->retire(old_pos)) { + destroy_block(old_blk); + } else { + // can not monitor all leak !!! + LIB_LOG(ERROR, "there was memory leak", K(old_blk->ref_)); + } + } + } + } + bsize_ = 0; + } void set_nway(int nway) { if (nway <= 0) { nway = 1; @@ -182,12 +203,13 @@ public: #ifdef OB_USE_ASAN ::free(p); #else - if (bsize_ > 0 && NULL != p) { + if (NULL != p) { Block::Item* item = (Block::Item*)p - 1; abort_unless(Block::ITEM_MAGIC_CODE == item->MAGIC_CODE_); Block* blk = item->host_; #ifndef NDEBUG abort_unless(blk->get_vslice_alloc() == this); + abort_unless(bsize_ != 0); #else if (this != blk->get_vslice_alloc()) { LIB_LOG(ERROR, "blk is freed or alloced by different vslice_alloc", K(this), K(blk->get_vslice_alloc())); diff --git a/deps/oblib/src/lib/checksum/ob_crc64.cpp b/deps/oblib/src/lib/checksum/ob_crc64.cpp index bab4b0ee8..ffdc22bd5 100644 --- a/deps/oblib/src/lib/checksum/ob_crc64.cpp +++ b/deps/oblib/src/lib/checksum/ob_crc64.cpp @@ -13,6 +13,8 @@ #include #include "lib/checksum/ob_crc64.h" #include "lib/ob_define.h" +#include "isa-l/crc64.h" +#include "isa-l/crc.h" namespace oceanbase { @@ -420,7 +422,7 @@ for RHEL4 support (GCC 3 doesn't support this instruction) */ #define crc32_sse42_byte crc = __crc32cb(crc, (uint8_t)*buf); len--, buf++ #endif /* defined(__GNUC__) && defined(__x86_64__) */ -inline static uint64_t crc64_sse42(uint64_t uCRC64, +uint64_t crc64_sse42(uint64_t uCRC64, const char *buf, int64_t len) { uint64_t crc = uCRC64; @@ -1102,6 +1104,14 @@ uint64_t fast_crc64_sse42_manually(uint64_t crc, const char *buf, int64_t len) return crc; } +//If the CPU is intel, ISA-L library for CRC can be used +uint64_t ob_crc64_isal(uint64_t uCRC64, const char* buf, int64_t cb) +{ + if (buf == NULL || cb <= 0){ + return uCRC64; + } + return crc32_iscsi((unsigned char*)(buf), cb, uCRC64); +} uint64_t crc64_sse42_dispatch(uint64_t crc, const char *buf, int64_t len) { @@ -1110,13 +1120,27 @@ uint64_t crc64_sse42_dispatch(uint64_t crc, const char *buf, int64_t len) uint32_t b = 0; uint32_t c = 0; uint32_t d = 0; - asm("cpuid": "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(1)); - if ((c & (1 << 20)) != 0) { - ob_crc64_sse42_func = &crc64_sse42; - _OB_LOG(INFO, "Use CPU crc32 instructs for crc64 calculate"); - } else { - ob_crc64_sse42_func = &fast_crc64_sse42_manually; - _OB_LOG(INFO, "Use manual crc32 table lookup for crc64 calculate"); + + uint32_t vendor_info[4]; + __asm__("mov $0x0, %eax\n\t"); + __asm__("cpuid\n\t"); + __asm__("mov %%ebx, %0\n\t":"=r" (vendor_info[0])); + __asm__("mov %%edx, %0\n\t":"=r" (vendor_info[1])); + __asm__("mov %%ecx, %0\n\t":"=r" (vendor_info[2])); + vendor_info[3]='\0'; + + if (strcmp((char*)vendor_info, "GenuineIntel") == 0) { + ob_crc64_sse42_func = &ob_crc64_isal; + _OB_LOG(WARN, "Use ISAL for crc64 calculate"); + } else{ + asm("cpuid" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(1)); + if ((c & (1 << 20)) != 0) { + ob_crc64_sse42_func = &crc64_sse42; + _OB_LOG(WARN, "Use CPU crc32 instructs for crc64 calculate"); + } else { + ob_crc64_sse42_func = &fast_crc64_sse42_manually; + _OB_LOG(WARN, "Use manual crc32 table lookup for crc64 calculate"); + } } #elif defined(__aarch64__) #if 1 diff --git a/deps/oblib/src/lib/checksum/ob_crc64.h b/deps/oblib/src/lib/checksum/ob_crc64.h index 8794202e9..d7ce4458a 100644 --- a/deps/oblib/src/lib/checksum/ob_crc64.h +++ b/deps/oblib/src/lib/checksum/ob_crc64.h @@ -68,6 +68,8 @@ inline uint64_t ob_crc64_sse42(const void *pv, int64_t cb) return (*ob_crc64_sse42_func)(0, static_cast(pv), cb); } +uint64_t ob_crc64_isal(uint64_t uCRC64, const char* buf, int64_t cb); +uint64_t crc64_sse42(uint64_t uCRC64, const char* buf, int64_t len); uint64_t crc64_sse42_manually(uint64_t crc, const char *buf, int64_t len); uint64_t fast_crc64_sse42_manually(uint64_t crc, const char *buf, int64_t len); diff --git a/deps/oblib/src/lib/hash/ob_linear_hash_map.h b/deps/oblib/src/lib/hash/ob_linear_hash_map.h index 18b7d9650..34ff84160 100644 --- a/deps/oblib/src/lib/hash/ob_linear_hash_map.h +++ b/deps/oblib/src/lib/hash/ob_linear_hash_map.h @@ -182,10 +182,18 @@ struct ShareMemMgrTag { }; /* Don't use this mode unless you know what it means. */ struct UniqueMemMgrTag { }; +// avoid allocator destructed before HashMap +template +struct ConstructGuard +{ +public: + ConstructGuard(); +}; template -class ObLinearHashMap +class ObLinearHashMap : public ConstructGuard { + friend class ConstructGuard; private: /* Entry. */ struct Node @@ -2372,6 +2380,12 @@ bool ObLinearHashMap::DoRemoveIfOnBkt::operator return true; } +template +ConstructGuard::ConstructGuard() +{ + auto& t = ObLinearHashMap::HashMapMemMgrCore::get_instance(); +} + } } diff --git a/deps/oblib/src/lib/json_type/ob_json_bin.cpp b/deps/oblib/src/lib/json_type/ob_json_bin.cpp index 93abbb365..cf9896171 100644 --- a/deps/oblib/src/lib/json_type/ob_json_bin.cpp +++ b/deps/oblib/src/lib/json_type/ob_json_bin.cpp @@ -1322,11 +1322,16 @@ int ObJsonBin::deserialize_json_object_v0(const char *data, uint64_t length, ObJ } else { // TODO if with key dict, read key from dict // to consider, add option to controll need alloc or not - void *key_buf = allocator_->alloc(key_len); - if (key_buf == NULL) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to alloc memory for data buf", K(ret)); - } else { + void *key_buf = nullptr; + if (key_len > 0) { + key_buf = allocator_->alloc(key_len); + if (key_buf == NULL) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memory for data buf", K(ret)); + } + } + + if (OB_SUCC(ret)) { MEMCPY(key_buf, data + key_offset, key_len); ObString key(key_len, reinterpret_cast(key_buf)); const char *val = data + value_offset; diff --git a/deps/oblib/src/lib/json_type/ob_json_path.cpp b/deps/oblib/src/lib/json_type/ob_json_path.cpp index 335d7060d..ec838b6f1 100644 --- a/deps/oblib/src/lib/json_type/ob_json_path.cpp +++ b/deps/oblib/src/lib/json_type/ob_json_path.cpp @@ -1049,7 +1049,7 @@ int ObJsonPath::parse_name_with_rapidjson(char*& str, uint64_t& len) ObJsonString *val = static_cast(dom); len = val->value().length(); str = static_cast (allocator_->alloc(len)); - if (OB_ISNULL(str)) { + if (len > 0 && OB_ISNULL(str)) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to allocate memory for member_name.", K(ret), K(len), K(val->value())); diff --git a/deps/oblib/src/lib/json_type/ob_json_tree.cpp b/deps/oblib/src/lib/json_type/ob_json_tree.cpp index 9bdb1e16b..87c1372fe 100644 --- a/deps/oblib/src/lib/json_type/ob_json_tree.cpp +++ b/deps/oblib/src/lib/json_type/ob_json_tree.cpp @@ -628,9 +628,6 @@ int ObJsonObject::add(const common::ObString &key, ObJsonNode *value) if (OB_ISNULL(value)) { // check param ret = OB_INVALID_ARGUMENT; LOG_WARN("param value is NULL", K(ret)); - } else if (key.empty()) { - ret = OB_ERR_JSON_DOCUMENT_NULL_KEY; - LOG_WARN("key is NULL", K(ret)); } else { value->set_parent(this); ObJsonObjectPair pair(key, value); diff --git a/deps/oblib/src/lib/lock/ob_bucket_lock.h b/deps/oblib/src/lib/lock/ob_bucket_lock.h index e81be24ef..26709d441 100644 --- a/deps/oblib/src/lib/lock/ob_bucket_lock.h +++ b/deps/oblib/src/lib/lock/ob_bucket_lock.h @@ -68,7 +68,7 @@ private: class ObBucketRLockGuard final { public: - ObBucketRLockGuard(ObBucketLock &lock, const uint64_t bucket_index) + [[nodiscard]] ObBucketRLockGuard(ObBucketLock &lock, const uint64_t bucket_index) : lock_(lock), index_(bucket_index), ret_(OB_SUCCESS), @@ -109,7 +109,7 @@ private: class ObBucketWLockGuard final { public: - ObBucketWLockGuard(ObBucketLock &lock, const uint64_t bucket_index) + [[nodiscard]] ObBucketWLockGuard(ObBucketLock &lock, const uint64_t bucket_index) : lock_(lock), index_(bucket_index), ret_(OB_SUCCESS), @@ -150,7 +150,7 @@ private: class ObBucketWLockAllGuard final { public: - ObBucketWLockAllGuard(ObBucketLock &lock) + [[nodiscard]] ObBucketWLockAllGuard(ObBucketLock &lock) : lock_(lock), ret_(OB_SUCCESS), lock_start_ts_(0) @@ -189,7 +189,7 @@ private: class ObBucketTryWLockAllGuard final { public: - ObBucketTryWLockAllGuard(ObBucketLock &lock) + [[nodiscard]] ObBucketTryWLockAllGuard(ObBucketLock &lock) : lock_(lock), ret_(OB_SUCCESS), lock_start_ts_(0) @@ -234,7 +234,7 @@ private: class ObBucketTryRLockAllGuard final { public: - ObBucketTryRLockAllGuard(ObBucketLock &lock) + [[nodiscard]] ObBucketTryRLockAllGuard(ObBucketLock &lock) : lock_(lock), ret_(OB_SUCCESS), lock_start_ts_(0) @@ -279,7 +279,7 @@ private: class ObMultiBucketLockGuard final { public: - ObMultiBucketLockGuard(ObBucketLock &lock, const bool is_write_lock); + [[nodiscard]] ObMultiBucketLockGuard(ObBucketLock &lock, const bool is_write_lock); ~ObMultiBucketLockGuard(); int lock_multi_buckets(ObIArray &hash_array); private: @@ -293,7 +293,7 @@ private: class ObBucketHashRLockGuard final { public: - ObBucketHashRLockGuard(ObBucketLock &lock, const uint64_t hash_value) + [[nodiscard]] ObBucketHashRLockGuard(ObBucketLock &lock, const uint64_t hash_value) : guard_(lock, lock.get_bucket_idx(hash_value)) { } @@ -307,7 +307,7 @@ private: class ObBucketHashWLockGuard final { public: - ObBucketHashWLockGuard(ObBucketLock &lock, const uint64_t hash_value) + [[nodiscard]] ObBucketHashWLockGuard(ObBucketLock &lock, const uint64_t hash_value) : guard_(lock, lock.get_bucket_idx(hash_value)) { } diff --git a/deps/oblib/src/lib/lock/ob_drw_lock.h b/deps/oblib/src/lib/lock/ob_drw_lock.h index 4f6278e0b..ad9b74a75 100644 --- a/deps/oblib/src/lib/lock/ob_drw_lock.h +++ b/deps/oblib/src/lib/lock/ob_drw_lock.h @@ -34,7 +34,7 @@ public: class RDLockGuard { public: - explicit RDLockGuard(DRWLock &rwlock): rwlock_(rwlock), ret_(OB_SUCCESS) + [[nodiscard]] explicit RDLockGuard(DRWLock &rwlock): rwlock_(rwlock), ret_(OB_SUCCESS) { if (OB_UNLIKELY(OB_SUCCESS != (ret_ = rwlock_.rdlock()))) { COMMON_LOG(WARN, "Fail to read lock, ", K_(ret)); @@ -58,7 +58,7 @@ public: class WRLockGuard { public: - explicit WRLockGuard(DRWLock &rwlock): rwlock_(rwlock), ret_(OB_SUCCESS) + [[nodiscard]] explicit WRLockGuard(DRWLock &rwlock): rwlock_(rwlock), ret_(OB_SUCCESS) { if (OB_UNLIKELY(OB_SUCCESS != (ret_ = rwlock_.wrlock()))) { COMMON_LOG(WARN, "Fail to write lock, ", K_(ret)); diff --git a/deps/oblib/src/lib/lock/ob_latch.h b/deps/oblib/src/lib/lock/ob_latch.h index 2e9bb41a3..d9930ad46 100644 --- a/deps/oblib/src/lib/lock/ob_latch.h +++ b/deps/oblib/src/lib/lock/ob_latch.h @@ -364,7 +364,7 @@ private: class ObLatchMutexGuard { public: - ObLatchMutexGuard(ObLatchMutex &lock, const uint32_t latch_id) + [[nodiscard]] ObLatchMutexGuard(ObLatchMutex &lock, const uint32_t latch_id) : lock_(lock), ret_(OB_SUCCESS) { if (OB_UNLIKELY(OB_SUCCESS != (ret_ = lock_.lock(latch_id)))) { @@ -390,7 +390,7 @@ private: class ObLatchRGuard { public: - ObLatchRGuard(ObLatch &lock, const uint32_t latch_id) + [[nodiscard]] ObLatchRGuard(ObLatch &lock, const uint32_t latch_id) : lock_(lock), ret_(OB_SUCCESS) { @@ -418,7 +418,7 @@ private: class ObLatchWGuard { public: - ObLatchWGuard(ObLatch &lock, const uint32_t latch_id) + [[nodiscard]] ObLatchWGuard(ObLatch &lock, const uint32_t latch_id) : lock_(lock), ret_(OB_SUCCESS) { diff --git a/deps/oblib/src/lib/lock/ob_lock_guard.h b/deps/oblib/src/lib/lock/ob_lock_guard.h index e4d80cac1..e907ba4bb 100644 --- a/deps/oblib/src/lib/lock/ob_lock_guard.h +++ b/deps/oblib/src/lib/lock/ob_lock_guard.h @@ -25,7 +25,7 @@ template class ObLockGuard { public: - explicit ObLockGuard(LockT &lock); + [[nodiscard]] explicit ObLockGuard(LockT &lock); ~ObLockGuard(); inline int get_ret() const { return ret_; } private: diff --git a/deps/oblib/src/lib/lock/ob_rwlock.h b/deps/oblib/src/lib/lock/ob_rwlock.h index 7c94c6937..9a785f5d9 100644 --- a/deps/oblib/src/lib/lock/ob_rwlock.h +++ b/deps/oblib/src/lib/lock/ob_rwlock.h @@ -23,7 +23,7 @@ template class ObLockGuardBase { public: - ObLockGuardBase(const T& lock, bool block = true) : lock_(lock) + [[nodiscard]] ObLockGuardBase(const T& lock, bool block = true) : lock_(lock) { acquired_ = !(block ? lock_.lock() : lock_.trylock()); } @@ -87,7 +87,7 @@ private: class ObRLockGuard { public: - ObRLockGuard(const ObRWLock& rwlock, bool block = true) : guard_((*rwlock.rlock()), block) {} + [[nodiscard]] ObRLockGuard(const ObRWLock& rwlock, bool block = true) : guard_((*rwlock.rlock()), block) {} ~ObRLockGuard(){} bool acquired() { return guard_.acquired(); } private: @@ -97,7 +97,7 @@ private: class ObWLockGuard { public: - ObWLockGuard(const ObRWLock& rwlock, bool block = true) : guard_((*rwlock.wlock()), block) {} + [[nodiscard]] ObWLockGuard(const ObRWLock& rwlock, bool block = true) : guard_((*rwlock.wlock()), block) {} ~ObWLockGuard(){} bool acquired() { return guard_.acquired(); } private: diff --git a/deps/oblib/src/lib/lock/ob_tc_rwlock.h b/deps/oblib/src/lib/lock/ob_tc_rwlock.h index a32a75886..6102efcc7 100644 --- a/deps/oblib/src/lib/lock/ob_tc_rwlock.h +++ b/deps/oblib/src/lib/lock/ob_tc_rwlock.h @@ -54,13 +54,13 @@ public: enum { WRITE_MASK = 1<<31 }; struct RLockGuard { - explicit RLockGuard(TCRWLock& lock): lock_(lock) { lock_.rdlock(); } + [[nodiscard]] explicit RLockGuard(TCRWLock& lock): lock_(lock) { lock_.rdlock(); } ~RLockGuard() { lock_.rdunlock(); } TCRWLock& lock_; }; struct RLockGuardWithTimeout { - explicit RLockGuardWithTimeout(TCRWLock &lock, const int64_t abs_timeout_us, int &ret): lock_(lock), need_unlock_(true) + [[nodiscard]] explicit RLockGuardWithTimeout(TCRWLock &lock, const int64_t abs_timeout_us, int &ret): lock_(lock), need_unlock_(true) { if (OB_FAIL(lock_.rdlock(abs_timeout_us))) { need_unlock_ = false; @@ -80,13 +80,13 @@ public: }; struct WLockGuard { - explicit WLockGuard(TCRWLock& lock): lock_(lock) { lock_.wrlock(); } + [[nodiscard]] explicit WLockGuard(TCRWLock& lock): lock_(lock) { lock_.wrlock(); } ~WLockGuard() { lock_.wrunlock(); } TCRWLock& lock_; }; struct WLockGuardWithTimeout { - explicit WLockGuardWithTimeout(TCRWLock &lock, const int64_t abs_timeout_us, int &ret): lock_(lock), need_unlock_(true) + [[nodiscard]] explicit WLockGuardWithTimeout(TCRWLock &lock, const int64_t abs_timeout_us, int &ret): lock_(lock), need_unlock_(true) { if (OB_FAIL(lock_.wrlock(abs_timeout_us))) { need_unlock_ = false; @@ -106,7 +106,7 @@ public: }; struct WLockGuardWithRetry { - explicit WLockGuardWithRetry(TCRWLock &lock, const int64_t abs_timeout_us) : lock_(lock) + [[nodiscard]] explicit WLockGuardWithRetry(TCRWLock &lock, const int64_t abs_timeout_us) : lock_(lock) { lock_.wrlock_with_retry(abs_timeout_us); } @@ -115,7 +115,7 @@ public: }; struct WLockGuardWithRetryInterval { - explicit WLockGuardWithRetryInterval(TCRWLock &lock, + [[nodiscard]] explicit WLockGuardWithRetryInterval(TCRWLock &lock, const int64_t try_thresold_us, const int64_t retry_interval_us) : lock_(lock) @@ -287,7 +287,7 @@ private: class TCWLockGuard { public: - explicit TCWLockGuard(const TCRWLock &lock) + [[nodiscard]] explicit TCWLockGuard(const TCRWLock &lock) : lock_(const_cast(lock)), ret_(OB_SUCCESS) { if (OB_UNLIKELY(OB_SUCCESS != (ret_ = lock_.wrlock()))) { @@ -313,7 +313,7 @@ private: class TCRLockGuard { public: - explicit TCRLockGuard(const TCRWLock &lock) + [[nodiscard]] explicit TCRLockGuard(const TCRWLock &lock) : lock_(const_cast(lock)), ret_(OB_SUCCESS), slot_id_(0) { if (OB_UNLIKELY(OB_SUCCESS != (ret_ = lock_.rdlock(INT64_MAX, slot_id_)))) { @@ -342,7 +342,7 @@ typedef TCRWLock RWLock; class RWLock { public: - struct RLockGuard + [[nodiscard]] struct RLockGuard { explicit RLockGuard(RWLock& lock): lock_(lock) { lock_.rdlock(); } ~RLockGuard() { lock_.rdunlock(); } @@ -350,7 +350,7 @@ public: }; struct RLockGuardWithTimeout { - explicit RLockGuardWithTimeout(RWLock &lock, const int64_t abs_timeout_us, int &ret): lock_(lock), need_unlock_(true) + [[nodiscard]] explicit RLockGuardWithTimeout(RWLock &lock, const int64_t abs_timeout_us, int &ret): lock_(lock), need_unlock_(true) { if (OB_FAIL(lock_.rdlock(abs_timeout_us))) { need_unlock_ = false; @@ -371,13 +371,13 @@ public: struct WLockGuard { - explicit WLockGuard(RWLock& lock): lock_(lock) { lock_.wrlock(); } + [[nodiscard]] explicit WLockGuard(RWLock& lock): lock_(lock) { lock_.wrlock(); } ~WLockGuard() { lock_.wrunlock(); } RWLock& lock_; }; struct WLockGuardWithTimeout { - explicit WLockGuardWithTimeout(RWLock &lock, const int64_t abs_timeout_us, int &ret): lock_(lock), need_unlock_(true) + [[nodiscard]] explicit WLockGuardWithTimeout(RWLock &lock, const int64_t abs_timeout_us, int &ret): lock_(lock), need_unlock_(true) { if (OB_FAIL(lock_.wrlock(abs_timeout_us))) { need_unlock_ = false; @@ -397,7 +397,7 @@ public: }; struct WLockGuardWithRetry { - explicit WLockGuardWithRetry(RWLock &lock, const int64_t abs_timeout_us) : lock_(lock) + [[nodiscard]] explicit WLockGuardWithRetry(RWLock &lock, const int64_t abs_timeout_us) : lock_(lock) { lock_.wrlock_with_retry(abs_timeout_us); } diff --git a/deps/oblib/src/lib/lock/threadmutex.h b/deps/oblib/src/lib/lock/threadmutex.h index 5fb844044..740203170 100644 --- a/deps/oblib/src/lib/lock/threadmutex.h +++ b/deps/oblib/src/lib/lock/threadmutex.h @@ -52,7 +52,7 @@ protected: class ThreadGuard { public: - ThreadGuard(ThreadMutex *mutex) + [[nodiscard]] ThreadGuard(ThreadMutex *mutex) { mutex_ = NULL; if (mutex) { diff --git a/deps/oblib/src/lib/statistic_event/ob_stat_event.h b/deps/oblib/src/lib/statistic_event/ob_stat_event.h index b9f150a83..638cdd871 100644 --- a/deps/oblib/src/lib/statistic_event/ob_stat_event.h +++ b/deps/oblib/src/lib/statistic_event/ob_stat_event.h @@ -114,6 +114,13 @@ STAT_EVENT_ADD_DEF(HA_GTS_HANDLE_GET_REQUEST_COUNT, "ha gts handle get request c STAT_EVENT_ADD_DEF(HA_GTS_SEND_GET_RESPONSE_COUNT, "ha gts send get response count", ObStatClassIds::TRANS, "ha gts send get response count", 30070, true, true) STAT_EVENT_ADD_DEF(HA_GTS_HANDLE_PING_REQUEST_COUNT, "ha gts handle ping request count", ObStatClassIds::TRANS, "ha gts handle ping request count", 30071, true, true) STAT_EVENT_ADD_DEF(HA_GTS_HANDLE_PING_RESPONSE_COUNT, "ha gts handle ping response count", ObStatClassIds::TRANS, "ha gts handle ping response count", 30072, true, true) +STAT_EVENT_ADD_DEF(HA_GTS_SOURCE_REQUEST_COUNT, "ha gts source request count", ObStatClassIds::TRANS, "ha gts source request count", 30073, true, true) +STAT_EVENT_ADD_DEF(HA_GTS_SOURCE_REQUEST_COST_TIME, "ha gts source request cost time", ObStatClassIds::TRANS, "ha gts source request cost time", 30074, true, true) +STAT_EVENT_ADD_DEF(HA_GTS_CACHE_HIT_COUNT, "ha gts cache hit count", ObStatClassIds::TRANS, "ha gts cache hit count", 30075, true, true) +STAT_EVENT_ADD_DEF(HA_GTS_CACHE_MISS_COUNT, "ha gts cache miss count", ObStatClassIds::TRANS, "ha gts cache miss count", 30076, true, true) +STAT_EVENT_ADD_DEF(TRANS_ELR_ENABLE_COUNT, "trans early lock release enable count", ObStatClassIds::TRANS, "trans early lock releaes enable count", 30077, true, true) +STAT_EVENT_ADD_DEF(TRANS_ELR_UNABLE_COUNT, "trans early lock release unable count", ObStatClassIds::TRANS, "trans early lock releaes unable count", 30078, true, true) +STAT_EVENT_ADD_DEF(READ_ELR_ROW_COUNT, "read elr row count", ObStatClassIds::TRANS, "read elr row count", 30079, true, true) // SQL //STAT_EVENT_ADD_DEF(PLAN_CACHE_HIT, "PLAN_CACHE_HIT", SQL, "PLAN_CACHE_HIT") diff --git a/deps/oblib/unittest/lib/checksum/test_crc64.cpp b/deps/oblib/unittest/lib/checksum/test_crc64.cpp index 9a6433672..53254cbd5 100644 --- a/deps/oblib/unittest/lib/checksum/test_crc64.cpp +++ b/deps/oblib/unittest/lib/checksum/test_crc64.cpp @@ -41,13 +41,16 @@ TEST(TestCrc64, common) const int64_t STR_LEN = 10240; char *tmp_str = new char[STR_LEN]; char *str = NULL; + uint64_t intermediate_hash = 0; for (int64_t i = 1; i < (STR_LEN - 1); ++i) { str = rand_str(tmp_str, i); - uint64_t manually_hash = crc64_sse42_manually(0, str, i); - uint64_t fast_manually_hash = fast_crc64_sse42_manually(0, str, i); - uint64_t sse42_hash = ob_crc64(0, str, i); + uint64_t manually_hash = crc64_sse42_manually(intermediate_hash, str, i); + uint64_t fast_manually_hash = fast_crc64_sse42_manually(intermediate_hash, str, i); + uint64_t sse42_hash = crc64_sse42(intermediate_hash, str, i); + uint64_t isal_hash = ob_crc64_isal(intermediate_hash, str, i); ASSERT_EQ(manually_hash, fast_manually_hash); ASSERT_EQ(manually_hash, sse42_hash); + ASSERT_EQ(manually_hash, isal_hash); //cout << "st = "<< tmp_str << endl; //cout << "crc64c = "<< manually_hash << endl; } @@ -77,11 +80,19 @@ TEST(TestCrc64, test_speed) start = get_current_time_us(); for (int64_t j = 0; j < COUNT; ++j) { - ob_crc64(0, tmp_str, i); + crc64_sse42(0, tmp_str, i); } end = get_current_time_us(); cout << " ob_crc64(sse42), execut_count = "<< COUNT << ", cost_us = " << end - start << " len = " << i << endl; + start = get_current_time_us(); + for (int64_t j = 0; j < COUNT; ++j) { + ob_crc64_isal(0, tmp_str, i); + } + end = get_current_time_us(); + cout << " ob_crc64(ob_crc64_isal), execut_count = " << COUNT << ", cost_us = " << end - start << " len = " << i + << endl; + cout << endl; cout << endl; } diff --git a/src/logservice/applyservice/ob_log_apply_service.cpp b/src/logservice/applyservice/ob_log_apply_service.cpp index 9b9d3ab73..16c11db7b 100644 --- a/src/logservice/applyservice/ob_log_apply_service.cpp +++ b/src/logservice/applyservice/ob_log_apply_service.cpp @@ -129,7 +129,8 @@ ObApplyServiceQueueTask::ObApplyServiceQueueTask() : ObApplyServiceTask(), total_submit_cb_cnt_(0), last_check_submit_cb_cnt_(0), - total_apply_cb_cnt_(0) + total_apply_cb_cnt_(0), + idx_(-1) { type_ = ObApplyServiceTaskType::APPLY_LOG_TASK; } @@ -149,21 +150,24 @@ void ObApplyServiceQueueTask::reset() total_submit_cb_cnt_ = 0; last_check_submit_cb_cnt_ = 0; total_apply_cb_cnt_ = 0; + idx_ = -1; } -int ObApplyServiceQueueTask::init(ObApplyStatus *apply_status) +int ObApplyServiceQueueTask::init(ObApplyStatus *apply_status, + const int64_t idx) { int ret = OB_SUCCESS; - if (OB_ISNULL(apply_status)) { + if (OB_ISNULL(apply_status) || idx < 0) { ret = OB_INVALID_ARGUMENT; - CLOG_LOG(WARN, "invalid argument", K(type_), K(apply_status), K(ret)); + CLOG_LOG(WARN, "invalid argument", K(type_), K(apply_status), K(idx), K(ret)); } else { apply_status_ = apply_status; type_ = ObApplyServiceTaskType::APPLY_LOG_TASK; total_submit_cb_cnt_ = 0; last_check_submit_cb_cnt_ = 0; total_apply_cb_cnt_ = 0; - CLOG_LOG(INFO, "ObApplyServiceQueueTask init success", K(type_), K(apply_status)); + idx_ = idx; + CLOG_LOG(INFO, "ObApplyServiceQueueTask init success", K(type_), K(apply_status), K(idx_)); } return ret; } @@ -236,6 +240,11 @@ int ObApplyServiceQueueTask::is_apply_done(bool &is_done) return ret; } +int64_t ObApplyServiceQueueTask::idx() const +{ + return idx_; +} + //---------------ObApplyStatus---------------// ObApplyStatus::ObApplyStatus() : is_inited_(false), @@ -285,7 +294,7 @@ int ObApplyStatus::init(const share::ObLSID &id, CLOG_LOG(WARN, "failed to init submit_task", K(ret), K(id)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < APPLY_TASK_QUEUE_SIZE; ++i) { - if (OB_FAIL(cb_queues_[i].init(this))) { + if (OB_FAIL(cb_queues_[i].init(this, i))) { CLOG_LOG(WARN, "failed to init cb_queues", K(ret), K(id)); } } @@ -451,6 +460,7 @@ int ObApplyStatus::try_handle_cb_queue(ObApplyServiceQueueTask *cb_queue, int64_t append_finish_time = OB_INVALID_TIMESTAMP; int64_t cb_first_handle_time = OB_INVALID_TIMESTAMP; int64_t cb_start_time = OB_INVALID_TIMESTAMP; + int64_t idx = cb_queue->idx(); RLockGuard guard(lock_); do { ObLink *link = NULL; @@ -476,7 +486,7 @@ int ObApplyStatus::try_handle_cb_queue(ObApplyServiceQueueTask *cb_queue, ret = OB_SUCCESS; } statistics_cb_cost_(lsn, scn, append_start_time, append_finish_time, - cb_first_handle_time, cb_start_time); + cb_first_handle_time, cb_start_time, idx); cb_queue->inc_total_apply_cb_cnt(); } } else if (FOLLOWER == role_) { @@ -492,7 +502,7 @@ int ObApplyStatus::try_handle_cb_queue(ObApplyServiceQueueTask *cb_queue, ret = OB_SUCCESS; } statistics_cb_cost_(lsn, scn, append_start_time, append_finish_time, - cb_first_handle_time, cb_start_time); + cb_first_handle_time, cb_start_time, idx); cb_queue->inc_total_apply_cb_cnt(); } } else { @@ -814,6 +824,7 @@ int ObApplyStatus::handle_drop_cb_queue_(ObApplyServiceQueueTask &cb_queue) int64_t append_finish_time = OB_INVALID_TIMESTAMP; int64_t cb_first_handle_time = OB_INVALID_TIMESTAMP; int64_t cb_start_time = OB_INVALID_TIMESTAMP; + int64_t idx = cb_queue.idx(); do { ObLink *link = NULL; AppendCb *cb = NULL; @@ -836,7 +847,7 @@ int ObApplyStatus::handle_drop_cb_queue_(ObApplyServiceQueueTask &cb_queue) ret = OB_SUCCESS; } statistics_cb_cost_(lsn, scn, append_start_time, append_finish_time, - cb_first_handle_time, cb_start_time); + cb_first_handle_time, cb_start_time, idx); cb_queue.inc_total_apply_cb_cnt(); } } while (OB_SUCC(ret) && (!is_queue_empty)); @@ -867,7 +878,8 @@ void ObApplyStatus::statistics_cb_cost_(const LSN &lsn, const int64_t append_start_time, const int64_t append_finish_time, const int64_t cb_first_handle_time, - const int64_t cb_start_time) + const int64_t cb_start_time, + const int64_t idx) { // no need to print debug log when config [default value is true] is false; if (GCONF.enable_record_trace_log) { @@ -882,7 +894,7 @@ void ObApplyStatus::statistics_cb_cost_(const LSN &lsn, cb_wait_commit_stat_.stat(cb_wait_commit_time); cb_execute_stat_.stat(cb_cost_time); if (total_cost_time > 1000 * 1000) { //1s - CLOG_LOG(WARN, "cb cost too much time", K(lsn), K(scn), K(total_cost_time), K(append_cost_time), + CLOG_LOG(WARN, "cb cost too much time", K(lsn), K(scn), K(idx), K(total_cost_time), K(append_cost_time), K(cb_wait_thread_time), K(cb_wait_commit_time), K(cb_cost_time), K(append_start_time), K(append_finish_time), K(cb_first_handle_time), K(cb_first_handle_time), K(cb_finish_time)); } diff --git a/src/logservice/applyservice/ob_log_apply_service.h b/src/logservice/applyservice/ob_log_apply_service.h index 6e851eba3..67a791760 100644 --- a/src/logservice/applyservice/ob_log_apply_service.h +++ b/src/logservice/applyservice/ob_log_apply_service.h @@ -107,7 +107,8 @@ public: public: ObApplyServiceQueueTask(); ~ObApplyServiceQueueTask(); - int init(ObApplyStatus *apply_status); + int init(ObApplyStatus *apply_status, + const int64_t idx); void reset() override; public: Link *top(); @@ -120,15 +121,18 @@ public: void set_snapshot_check_submit_cb_cnt(); int is_snapshot_apply_done(bool &is_done); int is_apply_done(bool &is_done); + int64_t idx() const; INHERIT_TO_STRING_KV("ObApplyServiceQueueTask", ObApplyServiceTask, K(total_submit_cb_cnt_), K(last_check_submit_cb_cnt_), - K(total_apply_cb_cnt_)); + K(total_apply_cb_cnt_), + K(idx_)); private: int64_t total_submit_cb_cnt_; int64_t last_check_submit_cb_cnt_; int64_t total_apply_cb_cnt_; common::ObSpLinkQueue queue_; + int64_t idx_; }; class ObApplyStatus @@ -185,7 +189,8 @@ private: const int64_t append_start_time, const int64_t append_finish_time, const int64_t cb_first_handle_time, - const int64_t cb_start_time); + const int64_t cb_start_time, + const int64_t idx); private: typedef common::RWLock RWLock; typedef RWLock::RLockGuard RLockGuard; diff --git a/src/logservice/logrpc/ob_log_request_handler.cpp b/src/logservice/logrpc/ob_log_request_handler.cpp index c56b18783..53854e4a4 100644 --- a/src/logservice/logrpc/ob_log_request_handler.cpp +++ b/src/logservice/logrpc/ob_log_request_handler.cpp @@ -13,20 +13,21 @@ #include "ob_log_request_handler.h" #include "ob_log_rpc_req.h" #include "logservice/ob_log_service.h" +#include "logservice/palf/log_define.h" namespace oceanbase { namespace logservice { - - -LogRequestHandler::LogRequestHandler(palf::PalfHandle *palf_handle) : palf_handle_(palf_handle) +LogRequestHandler::LogRequestHandler(palf::PalfHandle *palf_handle) : palf_handle_(palf_handle), + handle_request_print_time_(OB_INVALID_TIMESTAMP) { } LogRequestHandler::~LogRequestHandler() { palf_handle_ = NULL; + handle_request_print_time_ = OB_INVALID_TIMESTAMP; } template <> @@ -43,6 +44,7 @@ int LogRequestHandler::handle_sync_requestclose(palf_handle_); + } lc_cb_ = NULL; rpc_proxy_ = NULL; palf_env_ = NULL; diff --git a/src/logservice/ob_log_service.cpp b/src/logservice/ob_log_service.cpp index 4ffc64398..de9505246 100644 --- a/src/logservice/ob_log_service.cpp +++ b/src/logservice/ob_log_service.cpp @@ -311,7 +311,9 @@ int ObLogService::create_ls(const share::ObLSID &id, return ret; } -int ObLogService::remove_ls(const ObLSID &id) +int ObLogService::remove_ls(const ObLSID &id, + ObLogHandler &log_handler, + ObLogRestoreHandler &restore_handler) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { @@ -322,10 +324,14 @@ int ObLogService::remove_ls(const ObLSID &id) } else if (OB_FAIL(replay_service_.remove_ls(id))) { CLOG_LOG(WARN, "failed to remove from replay_service", K(ret), K(id)); // NB: remove palf_handle lastly. - } else if (OB_FAIL(palf_env_->remove(id.id()))) { - CLOG_LOG(WARN, "failed to remove from palf_env_", K(ret), K(id)); } else { - FLOG_INFO("ObLogService remove_ls success", K(ret), K(id)); + log_handler.destroy(); + restore_handler.destroy(); + if (OB_FAIL(palf_env_->remove(id.id()))) { + CLOG_LOG(WARN, "failed to remove from palf_env_", K(ret), K(id)); + } else { + FLOG_INFO("ObLogService remove_ls success", K(ret), K(id)); + } } return ret; diff --git a/src/logservice/ob_log_service.h b/src/logservice/ob_log_service.h index 9f9dd41c4..51bf25a3b 100644 --- a/src/logservice/ob_log_service.h +++ b/src/logservice/ob_log_service.h @@ -124,7 +124,9 @@ public: ObLogRestoreHandler &restore_handler); //删除日志流接口:外层调用create_ls()之后，后续流程失败，需要调用remove_ls() - int remove_ls(const share::ObLSID &id); + int remove_ls(const share::ObLSID &id, + ObLogHandler &log_handler, + ObLogRestoreHandler &restore_handler); int check_palf_exist(const share::ObLSID &id, bool &exist) const; //宕机重启恢复日志流接口，包括生成并初始化对应的ObReplayStatus结构 diff --git a/src/logservice/palf/election/algorithm/election_impl.cpp b/src/logservice/palf/election/algorithm/election_impl.cpp index 2e80c786d..6f9d2db22 100644 --- a/src/logservice/palf/election/algorithm/election_impl.cpp +++ b/src/logservice/palf/election/algorithm/election_impl.cpp @@ -398,16 +398,16 @@ int ElectionImpl::send_(const ElectionChangeLeaderMsg &msg) const #undef PRINT_WRAPPER } -int ElectionImpl::revoke() +int ElectionImpl::revoke(const RoleChangeReason &reason) { ELECT_TIME_GUARD(500_ms); int ret = OB_SUCCESS; LockGuard lock_guard(lock_); CHECK_ELECTION_INIT(); - ret = proposer_.revoke(); + ret = proposer_.revoke(reason); return ret; } } } -} \ No newline at end of file +} diff --git a/src/logservice/palf/election/algorithm/election_impl.h b/src/logservice/palf/election/algorithm/election_impl.h index 71d9f2565..634ba34eb 100644 --- a/src/logservice/palf/election/algorithm/election_impl.h +++ b/src/logservice/palf/election/algorithm/election_impl.h @@ -64,7 +64,7 @@ public: const int64_t restart_counter, const ObFunction &prepare_change_leader_cb, const ObFunction &role_change_cb = DefaultRoleChangeCallBack()); - int revoke(); + int revoke(const RoleChangeReason &reason) override; virtual void stop() override final; virtual int set_memberlist(const MemberList &new_memberlist) override final; virtual int change_leader_to(const common::ObAddr &dest_addr) override final; diff --git a/src/logservice/palf/election/algorithm/election_proposer.cpp b/src/logservice/palf/election/algorithm/election_proposer.cpp index 2c9b19123..9cfeb8afe 100644 --- a/src/logservice/palf/election/algorithm/election_proposer.cpp +++ b/src/logservice/palf/election/algorithm/election_proposer.cpp @@ -688,7 +688,7 @@ int64_t ElectionProposer::to_string(char *buf, const int64_t buf_len) const return pos; } -int ElectionProposer::revoke() +int ElectionProposer::revoke(const RoleChangeReason &reason) { ELECT_TIME_GUARD(500_ms); #define PRINT_WRAPPER K(*this) @@ -698,7 +698,7 @@ int ElectionProposer::revoke() LOG_NONE(WARN, "i am not leader, but someone ask me to revoke", K(lbt())); } leader_lease_and_epoch_.reset(); - if (!leader_revoke_if_lease_expired_(RoleChangeReason::AskToRevoke)) { + if (!leader_revoke_if_lease_expired_(reason)) { LOG_NONE(WARN, "somethig wrong when revoke", K(lbt())); } return ret; diff --git a/src/logservice/palf/election/algorithm/election_proposer.h b/src/logservice/palf/election/algorithm/election_proposer.h index ac179f33f..53eb30c92 100644 --- a/src/logservice/palf/election/algorithm/election_proposer.h +++ b/src/logservice/palf/election/algorithm/election_proposer.h @@ -81,7 +81,7 @@ public: } return ret; } - int revoke(); + int revoke(const RoleChangeReason &reason); public: // 发prepare请求 void prepare(const common::ObRole role); diff --git a/src/logservice/palf/election/interface/election.h b/src/logservice/palf/election/interface/election.h index 7178b1151..19852355f 100644 --- a/src/logservice/palf/election/interface/election.h +++ b/src/logservice/palf/election/interface/election.h @@ -42,6 +42,7 @@ enum class RoleChangeReason ChangeLeaderToRevoke = 4, // 切主流程旧主从Leader变为Follower StopToRevoke = 5,// 选举leader调用stop接口后leader卸任 AskToRevoke = 6,// 有人要求选举卸任(???) + PalfDisableVoteToRevoke = 7,// palf disable vote触发卸任 }; class ElectionProposer; @@ -72,6 +73,7 @@ public: int64_t &cur_leader_epoch) const = 0; // 供role change service使用 virtual int change_leader_to(const common::ObAddr &dest_addr) = 0; + virtual int revoke(const RoleChangeReason &reason) = 0; // 拿本机地址 virtual const common::ObAddr &get_self_addr() const = 0; // 打印日志 diff --git a/src/logservice/palf/log_config_mgr.cpp b/src/logservice/palf/log_config_mgr.cpp index e15a2f66f..2574bc8a2 100644 --- a/src/logservice/palf/log_config_mgr.cpp +++ b/src/logservice/palf/log_config_mgr.cpp @@ -48,6 +48,7 @@ LogConfigMgr::LogConfigMgr() persistent_config_version_(), barrier_print_log_time_(OB_INVALID_TIMESTAMP), last_check_state_ts_us_(OB_INVALID_TIMESTAMP), + check_config_print_time_(OB_INVALID_TIMESTAMP), parent_lock_(), register_time_us_(OB_INVALID_TIMESTAMP), parent_(), @@ -771,8 +772,10 @@ int LogConfigMgr::check_config_change_args_(const LogConfigChangeArgs &args, boo if (args.type_ == ADD_MEMBER_AND_NUM || new_replica_num == paxos_replica_num_) { // config change has finished successfully, do not need change again is_already_finished = true; - PALF_LOG(INFO, "member already exists, don't need add_member/replace_member", KR(ret), K_(palf_id), K_(self), - K_(log_ms_meta), K(member), K(new_replica_num), K_(paxos_replica_num)); + if (palf_reach_time_interval(100 * 1000, check_config_print_time_)) { + PALF_LOG(INFO, "member already exists, don't need add_member/replace_member", KR(ret), K_(palf_id), K_(self), + K_(log_ms_meta), K(member), K(new_replica_num), K_(paxos_replica_num)); + } } else { ret = OB_INVALID_ARGUMENT; PALF_LOG(INFO, "member already exists, but new_replica_num not equal to curr val", KR(ret), K_(palf_id), K_(self), diff --git a/src/logservice/palf/log_config_mgr.h b/src/logservice/palf/log_config_mgr.h index 500887e5a..2555a5b6c 100644 --- a/src/logservice/palf/log_config_mgr.h +++ b/src/logservice/palf/log_config_mgr.h @@ -422,6 +422,7 @@ private: LogConfigVersion persistent_config_version_; mutable int64_t barrier_print_log_time_; mutable int64_t last_check_state_ts_us_; + mutable int64_t check_config_print_time_; // ================= Config Change ================= // ==================== Child ======================== mutable common::ObSpinLock parent_lock_; diff --git a/src/logservice/palf/log_engine.cpp b/src/logservice/palf/log_engine.cpp index 9c23608da..b9e527e27 100644 --- a/src/logservice/palf/log_engine.cpp +++ b/src/logservice/palf/log_engine.cpp @@ -937,6 +937,31 @@ int LogEngine::submit_learner_keepalive_resp(const common::ObAddr &server, return ret; } +int LogEngine::submit_committed_info_req( + const common::ObAddr &server, + const int64_t &msg_proposal_id, + const int64_t prev_log_id, + const int64_t &prev_log_proposal_id, + const LSN &committed_end_lsn) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + PALF_LOG(ERROR, "LogEngine not init", K(ret), KPC(this)); + } else if (OB_FAIL(log_net_service_.submit_committed_info_req( + server, msg_proposal_id, + prev_log_id, prev_log_proposal_id, committed_end_lsn))) { + PALF_LOG(ERROR, "LogNetService submit_committed_info_req failed", K(ret), + KPC(this), K(server), + K(prev_log_id), K(prev_log_proposal_id), K(committed_end_lsn)); + } else { + PALF_LOG(TRACE, "submit_committed_info_req success", K(ret), KPC(this), + K(server), K(msg_proposal_id), K(prev_log_id), + K(prev_log_proposal_id), K(committed_end_lsn)); + } + return ret; +} + LogMeta LogEngine::get_log_meta() const { ObSpinLockGuard guard(log_meta_lock_); diff --git a/src/logservice/palf/log_engine.h b/src/logservice/palf/log_engine.h index eca3be60e..886e246ef 100644 --- a/src/logservice/palf/log_engine.h +++ b/src/logservice/palf/log_engine.h @@ -320,7 +320,12 @@ public: int submit_notify_rebuild_req(const ObAddr &server, const LSN &base_lsn, const LogInfo &base_prev_log_info); - + int submit_committed_info_req( + const ObAddr &server, + const int64_t &msg_proposal_id, + const int64_t prev_log_id, + const int64_t &prev_log_proposal_id, + const LSN &committed_end_lsn); // @brief: this function used to send committed_info to child replica // @param[in] member_list: current paxos member list // @param[in] msg_proposal_id: the current proposal_id diff --git a/src/logservice/palf/log_net_service.cpp b/src/logservice/palf/log_net_service.cpp index 7be708a18..7469809fb 100644 --- a/src/logservice/palf/log_net_service.cpp +++ b/src/logservice/palf/log_net_service.cpp @@ -94,6 +94,25 @@ int LogNetService::submit_push_log_req( return ret; } +int LogNetService::submit_committed_info_req( + const common::ObAddr &server, + const int64_t &msg_proposal_id, + const int64_t prev_log_id, + const int64_t &prev_log_proposal_id, + const LSN &committed_end_lsn) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + PALF_LOG(ERROR, "LogNetService has not inited!!!", K(ret)); + } else { + CommittedInfo committed_info_req(msg_proposal_id, prev_log_id, + prev_log_proposal_id, committed_end_lsn); + ret = post_request_to_server_(server, committed_info_req); + } + return ret; +} + int LogNetService::submit_push_log_resp( const ObAddr &server, const int64_t &msg_proposal_id, diff --git a/src/logservice/palf/log_net_service.h b/src/logservice/palf/log_net_service.h index 30d63de56..454c14e19 100644 --- a/src/logservice/palf/log_net_service.h +++ b/src/logservice/palf/log_net_service.h @@ -188,7 +188,11 @@ public: int submit_retire_child_req(const common::ObAddr &server, const LogLearner &parent_itself); int submit_learner_keepalive_req(const common::ObAddr &server, const LogLearner &sender_itself); int submit_learner_keepalive_resp(const common::ObAddr &server, const LogLearner &sender_itself); - + int submit_committed_info_req(const common::ObAddr &server, + const int64_t &msg_proposal_id, + const int64_t prev_log_id, + const int64_t &prev_log_proposal_id, + const LSN &committed_end_lsn); template int submit_committed_info_req( const List &member_list, diff --git a/src/logservice/palf/log_sliding_window.cpp b/src/logservice/palf/log_sliding_window.cpp index 84e64af81..e2bf3a78e 100644 --- a/src/logservice/palf/log_sliding_window.cpp +++ b/src/logservice/palf/log_sliding_window.cpp @@ -69,6 +69,7 @@ LogSlidingWindow::LogSlidingWindow() cannot_fetch_log_warn_time_(OB_INVALID_TIMESTAMP), cannot_freeze_log_warn_time_(OB_INVALID_TIMESTAMP), larger_log_warn_time_(OB_INVALID_TIMESTAMP), + log_life_long_warn_time_(OB_INVALID_TIMESTAMP), lc_cb_get_warn_time_(OB_INVALID_TIMESTAMP), fetch_dst_invalid_warn_time_(OB_INVALID_TIMESTAMP), commit_log_handling_lease_(), @@ -1769,8 +1770,10 @@ int LogSlidingWindow::sliding_cb(const int64_t sn, const FixedSlidingWindowSlot log_submit_to_slide_cost_stat_.stat(fs_cb_begin_ts - log_submit_ts); if (log_life_time > 100 * 1000) { - PALF_LOG(WARN, "log_task life cost too much time", K_(palf_id), K_(self), K(log_id), KPC(log_task), - K(fs_cb_begin_ts), K(log_life_time)); + if (palf_reach_time_interval(10 * 1000, log_life_long_warn_time_)) { + PALF_LOG(WARN, "log_task life cost too much time", K_(palf_id), K_(self), K(log_id), KPC(log_task), + K(fs_cb_begin_ts), K(log_life_time)); + } } if (OB_FAIL(checksum_.verify_accum_checksum(log_task_header.data_checksum_, @@ -2470,7 +2473,9 @@ int LogSlidingWindow::receive_log(const common::ObAddr &src_server, "start_id", get_start_id()); } else if (OB_FAIL(guard.get_log_task(log_id, log_task))) { if (OB_ERR_OUT_OF_LOWER_BOUND == ret) { - PALF_LOG(WARN, "this log has slide out, no need receive", K(ret), K(log_id), K_(palf_id), K_(self)); + if (REACH_TIME_INTERVAL(100 * 1000)) { + PALF_LOG(WARN, "this log has slide out, no need receive", K(ret), K(log_id), K_(palf_id), K_(self)); + } } else { PALF_LOG(ERROR, "get_log_task failed", K(ret), K(log_id), K_(palf_id), K_(self), K(group_entry_header)); } @@ -3273,7 +3278,60 @@ int LogSlidingWindow::try_update_match_lsn_map_(const common::ObAddr &server, co return ret; } -int LogSlidingWindow::leader_broadcast_committed_info_(const LSN &committed_end_lsn) +int LogSlidingWindow::try_send_committed_info(const common::ObAddr &server, + const LSN &log_lsn, + const LSN &log_end_lsn, + const int64_t &log_proposal_id) +{ + int ret = OB_SUCCESS; + LSN committed_end_lsn; + get_committed_end_lsn_(committed_end_lsn); + const int64_t curr_proposal_id = state_mgr_->get_proposal_id(); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else if (!log_lsn.is_valid() || !log_end_lsn.is_valid() || INVALID_PROPOSAL_ID == log_proposal_id) { + ret = OB_INVALID_ARGUMENT; + } else if (state_mgr_->is_leader_active()) { + // Leader + int64_t last_log_id = OB_INVALID_LOG_ID; + int64_t last_log_proposal_id = INVALID_PROPOSAL_ID; + if (OB_FAIL(leader_get_committed_log_info_(committed_end_lsn, last_log_id, last_log_proposal_id)) + || OB_INVALID_LOG_ID == last_log_id) { + // no need send committed_info + } else if (OB_FAIL(log_engine_->submit_committed_info_req(server, curr_proposal_id, + last_log_id, log_proposal_id, committed_end_lsn))) { + PALF_LOG(WARN, "submit_committed_info_req failed", K(ret), K_(palf_id), K_(self), K(server)); + } + } else { + // Follower + int64_t last_slide_log_id = OB_INVALID_LOG_ID; + SCN last_slide_scn; + LSN last_slide_lsn; + LSN last_slide_end_lsn; + int64_t last_slide_log_pid = INVALID_PROPOSAL_ID; + int64_t last_slide_accum_checksum = -1; + get_last_slide_log_info_(last_slide_log_id, last_slide_scn, last_slide_lsn, \ + last_slide_end_lsn, last_slide_log_pid, last_slide_accum_checksum); + if (log_lsn == last_slide_lsn + && log_proposal_id == last_slide_log_pid + && committed_end_lsn == log_end_lsn) { + // If arg log does match with last slide log, follower can send committed_info to server. + OB_ASSERT(log_end_lsn == last_slide_end_lsn); + if (OB_FAIL(log_engine_->submit_committed_info_req(server, curr_proposal_id, + last_slide_log_id, log_proposal_id, committed_end_lsn))) { + PALF_LOG(WARN, "submit_committed_info_req failed", K(ret), K_(palf_id), K_(self), K(server)); + } else { + PALF_LOG(TRACE, "follower try_send_committed_info success", K(ret), K_(palf_id), K_(self), + K(last_slide_log_id), K(log_proposal_id), K(committed_end_lsn)); + } + } + } + return ret; +} + +int LogSlidingWindow::leader_get_committed_log_info_(const LSN &committed_end_lsn, + int64_t &log_id, + int64_t &log_proposal_id) { int ret = OB_SUCCESS; const int64_t max_log_id = get_max_log_id(); @@ -3289,25 +3347,38 @@ int LogSlidingWindow::leader_broadcast_committed_info_(const LSN &committed_end_ // log_task is invalid or not freezed, that means there is maybe new log after committed_end_lsn. // No need broadcast commonitted_info. } else { - int64_t log_proposal_id = INVALID_PROPOSAL_ID; LSN log_end_lsn; log_task->lock(); log_proposal_id = log_task->get_proposal_id(); log_end_lsn = log_task->get_begin_lsn() + LogGroupEntryHeader::HEADER_SER_SIZE + log_task->get_data_len(); log_task->unlock(); if (log_end_lsn == committed_end_lsn) { - ObMemberList dst_member_list; - const int64_t curr_proposal_id = state_mgr_->get_proposal_id(); - if (OB_FAIL(mm_->get_curr_member_list(dst_member_list))) { - PALF_LOG(WARN, "get_curr_member_list failed", K(ret), K_(palf_id), K_(self)); - } else if (OB_FAIL(dst_member_list.remove_server(self_))) { - PALF_LOG(WARN, "dst_member_list remove_server failed", K(ret), K_(palf_id), K_(self)); - } else if (dst_member_list.is_valid() - && OB_FAIL(log_engine_->submit_committed_info_req(dst_member_list, curr_proposal_id, - max_log_id, log_proposal_id, committed_end_lsn))) { - } else {} + log_id = max_log_id; } - PALF_LOG(TRACE, "leader_broadcast_committed_info_", K(ret), K_(palf_id), K_(self), K(max_log_id)); + } + return ret; +} + +int LogSlidingWindow::leader_broadcast_committed_info_(const LSN &committed_end_lsn) +{ + int ret = OB_SUCCESS; + const int64_t curr_proposal_id = state_mgr_->get_proposal_id(); + int64_t log_id = OB_INVALID_LOG_ID; + int64_t log_proposal_id = INVALID_PROPOSAL_ID; + ObMemberList dst_member_list; + if (OB_FAIL(leader_get_committed_log_info_(committed_end_lsn, log_id, log_proposal_id)) + || OB_INVALID_LOG_ID == log_id) { + // no need send committed_info + } else if (OB_FAIL(mm_->get_curr_member_list(dst_member_list))) { + PALF_LOG(WARN, "get_curr_member_list failed", K(ret), K_(palf_id), K_(self)); + } else if (OB_FAIL(dst_member_list.remove_server(self_))) { + PALF_LOG(WARN, "dst_member_list remove_server failed", K(ret), K_(palf_id), K_(self)); + } else if (dst_member_list.is_valid() + && OB_FAIL(log_engine_->submit_committed_info_req(dst_member_list, curr_proposal_id, + log_id, log_proposal_id, committed_end_lsn))) { + PALF_LOG(WARN, "submit_committed_info_req failed", K(ret), K_(palf_id), K_(self), K(log_id)); + } else { + PALF_LOG(TRACE, "leader_broadcast_committed_info_", K(ret), K_(palf_id), K_(self), K(log_id)); } return ret; } diff --git a/src/logservice/palf/log_sliding_window.h b/src/logservice/palf/log_sliding_window.h index 635dfb5bb..f99eb40cb 100644 --- a/src/logservice/palf/log_sliding_window.h +++ b/src/logservice/palf/log_sliding_window.h @@ -191,6 +191,10 @@ public: virtual int set_location_cache_cb(PalfLocationCacheCb *lc_cb); virtual int reset_location_cache_cb(); virtual int advance_reuse_lsn(const LSN &flush_log_end_lsn); + virtual int try_send_committed_info(const common::ObAddr &server, + const LSN &log_lsn, + const LSN &log_end_lsn, + const int64_t &log_proposal_id); TO_STRING_KV(K_(palf_id), K_(self), K_(lsn_allocator), K_(group_buffer), \ K_(last_submit_lsn), K_(last_submit_end_lsn), K_(last_submit_log_id), K_(last_submit_log_pid), \ K_(max_flushed_lsn), K_(max_flushed_end_lsn), K_(max_flushed_log_pid), K_(committed_end_lsn), \ @@ -304,6 +308,9 @@ private: const char *buf, const int64_t buf_len, SCN &min_scn); + int leader_get_committed_log_info_(const LSN &committed_end_lsn, + int64_t &log_id, + int64_t &log_proposal_id); int leader_broadcast_committed_info_(const LSN &committed_end_lsn); int submit_push_log_resp_(const common::ObAddr &server, const int64_t &msg_proposal_id, const LSN &lsn); inline int try_push_log_to_paxos_follower_(const int64_t curr_proposal_id, @@ -412,6 +419,7 @@ private: mutable int64_t cannot_fetch_log_warn_time_; mutable int64_t cannot_freeze_log_warn_time_; mutable int64_t larger_log_warn_time_; + mutable int64_t log_life_long_warn_time_; mutable int64_t lc_cb_get_warn_time_; mutable int64_t fetch_dst_invalid_warn_time_; common::ObThreadLease commit_log_handling_lease_; // thread lease for handling committed logs diff --git a/src/logservice/palf/log_state_mgr.cpp b/src/logservice/palf/log_state_mgr.cpp index 802b0e963..be1f6a26f 100644 --- a/src/logservice/palf/log_state_mgr.cpp +++ b/src/logservice/palf/log_state_mgr.cpp @@ -217,9 +217,9 @@ bool LogStateMgr::is_state_changed() state_changed = follower_need_update_role_(new_leader, new_leader_epoch); } } else if (is_leader_reconfirm_()) { - state_changed = leader_reconfirm_need_switch_(); + state_changed = (leader_reconfirm_need_switch_() || false == is_allow_vote()); } else if (is_leader_active_()) { - state_changed = leader_active_need_switch_(is_error); + state_changed = (leader_active_need_switch_(is_error) || false == is_allow_vote()); } else {} return state_changed; } @@ -292,6 +292,10 @@ int LogStateMgr::switch_state() need_next_loop = true; // 1) drive reconfirm or 2) fetch log from leader } } else if (is_leader_reconfirm_()) { + if (false == is_allow_vote() + && OB_FAIL(election_->revoke(election::RoleChangeReason::PalfDisableVoteToRevoke))) { + PALF_LOG(WARN, "election revoke failed", K(ret), K_(palf_id)); + } if (is_reconfirm_need_start_()) { ret = reconfirm_->reconfirm(); if (OB_EAGAIN == ret) { @@ -313,12 +317,12 @@ int LogStateMgr::switch_state() // do nothing } } else if (is_leader_active_()) { + if (false == is_allow_vote() + && OB_FAIL(election_->revoke(election::RoleChangeReason::PalfDisableVoteToRevoke))) { + PALF_LOG(WARN, "election revoke failed", K(ret), K_(palf_id)); + } bool is_error = false; if (leader_active_need_switch_(is_error)) { - if (is_error) { - // const uint32_t revoke_type = RevokeType::CLOG_SW_TIMEOUT; - // revoke_leader_(revoke_type); - } ret = leader_active_to_follower_pending_(); need_next_loop = true; } @@ -578,7 +582,8 @@ int LogStateMgr::follower_pending_to_reconfirm_(const int64_t new_leader_epoch) int LogStateMgr::reconfirm_to_follower_pending_() { - PALF_EVENT("reconfirm_to_follower_pending", palf_id_, K_(self), K_(leader), K(lbt())); + PALF_EVENT("reconfirm_to_follower_pending", palf_id_, K_(self), K_(leader), "is_allow_vote", + is_allow_vote(), K(lbt())); int ret = OB_SUCCESS; if (OB_FAIL(to_follower_pending_())) { PALF_LOG(WARN, "to_follower_pending_ failed, try again", K(ret), K_(palf_id)); @@ -739,9 +744,7 @@ bool LogStateMgr::follower_active_need_switch_() bool state_changed = false; common::ObAddr new_leader; if (need_update_leader_(new_leader)) { - if (new_leader.is_valid() || !leader_.is_valid() || leader_epoch_ == OB_INVALID_TIMESTAMP) { - state_changed = true; - } + state_changed = true; } else if (new_leader.is_valid() && self_ == new_leader) { state_changed = true; @@ -902,7 +905,8 @@ bool LogStateMgr::follower_need_update_role_(common::ObAddr &new_leader, new_leader_epoch = OB_INVALID_TIMESTAMP; } else { bool_ret = (self_ == new_leader); - PALF_LOG(TRACE, "follower_need_update_role_", K(new_leader), K(new_leader_epoch)); + PALF_LOG(TRACE, "follower_need_update_role_", K(bool_ret), K(new_leader), K(new_leader_epoch), + "is_allow_vote", is_allow_vote()); } return bool_ret; } @@ -914,14 +918,6 @@ void LogStateMgr::set_leader_and_epoch_(const common::ObAddr &new_leader, const PALF_LOG(INFO, "set_leader_and_epoch_", K_(palf_id), K_(self), K(leader_), K(leader_epoch_), K(new_leader)); } -int LogStateMgr::revoke_leader_(const uint32_t revoke_type) -{ - int ret = OB_SUCCESS; - UNUSED(revoke_type); - // election_->leader_revoke(revoke_type); - return ret; -} - int LogStateMgr::get_elect_leader_(common::ObAddr &leader, int64_t &leader_epoch) const { @@ -1020,6 +1016,7 @@ int LogStateMgr::check_and_try_fetch_log_() } else { last_check_start_id_ = start_id; last_check_start_id_time_us_ = now_us; + PALF_LOG(TRACE, "sw try_fetch_log success", K(ret), K_(palf_id), K(start_id)); } } else if (last_check_start_id_ != start_id) { last_check_start_id_ = start_id; diff --git a/src/logservice/palf/log_state_mgr.h b/src/logservice/palf/log_state_mgr.h index f70feb90a..438b0336c 100644 --- a/src/logservice/palf/log_state_mgr.h +++ b/src/logservice/palf/log_state_mgr.h @@ -97,7 +97,7 @@ public: TO_STRING_KV(KP(this), K_(self), K_(palf_id), "role", role_to_string(role_), \ "state", replica_state_to_string(state_), K_(prepare_meta), K_(leader), K_(leader_epoch), \ K_(is_sync_enabled), K_(pending_end_lsn), K_(scan_disk_log_finished), K_(last_check_start_id), \ - K_(reconfirm_start_time_us), KP_(palf_role_change_cb)); + K_(reconfirm_start_time_us), KP_(palf_role_change_cb), K_(allow_vote)); private: bool check_role_and_state_(const common::ObRole &role, const ObReplicaState &state) const; void update_role_and_state_(const common::ObRole &new_role, const ObReplicaState &new_state); @@ -139,7 +139,6 @@ private: bool need_update_leader_(common::ObAddr &new_leader); bool follower_need_update_role_(common::ObAddr &new_leader, int64_t &new_leader_epoch); void set_leader_and_epoch_(const common::ObAddr &new_leader, const int64_t new_leader_epoch); - int revoke_leader_(const uint32_t revoke_type); int get_elect_leader_(common::ObAddr &leader, int64_t &leader_epoch) const; bool check_leader_log_sync_state_(); bool need_fetch_log_() const; diff --git a/src/logservice/palf/palf_handle_impl.cpp b/src/logservice/palf/palf_handle_impl.cpp index 4dc3c24e8..18058d695 100644 --- a/src/logservice/palf/palf_handle_impl.cpp +++ b/src/logservice/palf/palf_handle_impl.cpp @@ -57,6 +57,8 @@ PalfHandleImpl::PalfHandleImpl() last_check_parent_child_ts_us_(OB_INVALID_TIMESTAMP), wait_slide_print_time_us_(OB_INVALID_TIMESTAMP), append_size_stat_time_us_(OB_INVALID_TIMESTAMP), + replace_member_print_time_us_(OB_INVALID_TIMESTAMP), + config_change_print_time_us_(OB_INVALID_TIMESTAMP), last_rebuild_lsn_(), last_record_append_lsn_(PALF_INITIAL_LSN_VAL), has_set_deleted_(false), @@ -612,7 +614,9 @@ int PalfHandleImpl::replace_member( } else if (FALSE_IT(args.server_ = removed_member)) { } else if (FALSE_IT(args.type_ = REMOVE_MEMBER_AND_NUM)) { } else if (OB_FAIL(one_stage_config_change_(args, timeout_us + begin_time_us - common::ObTimeUtility::current_time()))) { - PALF_LOG(WARN, "remove_member in replace_member failed", KR(ret), K(args), KPC(this)); + if (palf_reach_time_interval(100 * 1000, replace_member_print_time_us_)) { + PALF_LOG(WARN, "remove_member in replace_member failed", KR(ret), K(args), KPC(this)); + } } else if (OB_FAIL(config_mgr_.get_curr_member_list(curr_member_list))) { PALF_LOG(WARN, "get_curr_member_list failed", KR(ret), KPC(this)); } else { @@ -1077,12 +1081,16 @@ int PalfHandleImpl::one_stage_config_change_(const LogConfigChangeArgs &args, ret = OB_INVALID_ARGUMENT; PALF_LOG(WARN, "invalid argument", KR(ret), KPC(this), K(args)); } else if (OB_FAIL(can_change_config_(args, curr_proposal_id))) { - PALF_LOG(WARN, "not active leader, can't change member", KR(ret), KPC(this), - "role", state_mgr_.get_role(), "state", state_mgr_.get_state()); + if (palf_reach_time_interval(100 * 1000, config_change_print_time_us_)) { + PALF_LOG(WARN, "not active leader, can't change member", KR(ret), KPC(this), + "role", state_mgr_.get_role(), "state", state_mgr_.get_state()); + } } else if (OB_FAIL(check_args_and_generate_config_(args, is_already_finished, new_log_sync_memberlist, new_log_sync_replica_num))) { PALF_LOG(WARN, "check_args_and_generate_config failed", KR(ret), KPC(this), K(args)); } else if (is_already_finished) { - PALF_LOG(INFO, "one_stage_config_change has already finished", K(ret), KPC(this), K(args)); + if (palf_reach_time_interval(100 * 1000, config_change_print_time_us_)) { + PALF_LOG(INFO, "one_stage_config_change has already finished", K(ret), KPC(this), K(args)); + } } else { PALF_LOG(INFO, "one_stage_config_change start", KPC(this), K(curr_proposal_id), K(args), K(timeout_us)); TimeoutChecker not_timeout(timeout_us); @@ -1350,7 +1358,11 @@ int PalfHandleImpl::set_allow_vote_flag_(const bool allow_vote) flush_meta_cb_ctx.allow_vote_ = allow_vote; LogReplicaPropertyMeta replica_property_meta = log_engine_.get_log_meta().get_log_replica_property_meta(); replica_property_meta.allow_vote_ = allow_vote; - if (OB_FAIL(log_engine_.submit_flush_replica_property_meta_task(flush_meta_cb_ctx, replica_property_meta))) { + if (false == allow_vote + && LEADER == state_mgr_.get_role() + && OB_FAIL(election_.revoke(RoleChangeReason::PalfDisableVoteToRevoke))) { + PALF_LOG(WARN, "election revoke failed", K(ret), K_(palf_id)); + } else if (OB_FAIL(log_engine_.submit_flush_replica_property_meta_task(flush_meta_cb_ctx, replica_property_meta))) { PALF_LOG(WARN, "submit_flush_replica_property_meta_task failed", K(ret), K(flush_meta_cb_ctx), K(replica_property_meta)); } } @@ -1945,6 +1957,7 @@ int PalfHandleImpl::set_location_cache_cb(PalfLocationCacheCb *lc_cb) int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; + PALF_LOG(WARN, "not initted", KR(ret), KPC(this)); } else if (OB_ISNULL(lc_cb)) { ret = OB_INVALID_ARGUMENT; PALF_LOG(WARN, "lc_cb is NULL, can't register", KR(ret), KPC(this)); @@ -2207,31 +2220,26 @@ int PalfHandleImpl::ack_mode_meta(const common::ObAddr &server, int PalfHandleImpl::handle_election_message(const election::ElectionPrepareRequestMsg &msg) { - RLockGuard guard(lock_); return election_.handle_message(msg); } int PalfHandleImpl::handle_election_message(const election::ElectionPrepareResponseMsg &msg) { - RLockGuard guard(lock_); return election_.handle_message(msg); } int PalfHandleImpl::handle_election_message(const election::ElectionAcceptRequestMsg &msg) { - RLockGuard guard(lock_); return election_.handle_message(msg); } int PalfHandleImpl::handle_election_message(const election::ElectionAcceptResponseMsg &msg) { - RLockGuard guard(lock_); return election_.handle_message(msg); } int PalfHandleImpl::handle_election_message(const election::ElectionChangeLeaderMsg &msg) { - RLockGuard guard(lock_); return election_.handle_message(msg); } @@ -2285,7 +2293,7 @@ int PalfHandleImpl::do_init_mem_( has_set_deleted_ = false; palf_env_impl_ = palf_env_impl; is_inited_ = true; - PALF_LOG(INFO, "PalfHandleImpl do_init_ success", K(ret), K(palf_id), K(log_dir), K(palf_base_info), + PALF_LOG(INFO, "PalfHandleImpl do_init_ success", K(ret), K(palf_id), K(self), K(log_dir), K(palf_base_info), K(log_meta), K(fetch_log_engine), K(alloc_mgr), K(log_rpc), K(log_io_worker)); } if (OB_FAIL(ret)) { @@ -2441,7 +2449,9 @@ int PalfHandleImpl::receive_log(const common::ObAddr &server, // rewrite -4023 to 0 ret = OB_SUCCESS; } else { - PALF_LOG(WARN, "sw_ receive_log failed", K(ret), KPC(this), K(server), K(msg_proposal_id), K(lsn)); + if (REACH_TIME_INTERVAL(100 * 1000)) { + PALF_LOG(WARN, "sw_ receive_log failed", K(ret), KPC(this), K(server), K(msg_proposal_id), K(lsn)); + } } } else { PALF_LOG(TRACE, "receive_log success", K(ret), KPC(this), K(server), K(msg_proposal_id), K(lsn), K(buf_len)); @@ -2482,7 +2492,9 @@ int PalfHandleImpl::receive_log(const common::ObAddr &server, if (OB_FAIL(ret)) { } else if (OB_FAIL(sw_.receive_log(server, push_log_type, prev_lsn, prev_log_proposal_id, lsn, buf, buf_len, false, truncate_log_info))) { - PALF_LOG(WARN, "sw_ receive_log failed", K(ret), KPC(this), K(server), K(msg_proposal_id), K(lsn)); + if (REACH_TIME_INTERVAL(100 * 1000)) { + PALF_LOG(WARN, "sw_ receive_log failed", K(ret), KPC(this), K(server), K(msg_proposal_id), K(lsn)); + } } else { PALF_LOG(INFO, "receive_log success", K(ret), KPC(this), K(server), K_(self), K(msg_proposal_id), K(prev_lsn), K(prev_log_proposal_id), K(lsn), K(truncate_log_info)); @@ -2756,13 +2768,36 @@ int PalfHandleImpl::fetch_log_from_storage(const common::ObAddr &server, return ret; } +int PalfHandleImpl::try_send_committed_info_(const ObAddr &server, + const LSN &log_lsn, + const LSN &log_end_lsn, + const int64_t &log_proposal_id) +{ + int ret = OB_SUCCESS; + AccessMode access_mode; + if (!log_lsn.is_valid() || !log_end_lsn.is_valid() || INVALID_PROPOSAL_ID == log_proposal_id) { + ret = OB_INVALID_ARGUMENT; + } else if (OB_FAIL(mode_mgr_.get_access_mode(access_mode))) { + PALF_LOG(WARN, "get_access_mode failed", K(ret), KPC(this)); + } else if (AccessMode::APPEND == access_mode) { + // No need send committed_info in APPEND mode, because leader will genenrate keeapAlive log periodically. + } else if (OB_FAIL(sw_.try_send_committed_info(server, log_lsn, log_end_lsn, log_proposal_id))) { + PALF_LOG(TRACE, "try_send_committed_info failed", K(ret), K_(palf_id), K_(self), + K(server), K(log_lsn), K(log_end_lsn), K(log_proposal_id)); + } else { + PALF_LOG(TRACE, "try_send_committed_info_ success", K(ret), K_(palf_id), K_(self), K(server), + K(log_lsn), K(log_end_lsn), K(log_proposal_id)); + } + return ret; +} + int PalfHandleImpl::fetch_log_from_storage_(const common::ObAddr &server, - const FetchLogType fetch_type, - const int64_t &msg_proposal_id, - const LSN &prev_lsn, - const LSN &fetch_start_lsn, - const int64_t fetch_log_size, - const int64_t fetch_log_count) + const FetchLogType fetch_type, + const int64_t &msg_proposal_id, + const LSN &prev_lsn, + const LSN &fetch_start_lsn, + const int64_t fetch_log_size, + const int64_t fetch_log_count) { int ret = OB_SUCCESS; PalfGroupBufferIterator iterator; @@ -2818,6 +2853,7 @@ int PalfHandleImpl::fetch_log_from_storage_(const common::ObAddr &server, bool is_reach_end = false; int64_t fetched_count = 0; LSN curr_log_end_lsn = curr_lsn + curr_group_entry.get_group_entry_size(); + LSN prev_log_end_lsn; int64_t prev_log_proposal_id = prev_log_info.log_proposal_id_; while (OB_SUCC(ret) && !is_reach_size_limit && !is_reach_count_limit && !is_reach_end && OB_SUCC(iterator.next())) { @@ -2849,12 +2885,18 @@ int PalfHandleImpl::fetch_log_from_storage_(const common::ObAddr &server, K(prev_log_proposal_id), K(fetch_end_lsn), K(curr_log_end_lsn), K(is_reach_size_limit), K(fetch_log_size), K(fetched_count), K(is_reach_count_limit)); each_round_prev_lsn = curr_lsn; + prev_log_end_lsn = curr_log_end_lsn; prev_log_proposal_id = curr_group_entry.get_header().get_log_proposal_id(); } } if (OB_ITER_END == ret) { ret = OB_SUCCESS; } + // try send committed_info to server + if (OB_SUCC(ret)) { + RLockGuard guard(lock_); + (void) try_send_committed_info_(server, each_round_prev_lsn, prev_log_end_lsn, prev_log_proposal_id); + } } if (OB_FAIL(ret) && OB_ERR_OUT_OF_LOWER_BOUND == ret) { @@ -3408,7 +3450,7 @@ int PalfHandleImpl::revoke_leader(const int64_t proposal_id) } else if (false == state_mgr_.can_revoke(proposal_id)) { ret = OB_NOT_MASTER; PALF_LOG(WARN, "revoke_leader failed, not master", K(ret), K_(palf_id), K(proposal_id)); - } else if (OB_FAIL(election_.revoke())) { + } else if (OB_FAIL(election_.revoke(RoleChangeReason::AskToRevoke))) { PALF_LOG(WARN, "PalfHandleImpl revoke leader failed", K(ret), K_(palf_id)); } else { PALF_LOG(INFO, "PalfHandleImpl revoke leader success", K(ret), K_(palf_id)); diff --git a/src/logservice/palf/palf_handle_impl.h b/src/logservice/palf/palf_handle_impl.h index 6e828503c..ccb7e7988 100644 --- a/src/logservice/palf/palf_handle_impl.h +++ b/src/logservice/palf/palf_handle_impl.h @@ -868,6 +868,10 @@ private: int construct_palf_base_info_(const LSN &max_committed_lsn, PalfBaseInfo &palf_base_info); int append_disk_log_to_sw_(const LSN &start_lsn); + int try_send_committed_info_(const common::ObAddr &server, + const LSN &log_lsn, + const LSN &log_end_lsn, + const int64_t &log_proposal_id); int fetch_log_from_storage_(const common::ObAddr &server, const FetchLogType fetch_type, const int64_t &msg_proposal_id, @@ -998,6 +1002,8 @@ private: int64_t last_check_parent_child_ts_us_; int64_t wait_slide_print_time_us_; int64_t append_size_stat_time_us_; + int64_t replace_member_print_time_us_; + int64_t config_change_print_time_us_; mutable SpinLock last_rebuild_lsn_lock_; LSN last_rebuild_lsn_; LSN last_record_append_lsn_; diff --git a/src/logservice/palf/scn.cpp b/src/logservice/palf/scn.cpp index 8f9c98936..e2a7e861d 100644 --- a/src/logservice/palf/scn.cpp +++ b/src/logservice/palf/scn.cpp @@ -317,7 +317,7 @@ int SCN::convert_for_sql(uint64_t value) void SCN::transform_max() { - if (share::ObScnRange::OLD_MAX_TS == val_) { + if (INT64_MAX == val_) { this->set_max(); } } diff --git a/src/logservice/palf/scn.h b/src/logservice/palf/scn.h index 4f5065eec..eed0469c5 100644 --- a/src/logservice/palf/scn.h +++ b/src/logservice/palf/scn.h @@ -80,7 +80,7 @@ public: // @param[in] convert for tx: INT64_MAX may convert to MAX_SCN int convert_for_tx(int64_t commit_trans_version); - // change val_ from ObScnRange::OLD_MAX_TS to OB_MAX_SCN_TS_NS for compitibility in + // change val_ from INT64_MAX to OB_MAX_SCN_TS_NS for compitibility in // deserialization void transform_max(); diff --git a/src/logservice/palf_handle_guard.h b/src/logservice/palf_handle_guard.h index ef8a7034e..80e398144 100644 --- a/src/logservice/palf_handle_guard.h +++ b/src/logservice/palf_handle_guard.h @@ -211,6 +211,7 @@ public: DELEGATE_WITH_RET(palf_handle_, switch_learner_to_acceptor, int); DELEGATE_WITH_RET(palf_handle_, switch_acceptor_to_learner, int); DELEGATE_WITH_RET(palf_handle_, set_region, int); + DELEGATE_WITH_RET(palf_handle_, set_location_cache_cb, int); DELEGATE_WITH_RET(palf_handle_, change_access_mode, int); DELEGATE_WITH_RET(palf_handle_, get_access_mode, int); private: diff --git a/src/logservice/replayservice/ob_replay_status.cpp b/src/logservice/replayservice/ob_replay_status.cpp index 7c4e25fec..aa7a023b4 100644 --- a/src/logservice/replayservice/ob_replay_status.cpp +++ b/src/logservice/replayservice/ob_replay_status.cpp @@ -551,7 +551,8 @@ ObReplayStatus::ObReplayStatus(): palf_handle_(), fs_cb_(), get_log_info_debug_time_(OB_INVALID_TIMESTAMP), - try_wrlock_debug_time_(OB_INVALID_TIMESTAMP) + try_wrlock_debug_time_(OB_INVALID_TIMESTAMP), + check_enable_debug_time_(OB_INVALID_TIMESTAMP) { } @@ -580,6 +581,7 @@ int ObReplayStatus::init(const share::ObLSID &id, ls_id_ = id; get_log_info_debug_time_ = OB_INVALID_TIMESTAMP; try_wrlock_debug_time_ = OB_INVALID_TIMESTAMP; + check_enable_debug_time_ = OB_INVALID_TIMESTAMP; palf_env_ = palf_env; rp_sv_ = rp_sv; fs_cb_ = ObReplayFsCb(this); @@ -628,6 +630,7 @@ void ObReplayStatus::destroy() fs_cb_.destroy(); get_log_info_debug_time_ = OB_INVALID_TIMESTAMP; try_wrlock_debug_time_ = OB_INVALID_TIMESTAMP; + check_enable_debug_time_ = OB_INVALID_TIMESTAMP; palf_env_ = NULL; rp_sv_ = NULL; } @@ -864,7 +867,9 @@ int ObReplayStatus::update_end_offset(const LSN &lsn) ret = OB_NOT_INIT; CLOG_LOG(ERROR, "replay status is not init", K(ls_id_), K(lsn), K(ret)); } else if (!is_enabled_) { - CLOG_LOG(INFO, "replay status is not enabled", K(this), K(ret), K(lsn)); + if (palf_reach_time_interval(100 * 1000, check_enable_debug_time_)) { + CLOG_LOG(INFO, "replay status is not enabled", K(this), K(ret), K(lsn)); + } } else if (OB_UNLIKELY(!lsn.is_valid())) { ret = OB_INVALID_ARGUMENT; CLOG_LOG(ERROR, "invalid arguments", K(ls_id_), K(lsn), K(ret)); diff --git a/src/logservice/replayservice/ob_replay_status.h b/src/logservice/replayservice/ob_replay_status.h index 2423c8a28..779ea70f6 100644 --- a/src/logservice/replayservice/ob_replay_status.h +++ b/src/logservice/replayservice/ob_replay_status.h @@ -597,6 +597,7 @@ private: ObReplayFsCb fs_cb_; mutable int64_t get_log_info_debug_time_; mutable int64_t try_wrlock_debug_time_; + mutable int64_t check_enable_debug_time_; DISALLOW_COPY_AND_ASSIGN(ObReplayStatus); }; diff --git a/src/objit/include/objit/common/ob_item_type.h b/src/objit/include/objit/common/ob_item_type.h index ef6c742f8..e20e7cbfc 100644 --- a/src/objit/include/objit/common/ob_item_type.h +++ b/src/objit/include/objit/common/ob_item_type.h @@ -2083,6 +2083,13 @@ typedef enum ObItemType T_OB_DDL_SCHEMA_VERSION, T_TG_ORDER, T_ODBC_ESCAPE_SEQUENCES, + + // column group + T_COLUMN_GROUP, + T_ALL_COLUMN_GROUP, + T_SINGLE_COLUMN_GROUP, + T_NORMAL_COLUMN_GROUP, + T_MAX //Attention: add a new type before T_MAX } ObItemType; diff --git a/src/observer/mysql/ob_async_cmd_driver.cpp b/src/observer/mysql/ob_async_cmd_driver.cpp index f8e4cd01a..1f46ec8fd 100644 --- a/src/observer/mysql/ob_async_cmd_driver.cpp +++ b/src/observer/mysql/ob_async_cmd_driver.cpp @@ -68,7 +68,7 @@ int ObAsyncCmdDriver::response_result(ObMySQLResultSet &result) LOG_WARN("close result set fail", K(close_ret)); } if (!result.is_async_end_trans_submitted()) { - retry_ctrl_.test_and_save_retry_state(gctx_, ctx_, result, ret, cli_ret); + retry_ctrl_.test_and_save_retry_state(gctx_, ctx_, result, ret, cli_ret, is_prexecute_); LOG_WARN("result set open failed, check if need retry", K(ret), K(cli_ret), K(retry_ctrl_.need_retry())); ret = cli_ret; diff --git a/src/observer/mysql/ob_async_plan_driver.cpp b/src/observer/mysql/ob_async_plan_driver.cpp index 3cfc19a78..a4a068877 100644 --- a/src/observer/mysql/ob_async_plan_driver.cpp +++ b/src/observer/mysql/ob_async_plan_driver.cpp @@ -79,7 +79,7 @@ int ObAsyncPlanDriver::response_result(ObMySQLResultSet &result) // 如果try_again为true，说明这条SQL需要重做。考虑到重做之前我们需要回滚整个事务，会调用EndTransCb // 所以这里设置一个标记，告诉EndTransCb这种情况下不要给客户端回包。 int cli_ret = OB_SUCCESS; - retry_ctrl_.test_and_save_retry_state(gctx_, ctx_, result, ret, cli_ret); + retry_ctrl_.test_and_save_retry_state(gctx_, ctx_, result, ret, cli_ret, is_prexecute_); if (retry_ctrl_.need_retry()) { result.set_end_trans_async(false); } diff --git a/src/observer/mysql/ob_sync_cmd_driver.cpp b/src/observer/mysql/ob_sync_cmd_driver.cpp index c5160acf4..c32ccb4c3 100644 --- a/src/observer/mysql/ob_sync_cmd_driver.cpp +++ b/src/observer/mysql/ob_sync_cmd_driver.cpp @@ -114,7 +114,7 @@ int ObSyncCmdDriver::response_result(ObMySQLResultSet &result) } // open失败，决定是否需要重试 - retry_ctrl_.test_and_save_retry_state(gctx_, ctx_, result, ret, cli_ret); + retry_ctrl_.test_and_save_retry_state(gctx_, ctx_, result, ret, cli_ret, is_prexecute_); LOG_WARN("result set open failed, check if need retry", K(ret), K(cli_ret), K(retry_ctrl_.need_retry())); ret = cli_ret; diff --git a/src/observer/mysql/ob_sync_plan_driver.cpp b/src/observer/mysql/ob_sync_plan_driver.cpp index b9bb6b5c4..a1ad23c0c 100644 --- a/src/observer/mysql/ob_sync_plan_driver.cpp +++ b/src/observer/mysql/ob_sync_plan_driver.cpp @@ -72,7 +72,8 @@ int ObSyncPlanDriver::response_result(ObMySQLResultSet &result) ctx_, result, ret, - cli_ret); + cli_ret, + is_prexecute_); if (OB_TRANSACTION_SET_VIOLATION != ret && OB_REPLICA_NOT_READABLE != ret) { if (OB_TRY_LOCK_ROW_CONFLICT == ret && retry_ctrl_.need_retry()) { //锁冲突重试不打印日志，避免刷屏 @@ -107,7 +108,8 @@ int ObSyncPlanDriver::response_result(ObMySQLResultSet &result) ctx_, result, ret, - cli_ret); + cli_ret, + is_prexecute_); LOG_WARN("result response failed, check if need retry", K(ret), K(cli_ret), K(retry_ctrl_.need_retry())); ret = cli_ret; diff --git a/src/pl/ob_pl_code_generator.cpp b/src/pl/ob_pl_code_generator.cpp index 37db545d1..abab41cf2 100644 --- a/src/pl/ob_pl_code_generator.cpp +++ b/src/pl/ob_pl_code_generator.cpp @@ -2284,19 +2284,36 @@ int ObPLCodeGenerateVisitor::visit(const ObPLSignalStmt &s) ObLLVMValue unwindException = generator_.get_saved_exception(); ObLLVMValue ob_error_code = generator_.get_saved_ob_error(); ObLLVMType unwind_exception_type, unwind_exception_pointer_type; + ObLLVMValue unwind_exception_header; + ObLLVMType condition_type; + ObLLVMType condition_pointer_type; + ObLLVMValue condition; + ObLLVMValue sql_state; + ObLLVMValue error_code; CK (OB_NOT_NULL(generator_.get_saved_exception().get_v())); CK (OB_NOT_NULL(generator_.get_saved_ob_error().get_v())); OZ (generator_.extract_status_from_context(generator_.get_vars().at(generator_.CTX_IDX), status)); OZ (generator_.get_helper().create_store(ob_error_code, status)); OZ (generator_.get_adt_service().get_unwind_exception(unwind_exception_type)); OZ (unwind_exception_type.get_pointer_to(unwind_exception_pointer_type)); + OZ (generator_.get_helper().create_const_gep1_64(ObString("extract_unwind_exception_header"), + unwindException, + generator_.get_eh_service().pl_exception_base_offset_, + unwind_exception_header)); + OZ (generator_.get_adt_service().get_pl_condition_value(condition_type)); + OZ (condition_type.get_pointer_to(condition_pointer_type)); + OZ (generator_.get_helper().create_pointer_cast(ObString("cast_header"), + unwind_exception_header, + condition_pointer_type, + condition)); + OZ (generator_.extract_name_from_condition_value(condition, sql_state)); + OZ (generator_.extract_code_from_condition_value(condition, error_code)); OZ (generator_.get_helper().create_pointer_cast( ObString("cast_unwind_exception"), unwindException, unwind_exception_pointer_type, unwindException)); OZ (generator_.get_helper().create_block(ObString("normal"), generator_.get_func(), normal)); - OZ (generator_.raise_exception(unwindException, null_sql_state, normal, - s.get_block()->in_notfound(), s.get_block()->in_warning(), - true/*is signal*/)); + OZ (generator_.raise_exception(unwindException, error_code, sql_state, normal, + s.get_block()->in_notfound(), s.get_block()->in_warning(), true)); OZ (generator_.set_current(normal)); } else { ObLLVMValue type, ob_err_code, err_code, sql_state, str_len, is_signal, stmt_id, loc; @@ -6509,7 +6526,7 @@ int ObPLCodeGenerator::generate_exception(ObLLVMValue &type, if (OB_FAIL(helper_.create_call(ObString("create_exception"), get_eh_service().eh_create_exception_, args, exception))) { LOG_WARN("failed to create_call", K(ret)); } else { - OZ (raise_exception(exception, sql_state, normal, in_notfound, in_warning, signal)); + OZ (raise_exception(exception, error_code, sql_state, normal, in_notfound, in_warning, signal)); } } } @@ -6518,6 +6535,7 @@ int ObPLCodeGenerator::generate_exception(ObLLVMValue &type, } int ObPLCodeGenerator::raise_exception(ObLLVMValue &exception, + ObLLVMValue &error_code, ObLLVMValue &sql_state, ObLLVMBasicBlock &normal, bool in_notfound, @@ -6587,26 +6605,33 @@ int ObPLCodeGenerator::raise_exception(ObLLVMValue &exception, } } if (OB_SUCC(ret)) { - ObLLVMValue exception_class; - ObLLVMSwitch switch_inst; - ObLLVMValue int_value; OZ (set_current(current)); - if (OB_SUCC(ret) && lib::is_oracle_mode()) { + if (OB_FAIL(ret)) { + } else if (lib::is_oracle_mode()) { OZ (helper_.create_br(raise_exception)); - } - if (OB_SUCC(ret) && lib::is_mysql_mode()) { - if (signal && OB_ISNULL(sql_state.get_v())) { - // resignal - // TODO: SQL_WARNING错误的resignal需要特别处理 - OZ (helper_.create_br(raise_exception)); - } else { - OZ (helper_.create_call(ObString("get_exception_class"), get_eh_service().eh_classify_exception, sql_state, exception_class)); - OZ (helper_.create_switch(exception_class, raise_exception, switch_inst)); - OZ (helper_.get_int64(SQL_WARNING, int_value)); - OZ (switch_inst.add_case(int_value, in_warning ? raise_exception : normal)); - OZ (helper_.get_int64(NOT_FOUND, int_value)); - OZ (switch_inst.add_case(int_value, signal || in_notfound ? raise_exception : normal)); - } + } else if (lib::is_mysql_mode()) { + ObLLVMBasicBlock normal_raise_block; + ObLLVMValue exception_class; + ObLLVMSwitch switch_inst1; + ObLLVMSwitch switch_inst2; + ObLLVMValue int_value; + + OZ (helper_.create_block(ObString("normal_raise_block"), get_func(), normal_raise_block)); + OZ (helper_.create_switch(error_code, normal_raise_block, switch_inst1)); + OZ (helper_.get_int64(ER_WARN_TOO_MANY_RECORDS, int_value)); + OZ (switch_inst1.add_case(int_value, raise_exception)); + OZ (helper_.get_int64(WARN_DATA_TRUNCATED, int_value)); + OZ (switch_inst1.add_case(int_value, raise_exception)); + OZ (helper_.get_int64(ER_SIGNAL_WARN, int_value)); + OZ (switch_inst1.add_case(int_value, raise_exception)); + + OZ (set_current(normal_raise_block)); + OZ (helper_.create_call(ObString("get_exception_class"), get_eh_service().eh_classify_exception, sql_state, exception_class)); + OZ (helper_.create_switch(exception_class, raise_exception, switch_inst2)); + OZ (helper_.get_int64(SQL_WARNING, int_value)); + OZ (switch_inst2.add_case(int_value, in_warning ? raise_exception : normal)); + OZ (helper_.get_int64(NOT_FOUND, int_value)); + OZ (switch_inst2.add_case(int_value, signal || in_notfound ? raise_exception : normal)); } } return ret; diff --git a/src/pl/ob_pl_code_generator.h b/src/pl/ob_pl_code_generator.h index 5e63a3766..d3e3b9c3e 100644 --- a/src/pl/ob_pl_code_generator.h +++ b/src/pl/ob_pl_code_generator.h @@ -243,6 +243,7 @@ public: bool in_warning, bool signal); int raise_exception(jit::ObLLVMValue &exception, + jit::ObLLVMValue &error_code, jit::ObLLVMValue &sql_staten, jit::ObLLVMBasicBlock &normal, bool in_notfound, diff --git a/src/pl/ob_pl_resolver.cpp b/src/pl/ob_pl_resolver.cpp index ff0ce5902..0ae363455 100644 --- a/src/pl/ob_pl_resolver.cpp +++ b/src/pl/ob_pl_resolver.cpp @@ -421,7 +421,12 @@ int ObPLResolver::resolve(const ObStmtNodeTree *parse_tree, ObPLFunctionAST &fun LOG_WARN("fail to check decalre order", K(ret)); } else if (OB_FAIL(resolve_declare_handler(parse_tree, static_cast(NULL == stmt ? current_block_->get_eh() : stmt), func))) { LOG_WARN("failed to resolve declare handler stmt", K(parse_tree), K(stmt), K(ret)); - } else { + } else if (stmt != NULL && static_cast(stmt)->get_handlers().count() <= 0) { + // continue handler will record into handler_analyzer_, + // so here handlers may null, do not add null handler to current block. + stmt = NULL; + } + if (OB_SUCC(ret)) { func.set_is_all_sql_stmt(false); } } @@ -645,7 +650,7 @@ int ObPLResolver::resolve(const ObStmtNodeTree *parse_tree, ObPLFunctionAST &fun } if (OB_SUCC(ret) && handler_analyzer_.in_continue() && NULL != stmt && !NO_EXCEPTION_STMT(stmt->get_type())) { - //如果自己在continue里，那么把栈里的handler都加到block里 + // for continue handler, should add handler to every stmt, here, make a new block, consturct new block with handler and stmt. ObPLStmtBlock *block = NULL; if (OB_FAIL(make_block(func, current_block_, block))) { LOG_WARN("failed to make block", K(current_block_), K(ret)); @@ -990,10 +995,22 @@ int ObPLResolver::check_declare_order(ObPLStmtType type) CK(PL_VAR == type || PL_COND == type || PL_HANDLER == type || PL_CURSOR == type); if (is_oracle_mode()) { // oracle compatible, do nothing ... - } else if (OB_NOT_NULL(current_block_) && current_block_->get_stmts().count() != 0) { - ObPLStmtType pre_type = - current_block_->get_stmts().at(current_block_->get_stmts().count() - 1)->get_type(); - if ((PL_VAR == type || PL_COND == type) + } else { + ObPLStmtType pre_type = INVALID_PL_STMT; + if (OB_NOT_NULL(current_block_) && (current_block_->get_stmts().count() != 0)) { + pre_type = + current_block_->get_stmts().at(current_block_->get_stmts().count() - 1)->get_type(); + } + if (handler_analyzer_.get_stack_depth() > 0) { + ObPLDeclareHandlerStmt::DeclareHandler info; + if (OB_FAIL(handler_analyzer_.get_handler(handler_analyzer_.get_stack_depth() - 1, info))) { + LOG_WARN("failed to get last handler", K(ret)); + } else if (info.get_level() == current_level_) { + pre_type = PL_HANDLER; + } + } + if (OB_FAIL(ret)) { + } else if ((PL_VAR == type || PL_COND == type) && (PL_HANDLER == pre_type || PL_CURSOR == pre_type)) { ret = OB_ER_SP_VARCOND_AFTER_CURSHNDLR; LOG_USER_ERROR(OB_ER_SP_VARCOND_AFTER_CURSHNDLR); @@ -5202,15 +5219,16 @@ int ObPLResolver::resolve_declare_handler(const ObStmtNodeTree *parse_tree, ObPL ret = OB_ERR_NO_CHOICES; LOG_WARN("no choices may appear with choice OTHERS in an exception handler", K(ret)); + } else if (OB_FAIL(desc->add_condition(value))) { + LOG_WARN("failed to add condition for delcare handler desc", K(ret), K(value)); } else { - desc->add_condition(value); if (NOT_FOUND == actual_type && !handler_analyzer_.in_notfound()) { handler_analyzer_.set_notfound(current_level_); current_block_->set_notfound(); } else if (SQL_WARNING == actual_type && !handler_analyzer_.in_warning()) { handler_analyzer_.set_warning(current_level_); current_block_->set_warning(); - } else { /*do nothing*/ } + } } } } @@ -5235,17 +5253,19 @@ int ObPLResolver::resolve_declare_handler(const ObStmtNodeTree *parse_tree, ObPL } } - if (OB_SUCC(ret)) { + if (OB_SUCC(ret) && !desc->is_continue()) { ObPLDeclareHandlerStmt::DeclareHandler handler; handler.set_desc(desc); - OZ (stmt->add_handler(handler)); + if (OB_FAIL(stmt->add_handler(handler))) { + LOG_WARN("failed to add handler", K(ret)); + } } if (OB_SUCC(ret)) { if (OB_ISNULL(current_block_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("Symbol table is NULL", K(current_block_), K(ret)); - } else { + } else if (stmt->get_handlers().count() > 0) { current_block_->set_eh(stmt); } } @@ -5744,7 +5764,7 @@ int ObPLResolver::resolve_cparam_with_assign(ObRawExpr* expr, ret = OB_ERR_SP_UNDECLARED_VAR; LOG_WARN("can not find param in param list", K(ret), K(position), K(name)); } - OZ (resolve_cparam_without_assign(call_expr, position, func, params, expr_idx)); + OZ (resolve_cparam_without_assign(call_expr->get_expr(), position, func, params, expr_idx)); } return ret; } @@ -10497,6 +10517,13 @@ int ObPLResolver::resolve_sf_clause( } OX (routine_info->set_pipelined()); } + } else if (T_COMMENT == child->type_) { + if (lib::is_mysql_mode()) { + ObString routine_comment; + CK (OB_NOT_NULL(dynamic_cast(routine_info))); + OX (routine_comment = ObString(child->str_len_, child->str_value_)); + OZ (dynamic_cast(routine_info)->set_comment(routine_comment)); + } } } } @@ -12187,7 +12214,7 @@ int ObPLResolver::check_duplicate_condition(const ObPLDeclareHandlerStmt &stmt, { int ret = OB_SUCCESS; dup = false; - for (int64_t i = 0; !dup && i < stmt.get_handlers().count(); ++i) { + for (int64_t i = 0; OB_SUCC(ret) && !dup && i < stmt.get_handlers().count(); ++i) { ObPLDeclareHandlerStmt::DeclareHandler::HandlerDesc *desc = stmt.get_handler(i).get_desc(); if (OB_ISNULL(desc)) { ret = OB_ERR_UNEXPECTED; @@ -12203,6 +12230,29 @@ int ObPLResolver::check_duplicate_condition(const ObPLDeclareHandlerStmt &stmt, } } } + for (int64_t i = handler_analyzer_.get_stack_depth() - 1; OB_SUCC(ret) && !dup && i >= 0; --i) { + ObPLDeclareHandlerStmt::DeclareHandler handler; + if (OB_FAIL(handler_analyzer_.get_handler(i, handler))) { + LOG_WARN("failed to get handler from handler analyzer", K(ret), K(i)); + } else if (handler.get_level() == current_level_) { + ObPLDeclareHandlerStmt::DeclareHandler::HandlerDesc *desc = handler.get_desc(); + if (OB_ISNULL(desc)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Handler desc is NULL", K(ret), K(i), K(handler)); + } else if (desc->is_continue()) { + for (int64_t j = 0; !dup && j < desc->get_conditions().count(); ++j) { + if (value.type_ == desc->get_condition(j).type_ && + value.error_code_ == desc->get_condition(j).error_code_ && + value.str_len_ == desc->get_condition(j).str_len_ && + 0 == STRNCMP(value.sql_state_, desc->get_condition(j).sql_state_, value.str_len_)) { + dup = true; + } + } + } + } else { + break; + } + } return ret; } @@ -12424,7 +12474,12 @@ int ObPLResolver::HandlerAnalyzer::reset_handlers(int64_t level) int ret = OB_SUCCESS; for (int64_t i = handler_stack_.count() - 1; OB_SUCC(ret) && i >= 0; --i) { if (handler_stack_.at(i).get_level() == level) { - handler_stack_.pop_back(); + if (i != handler_stack_.count() -1) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to reset handlers with level", K(ret), K(level), K(i), K(handler_stack_.at(i))); + } else { + handler_stack_.pop_back(); + } } } if (OB_SUCC(ret) && handler_stack_.count() - 1 < top_continue_) { diff --git a/src/pl/ob_pl_stmt.cpp b/src/pl/ob_pl_stmt.cpp index 66eb3e4c3..5f2a9e2cf 100644 --- a/src/pl/ob_pl_stmt.cpp +++ b/src/pl/ob_pl_stmt.cpp @@ -1200,6 +1200,10 @@ int ObPLBlockNS::check_dup_symbol(const ObString &name, const ObPLDataType &type is_dup = true; ObPLVar *pl_var = const_cast(symbol_table_->get_symbol(symbols_.at(i))); pl_var->set_dup_declare(is_dup); + if (pl_var->is_referenced()) { + ret = OB_ERR_DECL_MORE_THAN_ONCE; + LOG_USER_ERROR(OB_ERR_DECL_MORE_THAN_ONCE, name.length(), name.ptr()); + } } } else { /*do nothing*/ } } @@ -2371,11 +2375,13 @@ int ObPLBlockNS::resolve_symbol(const ObString &var_name, && OB_INVALID_INDEX == var_idx && OB_INVALID_INDEX == parent_id && i < get_symbols().count(); ++i) { - const ObPLVar *pl_var = symbol_table_->get_symbol(get_symbols().at(i)); + ObPLVar *pl_var = const_cast(symbol_table_->get_symbol(get_symbols().at(i))); if (OB_ISNULL(pl_var)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("PL var ns is null", K(i), K(get_symbols().at(i)), K(ret)); } else if (ObCharset::case_compat_mode_equal(var_name, pl_var->get_name())) { + bool is_referenced = true; + pl_var->set_is_referenced(is_referenced); if (pl_var->is_dup_declare()) { ret = OB_ERR_DECL_MORE_THAN_ONCE; LOG_USER_ERROR(OB_ERR_DECL_MORE_THAN_ONCE, var_name.length(), var_name.ptr()); diff --git a/src/pl/ob_pl_stmt.h b/src/pl/ob_pl_stmt.h index 62e48251e..8351493af 100644 --- a/src/pl/ob_pl_stmt.h +++ b/src/pl/ob_pl_stmt.h @@ -121,7 +121,8 @@ public: is_readonly_(false), is_not_null_(false), is_default_construct_(false), - is_formal_param_(false) {} + is_formal_param_(false), + is_referenced_(false) {} virtual ~ObPLVar() {} inline const common::ObString &get_name() const { return name_; } @@ -146,6 +147,8 @@ public: int deep_copy(const ObPLVar &var, common::ObIAllocator &allocator); inline void set_dup_declare(bool dup_declare) { is_dup_declare_ = dup_declare; } inline bool is_dup_declare() const { return is_dup_declare_; } + inline void set_is_referenced(bool is_referenced) { is_referenced_ = is_referenced; } + inline bool is_referenced() const { return is_referenced_; } TO_STRING_KV(K_(name), K_(type), @@ -164,6 +167,7 @@ private: bool is_default_construct_; //默认值是否是该变量的构造函数 bool is_formal_param_; // this is formal param of a routine bool is_dup_declare_; + bool is_referenced_; }; class ObPLSymbolTable diff --git a/src/pl/parser/pl_parser_mysql_mode.y b/src/pl/parser/pl_parser_mysql_mode.y index 482e41063..c2162d115 100644 --- a/src/pl/parser/pl_parser_mysql_mode.y +++ b/src/pl/parser/pl_parser_mysql_mode.y @@ -1137,7 +1137,12 @@ sp_create_chistic: ; sp_chistic: - COMMENT STRING { } + COMMENT STRING + { + malloc_terminal_node($$, parse_ctx->mem_pool_, T_COMMENT); + $$->str_value_ = $2->str_value_; + $$->str_len_ = $2->str_len_; + } | LANGUAGE SQL { /* Just parse it, we only have one language for now. */ $$ = NULL; } | NO SQL {} | CONTAINS SQL {} diff --git a/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp b/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp index 4fb000d26..62d7e1784 100644 --- a/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp +++ b/src/rootserver/ddl_task/ob_ddl_redefinition_task.cpp @@ -1847,7 +1847,9 @@ int ObSyncTabletAutoincSeqCtx::sync() LOG_WARN("failed to call and process", K(ret)); } } - is_synced_ = true; + if (OB_SUCC(ret)) { + is_synced_ = true; + } } } return ret; diff --git a/src/rootserver/ddl_task/ob_ddl_scheduler.cpp b/src/rootserver/ddl_task/ob_ddl_scheduler.cpp index 76c62b11f..a643722c1 100644 --- a/src/rootserver/ddl_task/ob_ddl_scheduler.cpp +++ b/src/rootserver/ddl_task/ob_ddl_scheduler.cpp @@ -315,6 +315,8 @@ void ObDDLScheduler::run1() ObDDLTask *first_retry_task = nullptr; (void)prctl(PR_SET_NAME, "DDLTaskExecutor", 0, 0, 0); while (!has_set_stop()) { + THIS_WORKER.set_worker_level(1); + THIS_WORKER.set_curr_request_level(1); while (!has_set_stop()) { if (OB_FAIL(task_queue_.get_next_task(task))) { if (common::OB_ENTRY_NOT_EXIST == ret) { diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index 300548d8e..910243f1e 100644 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -13607,12 +13607,22 @@ int ObDDLService::add_new_index_schema(obrpc::ObAlterTableArg &alter_table_arg, } else if (OB_FAIL(generate_tablet_id(index_schema))) { LOG_WARN("fail to generate tablet id for hidden table", K(ret), K(index_schema)); } else { + bool is_exist = false; index_schema.set_table_id(new_idx_tid); index_schema.set_data_table_id(new_table_schema.get_table_id()); index_schema.set_index_status(INDEX_STATUS_UNAVAILABLE); // set the hidden attributes of the table index_schema.set_table_state_flag(ObTableStateFlag::TABLE_STATE_HIDDEN_OFFLINE_DDL); if (OB_FAIL(ret)) { + } else if (OB_FAIL(schema_guard.check_table_exist(index_schema.get_tenant_id(), + index_schema.get_database_id(), + index_schema.get_table_name_str(), + true/*is_index*/, + ObSchemaGetterGuard::USER_HIDDEN_TABLE_TYPE/*check_type*/, + is_exist))) { + LOG_WARN("failed to check table exist", K(ret)); + } else if (is_exist) { + LOG_INFO("index already rebuilt, skip", K(index_schema.get_table_id()), K(index_schema.get_table_name_str())); } else if (OB_FAIL(new_table_schemas.push_back(index_schema))) { LOG_WARN("failed to add table schema!", K(ret)); } else if (OB_FAIL(index_ids.push_back(index_schema.get_table_id()))) { @@ -13809,6 +13819,7 @@ int ObDDLService::reconstruct_index_schema(const ObTableSchema &orig_table_schem new_index_schema.set_table_name(new_index_table_name); // set the hidden attributes of the table new_index_schema.set_table_state_flag(ObTableStateFlag::TABLE_STATE_HIDDEN_OFFLINE_DDL); + bool is_exist = false; if (OB_FAIL(ret)) { } else if (OB_FAIL(new_table_schemas.push_back(new_index_schema))) { LOG_WARN("failed to add table schema!", K(ret)); diff --git a/src/rootserver/ob_tenant_role_transition_service.cpp b/src/rootserver/ob_tenant_role_transition_service.cpp index 05cc4475b..b22a4a3ef 100644 --- a/src/rootserver/ob_tenant_role_transition_service.cpp +++ b/src/rootserver/ob_tenant_role_transition_service.cpp @@ -409,7 +409,10 @@ int ObTenantRoleTransitionService::get_ls_access_mode_(ObIArray return_code_array; + int tmp_ret = OB_SUCCESS; for (int64_t i = 0; OB_SUCC(ret) && i < status_info_array.count(); ++i) { + return_code_array.reset(); const ObLSStatusInfo &info = status_info_array.at(i); const int64_t timeout = ctx.get_timeout(); if (info.ls_is_create_abort()) { @@ -426,7 +429,6 @@ int ObTenantRoleTransitionService::get_ls_access_mode_(ObIArraynonblock_renew( GCONF.cluster_id, tenant_id_, info.ls_id_))) { @@ -435,9 +437,12 @@ int ObTenantRoleTransitionService::get_ls_access_mode_(ObIArray return_code_array; - if (FAILEDx(proxy.wait_all(return_code_array))) { - LOG_WARN("wait all batch result failed", KR(ret)); + //need to wait all result whether success or fail + if (OB_SUCCESS != (tmp_ret = proxy.wait_all(return_code_array))) { + LOG_WARN("wait all batch result failed", KR(ret), KR(tmp_ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } else if (OB_FAIL(ret)) { + //no need to process return code } else if (rpc_count != return_code_array.count() || rpc_count != proxy.get_args().count() || rpc_count != proxy.get_results().count()) { diff --git a/src/share/cache/ob_kv_storecache.cpp b/src/share/cache/ob_kv_storecache.cpp index 0fd2632c9..6c6a514a6 100644 --- a/src/share/cache/ob_kv_storecache.cpp +++ b/src/share/cache/ob_kv_storecache.cpp @@ -232,6 +232,12 @@ int ObKVGlobalCache::init( map_once_clean_num_ = min(MAX_MAP_ONCE_CLEAN_NUM, bucket_num / MAP_ONCE_CLEAN_RATIO); map_once_replace_num_ = min(MAX_MAP_ONCE_REPLACE_NUM, bucket_num / MAP_ONCE_REPLACE_RATIO); inited_ = true; +#ifdef ENABLE_DEBUG_LOG + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(set_checker_cache_name(GCONF.leak_mod_to_check.str()))) { + COMMON_LOG(WARN, "[CACHE-HANDLE-CHECKER] Fail to set check cache name", K(tmp_ret)); + } +#endif } if (OB_UNLIKELY(!inited_)) { @@ -853,10 +859,10 @@ int ObKVGlobalCache::set_checker_cache_name(const char *cache_name) int64_t cache_id = INVALID_CACHE_ID; if (OB_UNLIKELY(!inited_)) { ret = OB_NOT_INIT; - COMMON_LOG(WARN, "The ObKVGlobalCache has not been inited", K(ret)); + COMMON_LOG(WARN, "[CACHE-HANDLE-CHECKER] The ObKVGlobalCache has not been inited", K(ret)); } else if (nullptr == cache_name) { ret = OB_INVALID_ARGUMENT; - COMMON_LOG(WARN, "Invalid argument", K(ret), KP(cache_name)); + COMMON_LOG(WARN, "[CACHE-HANDLE-CHECKER] Invalid argument", K(ret), KP(cache_name)); } else { if (0 == STRNCMP(cache_name, ObKVCacheHandleRefChecker::ALL_CACHE_NAME, MAX_CACHE_NAME_LENGTH)) { cache_id = MAX_CACHE_NUM; diff --git a/src/share/cache/ob_kv_storecache.h b/src/share/cache/ob_kv_storecache.h index 07e428966..b103e8e42 100644 --- a/src/share/cache/ob_kv_storecache.h +++ b/src/share/cache/ob_kv_storecache.h @@ -583,6 +583,10 @@ int ObKVCache::put_and_fetch( if (OB_ENTRY_EXIST != ret) { COMMON_LOG(WARN, "Fail to put kv to ObKVGlobalCache, ", K_(cache_id), K(ret)); } + } else { +#ifdef ENABLE_DEBUG_LOG + ObKVCacheHandleRefChecker::get_instance().handle_ref_inc(handle); +#endif } return ret; } diff --git a/src/share/cache/ob_kvcache_handle_ref_checker.cpp b/src/share/cache/ob_kvcache_handle_ref_checker.cpp index 696901b38..ae42b169b 100644 --- a/src/share/cache/ob_kvcache_handle_ref_checker.cpp +++ b/src/share/cache/ob_kvcache_handle_ref_checker.cpp @@ -12,21 +12,27 @@ #include "share/cache/ob_kvcache_handle_ref_checker.h" #include "share/cache/ob_kv_storecache.h" + + namespace oceanbase { namespace common { + /* * ---------------------------------------------- ObKVCacheHandleRefChecker::Key ---------------------------------------------- */ + ObKVCacheHandleRefChecker::Key::Key() : handle_(nullptr) { } + ObKVCacheHandleRefChecker::Key::Key(const ObKVCacheHandle &cache_handle) : handle_(&cache_handle) { } + uint64_t ObKVCacheHandleRefChecker::Key::hash() const { uint64_t hash = 0; @@ -35,13 +41,16 @@ uint64_t ObKVCacheHandleRefChecker::Key::hash() const } return hash; } + bool ObKVCacheHandleRefChecker::Key::operator== (const Key &other) const { return handle_ == other.handle_; } + /* * ---------------------------------------------- ObKVCacheHandleRefChecker::Value ---------------------------------------------- */ + ObKVCacheHandleRefChecker::Value::Value() : tenant_id_(OB_INVALID_TENANT_ID), cache_id_(INVALID_CACHE_ID), @@ -49,20 +58,24 @@ ObKVCacheHandleRefChecker::Value::Value() { MEMSET(bt_, 0, sizeof(bt_)); } + ObKVCacheHandleRefChecker::Value::Value(const Value &other) { *this = other; } + uint64_t ObKVCacheHandleRefChecker::Value::hash() const { uint64_t hash = 0; hash = murmurhash(bt_, sizeof(bt_), hash); return hash; } + bool ObKVCacheHandleRefChecker::Value::operator== (const Value &other) const { return (tenant_id_ == other.tenant_id_) && (cache_id_ == other.cache_id_) && (0 == STRNCMP(bt_, other.bt_, sizeof(bt_))); } + ObKVCacheHandleRefChecker::Value & ObKVCacheHandleRefChecker::Value::operator= (const Value &other) { if (this != &other) { @@ -72,10 +85,13 @@ ObKVCacheHandleRefChecker::Value & ObKVCacheHandleRefChecker::Value::operator= ( } return *this; } + /* * ---------------------------------------------- ObKVCacheHandleRefChecker ---------------------------------------------- */ + const char ObKVCacheHandleRefChecker::ALL_CACHE_NAME[MAX_CACHE_NAME_LENGTH] = "all_cache"; + ObKVCacheHandleRefChecker::ObKVCacheHandleRefChecker() : cache_id_(INVALID_CACHE_ID), handle_ref_bt_info_(), @@ -88,21 +104,25 @@ ObKVCacheHandleRefChecker::ObKVCacheHandleRefChecker() is_inited_ = true; } } + ObKVCacheHandleRefChecker::~ObKVCacheHandleRefChecker() { reset(); } + ObKVCacheHandleRefChecker &ObKVCacheHandleRefChecker::get_instance() { static ObKVCacheHandleRefChecker cache_handle_ref_checker_; return cache_handle_ref_checker_; } + void ObKVCacheHandleRefChecker::reset() { cache_id_ = INVALID_CACHE_ID; handle_ref_bt_info_.reuse(); is_inited_ = false; } + void ObKVCacheHandleRefChecker::handle_ref_inc(const ObKVCacheHandle &cache_handle) { INIT_SUCC(ret); @@ -122,6 +142,7 @@ void ObKVCacheHandleRefChecker::handle_ref_inc(const ObKVCacheHandle &cache_hand } } } + void ObKVCacheHandleRefChecker::handle_ref_de(const ObKVCacheHandle &cache_handle) { INIT_SUCC(ret); @@ -138,6 +159,7 @@ void ObKVCacheHandleRefChecker::handle_ref_de(const ObKVCacheHandle &cache_handl } } } + int ObKVCacheHandleRefChecker::set_cache_id(const int64_t cache_id) { INIT_SUCC(ret); @@ -153,6 +175,7 @@ int ObKVCacheHandleRefChecker::set_cache_id(const int64_t cache_id) } return ret; } + int ObKVCacheHandleRefChecker::get_aggregate_bt_info(hash::ObHashMap &bt_info) { INIT_SUCC(ret); @@ -186,5 +209,7 @@ int ObKVCacheHandleRefChecker::get_aggregate_bt_info(hash::ObHashMap> 30; + if (memory_limit_g < 4) { + OB_LOG(ERROR, "memory_limit with unexpected value", K(memory_limit_g)); + } else if (memory_limit_g <= 8) { + system_memory_b = 2LL << 30; + } else if (memory_limit_g <= 16) { + system_memory_b = 3LL << 30; + } else if (memory_limit_g <= 32) { + system_memory_b = 5LL << 30; + } else if (memory_limit_g <= 48) { + system_memory_b = 7LL << 30; + } else if (memory_limit_g <= 64) { + system_memory_b = 10LL << 30; + } else { + system_memory_b = int64_t(15 + 3 * (sqrt(memory_limit_g) - 8)) << 30; + } + } + return system_memory_b; } int ObServerConfig::read_config() diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index f7388aec6..233f7f31b 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -5111,6 +5111,8 @@ def_table_schema( # 427 : __all_ls_balance_job # 428 : __all_ls_balance_job_history +# 429 : __all_arbitration_service +# 430 : __all_arbitration_service_replica_task ################################################################################ # Virtual Table (10000, 20000] # Normally, virtual table's index_using_type should be USING_HASH. diff --git a/src/share/location_cache/ob_ls_location_map.cpp b/src/share/location_cache/ob_ls_location_map.cpp index cb1894625..27851839a 100644 --- a/src/share/location_cache/ob_ls_location_map.cpp +++ b/src/share/location_cache/ob_ls_location_map.cpp @@ -82,6 +82,11 @@ void ObLSLocationMap::destroy() ls_buckets_ = NULL; } if (OB_NOT_NULL(buckets_lock_)) { + for (int64_t i = 0; i < BUCKETS_CNT; ++i) { + if (OB_NOT_NULL(buckets_lock_ + i)) { + (buckets_lock_ + i)->~ObQSyncLock(); + } + } ob_free(buckets_lock_); buckets_lock_ = nullptr; } diff --git a/src/share/location_cache/ob_ls_location_service.cpp b/src/share/location_cache/ob_ls_location_service.cpp index dde66ad44..e989ae706 100644 --- a/src/share/location_cache/ob_ls_location_service.cpp +++ b/src/share/location_cache/ob_ls_location_service.cpp @@ -789,7 +789,7 @@ int ObLSLocationService::renew_location( // get from cache just for printing cache changes log if (OB_FAIL(ret)) { } else if (OB_SUCCESS != (tmp_ret = get_from_cache(cluster_id, tenant_id, ls_id, old_location))) { - if (OB_ENTRY_NOT_EXIST == tmp_ret) { + if (OB_CACHE_NOT_HIT == tmp_ret) { tmp_ret = OB_SUCCESS; } else { LOG_WARN("fail to get from cache", KR(tmp_ret), K(cluster_id), K(tenant_id), K(ls_id)); @@ -813,7 +813,7 @@ int ObLSLocationService::renew_location( LOG_WARN("get empty location from meta table", KR(ret), K(location)); } // print cache changes - if (OB_SUCC(ret) && OB_SUCC(tmp_ret) && !location.is_same_with(old_location)) { + if (OB_SUCC(ret) && (OB_SUCCESS == tmp_ret) && !location.is_same_with(old_location)) { FLOG_INFO("[LOCATION_CACHE]ls location cache has changed", KR(ret), K(old_location), "new_location", location); } return ret; diff --git a/src/share/ob_debug_sync_point.h b/src/share/ob_debug_sync_point.h index d8cf21903..5ab7454ff 100644 --- a/src/share/ob_debug_sync_point.h +++ b/src/share/ob_debug_sync_point.h @@ -417,6 +417,9 @@ class ObString; ACT(BEFORE_BACKUP_TASK_FINISH,)\ ACT(BEFORE_UPDATE_LS_META_TABLE,)\ ACT(BEFORE_LOAD_ARCHIVE_ROUND,)\ + ACT(BEFORE_MIG_DDL_TABLE_MERGE_TASK,)\ + ACT(BEFORE_COPY_DDL_SSTABLE,)\ + ACT(BEFORE_DDL_WRITE_PREPARE_LOG,)\ ACT(MAX_DEBUG_SYNC_POINT,) DECLARE_ENUM(ObDebugSyncPoint, debug_sync_point, OB_DEBUG_SYNC_POINT_DEF); diff --git a/src/share/ob_errno.cpp b/src/share/ob_errno.cpp index 4e2d82690..9097b8f91 100644 --- a/src/share/ob_errno.cpp +++ b/src/share/ob_errno.cpp @@ -10725,6 +10725,18 @@ static const _error _error_OB_DDL_SCHEMA_VERSION_NOT_MATCH = { .oracle_str_error = "ORA-00600: internal error code, arguments: -5403, ddl schema version not match", .oracle_str_user_error = "ORA-00600: internal error code, arguments: -5403, ddl schema version not match" }; +static const _error _error_OB_ERR_COLUMN_GROUP_DUPLICATE = { + .error_name = "OB_ERR_COLUMN_GROUP_DUPLICATE", + .error_cause = "Internal Error", + .error_solution = "Contact OceanBase Support", + .mysql_errno = ER_DUP_FIELDNAME, + .sqlstate = "42S21", + .str_error = "Duplicate column group name", + .str_user_error = "Duplicate column group name '%.*s'", + .oracle_errno = 600, + .oracle_str_error = "ORA-00600: internal error code, arguments: -5404, Duplicate column group name", + .oracle_str_user_error = "ORA-00600: internal error code, arguments: -5404, Duplicate column group name '%.*s'" +}; static const _error _error_OB_ERR_INVALID_JSON_TEXT = { .error_name = "OB_ERR_INVALID_JSON_TEXT", .error_cause = "Internal Error", @@ -22856,6 +22868,7 @@ struct ObStrErrorInit _errors[-OB_ERR_RESULTANT_DATA_TYPE_OF_VIRTUAL_COLUMN_IS_NOT_SUPPORTED] = &_error_OB_ERR_RESULTANT_DATA_TYPE_OF_VIRTUAL_COLUMN_IS_NOT_SUPPORTED; _errors[-OB_ERR_GET_STACKED_DIAGNOSTICS] = &_error_OB_ERR_GET_STACKED_DIAGNOSTICS; _errors[-OB_DDL_SCHEMA_VERSION_NOT_MATCH] = &_error_OB_DDL_SCHEMA_VERSION_NOT_MATCH; + _errors[-OB_ERR_COLUMN_GROUP_DUPLICATE] = &_error_OB_ERR_COLUMN_GROUP_DUPLICATE; _errors[-OB_ERR_INVALID_JSON_TEXT] = &_error_OB_ERR_INVALID_JSON_TEXT; _errors[-OB_ERR_INVALID_JSON_TEXT_IN_PARAM] = &_error_OB_ERR_INVALID_JSON_TEXT_IN_PARAM; _errors[-OB_ERR_INVALID_JSON_BINARY_DATA] = &_error_OB_ERR_INVALID_JSON_BINARY_DATA; diff --git a/src/share/ob_errno.def b/src/share/ob_errno.def index 859c6baaa..98bd1ca5d 100644 --- a/src/share/ob_errno.def +++ b/src/share/ob_errno.def @@ -624,6 +624,8 @@ DEFINE_ERROR(OB_FREEZE_SERVICE_EPOCH_MISMATCH, -4743, -1, "HY000", "freeze servi DEFINE_ERROR_EXT_DEP(OB_FROZEN_INFO_ALREADY_EXIST, -4744, -1, "HY000", "already exist larger frozen_scn in __all_freeze_info", "%s"); DEFINE_ERROR_EXT(OB_DELETE_SERVER_NOT_ALLOWED, -4745, -1, "HY000", "delete server not allowed", "%s"); DEFINE_ERROR(OB_PACKET_STATUS_UNKNOWN, -4746, -1, "HY000", "Network error and packet status unknown. Abort auto retry."); +//4747: OB_ARBITRATION_SERVICE_NOT_EXIST +//4748: OB_ARBITRATION_SERVICE_ALREADY_EXIST //////////////////////////////////////////////////////////////// // SQL & Schema specific error code, -5000 ~ -6000 @@ -995,6 +997,7 @@ DEFINE_ORACLE_ERROR_EXT(OB_ERR_OBJECT_STRING_DOES_NOT_EXIST, -5400, -1, "HY000", DEFINE_ORACLE_ERROR(OB_ERR_RESULTANT_DATA_TYPE_OF_VIRTUAL_COLUMN_IS_NOT_SUPPORTED, -5401, -1, "HY000", "resultant data type of virtual column is not supported", 54004, "resultant data type of virtual column is not supported"); DEFINE_ERROR(OB_ERR_GET_STACKED_DIAGNOSTICS, -5402, ER_GET_STACKED_DIAGNOSTICS_INACTIVE, "0Z002", "GET STACKED DIAGNOSTICS when handler not active"); DEFINE_ERROR(OB_DDL_SCHEMA_VERSION_NOT_MATCH, -5403, -1, "HY000", "ddl schema version not match"); +DEFINE_ERROR_EXT(OB_ERR_COLUMN_GROUP_DUPLICATE, -5404, ER_DUP_FIELDNAME, "42S21", "Duplicate column group name", "Duplicate column group name '%.*s'"); ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //error code for json -5410 ---- -5433, as for json only support mysql mode, error code for oracle use special values diff --git a/src/share/ob_errno.h b/src/share/ob_errno.h index 524922a15..8def28142 100644 --- a/src/share/ob_errno.h +++ b/src/share/ob_errno.h @@ -763,6 +763,7 @@ constexpr int OB_ERR_OBJECT_STRING_DOES_NOT_EXIST = -5400; constexpr int OB_ERR_RESULTANT_DATA_TYPE_OF_VIRTUAL_COLUMN_IS_NOT_SUPPORTED = -5401; constexpr int OB_ERR_GET_STACKED_DIAGNOSTICS = -5402; constexpr int OB_DDL_SCHEMA_VERSION_NOT_MATCH = -5403; +constexpr int OB_ERR_COLUMN_GROUP_DUPLICATE = -5404; constexpr int OB_SQL_RETRY_SPM = -5434; constexpr int OB_OUTLINE_NOT_REPRODUCIBLE = -5435; constexpr int OB_ERR_SP_ALREADY_EXISTS = -5541; @@ -2491,6 +2492,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_RESULTANT_DATA_TYPE_OF_VIRTUAL_COLUMN_IS_NOT_SUPPORTED__USER_ERROR_MSG "resultant data type of virtual column is not supported" #define OB_ERR_GET_STACKED_DIAGNOSTICS__USER_ERROR_MSG "GET STACKED DIAGNOSTICS when handler not active" #define OB_DDL_SCHEMA_VERSION_NOT_MATCH__USER_ERROR_MSG "ddl schema version not match" +#define OB_ERR_COLUMN_GROUP_DUPLICATE__USER_ERROR_MSG "Duplicate column group name '%.*s'" #define OB_ERR_INVALID_JSON_TEXT__USER_ERROR_MSG "Invalid JSON text." #define OB_ERR_INVALID_JSON_TEXT_IN_PARAM__USER_ERROR_MSG "Invalid JSON text in argument." #define OB_ERR_INVALID_JSON_BINARY_DATA__USER_ERROR_MSG "The JSON binary value contains invalid data." @@ -4322,6 +4324,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_RESULTANT_DATA_TYPE_OF_VIRTUAL_COLUMN_IS_NOT_SUPPORTED__ORA_USER_ERROR_MSG "ORA-54004: resultant data type of virtual column is not supported" #define OB_ERR_GET_STACKED_DIAGNOSTICS__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -5402, GET STACKED DIAGNOSTICS when handler not active" #define OB_DDL_SCHEMA_VERSION_NOT_MATCH__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -5403, ddl schema version not match" +#define OB_ERR_COLUMN_GROUP_DUPLICATE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -5404, Duplicate column group name '%.*s'" #define OB_ERR_INVALID_JSON_TEXT__ORA_USER_ERROR_MSG "ORA-00600: Invalid JSON text." #define OB_ERR_INVALID_JSON_TEXT_IN_PARAM__ORA_USER_ERROR_MSG "ORA-00600: Invalid JSON text in argument." #define OB_ERR_INVALID_JSON_BINARY_DATA__ORA_USER_ERROR_MSG "ORA-00600: The JSON binary value contains invalid data." diff --git a/src/share/ob_rpc_struct.cpp b/src/share/ob_rpc_struct.cpp index 0165e243f..763469a6e 100644 --- a/src/share/ob_rpc_struct.cpp +++ b/src/share/ob_rpc_struct.cpp @@ -3715,8 +3715,9 @@ bool ObPhysicalRestoreTenantArg::is_valid() const && uri_.length() < share::OB_MAX_RESTORE_DEST_LENGTH && !restore_option_.empty() && restore_option_.length() < common::OB_INNER_TABLE_DEFAULT_VALUE_LENTH - && restore_scn_.is_valid(); + && (!with_restore_scn_ || restore_scn_.is_valid()); } + OB_SERIALIZE_MEMBER((ObPhysicalRestoreTenantArg, ObCmdArg), tenant_name_, uri_, diff --git a/src/share/ob_snapshot_table_proxy.cpp b/src/share/ob_snapshot_table_proxy.cpp index c0f0821a0..3eea8ad3d 100644 --- a/src/share/ob_snapshot_table_proxy.cpp +++ b/src/share/ob_snapshot_table_proxy.cpp @@ -194,6 +194,7 @@ int ObSnapshotTableProxy::batch_add_snapshot( ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(tenant_id), K(schema_version), K(snapshot_scn), K(tablet_id_array)); } else { + palf::SCN snapshot_gc_scn = palf::SCN::min_scn(); int64_t report_idx = 0; const int64_t exec_tenant_id = ObSchemaUtils::get_exec_tenant_id(tenant_id); ObSnapshotInfo info; @@ -203,6 +204,9 @@ int ObSnapshotTableProxy::batch_add_snapshot( info.snapshot_scn_ = snapshot_scn; info.schema_version_ = schema_version; info.comment_ = comment; + if (OB_FAIL(ObGlobalStatProxy::select_snapshot_gc_scn_for_update(trans, tenant_id, snapshot_gc_scn))) { + LOG_WARN("fail to select gc timstamp for update", KR(ret), K(info), K(tenant_id)); + } while (OB_SUCC(ret) && report_idx < tablet_id_array.count()) { sql.reuse(); columns.reuse(); @@ -211,7 +215,7 @@ int ObSnapshotTableProxy::batch_add_snapshot( for (int64_t i = 0; OB_SUCC(ret) && i < cur_batch_cnt; ++i) { info.tablet_id_ = tablet_id_array.at(report_idx + i).id(); dml.reuse(); - if (OB_FAIL(check_snapshot_valid(trans, info.tenant_id_, info, is_valid))) { + if (OB_FAIL(check_snapshot_valid(snapshot_gc_scn, info, is_valid))) { LOG_WARN("fail to check snapshot valid", KR(ret), K(info), K(tenant_id)); } else if (!is_valid) { ret = OB_SNAPSHOT_DISCARDED; @@ -428,19 +432,15 @@ int ObSnapshotTableProxy::get_all_snapshots( } int ObSnapshotTableProxy::check_snapshot_valid( - ObISQLClient &client, - const uint64_t tenant_id, + const palf::SCN &snapshot_gc_scn, const ObSnapshotInfo &info, - bool &is_valid) + bool &is_valid) const { int ret = OB_SUCCESS; - palf::SCN snapshot_gc_scn; is_valid = false; if (!info.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(info)); - } else if (OB_FAIL(ObGlobalStatProxy::select_snapshot_gc_scn_for_update(client, tenant_id, snapshot_gc_scn))) { - LOG_WARN("fail to select gc timstamp for update", KR(ret), K(info), K(tenant_id)); } else if (info.snapshot_scn_ <= snapshot_gc_scn) { is_valid = false; LOG_WARN("invalid snapshot info", KR(ret), K(info), K(snapshot_gc_scn)); diff --git a/src/share/ob_snapshot_table_proxy.h b/src/share/ob_snapshot_table_proxy.h index f16698357..d91bdef8d 100644 --- a/src/share/ob_snapshot_table_proxy.h +++ b/src/share/ob_snapshot_table_proxy.h @@ -148,10 +148,9 @@ public: int64_t &count); private: int gen_event_ts(int64_t &event_ts); - int check_snapshot_valid(common::ObISQLClient &client, - const uint64_t tenant_id, + int check_snapshot_valid(const palf::SCN &snapshot_gc_scn, const ObSnapshotInfo &info, - bool &is_valid); + bool &is_valid) const; int fill_snapshot_item(const ObSnapshotInfo &info, share::ObDMLSqlSplicer &dml); diff --git a/src/share/ob_table_range.cpp b/src/share/ob_table_range.cpp index ae1b2809d..ed65a6941 100644 --- a/src/share/ob_table_range.cpp +++ b/src/share/ob_table_range.cpp @@ -16,10 +16,6 @@ namespace share const palf::SCN ObScnRange::MIN_SCN = palf::SCN::min_scn(); const palf::SCN ObScnRange::MAX_SCN = palf::SCN::max_scn(); -const int64_t ObScnRange::MIN_TS = palf::OB_MIN_SCN_TS_NS; -const int64_t ObScnRange::MAX_TS = palf::OB_MAX_SCN_TS_NS; -const int64_t ObScnRange::OLD_MAX_TS = INT64_MAX; - ObScnRange::ObScnRange() : start_scn_(MIN_SCN), end_scn_(MIN_SCN) diff --git a/src/share/ob_table_range.h b/src/share/ob_table_range.h index e7a05f9a0..d5b729129 100644 --- a/src/share/ob_table_range.h +++ b/src/share/ob_table_range.h @@ -25,9 +25,6 @@ struct ObScnRange public: static const palf::SCN MIN_SCN; static const palf::SCN MAX_SCN; - static const int64_t MIN_TS; - static const int64_t MAX_TS; - static const int64_t OLD_MAX_TS; ObScnRange(); int64_t hash() const; diff --git a/src/share/ob_tablet_autoincrement_service.cpp b/src/share/ob_tablet_autoincrement_service.cpp index b5b01d1a8..f2204e8aa 100644 --- a/src/share/ob_tablet_autoincrement_service.cpp +++ b/src/share/ob_tablet_autoincrement_service.cpp @@ -508,5 +508,37 @@ int ObTabletAutoincSeqRpcHandler::batch_set_tablet_autoinc_seq( return ret; } +int ObTabletAutoincSeqRpcHandler::replay_update_tablet_autoinc_seq( + const ObLS *ls, + const ObTabletID &tablet_id, + const uint64_t autoinc_seq, + const palf::SCN &replay_scn) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(ls == nullptr || !tablet_id.is_valid() || autoinc_seq == 0 || !replay_scn.is_valid_and_not_min())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(tablet_id), K(autoinc_seq), K(replay_scn)); + } else { + ObTabletHandle tablet_handle; + ObBucketHashWLockGuard guard(bucket_lock_, tablet_id.hash()); + if (OB_FAIL(ls->replay_get_tablet(tablet_id, + replay_scn, + tablet_handle))) { + if (OB_TABLET_NOT_EXIST == ret) { + LOG_INFO("tablet may be deleted, skip this log", K(ret), K(tablet_id), K(replay_scn)); + ret = OB_SUCCESS; + } else if (OB_EAGAIN == ret) { + // retry replay again + } else { + LOG_WARN("fail to replay get tablet, retry again", K(ret), K(tablet_id), K(replay_scn)); + ret = OB_EAGAIN; + } + } else if (OB_FAIL(tablet_handle.get_obj()->update_tablet_autoinc_seq(autoinc_seq, replay_scn))) { + LOG_WARN("failed to update tablet auto inc seq", K(ret), K(autoinc_seq), K(replay_scn)); + } + } + return ret; +} + } } diff --git a/src/share/ob_tablet_autoincrement_service.h b/src/share/ob_tablet_autoincrement_service.h index 423057c68..b8dd25592 100644 --- a/src/share/ob_tablet_autoincrement_service.h +++ b/src/share/ob_tablet_autoincrement_service.h @@ -131,11 +131,16 @@ public: int batch_set_tablet_autoinc_seq( const obrpc::ObBatchSetTabletAutoincSeqArg &arg, obrpc::ObBatchSetTabletAutoincSeqRes &res); + int replay_update_tablet_autoinc_seq( + const ObLS *ls, + const ObTabletID &tablet_id, + const uint64_t autoinc_seq, + const palf::SCN &replay_scn); private: ObTabletAutoincSeqRpcHandler(); ~ObTabletAutoincSeqRpcHandler(); private: - static const int64_t BUCKET_LOCK_BUCKET_CNT = 100; + static const int64_t BUCKET_LOCK_BUCKET_CNT = 10243L; bool is_inited_; common::ObBucketLock bucket_lock_; }; diff --git a/src/share/parameter/ob_parameter_seed.ipp b/src/share/parameter/ob_parameter_seed.ipp index d0fee577d..14a420625 100644 --- a/src/share/parameter/ob_parameter_seed.ipp +++ b/src/share/parameter/ob_parameter_seed.ipp @@ -80,7 +80,7 @@ DEF_CAP_WITH_CHECKER(memory_limit, OB_CLUSTER_PARAMETER, "0", DEF_CAP(rootservice_memory_limit, OB_CLUSTER_PARAMETER, "2G", "[2G,)", "max memory size which can be used by rs tenant The default value is 2G. Range: [2G,)", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); -DEF_CAP(system_memory, OB_CLUSTER_PARAMETER, "30G", "[0M,)", +DEF_CAP(system_memory, OB_CLUSTER_PARAMETER, "0M", "[0M,)", "the memory reserved for internal use which cannot be allocated to any outer-tenant, " "and should be determined to guarantee every server functions normally. Range: [0M,)", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); @@ -1291,6 +1291,21 @@ DEF_STR_LIST(sanity_whitelist, OB_CLUSTER_PARAMETER, "", "vip who wouldn't leadi DEF_TIME(_advance_checkpoint_timeout, OB_CLUSTER_PARAMETER, "30m", "[10s,180m]", "the timeout for backup/migrate advance checkpoint Range: [10s,180m]", ObParameterAttr(Section::ROOT_SERVICE, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +ERRSIM_DEF_INT(errsim_migration_tablet_id, OB_CLUSTER_PARAMETER, "0", "[0,)", + "the tablet id that migration want to insert error" + "Range: [0,) in integer", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +ERRSIM_DEF_INT(errsim_max_ddl_sstable_count, OB_CLUSTER_PARAMETER, "0", "[0,)", + "max ddl sstable count in errsim mode" + "Range: [0,) in integer", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +ERRSIM_DEF_INT(errsim_max_ddl_block_count, OB_CLUSTER_PARAMETER, "0", "[0,)", + "max ddl block count in errsim mode" + "Range: [0,) in integer", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); +ERRSIM_DEF_STR(errsim_migration_src_server_addr, OB_CLUSTER_PARAMETER, "", + "the server dest ls choose as src when in errsim mode", + ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); DEF_BOOL(enable_cgroup, OB_CLUSTER_PARAMETER, "True", "when set to false, cgroup will not init; when set to true but cgroup root dir is not ready, print ERROR", diff --git a/src/share/schema/ob_schema_getter_guard.cpp b/src/share/schema/ob_schema_getter_guard.cpp index f9dda55cc..08f642bea 100644 --- a/src/share/schema/ob_schema_getter_guard.cpp +++ b/src/share/schema/ob_schema_getter_guard.cpp @@ -3332,8 +3332,131 @@ int ObSchemaGetterGuard::check_priv(const ObSessionPrivInfo &session_priv, return ret; } +int ObSchemaGetterGuard::check_priv_db_or_(const ObSessionPrivInfo &session_priv, + const ObNeedPriv &need_priv, + const ObPrivMgr &priv_mgr, + const uint64_t tenant_id, + const uint64_t user_id, + bool& pass) { + int ret = OB_SUCCESS; + int64_t total_db_priv_set_role = 0; + ObString db = need_priv.db_; + ObPrivSet db_priv_set = 0; + if (session_priv.db_.length() != 0 && (session_priv.db_ == db || 0 == db.length())) { + db_priv_set = session_priv.db_priv_set_; + } else { + ObOriginalDBKey db_priv_key(tenant_id, user_id, db); + if (OB_FAIL(priv_mgr.get_db_priv_set(db_priv_key, db_priv_set))) { + LOG_WARN("get db priv set failed", K(db_priv_key), KR(ret)); + } + } + + /* load role db privs */ + if (OB_SUCC(ret)) { + const ObUserInfo *user_info = NULL; + //bool is_grant_role = false; + if (OB_FAIL(get_user_info(tenant_id, user_id, user_info))) { + LOG_WARN("failed to get user info", KR(ret), K(tenant_id), K(user_id)); + } else if (OB_ISNULL(user_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("user info is null", KR(ret), K(tenant_id), K(user_id)); + } else { + const ObIArray &role_id_array = user_info->get_role_id_array(); + for (int64_t i = 0; OB_SUCC(ret) && i < role_id_array.count(); ++i) { + const ObUserInfo *role_info = NULL; + if (OB_FAIL(get_user_info(tenant_id, role_id_array.at(i), role_info))) { + LOG_WARN("failed to get role ids", KR(ret), K(tenant_id), K(role_id_array.at(i))); + } else if (OB_ISNULL(role_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("role info is null", KR(ret), K(role_id_array.at(i))); + } else { + ObPrivSet db_priv_set_role = OB_PRIV_SET_EMPTY; + ObOriginalDBKey db_priv_key_role(tenant_id, role_info->get_user_id(), db); + if (OB_FAIL(priv_mgr.get_db_priv_set(db_priv_key_role, db_priv_set_role))) { + LOG_WARN("get db priv set failed", KR(ret), K(db_priv_key_role)); + } else { + db_priv_set |= db_priv_set_role; + } + } + } + } + } + + if (OB_SUCC(ret)) { + pass = OB_PRIV_HAS_ANY(db_priv_set, need_priv.priv_set_); + } + + return ret; +} + +int ObSchemaGetterGuard::check_priv_table_or_(const ObNeedPriv &need_priv, + const ObPrivMgr &priv_mgr, + const uint64_t tenant_id, + const uint64_t user_id, + bool& pass) { + int ret = OB_SUCCESS; + //1. fetch table priv + const ObTablePriv *table_priv = NULL; + ObPrivSet table_priv_set = 0; + ObTablePrivSortKey table_priv_key(tenant_id, + user_id, + need_priv.db_, + need_priv.table_); + if (OB_FAIL(priv_mgr.get_table_priv(table_priv_key, table_priv))) { + LOG_WARN("get table priv failed", KR(ret), K(table_priv_key)); + } else if (OB_ISNULL(table_priv)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table priv is null", KR(ret), K(table_priv_key)); + } else { + table_priv_set = table_priv->get_priv_set(); + } + + if (OB_SUCC(ret)) { + //2. fetch roles privs + const ObUserInfo *user_info = NULL; + if (OB_FAIL(get_user_info(tenant_id, user_id, user_info))) { + LOG_WARN("failed to get user info", KR(ret), K(tenant_id), K(user_id)); + } else if (OB_ISNULL(user_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("user info is null", KR(ret), K(tenant_id), K(user_id)); + } else { + const ObIArray &role_id_array = user_info->get_role_id_array(); + for (int64_t i = 0; OB_SUCC(ret) && i < role_id_array.count(); ++i) { + const ObUserInfo *role_info = NULL; + const ObTablePriv *role_table_priv = NULL; + if (OB_FAIL(get_user_info(tenant_id, role_id_array.at(i), role_info))) { + LOG_WARN("failed to get role ids", KR(ret), K(tenant_id), K(role_id_array.at(i))); + } else if (OB_ISNULL(role_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("role info is null", KR(ret), K(role_id_array.at(i))); + } else { + ObTablePrivSortKey role_table_priv_key(tenant_id, + role_info->get_user_id(), + need_priv.db_, + need_priv.table_); + if (OB_FAIL(priv_mgr.get_table_priv(role_table_priv_key, role_table_priv))) { + LOG_WARN("get table priv failed", KR(ret), K(role_table_priv_key) ); + } else if (OB_ISNULL(role_table_priv)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("role table priv is null", KR(ret), K(role_table_priv_key)); + } else { + table_priv_set |= role_table_priv->get_priv_set(); + } + } + } + } + } + + //3. check privs + if (OB_SUCC(ret)) { + pass = OB_PRIV_HAS_ANY(table_priv_set, need_priv.priv_set_); + } + + return ret; +} + int ObSchemaGetterGuard::check_priv_or(const ObSessionPrivInfo &session_priv, - const ObStmtNeedPrivs &stmt_need_privs) + const ObStmtNeedPrivs &stmt_need_privs) { int ret = OB_SUCCESS; @@ -3341,13 +3464,18 @@ int ObSchemaGetterGuard::check_priv_or(const ObSessionPrivInfo &session_priv, bool pass = false; ObPrivLevel max_priv_level = OB_PRIV_INVALID_LEVEL; uint64_t tenant_id = session_priv.tenant_id_; + uint64_t user_id = session_priv.user_id_; const ObSchemaMgr *mgr = NULL; if (OB_FAIL(check_tenant_schema_guard(tenant_id))) { LOG_WARN("fail to check tenant schema guard", KR(ret), K(tenant_id), K_(tenant_id)); } else if (OB_FAIL(check_lazy_guard(tenant_id, mgr))) { LOG_WARN("fail to check lazy guard", KR(ret), K(tenant_id)); + } else if (OB_ISNULL(mgr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("mgr is NULL", KR(ret), K(tenant_id)); } else if (session_priv.is_valid()) { - for (int64_t i = 0; !pass && OB_SUCCESS == ret && i < need_privs.count(); ++i) { + const ObPrivMgr &priv_mgr = mgr->priv_mgr_; + for (int64_t i = 0; !pass && OB_SUCC(ret) && i < need_privs.count(); ++i) { const ObNeedPriv &need_priv = need_privs.at(i); if (need_priv.priv_level_ > max_priv_level) { max_priv_level = need_priv.priv_level_; @@ -3358,20 +3486,14 @@ int ObSchemaGetterGuard::check_priv_or(const ObSessionPrivInfo &session_priv, break; } case OB_PRIV_DB_LEVEL: { - pass = OB_PRIV_HAS_ANY(session_priv.db_priv_set_, need_priv.priv_set_); + if (OB_FAIL(check_priv_db_or_(session_priv, need_priv, priv_mgr, tenant_id, user_id, pass))) { + LOG_WARN("fail to check priv db only", KR(ret), K(tenant_id), K(user_id), K(need_priv.db_)); + } break; } case OB_PRIV_TABLE_LEVEL: { - const ObPrivMgr &priv_mgr = mgr->priv_mgr_; - const ObTablePriv *table_priv = NULL; - ObTablePrivSortKey table_priv_key(session_priv.tenant_id_, - session_priv.user_id_, - need_priv.db_, - need_priv.table_); - if (OB_FAIL(priv_mgr.get_table_priv(table_priv_key, table_priv))) { - LOG_WARN("get table priv failed", KR(ret), K(table_priv_key)); - } else if (NULL != table_priv) { - pass = OB_PRIV_HAS_ANY(table_priv->get_priv_set(), need_priv.priv_set_); + if (OB_FAIL(check_priv_table_or_(need_priv, priv_mgr, tenant_id, user_id, pass))) { + LOG_WARN("fail to check priv table only", KR(ret), K(tenant_id), K(user_id), K(need_priv.db_), K(need_priv.table_)); } break; } diff --git a/src/share/schema/ob_schema_getter_guard.h b/src/share/schema/ob_schema_getter_guard.h index 1824e6667..17fc36fb9 100644 --- a/src/share/schema/ob_schema_getter_guard.h +++ b/src/share/schema/ob_schema_getter_guard.h @@ -48,6 +48,7 @@ class ObColumnSchemaV2; class ObDBPriv; class ObDatabaseSchema; class ObMultiVersionSchemaService; +class ObPrivMgr; class ObSimpleDatabaseSchema; class ObSimplePackageSchema; class ObSimpleRoutineSchema; @@ -72,6 +73,7 @@ struct ObSessionPrivInfo; struct ObStmtNeedPrivs; struct ObUserLoginInfo; + class ObSchemaMgrInfo { public: @@ -1087,6 +1089,17 @@ private: const ObIArray &role_id_array); bool ignore_tenant_not_exist_error(const uint64_t tenant_id); + int check_priv_db_or_(const ObSessionPrivInfo &session_priv, + const ObNeedPriv &need_priv, + const ObPrivMgr &priv_mgr, + const uint64_t tenant_id, + const uint64_t user_id, + bool& pass); + int check_priv_table_or_(const ObNeedPriv &need_priv, + const ObPrivMgr &priv_mgr, + const uint64_t tenant_id, + const uint64_t user_id, + bool& pass); private: common::ObArenaAllocator local_allocator_; ObMultiVersionSchemaService *schema_service_; diff --git a/src/share/schema/ob_schema_printer.cpp b/src/share/schema/ob_schema_printer.cpp index 07920898c..50c08f60b 100644 --- a/src/share/schema/ob_schema_printer.cpp +++ b/src/share/schema/ob_schema_printer.cpp @@ -3840,6 +3840,16 @@ int ObSchemaPrinter::print_routine_definition(const ObRoutineInfo *routine_info, OX (routine_param = static_cast(routine_info->get_ret_info())); OZ (print_routine_param_type(routine_param, return_type, buf, buf_len, pos, tz_info)); } + if (OB_SUCC(ret) && lib::is_mysql_mode()) { + OZ (databuff_printf(buf, buf_len, pos, "%s", + routine_info->is_deterministic() ? "\nDETERMINISTIC\n" : "")); + if (OB_SUCC(ret) && OB_NOT_NULL(routine_info->get_comment())) { + OZ (databuff_printf(buf, buf_len, pos, "%s%.*s%s\n","COMMENT `", + routine_info->get_comment().length(), + routine_info->get_comment().ptr(), + "`")); + } + } OZ (databuff_printf(buf, buf_len, pos, lib::is_oracle_mode() ? " IS\n%.*s" : " %.*s", body.length(), body.ptr())); return ret; } diff --git a/src/sql/das/ob_das_group_scan_op.cpp b/src/sql/das/ob_das_group_scan_op.cpp index 8d8bdfeec..808a552bd 100644 --- a/src/sql/das/ob_das_group_scan_op.cpp +++ b/src/sql/das/ob_das_group_scan_op.cpp @@ -30,6 +30,14 @@ ObDASGroupScanOp::ObDASGroupScanOp(ObIAllocator &op_alloc) { } +ObDASGroupScanOp::~ObDASGroupScanOp() +{ + if (result_iter_ != nullptr && result_iter_->get_type() == ObNewRowIterator::ObTableScanIterator) { + LOG_ERROR("table group scan iter is not released, maybe some bug occured", + KPC(scan_ctdef_), K(scan_param_), KPC(scan_rtdef_)); + } +} + int ObDASGroupScanOp::rescan() { int &ret = errcode_; @@ -223,6 +231,21 @@ int ObDASGroupScanOp::decode_task_result(ObIDASTaskResult *task_result) } return ret; } +ObGroupLookupOp::~ObGroupLookupOp() +{ + const ObNewRowIterator *lookup_iter = get_lookup_storage_iter(); + if (lookup_iter != nullptr && lookup_iter->get_type() == ObNewRowIterator::ObTableScanIterator) { + LOG_ERROR("lookup_iter iter is not released, maybe some bug occured", + KPC(lookup_ctdef_), K(scan_param_), KPC(index_ctdef_), + K(lookup_rowkey_cnt_), K(lookup_row_cnt_)); + } + const ObNewRowIterator *rowkey_iter = static_cast(get_rowkey_iter())->get_iter(); + if (rowkey_iter != nullptr && rowkey_iter->get_type() == ObNewRowIterator::ObTableScanIterator) { + LOG_ERROR("rowkey_iter iter is not released, maybe some bug occured", + KPC(lookup_ctdef_), K(scan_param_), KPC(index_ctdef_), + K(lookup_rowkey_cnt_), K(lookup_row_cnt_)); + } +} int ObGroupLookupOp::init_group_range(int64_t cur_group_idx, int64_t group_size) { diff --git a/src/sql/das/ob_das_group_scan_op.h b/src/sql/das/ob_das_group_scan_op.h index 9bafddb50..af737327d 100644 --- a/src/sql/das/ob_das_group_scan_op.h +++ b/src/sql/das/ob_das_group_scan_op.h @@ -25,6 +25,7 @@ public: index_group_cnt_(1), lookup_group_cnt_(1) {} + virtual ~ObGroupLookupOp(); virtual void reset() override { ObLocalIndexLookupOp::reset(); @@ -57,6 +58,7 @@ class ObDASGroupScanOp : public ObDASScanOp OB_UNIS_VERSION(1); public: ObDASGroupScanOp(common::ObIAllocator &op_alloc); + virtual ~ObDASGroupScanOp(); int open_op() override; int release_op() override; virtual int rescan() override; diff --git a/src/sql/das/ob_das_rpc_processor.cpp b/src/sql/das/ob_das_rpc_processor.cpp index 06073275f..3d44f747b 100644 --- a/src/sql/das/ob_das_rpc_processor.cpp +++ b/src/sql/das/ob_das_rpc_processor.cpp @@ -168,7 +168,14 @@ int ObDASSyncFetchP::process() } else if (OB_FAIL(das->get_task_res_mgr().iterator_task_result(task_id, datum_store, has_more))) { - LOG_WARN("get task result failed", KR(ret), K(res)); + if (OB_UNLIKELY(OB_ENTRY_NOT_EXIST == ret)) { + // After server reboot, the hash map containing task results was gone. + // We need to retry for such cases. + LOG_WARN("task result was gone due to server reboot, will retry", KR(ret), K(res)); + ret = OB_RPC_SEND_ERROR; + } else { + LOG_WARN("get task result failed", KR(ret), K(res)); + } } else { res.set_has_more(has_more); } diff --git a/src/sql/das/ob_das_task_result.cpp b/src/sql/das/ob_das_task_result.cpp index 3d2105784..01624f104 100644 --- a/src/sql/das/ob_das_task_result.cpp +++ b/src/sql/das/ob_das_task_result.cpp @@ -361,7 +361,11 @@ int ObDASTaskResultMgr::iterator_task_result(int64_t task_id, ret = OB_ERR_UNEXPECTED; LOG_WARN("das tcb is null", KR(ret), K(task_id)); } else if (tcb->is_exiting_) { - ret = OB_INVALID_ARGUMENT; + // The background GC thread is already cleaning up this tcb, + // and it will be removed from the hash map shortly. + // This happens when the task result has expired meaning the + // SQL has timed out. + ret = OB_TIMEOUT; LOG_WARN("das tcb is exiting", KR(ret), K(task_id)); } else if (tcb->is_reading_) { ret = OB_EAGAIN; diff --git a/src/sql/engine/cmd/ob_variable_set_executor.cpp b/src/sql/engine/cmd/ob_variable_set_executor.cpp index 38aeea932..4adba2423 100644 --- a/src/sql/engine/cmd/ob_variable_set_executor.cpp +++ b/src/sql/engine/cmd/ob_variable_set_executor.cpp @@ -414,26 +414,18 @@ int ObVariableSetExecutor::execute_subquery_expr(ObExecContext &ctx, LOG_WARN("failed to get obj", K(ret), K(idx)); } } - if (OB_FAIL(ret)) { - } else if (OB_NOT_NULL(conn) && OB_NOT_NULL(sql_proxy) && - OB_FAIL(sql_proxy->close(conn, true))) { - LOG_WARN("failed to close connection", K(ret)); - } else if (tmp_value.need_deep_copy()) { - char *copy_data = NULL; - int64_t copy_size = tmp_value.get_deep_copy_size(); - int64_t copy_pos = 0; - if (OB_ISNULL(copy_data = static_cast(ctx.get_allocator().alloc(copy_size)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("memory allocate failed", K(ret)); - } else if (OB_FAIL(value_obj.deep_copy(tmp_value, copy_data, copy_size, copy_pos))) { - LOG_WARN("obj deep copy failed", K(ret)); - } - LOG_TRACE("succeed to deep copy current value", K(ret)); - } else { - value_obj = tmp_value; + if (OB_SUCC(ret) && (OB_FAIL(ob_write_obj(ctx.get_allocator(), tmp_value, value_obj)))) { + LOG_WARN("failed to write value", K(ret)); } LOG_TRACE("succ to calculate value by executing inner sql", K(ret), K(value_obj), K(subquery_expr)); } + if (OB_NOT_NULL(conn) && OB_NOT_NULL(sql_proxy)) { + int tmp_ret = sql_proxy->close(conn, true); + if (OB_UNLIKELY(tmp_ret != OB_SUCCESS)) { + LOG_WARN("failed to close sql connection", K(tmp_ret)); + } + ret = ret == OB_SUCCESS ? tmp_ret : ret; + } return ret; } diff --git a/src/sql/engine/expr/ob_datum_cast.cpp b/src/sql/engine/expr/ob_datum_cast.cpp index 7b6b83d8c..529705278 100644 --- a/src/sql/engine/expr/ob_datum_cast.cpp +++ b/src/sql/engine/expr/ob_datum_cast.cpp @@ -250,7 +250,7 @@ int ObDatumHexUtils::hex(const ObExpr &expr, const ObString &in_str, ObEvalCtx & ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc memory failed", K(ret), K(alloc_length)); } else { - static const char *HEXCHARS = upper_case ? "0123456789ABCDEF" : "0123456789abcdef"; + const char *HEXCHARS = upper_case ? "0123456789ABCDEF" : "0123456789abcdef"; int32_t pos = 0; for (int32_t i = 0; i < in_str.length(); ++i) { buf[pos++] = HEXCHARS[in_str[i] >> 4 & 0xF]; diff --git a/src/sql/engine/expr/ob_expr_json_search.cpp b/src/sql/engine/expr/ob_expr_json_search.cpp index 27a96d746..1abf36aa9 100644 --- a/src/sql/engine/expr/ob_expr_json_search.cpp +++ b/src/sql/engine/expr/ob_expr_json_search.cpp @@ -262,7 +262,7 @@ int ObExprJsonSearch::eval_json_search(const ObExpr &expr, ObEvalCtx &ctx, ObDat } // check one_or_all flag - bool one_flag; + bool one_flag = false; if (OB_SUCC(ret) && !is_null) { json_arg = expr.args_[1]; val_type = json_arg->datum_meta_.type_; diff --git a/src/sql/engine/expr/ob_expr_json_value.cpp b/src/sql/engine/expr/ob_expr_json_value.cpp index db502d76a..6cc0dabce 100644 --- a/src/sql/engine/expr/ob_expr_json_value.cpp +++ b/src/sql/engine/expr/ob_expr_json_value.cpp @@ -1151,7 +1151,7 @@ int ObExprJsonValue::cast_to_res(common::ObIAllocator *allocator, case ObMediumIntType: case ObInt32Type: case ObIntType: { - int64_t val; + int64_t val = 0; ret = cast_to_int(j_base, dst_type, val); if (!try_set_error_val(res, ret, error_type, error_val, "SIGNED")) { res.set_int(val); @@ -1163,7 +1163,7 @@ int ObExprJsonValue::cast_to_res(common::ObIAllocator *allocator, case ObUMediumIntType: case ObUInt32Type: case ObUInt64Type: { - uint64_t val; + uint64_t val = 0; ret = cast_to_uint(j_base, dst_type, val); if (!try_set_error_val(res, ret, error_type, error_val, "UNSIGNED")) { res.set_uint(val); @@ -1171,7 +1171,7 @@ int ObExprJsonValue::cast_to_res(common::ObIAllocator *allocator, break; } case ObDateTimeType: { - int64_t val; + int64_t val = 0; ret = cast_to_datetime(j_base, accuracy, val); if (ret == OB_ERR_NULL_VALUE) { res.set_null(); @@ -1192,7 +1192,7 @@ int ObExprJsonValue::cast_to_res(common::ObIAllocator *allocator, break; } case ObDateType: { - int32_t val; + int32_t val = 0; ret = cast_to_date(j_base, val); if (!try_set_error_val(res, ret, error_type, error_val, "DATE")) { res.set_date(val); @@ -1200,7 +1200,7 @@ int ObExprJsonValue::cast_to_res(common::ObIAllocator *allocator, break; } case ObTimeType: { - int64_t val; + int64_t val = 0; ret = cast_to_time(j_base, accuracy, val); if (!try_set_error_val(res, ret, error_type, error_val, "TIME")) { res.set_time(val); @@ -1208,7 +1208,7 @@ int ObExprJsonValue::cast_to_res(common::ObIAllocator *allocator, break; } case ObYearType: { - uint8_t val; + uint8_t val = 0; ret = cast_to_year(j_base, val); if (!try_set_error_val(res, ret, error_type, error_val, "YEAR")) { res.set_year(val); @@ -1217,7 +1217,7 @@ int ObExprJsonValue::cast_to_res(common::ObIAllocator *allocator, } case ObFloatType: case ObUFloatType: { - float out_val; + float out_val = 0; ret = cast_to_float(j_base, dst_type, out_val); if (!try_set_error_val(res, ret, error_type, error_val, "FLOAT")) { res.set_float(out_val); @@ -1226,7 +1226,7 @@ int ObExprJsonValue::cast_to_res(common::ObIAllocator *allocator, } case ObDoubleType: case ObUDoubleType: { - double out_val; + double out_val = 0; ret = cast_to_double(j_base, dst_type, out_val); if (!try_set_error_val(res, ret, error_type, error_val, "DOUBLE")) { res.set_double(out_val); @@ -1268,7 +1268,7 @@ int ObExprJsonValue::cast_to_res(common::ObIAllocator *allocator, break; } case ObBitType: { - uint64_t out_val; + uint64_t out_val = 0; ret = cast_to_bit(j_base, out_val); if (!try_set_error_val(res, ret, error_type, error_val, "BIT")) { res.set_bit(out_val); diff --git a/src/sql/engine/expr/ob_expr_last_trace_id.cpp b/src/sql/engine/expr/ob_expr_last_trace_id.cpp index 8be97f4de..c69b3d704 100644 --- a/src/sql/engine/expr/ob_expr_last_trace_id.cpp +++ b/src/sql/engine/expr/ob_expr_last_trace_id.cpp @@ -45,12 +45,12 @@ int ObExprLastTraceId::eval_last_trace_id(const ObExpr &expr, ObEvalCtx &ctx, { int ret = OB_SUCCESS; UNUSED(expr); - const ObSQLSessionInfo *session_info = NULL; - if (OB_ISNULL(session_info = ctx.exec_ctx_.get_my_session())) { + const ObPhysicalPlanCtx *phy_plan_ctx = NULL; + if (OB_ISNULL(phy_plan_ctx = ctx.exec_ctx_.get_physical_plan_ctx())) { ret = OB_ERR_UNEXPECTED; SQL_ENG_LOG(WARN, "session info is null", K(ret)); } else { - const ObCurTraceId::TraceId &trace_id = session_info->get_last_trace_id(); + const ObCurTraceId::TraceId &trace_id = phy_plan_ctx->get_last_trace_id(); if (trace_id.is_invalid()) { expr_datum.set_null(); } else { diff --git a/src/sql/engine/expr/ob_expr_merge_result_type.map b/src/sql/engine/expr/ob_expr_merge_result_type.map index 6d48be03c..f50f90bcc 100644 --- a/src/sql/engine/expr/ob_expr_merge_result_type.map +++ b/src/sql/engine/expr/ob_expr_merge_result_type.map @@ -1172,9 +1172,9 @@ static ObObjType MERGE_RESULT_TYPE[ObMaxType][ObMaxType] = { ObVarcharType, /*VarcharType=>HexStringType */ ObMaxType, /*VarcharType=>ExtendType */ ObVarcharType, /*VarcharType=>UnknownType */ - ObTinyTextType, /*VarcharType=>ObTinyTextType */ - ObTextType, /* VarcharType=>ObTextType */ - ObMediumTextType, /* VarcharType=>ObMediumTextType */ + ObLongTextType, /*VarcharType=>ObTinyTextType */ + ObLongTextType, /* VarcharType=>ObTextType */ + ObLongTextType, /* VarcharType=>ObMediumTextType */ ObLongTextType, /* VarcharType=>ObLongTextType */ ObVarcharType, /* VarcharType=>ObBitType */ ObVarcharType, /* VarcharType=>ObEnumType */ @@ -1224,9 +1224,9 @@ static ObObjType MERGE_RESULT_TYPE[ObMaxType][ObMaxType] = { ObVarcharType, /*CharType=>HexStringType */ ObMaxType, /*CharType=>ExtendType */ ObVarcharType, /*CharType=>UnknownType */ - ObTinyTextType, /*CharType=>ObTinyTextType */ - ObTextType, /* CharType=>ObTextType */ - ObMediumTextType, /* CharType=>ObMediumTextType */ + ObLongTextType, /*CharType=>ObTinyTextType */ + ObLongTextType, /* CharType=>ObTextType */ + ObLongTextType, /* CharType=>ObMediumTextType */ ObLongTextType, /* CharType=>ObLongTextType */ ObCharType, /* CharType=>ObBitType */ ObCharType, /* CharType=>ObEnumType */ @@ -1426,8 +1426,8 @@ static ObObjType MERGE_RESULT_TYPE[ObMaxType][ObMaxType] = { ObTinyTextType, /*ObTinyTextType=>DateType */ ObTinyTextType, /*ObTinyTextType=>TimeType */ ObTinyTextType, /*ObTinyTextType=>YearType */ - ObTinyTextType, /*ObTinyTextType=>VarcharType */ - ObTinyTextType, /*ObTinyTextType=>CharType */ + ObLongTextType, /*ObTinyTextType=>VarcharType */ + ObLongTextType, /*ObTinyTextType=>CharType */ ObTinyTextType, /*ObTinyTextType=>HexStringType */ ObMaxType, /*ObTinyTextType=>ExtendType */ ObTinyTextType, /*ObTinyTextType=>UnknownType */ @@ -1435,9 +1435,9 @@ static ObObjType MERGE_RESULT_TYPE[ObMaxType][ObMaxType] = { ObTextType, /* ObTinyTextType=>ObTextType */ ObMediumTextType, /* ObTinyTextType=>ObMediumTextType */ ObLongTextType, /* ObTinyTextType=>ObLongTextType */ - ObTextType, /* ObTinyTextType=>ObBitType */ - ObTextType, /* ObTinyTextType=>ObEnumType */ - ObTextType, /* ObTinyTextType=>ObSetType */ + ObLongTextType, /* ObTinyTextType=>ObBitType */ + ObLongTextType, /* ObTinyTextType=>ObEnumType */ + ObLongTextType, /* ObTinyTextType=>ObSetType */ ObMaxType, /* ObTinyTextType=>ObEnumInnerType */ ObMaxType, /* ObTinyTextType=>ObSetInnerType */ ObTinyTextType, /*ObTinyTextType=>ObTimestampTZType */ @@ -1477,8 +1477,8 @@ static ObObjType MERGE_RESULT_TYPE[ObMaxType][ObMaxType] = { ObTextType, /*ObTextType=>DateType */ ObTextType, /*ObTextType=>TimeType */ ObTextType, /*ObTextType=>YearType */ - ObTextType, /*ObTextType=>VarcharType */ - ObTextType, /*ObTextType=>CharType */ + ObLongTextType, /*ObTextType=>VarcharType */ + ObLongTextType, /*ObTextType=>CharType */ ObTextType, /*ObTextType=>HexStringType */ ObMaxType, /*ObTextType=>ExtendType */ ObTextType, /*ObTextType=>UnknownType */ @@ -1486,9 +1486,9 @@ static ObObjType MERGE_RESULT_TYPE[ObMaxType][ObMaxType] = { ObTextType, /* ObTextType=>ObTextType */ ObMediumTextType, /* ObTextType=>ObMediumTextType */ ObLongTextType, /* ObTextType=>ObLongTextType */ - ObMediumTextType, /* ObTextType=>ObBitType */ - ObMediumTextType, /* ObTextType=>ObEnumType */ - ObMediumTextType, /* ObTextType=>ObSetType */ + ObLongTextType, /* ObTextType=>ObBitType */ + ObLongTextType, /* ObTextType=>ObEnumType */ + ObLongTextType, /* ObTextType=>ObSetType */ ObMaxType, /* ObTextType=>ObEnumInnerType */ ObMaxType, /* ObTextType=>ObSetInnerType */ ObTextType, /*ObTextType=>ObTimestampTZType */ @@ -1529,8 +1529,8 @@ static ObObjType MERGE_RESULT_TYPE[ObMaxType][ObMaxType] = { ObMediumTextType, /*ObMediumTextType=>DateType */ ObMediumTextType, /*ObMediumTextType=>TimeType */ ObMediumTextType, /*ObMediumTextType=>YearType */ - ObMediumTextType, /*ObMediumTextType=>VarcharType */ - ObMediumTextType, /*ObMediumTextType=>CharType */ + ObLongTextType, /*ObMediumTextType=>VarcharType */ + ObLongTextType, /*ObMediumTextType=>CharType */ ObMediumTextType, /*ObMediumTextType=>HexStringType */ ObMaxType, /*ObMediumTextType=>ExtendType */ ObMediumTextType, /*ObMediumTextType=>UnknownType */ @@ -1636,8 +1636,8 @@ static ObObjType MERGE_RESULT_TYPE[ObMaxType][ObMaxType] = { ObVarcharType, /*ObBitType=>HexStringType */ ObMaxType, /*ObBitType=>ExtendType */ ObVarcharType, /*ObBitType=>UnknownType */ - ObTextType, /*ObBitType=>ObTinyTextType */ - ObMediumTextType, /* ObBitType=>ObTextType */ + ObLongTextType, /*ObBitType=>ObTinyTextType */ + ObLongTextType, /* ObBitType=>ObTextType */ ObLongTextType, /* ObBitType=>ObMediumTextType */ ObLongTextType, /* ObBitType=>ObLongTextType */ ObBitType, /* ObBitType=>ObBitType */ @@ -1687,8 +1687,8 @@ static ObObjType MERGE_RESULT_TYPE[ObMaxType][ObMaxType] = { ObVarcharType, /*ObEnumType=>HexStringType */ ObMaxType, /*ObEnumType=>ExtendType */ ObVarcharType, /*ObEnumType=>UnknownType */ - ObTextType, /*ObEnumType=>ObTinyTextType */ - ObMediumTextType, /* ObEnumType=>ObTextType */ + ObLongTextType, /*ObEnumType=>ObTinyTextType */ + ObLongTextType, /* ObEnumType=>ObTextType */ ObLongTextType, /* ObEnumType=>ObMediumTextType */ ObLongTextType, /* ObEnumType=>ObLongTextType */ ObVarcharType, /* ObEnumType=>ObBitType */ @@ -1738,8 +1738,8 @@ static ObObjType MERGE_RESULT_TYPE[ObMaxType][ObMaxType] = { ObVarcharType, /*ObSetType=>HexStringType */ ObMaxType, /*ObSetType=>ExtendType */ ObVarcharType, /*ObSetType=>UnknownType */ - ObTextType, /*ObSetType=>ObTinyTextType */ - ObMediumTextType, /* ObSetType=>ObTextType */ + ObLongTextType, /*ObSetType=>ObTinyTextType */ + ObLongTextType, /* ObSetType=>ObTextType */ ObLongTextType, /* ObSetType=>ObMediumTextType */ ObLongTextType, /* ObSetType=>ObLongTextType */ ObVarcharType, /* ObSetType=>ObBitType */ diff --git a/src/sql/engine/expr/ob_expr_repeat.cpp b/src/sql/engine/expr/ob_expr_repeat.cpp index 14f343814..5307e2b0d 100644 --- a/src/sql/engine/expr/ob_expr_repeat.cpp +++ b/src/sql/engine/expr/ob_expr_repeat.cpp @@ -147,8 +147,8 @@ int ObExprRepeat::repeat(ObString &output, if ((length > max_result_size / count) || (length > INT_MAX / count)) { LOG_WARN("Result of repeat was larger than max_allow_packet_size", K(ret), K(length), K(count), K(max_result_size)); - LOG_USER_WARN(OB_ERR_FUNC_RESULT_TOO_LARGE, "repeat", static_cast(max_result_size)); - is_null = true; + ret = OB_ERR_FUNC_RESULT_TOO_LARGE; + LOG_USER_ERROR(OB_ERR_FUNC_RESULT_TOO_LARGE, "repeat", static_cast(max_result_size)); } else { //avoid realloc if (1 == count) { diff --git a/src/sql/engine/ob_physical_plan_ctx.cpp b/src/sql/engine/ob_physical_plan_ctx.cpp index 8a71724b7..aedbcb01f 100644 --- a/src/sql/engine/ob_physical_plan_ctx.cpp +++ b/src/sql/engine/ob_physical_plan_ctx.cpp @@ -681,6 +681,7 @@ OB_DEF_SERIALIZE(ObPhysicalPlanCtx) OB_UNIS_ENCODE(tenant_schema_version_); OB_UNIS_ENCODE(cursor_count); OB_UNIS_ENCODE(plan_start_time_); + OB_UNIS_ENCODE(last_trace_id_); return ret; } @@ -762,6 +763,7 @@ OB_DEF_SERIALIZE_SIZE(ObPhysicalPlanCtx) OB_UNIS_ADD_LEN(tenant_schema_version_); OB_UNIS_ADD_LEN(cursor_count); OB_UNIS_ADD_LEN(plan_start_time_); + OB_UNIS_ADD_LEN(last_trace_id_); return len; } @@ -842,6 +844,7 @@ OB_DEF_DESERIALIZE(ObPhysicalPlanCtx) (void)ObSQLUtils::adjust_time_by_ntp_offset(plan_start_time_); (void)ObSQLUtils::adjust_time_by_ntp_offset(ts_timeout_us_); } + OB_UNIS_DECODE(last_trace_id_); return ret; } diff --git a/src/sql/engine/ob_physical_plan_ctx.h b/src/sql/engine/ob_physical_plan_ctx.h index 09d0342b4..037068b41 100644 --- a/src/sql/engine/ob_physical_plan_ctx.h +++ b/src/sql/engine/ob_physical_plan_ctx.h @@ -419,6 +419,12 @@ public: void set_plan_start_time(int64_t t) { plan_start_time_ = t; } int64_t get_plan_start_time() const { return plan_start_time_; } int replace_batch_param_datum(int64_t cur_group_id); + void set_last_trace_id(const common::ObCurTraceId::TraceId &trace_id) + { + last_trace_id_ = trace_id; + } + const common::ObCurTraceId::TraceId &get_last_trace_id() const { return last_trace_id_; } + common::ObCurTraceId::TraceId &get_last_trace_id() { return last_trace_id_; } private: void reset_datum_frame(char *frame, int64_t expr_cnt); @@ -481,6 +487,7 @@ private: //在存储层，如果table_id是系统表，则会跳过对tenant_schema_version的检查，还是使用原来的办法获取table schema version（见ObRelativeTables::check_schema_version） int64_t tenant_schema_version_; int64_t orig_question_mark_cnt_; + common::ObCurTraceId::TraceId last_trace_id_; private: /** diff --git a/src/sql/engine/px/ob_px_bloom_filter.cpp b/src/sql/engine/px/ob_px_bloom_filter.cpp index 80a20e740..29ebb74bf 100644 --- a/src/sql/engine/px/ob_px_bloom_filter.cpp +++ b/src/sql/engine/px/ob_px_bloom_filter.cpp @@ -551,7 +551,6 @@ int ObSendBloomFilterP::process_px_bloom_filter_data() phase_end))) { LOG_WARN("fail to process recieve count", K(ret)); } - (void)filter->dec_merge_filter_count(); } if (OB_SUCC(ret) && phase_end && arg_.is_first_phase() && !arg_.next_peer_addrs_.empty()) { @@ -584,6 +583,9 @@ int ObSendBloomFilterP::process_px_bloom_filter_data() } } } + if (OB_NOT_NULL(filter)) { + (void)filter->dec_merge_filter_count(); + } return ret; } diff --git a/src/sql/engine/px/ob_px_util.cpp b/src/sql/engine/px/ob_px_util.cpp index 6bb3c9d7a..2d15cb790 100644 --- a/src/sql/engine/px/ob_px_util.cpp +++ b/src/sql/engine/px/ob_px_util.cpp @@ -37,7 +37,7 @@ using namespace oceanbase::sql; using namespace oceanbase::sql::dtl; using namespace oceanbase::share; -OB_SERIALIZE_MEMBER(ObExprExtraSerializeInfo, *current_time_); +OB_SERIALIZE_MEMBER(ObExprExtraSerializeInfo, *current_time_, *last_trace_id_); // 物理分布策略：对于叶子节点，dfo 分布一般直接按照数据分布来 // Note：如果 dfo 中有两个及以上的 scan，仅仅考虑第一个。并且，要求其余 scan @@ -1347,6 +1347,7 @@ int ObPxTreeSerializer::serialize_expr_frame_info(char *buf, OB_UNIS_ENCODE(expr_frame_info.need_ctx_cnt_); ObPhysicalPlanCtx *plan_ctx = ctx.get_physical_plan_ctx(); expr_info.current_time_ = &plan_ctx->get_cur_time(); + expr_info.last_trace_id_ = &plan_ctx->get_last_trace_id(); // rt exprs ObExpr::get_serialize_array() = &exprs; @@ -1507,6 +1508,7 @@ int ObPxTreeSerializer::deserialize_expr_frame_info(const char *buf, ObExprExtraSerializeInfo expr_info; ObPhysicalPlanCtx *plan_ctx = ctx.get_physical_plan_ctx(); expr_info.current_time_ = &plan_ctx->get_cur_time(); + expr_info.last_trace_id_ = &plan_ctx->get_last_trace_id(); if (OB_FAIL(expr_info.deserialize(buf, data_len, pos))) { LOG_WARN("fail to deserialize expr extra info", K(ret)); } else if (OB_FAIL(serialization::decode_i32(buf, data_len, pos, &expr_cnt))) { @@ -1622,6 +1624,7 @@ int64_t ObPxTreeSerializer::get_serialize_expr_frame_info_size( ObExprExtraSerializeInfo expr_info; ObPhysicalPlanCtx *plan_ctx = ctx.get_physical_plan_ctx(); expr_info.current_time_ = &plan_ctx->get_cur_time(); + expr_info.last_trace_id_ = &plan_ctx->get_last_trace_id(); ObIArray &exprs = expr_frame_info.rt_exprs_; int32_t expr_cnt = expr_frame_info.is_mark_serialize() ? expr_frame_info.ser_expr_marks_.count() diff --git a/src/sql/engine/px/ob_px_util.h b/src/sql/engine/px/ob_px_util.h index ddb99f75b..ae2e76c42 100644 --- a/src/sql/engine/px/ob_px_util.h +++ b/src/sql/engine/px/ob_px_util.h @@ -46,8 +46,9 @@ struct ObExprExtraSerializeInfo { OB_UNIS_VERSION(1); public: - ObExprExtraSerializeInfo() : current_time_(nullptr) { } + ObExprExtraSerializeInfo() : current_time_(nullptr), last_trace_id_(nullptr) { } common::ObObj *current_time_; + common::ObCurTraceId::TraceId *last_trace_id_; }; class ObPxSqcUtil diff --git a/src/sql/engine/sort/ob_sort_op_impl.cpp b/src/sql/engine/sort/ob_sort_op_impl.cpp index 8c2e421d3..c27625697 100644 --- a/src/sql/engine/sort/ob_sort_op_impl.cpp +++ b/src/sql/engine/sort/ob_sort_op_impl.cpp @@ -617,8 +617,8 @@ int ObSortOpImpl::init( default_block_size))) { LOG_WARN("init row store failed", K(ret)); } else if (batch_size > 0 - && NULL == (stored_rows_ = static_cast( - mem_context_->get_arena_allocator().alloc( + && OB_ISNULL(stored_rows_ = static_cast( + mem_context_->get_malloc_allocator().alloc( sizeof(*stored_rows_) * batch_size)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("allocate memory failed", K(ret)); @@ -699,6 +699,10 @@ void ObSortOpImpl::reset() mem_context_->get_malloc_allocator().free(ems_heap_); ems_heap_ = NULL; } + if (NULL != stored_rows_) { + mem_context_->get_malloc_allocator().free(stored_rows_); + stored_rows_ = NULL; + } if (NULL != buckets_) { mem_context_->get_malloc_allocator().free(buckets_); buckets_ = NULL; diff --git a/src/sql/ob_dml_stmt_printer.cpp b/src/sql/ob_dml_stmt_printer.cpp index 37923c543..2c1c1ac7d 100644 --- a/src/sql/ob_dml_stmt_printer.cpp +++ b/src/sql/ob_dml_stmt_printer.cpp @@ -546,7 +546,8 @@ int ObDMLStmtPrinter::print_fetch() } } //fetch only/with ties - if (OB_SUCC(ret)) { + if (OB_SUCC(ret) && + (NULL != stmt_->get_limit_expr() || NULL != stmt_->get_limit_percent_expr()) ) { if (stmt_->is_fetch_with_ties()) { DATA_PRINTF(" with ties"); } else { diff --git a/src/sql/ob_spi.cpp b/src/sql/ob_spi.cpp index bb2e7a3eb..a64e1101f 100644 --- a/src/sql/ob_spi.cpp +++ b/src/sql/ob_spi.cpp @@ -1032,22 +1032,41 @@ int ObSPIService::set_variable(ObPLExecCtx *ctx, tmp_pos, print_params))) { if (OB_SIZE_OVERFLOW == ret) { - ret = OB_SUCCESS; - new_buf = static_cast(allocator.alloc(OB_MAX_SQL_LENGTH)); - if (OB_ISNULL(new_buf)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memory for set sql", K(ret), K(OB_MAX_SQL_LENGTH)); - } else { - tmp_pos = 0; - MEMCPY(new_buf, buf, pos); - if (OB_FAIL(value.print_sql_literal(new_buf + pos, - OB_MAX_SQL_LENGTH - pos, - tmp_pos, - print_params))) { - LOG_WARN("failed to print_plain_str_literal", - K(buf), K(new_buf), K(pos), K(tmp_pos), K(ret)); + int64_t alloc_len = OB_MAX_SQL_LENGTH; + while (OB_SIZE_OVERFLOW == ret) { + ret = OB_SUCCESS; + if (OB_ISNULL(new_buf)) { + new_buf = static_cast(allocator.alloc(alloc_len)); } else { - sql = new_buf; + if (alloc_len < (1L << 20)) { + alloc_len *= 2; + allocator.free(new_buf); + new_buf = NULL; + new_buf = static_cast(allocator.alloc(alloc_len)); + } else { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("failed to print_plain_str_literal", + K(buf), K(new_buf), K(pos), K(tmp_pos), K(ret)); + break; + } + } + if (OB_ISNULL(new_buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for set sql", K(ret), K(OB_MAX_SQL_LENGTH)); + } else { + tmp_pos = 0; + MEMCPY(new_buf, buf, pos); + if (OB_FAIL(value.print_sql_literal(new_buf + pos, + alloc_len - pos, + tmp_pos, + print_params))) { + if (OB_SIZE_OVERFLOW != ret) { + LOG_WARN("failed to print_plain_str_literal", + K(buf), K(new_buf), K(pos), K(tmp_pos), K(ret)); + } + } else { + sql = new_buf; + } } } } else { diff --git a/src/sql/ob_sql.cpp b/src/sql/ob_sql.cpp index 01f4fe004..bc5355b99 100644 --- a/src/sql/ob_sql.cpp +++ b/src/sql/ob_sql.cpp @@ -2409,6 +2409,7 @@ int ObSql::generate_physical_plan(ParseResult &parse_result, if (phy_plan->get_fetch_cur_time() && !pctx->has_cur_time()) { pctx->set_cur_time(ObTimeUtility::current_time(), *(sql_ctx.session_info_)); } + pctx->set_last_trace_id(sql_ctx.session_info_->get_last_trace_id()); } if (OB_FAIL(ret)) { diff --git a/src/sql/ob_sql_utils.cpp b/src/sql/ob_sql_utils.cpp index 980c6f6ae..a70c4d3d8 100644 --- a/src/sql/ob_sql_utils.cpp +++ b/src/sql/ob_sql_utils.cpp @@ -707,6 +707,8 @@ int ObSQLUtils::se_calc_const_expr(ObSQLSessionInfo *session, // set current time before do pre calculation } else if (FALSE_IT(phy_plan_ctx.set_cur_time(ObTimeUtility::current_time(), *session))) { // do nothing + } else if (FALSE_IT(phy_plan_ctx.set_last_trace_id(session->get_last_trace_id()))) { + // do nothing } else if (OB_FAIL(ObPlanCacheObject::pre_calculation(false, *pre_calc_frame, exec_ctx))) { diff --git a/src/sql/optimizer/ob_join_order.cpp b/src/sql/optimizer/ob_join_order.cpp index 0a6cee2ac..10e5cc2f7 100644 --- a/src/sql/optimizer/ob_join_order.cpp +++ b/src/sql/optimizer/ob_join_order.cpp @@ -11385,10 +11385,10 @@ int ObJoinOrder::extract_pushdown_quals(const ObIArray &quals, ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexplected null", K(qual), K(ret)); // can not push down expr with subquery - } else if (qual->has_flag(CNT_ONETIME) || - qual->has_flag(CNT_PSEUDO_COLUMN) || - qual->has_flag(CNT_PRIOR) || - qual->has_flag(CNT_ROWNUM)) { + } else if (qual->has_flag(CNT_SUB_QUERY) && + (qual->has_flag(CNT_PSEUDO_COLUMN) || + qual->has_flag(CNT_PRIOR) || + qual->has_flag(CNT_ROWNUM))) { if (force_inner_nl) { ret = OB_ERR_UNEXPECTED; LOG_WARN("can not push down special qual", KPC(qual), K(ret)); diff --git a/src/sql/optimizer/ob_log_plan.cpp b/src/sql/optimizer/ob_log_plan.cpp index e7f14f202..a1520482b 100644 --- a/src/sql/optimizer/ob_log_plan.cpp +++ b/src/sql/optimizer/ob_log_plan.cpp @@ -5231,10 +5231,6 @@ int ObLogPlan::create_plan_tree_from_path(Path *path, JoinPath *join_path = static_cast(path); if (OB_FAIL(allocate_join_path(join_path, op))) { LOG_WARN("failed to allocate join path", K(ret)); - } else if (lib::is_mysql_mode() && - OB_FAIL(allocate_for_update_for_semi_anti_join(join_path, - static_cast(op)))) { - LOG_WARN("failed to allocate allocate for update for semi anti join", K(ret)); } else {/* do nothing */ } } else if (path->is_subquery_path()) { SubQueryPath *subquery_path = static_cast(path); @@ -11757,78 +11753,6 @@ int ObLogPlan::generate_column_expr(ObRawExprFactory &expr_factory, return ret; } -int ObLogPlan::allocate_for_update_for_semi_anti_join(JoinPath *join_path, - ObLogJoin *join_op) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(join_path) || OB_ISNULL(join_op)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(join_path), K(join_op)); - } else if (IS_SEMI_ANTI_JOIN(join_path->join_type_)) { - const Path *left_path = join_path->left_path_; - const Path *right_path = join_path->right_path_; - ObLogicalOperator *left_child = join_op->get_left_table(); - ObLogicalOperator *right_child = join_op->get_right_table(); - if (OB_ISNULL(left_path) || OB_ISNULL(right_path)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret), K(left_path), K(right_path)); - } else if (IS_LEFT_SEMI_ANTI_JOIN(join_path->join_type_)) { - //for update need allocate before rigth table which in left semi/anti join becauese - //it isn't pk preserving. - ObSEArray sfu_table_list; - if (OB_FAIL(recursive_collect_sfu_table_ids(right_path, sfu_table_list))) { - LOG_WARN("failed to recursive collect sfu table ids", K(ret)); - } else if (sfu_table_list.empty()) { - //do nothing - } else if (OB_FAIL(allocate_for_update_as_top(right_child, sfu_table_list))) { - LOG_WARN("failed to allocate for update as top", K(ret)); - } else { - join_op->set_right_child(right_child); - } - } else if (IS_RIGHT_SEMI_ANTI_JOIN(join_path->join_type_)) { - //for update need allocate before left table which in right semi/anti join becauese - //it isn't pk preserving. - ObSEArray sfu_table_list; - if (OB_FAIL(recursive_collect_sfu_table_ids(left_path, sfu_table_list))) { - LOG_WARN("failed to recursive collect sfu table ids", K(ret)); - } else if (sfu_table_list.empty()) { - //do nothing - } else if (OB_FAIL(allocate_for_update_as_top(left_child, sfu_table_list))) { - LOG_WARN("failed to allocate for update as top", K(ret)); - } else { - join_op->set_left_child(left_child); - } - } else {/*do nothing*/} - } - return ret; -} - -int ObLogPlan::recursive_collect_sfu_table_ids(const Path *path, ObIArray &sfu_table_list) -{ - int ret = OB_SUCCESS; - if (OB_ISNULL(path)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpecpted null", K(ret), K(path)); - } else if (path->is_access_path()) { - const AccessPath* access_path = static_cast(path); - if (access_path->for_update_ && !table_is_allocated_for_update(access_path->table_id_)) { - if (OB_FAIL(sfu_table_list.push_back(access_path->table_id_))) { - LOG_WARN("failed to push back", K(ret)); - } else {/*do nothing*/} - } - } else if (path->is_join_path()) { - const JoinPath *join_path = static_cast(path); - if (OB_FAIL(SMART_CALL(recursive_collect_sfu_table_ids(join_path->left_path_, - sfu_table_list)))) { - LOG_WARN("failed to recursive collect sfu table ids", K(ret)); - } else if (OB_FAIL(SMART_CALL(recursive_collect_sfu_table_ids(join_path->right_path_, - sfu_table_list)))) { - LOG_WARN("failed to recursive collect sfu table ids", K(ret)); - } else {/*do nothing*/} - } else {/*do nothing*/} - return ret; -} - //mysql mode need distinguish different of for update, eg: /* * create table t1(c1 int primary key, c2 int); diff --git a/src/sql/optimizer/ob_log_plan.h b/src/sql/optimizer/ob_log_plan.h index 3c4256650..30a6622bf 100644 --- a/src/sql/optimizer/ob_log_plan.h +++ b/src/sql/optimizer/ob_log_plan.h @@ -1287,10 +1287,6 @@ public: common::ObIArray &get_alloc_sfu_list() { return alloc_sfu_list_; } - int allocate_for_update_for_semi_anti_join(JoinPath *join_path, ObLogJoin *join_op); - - int recursive_collect_sfu_table_ids(const Path *path, ObIArray &sfu_table_list); - int merge_same_sfu_table_list(uint64_t target_id, int64_t begin_idx, ObIArray &src_table_list, diff --git a/src/sql/parser/ob_fast_parser.cpp b/src/sql/parser/ob_fast_parser.cpp index 304755b77..2eb66cce4 100644 --- a/src/sql/parser/ob_fast_parser.cpp +++ b/src/sql/parser/ob_fast_parser.cpp @@ -1892,11 +1892,16 @@ int ObFastParserMysql::process_string(const char quote) } } // end while if (OB_SUCC(ret)) { + // in ansi_quotes sql_mode, the "" is treated as `, shouldn't parameterize it. + bool is_ansi_quotes = false; + IS_ANSI_QUOTES(sql_mode_, is_ansi_quotes); raw_sql_.scan(); if (!is_quote_end) { cur_token_type_ = IGNORE_TOKEN; ret = OB_ERR_PARSER_SYNTAX; LOG_WARN("parser syntax error", K(ret), K(raw_sql_.to_string()), K_(raw_sql_.cur_pos)); + } else if (is_ansi_quotes && quote == '"') { + cur_token_type_ = IGNORE_TOKEN; } else { char *buf = nullptr; cur_token_type_ = PARAM_TOKEN; diff --git a/src/sql/plan_cache/ob_plan_set.cpp b/src/sql/plan_cache/ob_plan_set.cpp index e3e71665f..eaabd375d 100644 --- a/src/sql/plan_cache/ob_plan_set.cpp +++ b/src/sql/plan_cache/ob_plan_set.cpp @@ -144,6 +144,7 @@ int ObPlanSet::match_params_info(const ParamStore *params, } else if (fetch_cur_time_ && FALSE_IT(plan_ctx->set_cur_time( ObClockGenerator::getClock(), *session))) { // never reach + } else if (FALSE_IT(plan_ctx->set_last_trace_id(session->get_last_trace_id()))) { } else if (params->count() != params_info_.count()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("param info count is different", K(params_info_), K(*params), K(ret)); diff --git a/src/sql/privilege_check/ob_privilege_check.cpp b/src/sql/privilege_check/ob_privilege_check.cpp index 5d97761c0..ebbe84c6b 100644 --- a/src/sql/privilege_check/ob_privilege_check.cpp +++ b/src/sql/privilege_check/ob_privilege_check.cpp @@ -460,7 +460,7 @@ int get_proc_db_name( } else { uint64_t tenant_id = ctx.session_info_->get_login_tenant_id(); ObSchemaGetterGuard &schema_guard = *ctx.schema_guard_; - uint64_t db_id; + uint64_t db_id = OB_INVALID_ID; const ObDatabaseSchema *db_schema = NULL; if (static_cast(ObObjectType::FUNCTION) == need_priv.obj_type_) { const ObRoutineInfo *routine_schema = NULL; @@ -494,7 +494,7 @@ int get_seq_db_name( ObString &db_name) { int ret = OB_SUCCESS; - uint64_t db_id; + uint64_t db_id = OB_INVALID_ID; const ObSequenceSchema *seq_schema = NULL; const ObDatabaseSchema *db_schema = NULL; OZ (schema_guard.get_sequence_schema(tenant_id, obj_id, seq_schema)); diff --git a/src/sql/resolver/cmd/ob_show_resolver.cpp b/src/sql/resolver/cmd/ob_show_resolver.cpp index bb4ac61a0..a419bc8f7 100644 --- a/src/sql/resolver/cmd/ob_show_resolver.cpp +++ b/src/sql/resolver/cmd/ob_show_resolver.cpp @@ -135,6 +135,7 @@ int ObShowResolver::resolve(const ParseNode &parse_tree) show_resv_ctx.condition_node_ = parse_tree.children_[1]; show_resv_ctx.stmt_type_ = stmt::T_SHOW_TABLES; ParseNode *condition_node = show_resv_ctx.condition_node_; + ObString show_db_name; uint64_t show_db_id = OB_INVALID_ID; if (OB_FAIL(get_database_info(parse_tree.children_[0], database_name, @@ -146,65 +147,76 @@ int ObShowResolver::resolve(const ParseNode &parse_tree) ret = OB_ERR_UNEXPECTED; LOG_WARN("database id is invalid", K(ret), K(show_db_id)); } else { - if (0 == parse_tree.children_[2]->value_) { - if (NULL != condition_node && T_LIKE_CLAUSE == condition_node->type_) { - if (OB_UNLIKELY(condition_node->num_child_ != 2 - || NULL == condition_node->children_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid like parse node", - K(ret), - K(condition_node->num_child_), - K(condition_node->children_)); - } else if (OB_UNLIKELY(NULL == condition_node->children_[0] - || NULL == condition_node->children_[1])) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid like parse node", - K(ret), - K(condition_node->num_child_), - K(condition_node->children_[0]), - K(condition_node->children_[1])); - - } else { - GEN_SQL_STEP_1(ObShowSqlSet::SHOW_TABLES_LIKE, - show_resv_ctx.show_database_name_.length(), - show_resv_ctx.show_database_name_.ptr(), - static_cast(condition_node->children_[0]->str_len_),//cast int64_t to obstr_size_t - condition_node->children_[0]->str_value_); - GEN_SQL_STEP_2(ObShowSqlSet::SHOW_TABLES_LIKE, OB_SYS_DATABASE_NAME, OB_TENANT_VIRTUAL_SHOW_TABLES_TNAME, show_db_id); - } + show_db_name = show_resv_ctx.show_database_name_; + if (OB_FAIL(schema_checker_->check_db_access(session_priv, show_db_name))) { + if (OB_ERR_NO_DB_PRIVILEGE == ret) { + LOG_USER_ERROR(OB_ERR_NO_DB_PRIVILEGE, session_priv.user_name_.length(), session_priv.user_name_.ptr(), + session_priv.host_name_.length(),session_priv.host_name_.ptr(), + show_db_name.length(), show_db_name.ptr()); } else { - GEN_SQL_STEP_1(ObShowSqlSet::SHOW_TABLES, show_resv_ctx.show_database_name_.length(), - show_resv_ctx.show_database_name_.ptr()); - GEN_SQL_STEP_2(ObShowSqlSet::SHOW_TABLES, OB_SYS_DATABASE_NAME, OB_TENANT_VIRTUAL_SHOW_TABLES_TNAME, show_db_id); - } - } else if (1 == parse_tree.children_[2]->value_) { - if (NULL != condition_node && T_LIKE_CLAUSE == condition_node->type_) { - if (OB_UNLIKELY(condition_node->num_child_ != 2 - || NULL == condition_node->children_[0] - || NULL == condition_node->children_[1])) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid like parse node", - K(ret), - K(condition_node->num_child_), - K(condition_node->children_[0]), - K(condition_node->children_[1])); - } else { - GEN_SQL_STEP_1(ObShowSqlSet::SHOW_FULL_TABLES_LIKE, - show_resv_ctx.show_database_name_.length(), - show_resv_ctx.show_database_name_.ptr(), - static_cast(condition_node->children_[0]->str_len_),//cast int64_t to obstr_size_t - condition_node->children_[0]->str_value_); - GEN_SQL_STEP_2(ObShowSqlSet::SHOW_FULL_TABLES_LIKE, OB_SYS_DATABASE_NAME, OB_TENANT_VIRTUAL_SHOW_TABLES_TNAME, show_db_id); - } - } else { - GEN_SQL_STEP_1(ObShowSqlSet::SHOW_FULL_TABLES, show_resv_ctx.show_database_name_.length(), - show_resv_ctx.show_database_name_.ptr()); - GEN_SQL_STEP_2(ObShowSqlSet::SHOW_FULL_TABLES, OB_SYS_DATABASE_NAME, OB_TENANT_VIRTUAL_SHOW_TABLES_TNAME, show_db_id); + LOG_WARN("fail to check priv", K(ret)); } } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("node value unexpected", K(parse_tree.value_)); - break; + if (0 == parse_tree.children_[2]->value_) { + if (NULL != condition_node && T_LIKE_CLAUSE == condition_node->type_) { + if (OB_UNLIKELY(condition_node->num_child_ != 2 + || NULL == condition_node->children_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid like parse node", + K(ret), + K(condition_node->num_child_), + K(condition_node->children_)); + } else if (OB_UNLIKELY(NULL == condition_node->children_[0] + || NULL == condition_node->children_[1])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid like parse node", + K(ret), + K(condition_node->num_child_), + K(condition_node->children_[0]), + K(condition_node->children_[1])); + + } else { + GEN_SQL_STEP_1(ObShowSqlSet::SHOW_TABLES_LIKE, + show_resv_ctx.show_database_name_.length(), + show_resv_ctx.show_database_name_.ptr(), + static_cast(condition_node->children_[0]->str_len_),//cast int64_t to obstr_size_t + condition_node->children_[0]->str_value_); + GEN_SQL_STEP_2(ObShowSqlSet::SHOW_TABLES_LIKE, OB_SYS_DATABASE_NAME, OB_TENANT_VIRTUAL_SHOW_TABLES_TNAME, show_db_id); + } + } else { + GEN_SQL_STEP_1(ObShowSqlSet::SHOW_TABLES, show_resv_ctx.show_database_name_.length(), + show_resv_ctx.show_database_name_.ptr()); + GEN_SQL_STEP_2(ObShowSqlSet::SHOW_TABLES, OB_SYS_DATABASE_NAME, OB_TENANT_VIRTUAL_SHOW_TABLES_TNAME, show_db_id); + } + } else if (1 == parse_tree.children_[2]->value_) { + if (NULL != condition_node && T_LIKE_CLAUSE == condition_node->type_) { + if (OB_UNLIKELY(condition_node->num_child_ != 2 + || NULL == condition_node->children_[0] + || NULL == condition_node->children_[1])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid like parse node", + K(ret), + K(condition_node->num_child_), + K(condition_node->children_[0]), + K(condition_node->children_[1])); + } else { + GEN_SQL_STEP_1(ObShowSqlSet::SHOW_FULL_TABLES_LIKE, + show_resv_ctx.show_database_name_.length(), + show_resv_ctx.show_database_name_.ptr(), + static_cast(condition_node->children_[0]->str_len_),//cast int64_t to obstr_size_t + condition_node->children_[0]->str_value_); + GEN_SQL_STEP_2(ObShowSqlSet::SHOW_FULL_TABLES_LIKE, OB_SYS_DATABASE_NAME, OB_TENANT_VIRTUAL_SHOW_TABLES_TNAME, show_db_id); + } + } else { + GEN_SQL_STEP_1(ObShowSqlSet::SHOW_FULL_TABLES, show_resv_ctx.show_database_name_.length(), + show_resv_ctx.show_database_name_.ptr()); + GEN_SQL_STEP_2(ObShowSqlSet::SHOW_FULL_TABLES, OB_SYS_DATABASE_NAME, OB_TENANT_VIRTUAL_SHOW_TABLES_TNAME, show_db_id); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("node value unexpected", K(parse_tree.value_)); + break; + } } //change where condition :Tables_in_xxx=>table_name @@ -352,7 +364,41 @@ int ObShowResolver::resolve(const ParseNode &parse_tree) show_db_name, show_db_id, show_table_name, show_table_id, is_view, synonym_checker))) { LOG_WARN("fail to resolve show from table", K(ret)); - } else { + } else if (!is_oracle_mode) { + if (OB_FAIL(stmt_need_privs.need_privs_.init(3))) { + LOG_WARN("fail to init need privs array", K(ret)); + } else { + ObNeedPriv need_priv; + //Priv check: global select || db select || table acc + need_priv.priv_level_ = OB_PRIV_USER_LEVEL; + need_priv.priv_set_ = OB_PRIV_SELECT; + stmt_need_privs.need_privs_.push_back(need_priv); + + need_priv.priv_level_ = OB_PRIV_DB_LEVEL; + need_priv.priv_set_ = OB_PRIV_SELECT; + need_priv.db_ = show_db_name; + stmt_need_privs.need_privs_.push_back(need_priv); + + need_priv.priv_level_ = OB_PRIV_TABLE_LEVEL; + need_priv.priv_set_ = OB_PRIV_TABLE_ACC; + need_priv.db_ = show_db_name; + need_priv.table_ = show_table_name; + stmt_need_privs.need_privs_.push_back(need_priv); + + if (OB_FAIL(schema_checker_->check_priv_or(session_priv, stmt_need_privs))) { + if (OB_ERR_NO_TABLE_PRIVILEGE == ret) { + LOG_USER_ERROR(OB_ERR_NO_TABLE_PRIVILEGE, (int)strlen("SELECT"), "SELECT", + session_priv.user_name_.length(), session_priv.user_name_.ptr(), + session_priv.host_name_.length(),session_priv.host_name_.ptr(), + show_table_name.length(), show_table_name.ptr()); + } else { + LOG_WARN("fail to check priv", K(ret)); + } + } + } + } + + if (OB_SUCC(ret)) { if (1 == parse_tree.children_[0]->value_) { GEN_SQL_STEP_1(ObShowSqlSet::SHOW_FULL_COLUMNS); GEN_SQL_STEP_2(ObShowSqlSet::SHOW_FULL_COLUMNS, REAL_NAME(OB_SYS_DATABASE_NAME, OB_ORA_SYS_SCHEMA_NAME), REAL_NAME(OB_TENANT_VIRTUAL_TABLE_COLUMN_TNAME, OB_TENANT_VIRTUAL_TABLE_COLUMN_ORA_TNAME), show_table_id); @@ -387,13 +433,35 @@ int ObShowResolver::resolve(const ParseNode &parse_tree) show_db_id, show_db_name))) { LOG_WARN("fail to resolve show database", K(ret), K(real_tenant_id)); + } else if (OB_FAIL(stmt_need_privs.need_privs_.init(2))) { + LOG_WARN("fail to init need privs array", K(ret)); } else { - if (NULL != parse_tree.children_[0]) { - GEN_SQL_STEP_1(ObShowSqlSet::SHOW_CREATE_DATABASE_EXISTS); - GEN_SQL_STEP_2(ObShowSqlSet::SHOW_CREATE_DATABASE_EXISTS, OB_SYS_DATABASE_NAME, OB_TENANT_VIRTUAL_SHOW_CREATE_DATABASE_TNAME, show_db_id); + ObNeedPriv need_priv; + need_priv.priv_level_ = OB_PRIV_USER_LEVEL; + need_priv.priv_set_ = OB_PRIV_DB_ACC; + stmt_need_privs.need_privs_.push_back(need_priv); + + need_priv.priv_level_ = OB_PRIV_DB_LEVEL; + need_priv.priv_set_ = OB_PRIV_DB_ACC; + need_priv.db_ = show_db_name; + stmt_need_privs.need_privs_.push_back(need_priv); + + if (OB_FAIL(schema_checker_->check_priv_or(session_priv, stmt_need_privs))) { + if (OB_ERR_NO_DB_PRIVILEGE == ret) { + LOG_USER_ERROR(OB_ERR_NO_DB_PRIVILEGE, session_priv.user_name_.length(), session_priv.user_name_.ptr(), + session_priv.host_name_.length(),session_priv.host_name_.ptr(), + show_db_name.length(), show_db_name.ptr()); + } else { + LOG_WARN("fail to check priv", K(ret)); + } } else { - GEN_SQL_STEP_1(ObShowSqlSet::SHOW_CREATE_DATABASE); - GEN_SQL_STEP_2(ObShowSqlSet::SHOW_CREATE_DATABASE, OB_SYS_DATABASE_NAME, OB_TENANT_VIRTUAL_SHOW_CREATE_DATABASE_TNAME, show_db_id); + if (NULL != parse_tree.children_[0]) { + GEN_SQL_STEP_1(ObShowSqlSet::SHOW_CREATE_DATABASE_EXISTS); + GEN_SQL_STEP_2(ObShowSqlSet::SHOW_CREATE_DATABASE_EXISTS, OB_SYS_DATABASE_NAME, OB_TENANT_VIRTUAL_SHOW_CREATE_DATABASE_TNAME, show_db_id); + } else { + GEN_SQL_STEP_1(ObShowSqlSet::SHOW_CREATE_DATABASE); + GEN_SQL_STEP_2(ObShowSqlSet::SHOW_CREATE_DATABASE, OB_SYS_DATABASE_NAME, OB_TENANT_VIRTUAL_SHOW_CREATE_DATABASE_TNAME, show_db_id); + } } } } @@ -565,11 +633,7 @@ int ObShowResolver::resolve(const ParseNode &parse_tree) T_SHOW_INDEXES, real_tenant_id, show_db_name, show_db_id, show_table_name, show_table_id, is_view, synonym_checker))) { LOG_WARN("fail to resolve show from table", K(ret)); - } else { - GEN_SQL_STEP_1(ObShowSqlSet::SHOW_INDEXES); - GEN_SQL_STEP_2(ObShowSqlSet::SHOW_INDEXES, OB_SYS_DATABASE_NAME, OB_TENANT_VIRTUAL_TABLE_INDEX_TNAME, show_table_id); - } - if (OB_SUCC(ret)) { + } else if (!is_oracle_mode) { if (OB_FAIL(stmt_need_privs.need_privs_.init(3))) { LOG_WARN("fail to init need privs array", K(ret)); } else { @@ -580,6 +644,8 @@ int ObShowResolver::resolve(const ParseNode &parse_tree) stmt_need_privs.need_privs_.push_back(need_priv); need_priv.priv_level_ = OB_PRIV_DB_LEVEL; + need_priv.priv_set_ = OB_PRIV_SELECT; + need_priv.db_ = show_db_name; stmt_need_privs.need_privs_.push_back(need_priv); need_priv.priv_level_ = OB_PRIV_TABLE_LEVEL; @@ -589,14 +655,22 @@ int ObShowResolver::resolve(const ParseNode &parse_tree) stmt_need_privs.need_privs_.push_back(need_priv); if (OB_FAIL(schema_checker_->check_priv_or(session_priv, stmt_need_privs))) { - ret = OB_ERR_NO_TABLE_PRIVILEGE; - LOG_USER_ERROR(OB_ERR_NO_TABLE_PRIVILEGE, (int)strlen("SELECT"), "SELECT", - session_priv.user_name_.length(), session_priv.user_name_.ptr(), - session_priv.host_name_.length(),session_priv.host_name_.ptr(), - show_table_name.length(), show_table_name.ptr()); + if (OB_ERR_NO_TABLE_PRIVILEGE == ret) { + LOG_USER_ERROR(OB_ERR_NO_TABLE_PRIVILEGE, (int)strlen("SELECT"), "SELECT", + session_priv.user_name_.length(), session_priv.user_name_.ptr(), + session_priv.host_name_.length(),session_priv.host_name_.ptr(), + show_table_name.length(), show_table_name.ptr()); + } else { + LOG_WARN("fail to check priv", K(ret)); + } } } } + + if (OB_SUCC(ret)) { + GEN_SQL_STEP_1(ObShowSqlSet::SHOW_INDEXES); + GEN_SQL_STEP_2(ObShowSqlSet::SHOW_INDEXES, OB_SYS_DATABASE_NAME, OB_TENANT_VIRTUAL_TABLE_INDEX_TNAME, show_table_id); + } } }(); break; @@ -2697,12 +2771,12 @@ DEFINE_SHOW_CLAUSE_SET(SHOW_CREATE_TENANT, NULL); DEFINE_SHOW_CLAUSE_SET(SHOW_DATABASES, NULL, - "SELECT `database_name` AS `Database` FROM %s.%s WHERE tenant_id = %ld and in_recyclebin = 0 and database_name not in('%s', '%s', '%s') and 0 = sys_privilege_check(\'db_acc\', `tenant_id`, `database_name`, \'\')", + "SELECT `database_name` AS `Database` FROM %s.%s WHERE tenant_id = %ld and in_recyclebin = 0 and database_name not in('%s', '%s', '%s') and 0 = sys_privilege_check(\'db_acc\', `tenant_id`, `database_name`, \'\') order by database_name asc", NULL, "Database"); DEFINE_SHOW_CLAUSE_SET(SHOW_DATABASES_LIKE, "SELECT `Database` AS `Database (%.*s)` ", - "SELECT `database_name` AS `Database` FROM %s.%s WHERE tenant_id = %ld and in_recyclebin = 0 and database_name not in ('%s', '%s', '%s') and 0 = sys_privilege_check(\'db_acc\', `tenant_id`, `database_name`, \'\')", + "SELECT `database_name` AS `Database` FROM %s.%s WHERE tenant_id = %ld and in_recyclebin = 0 and database_name not in ('%s', '%s', '%s') and 0 = sys_privilege_check(\'db_acc\', `tenant_id`, `database_name`, \'\') order by database_name asc", NULL, "Database"); DEFINE_SHOW_CLAUSE_SET(SHOW_DATABASES_STATUS, diff --git a/src/sql/resolver/cmd/ob_show_resolver.h b/src/sql/resolver/cmd/ob_show_resolver.h index 05411d1d6..0e8114141 100644 --- a/src/sql/resolver/cmd/ob_show_resolver.h +++ b/src/sql/resolver/cmd/ob_show_resolver.h @@ -40,17 +40,18 @@ private: const ObString &database_name, bool is_sys_view); + // in oracle mode, check_desc_priv_if_ness is called inside int resolve_show_from_table(const ParseNode *from_table_node, - const ParseNode *from_database_clause_node, - bool is_database_unselected, - ObItemType node_type, - uint64_t real_tenant_id, - common::ObString &show_database_name, - uint64_t &show_database_id, - common::ObString &show_table_name, - uint64_t &show_table_id, - bool &is_view, - ObSynonymChecker &synonym_checker); + const ParseNode *from_database_clause_node, + bool is_database_unselected, + ObItemType node_type, + uint64_t real_tenant_id, + common::ObString &show_database_name, + uint64_t &show_database_id, + common::ObString &show_table_name, + uint64_t &show_table_id, + bool &is_view, + ObSynonymChecker &synonym_checker); int resolve_show_from_database(const ParseNode &from_db_node, uint64_t real_tenant_id, uint64_t &show_database_id, diff --git a/src/sql/resolver/dml/ob_dml_resolver.cpp b/src/sql/resolver/dml/ob_dml_resolver.cpp index 3270d7574..4987af2e6 100644 --- a/src/sql/resolver/dml/ob_dml_resolver.cpp +++ b/src/sql/resolver/dml/ob_dml_resolver.cpp @@ -5437,7 +5437,7 @@ int ObDMLResolver::resolve_table_relation_factor(const ParseNode *node, table_name, synonym_name, synonym_db_name, db_name, is_db_explicit, synonym_checker))) { - LOG_WARN("resolve table relation factor failed", K(ret)); + LOG_WARN("resolve table relation factor failed", K(ret), K(table_name)); // table_name may be dblink table, here to test is, if (OB_ERR_SYNONYM_TRANSLATION_INVALID == ret || OB_TABLE_NOT_EXIST == ret) { @@ -5489,7 +5489,7 @@ int ObDMLResolver::resolve_dblink_with_synonym(uint64_t tenant_id, ObString &tab int ret = OB_SUCCESS; // dblink name must be something like 'db_name.tbl_name@dblink', or 'tbl_name@dblink' ObString tmp_table_name; - ObString tmp_db_name; + ObString dblink_user_name; CK (OB_NOT_NULL(allocator_)); OZ (ob_write_string(*allocator_, table_name, tmp_table_name)); ObString tbl_sch_name = tmp_table_name.split_on('@'); @@ -5497,13 +5497,16 @@ int ObDMLResolver::resolve_dblink_with_synonym(uint64_t tenant_id, ObString &tab // do nothing; not a valid dblink name format } else if (tmp_table_name.empty()) { ret = OB_ERR_UNEXPECTED; + LOG_WARN("tmp_table_name is empty", K(ret)); } else { OZ (schema_checker_->get_dblink_id(tenant_id, tmp_table_name, dblink_id)); - OZ (schema_checker_->get_dblink_user(tenant_id, tmp_table_name, tmp_db_name, *allocator_)); + OZ (schema_checker_->get_dblink_user(tenant_id, tmp_table_name, dblink_user_name, *allocator_)); if (OB_FAIL(ret)) { ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error", K(ret)); } else if (OB_INVALID_ID == dblink_id) { ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalide dblink_id", K(ret)); } else { OZ (ob_write_string(*allocator_, tmp_table_name, dblink_name)); ObString remote_schema_name; @@ -5511,11 +5514,11 @@ int ObDMLResolver::resolve_dblink_with_synonym(uint64_t tenant_id, ObString &tab OX (remote_schema_name = tbl_sch_name.split_on('.')); if (OB_FAIL(ret)) { ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error", K(ret)); } else { - if (!remote_schema_name.empty()) { - if (0 != tmp_db_name.case_compare(remote_schema_name)) { - ret = OB_ERR_UNEXPECTED; - } else { /*do nothing*/ } + ObString &tmp_db_name = dblink_user_name; + if (!remote_schema_name.empty() && (0 != dblink_user_name.case_compare(remote_schema_name))) { + tmp_db_name = remote_schema_name; } // convert db_name to upper, for the field in all_object is upper if (OB_SUCC(ret)) { diff --git a/src/sql/resolver/dml/ob_dml_stmt.h b/src/sql/resolver/dml/ob_dml_stmt.h index 334280f88..dbd2d1a50 100644 --- a/src/sql/resolver/dml/ob_dml_stmt.h +++ b/src/sql/resolver/dml/ob_dml_stmt.h @@ -266,6 +266,7 @@ struct TableItem return (is_generated_table() || is_temp_table()) && view_base_item_ != NULL ? view_base_item_->get_base_table_item() : *this; } + virtual bool has_for_update() const { return for_update_; } // if real table id, it is valid for all threads, // else if generated id, it is unique just during the thread session uint64_t table_id_; @@ -443,6 +444,11 @@ struct JoinedTable : public TableItem bool is_left_join() const { return LEFT_OUTER_JOIN == joined_type_; } bool is_right_join() const { return RIGHT_OUTER_JOIN == joined_type_; } bool is_full_join() const { return FULL_OUTER_JOIN == joined_type_; } + virtual bool has_for_update() const + { + return (left_table_ != NULL && left_table_->has_for_update()) + || (right_table_ != NULL && right_table_->has_for_update()); + } common::ObIArray &get_join_conditions() { return join_conditions_; } const common::ObIArray &get_join_conditions() const { return join_conditions_; } TO_STRING_KV(N_TID, table_id_, diff --git a/src/sql/resolver/expr/ob_raw_expr_util.cpp b/src/sql/resolver/expr/ob_raw_expr_util.cpp index b33369648..f20a5b180 100644 --- a/src/sql/resolver/expr/ob_raw_expr_util.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_util.cpp @@ -892,7 +892,10 @@ int ObRawExprUtils::resolve_udf_param_exprs(ObResolverParams ¶ms, SQL_LOG(WARN, "names array not equle to exprs array count", K(ret), K(udf_info.param_names_.count()), K(udf_info.param_exprs_.count())); } else if ((udf_info.udf_param_num_ + udf_info.param_names_.count()) > func_info->get_param_count()) { - ret = OB_ERR_PARAM_SIZE; + ret = OB_ERR_SP_WRONG_ARG_NUM; + LOG_USER_ERROR(OB_ERR_SP_WRONG_ARG_NUM, "FUNCTION", udf_info.udf_name_.ptr(), + static_cast(func_info->get_param_count()), + static_cast(udf_info.udf_param_num_ + udf_info.param_names_.count())); SQL_LOG(WARN, "params count mismatch", K(ret), K(udf_info.udf_name_), K(func_info->get_param_count()), K(udf_info)); } else { @@ -941,7 +944,10 @@ int ObRawExprUtils::resolve_udf_param_exprs(ObResolverParams ¶ms, *(params.allocator_), params.session_info_->get_dtc_params(), default_val))) { LOG_WARN("fail to get default value", K(ret)); } else if (OB_UNLIKELY(default_val.empty())) { - ret = OB_ERR_PARAM_SIZE; + ret = OB_ERR_SP_WRONG_ARG_NUM; + LOG_USER_ERROR(OB_ERR_SP_WRONG_ARG_NUM, "FUNCTION", udf_info.udf_name_.ptr(), + static_cast(func_info->get_param_count()), + static_cast(udf_info.udf_param_num_ + udf_info.param_names_.count())); SQL_LOG(WARN, "param count mismatch", K(ret), K(i), K(default_val)); } else if (OB_FAIL(ObRawExprUtils::parse_default_expr_from_str( default_val, params.session_info_->get_local_collation_connection(), @@ -977,7 +983,10 @@ int ObRawExprUtils::resolve_udf_param_exprs(ObResolverParams ¶ms, } if (OB_SUCC(ret) && (func_info->get_param_count() != udf_info.udf_param_num_ + param_exprs.count())) { - ret = OB_ERR_PARAM_SIZE; + ret = OB_ERR_SP_WRONG_ARG_NUM; + LOG_USER_ERROR(OB_ERR_SP_WRONG_ARG_NUM, "FUNCTION", udf_info.udf_name_.ptr(), + static_cast(func_info->get_param_count()), + static_cast(udf_info.udf_param_num_ + udf_info.param_names_.count())); SQL_LOG(WARN, "params count mismatch", K(ret), K(udf_info.udf_name_), K(func_info->get_param_count()), K(udf_info)); diff --git a/src/sql/resolver/ob_schema_checker.h b/src/sql/resolver/ob_schema_checker.h index ea7c33760..11563f512 100644 --- a/src/sql/resolver/ob_schema_checker.h +++ b/src/sql/resolver/ob_schema_checker.h @@ -73,7 +73,7 @@ public: int init(ObSqlSchemaGuard &schema_guard, uint64_t session_id = common::OB_INVALID_ID); ObSqlSchemaGuard *get_sql_schema_guard() { return sql_schema_mgr_; } share::schema::ObSchemaGetterGuard *get_schema_guard() { return schema_mgr_; } - + // need satifing each priv in stmt_need_privs int check_priv(const share::schema::ObSessionPrivInfo &session_priv, const share::schema::ObStmtNeedPrivs &stmt_need_privs) const; @@ -81,7 +81,7 @@ public: const uint64_t uid, const share::schema::ObStmtOraNeedPrivs &stmt_need_privs, const ObIArray &role_id_array) const; - + // need satifing one of stmt_need_privs int check_priv_or(const share::schema::ObSessionPrivInfo &session_priv, const share::schema::ObStmtNeedPrivs &stmt_need_privs); diff --git a/src/sql/rewrite/ob_transform_join_elimination.cpp b/src/sql/rewrite/ob_transform_join_elimination.cpp index c8963c173..a6c3e27fd 100644 --- a/src/sql/rewrite/ob_transform_join_elimination.cpp +++ b/src/sql/rewrite/ob_transform_join_elimination.cpp @@ -698,29 +698,70 @@ int ObTransformJoinElimination::check_transform_validity_outer_join( } if (OB_SUCC(ret) && need_check_relation_exprs) { - ObSqlBitSet<8, int64_t> rel_ids; + ObSqlBitSet<> semi_left_rel_ids; + ObSqlBitSet<8, int64_t> right_rel_ids; ObSEArray join_conditions; if (OB_FAIL(join_conditions.assign(joined_table->join_conditions_))) { LOG_WARN("failed to push back to join conditions.", K(ret)); + } else if (OB_FAIL(extract_semi_left_rel_ids(stmt, + joined_table->right_table_, + semi_left_rel_ids))) { + LOG_WARN("failed to extract semi info left table rel ids", K(ret)); } else if (OB_FAIL(extract_child_conditions(stmt, - joined_table->right_table_, - join_conditions, - rel_ids))) { + joined_table->right_table_, + join_conditions, + right_rel_ids))) { LOG_WARN("failed to extract right child exprs.", K(ret)); - } else if (OB_FAIL(adjust_relation_exprs(rel_ids,// right table not use by any other expr - join_conditions, - relation_exprs, - is_valid))) { + } else if (OB_FAIL(adjust_relation_exprs(semi_left_rel_ids, + right_rel_ids,// right table not use by any other expr + join_conditions, + relation_exprs, + is_valid))) { LOG_WARN("failed to check expr in select items.", K(ret)); } } return ret; } +int ObTransformJoinElimination::extract_semi_left_rel_ids(ObDMLStmt *stmt, + TableItem *table_item, + ObSqlBitSet<> &semi_left_rel_ids) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(stmt) || OB_ISNULL(table_item)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < stmt->get_semi_infos().count(); ++i) { + SemiInfo *semi = stmt->get_semi_infos().at(i); + ObSEArray r_table_ids; + ObSEArray common_table_ids; + if (OB_ISNULL(semi)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("semi info is null", K(ret)); + } else if (table_item->is_joined_table() && + OB_FAIL(r_table_ids.assign(static_cast(table_item)->single_table_ids_))) { + } else if (!table_item->is_joined_table() && + OB_FAIL(r_table_ids.push_back(table_item->table_id_))) { + LOG_WARN("failed to push back joined table", K(ret)); + } else if (OB_FAIL(ObOptimizerUtil::intersect(semi->left_table_ids_, r_table_ids, common_table_ids))) { + LOG_WARN("failed to intersect table ids", K(ret)); + } else if (!common_table_ids.empty()) { + for (int64_t j = 0; OB_SUCC(ret) && j < common_table_ids.count(); ++j) { + if (OB_FAIL(semi_left_rel_ids.add_member(stmt->get_table_bit_index(common_table_ids.at(j))))) { + LOG_WARN("failed to add semi join left table ids", K(ret)); + } + } + } + } + } + return ret; +} + int ObTransformJoinElimination::extract_child_conditions(ObDMLStmt *stmt, TableItem *table_item, ObIArray &join_conditions, - ObSqlBitSet<8, int64_t> &rel_ids) + ObSqlBitSet<8, int64_t> &right_rel_ids) { int ret = OB_SUCCESS; bool is_stack_overflow = false; @@ -737,12 +778,12 @@ int ObTransformJoinElimination::extract_child_conditions(ObDMLStmt *stmt, if (OB_FAIL(SMART_CALL(extract_child_conditions(stmt, joined_table->left_table_, join_conditions, - rel_ids)))) { + right_rel_ids)))) { LOG_WARN("failed to remove right tables from stmt", K(ret)); } else if (OB_FAIL(SMART_CALL(extract_child_conditions(stmt, joined_table->right_table_, join_conditions, - rel_ids)))) { + right_rel_ids)))) { LOG_WARN("failed to remove right tables from stmt", K(ret)); } else { int64_t N = joined_table->join_conditions_.count(); @@ -752,13 +793,14 @@ int ObTransformJoinElimination::extract_child_conditions(ObDMLStmt *stmt, } else { /* do nothing. */ } } } - } else if (OB_FAIL(rel_ids.add_member(stmt->get_table_bit_index(table_item->table_id_)))) { + } else if (OB_FAIL(right_rel_ids.add_member(stmt->get_table_bit_index(table_item->table_id_)))) { LOG_WARN("failed to add member to rel ids.", K(ret)); } else { /* do nothing. */ } return ret; } -int ObTransformJoinElimination::adjust_relation_exprs(const ObSqlBitSet<8, int64_t> &rel_ids, +int ObTransformJoinElimination::adjust_relation_exprs(const ObSqlBitSet<> &semi_left_rel_ids, + const ObSqlBitSet<8, int64_t> &right_rel_ids, const ObIArray &join_conditions, ObIArray &relation_exprs, bool &is_valid) @@ -771,6 +813,8 @@ int ObTransformJoinElimination::adjust_relation_exprs(const ObSqlBitSet<8, int64 LOG_WARN("failed to assign to all exprs.", K(ret)); } else if (OB_FAIL(ObOptimizerUtil::remove_item(all_exprs, join_conditions))) { LOG_WARN("faile to remove item from all stmt exprs.", K(ret)); + } else if (OB_FAIL(select_rel_ids.add_members2(semi_left_rel_ids))) { + LOG_WARN("failed to add rel ids", K(ret)); } for (int64_t i = 0; OB_SUCC(ret) && i < all_exprs.count(); i++) { if (OB_ISNULL(all_exprs.at(i))) { @@ -781,7 +825,7 @@ int ObTransformJoinElimination::adjust_relation_exprs(const ObSqlBitSet<8, int64 } else { /*do nothing.*/ } } if (OB_SUCC(ret)) { - is_valid = select_rel_ids.overlap2(rel_ids) ? false : true; + is_valid = select_rel_ids.overlap2(right_rel_ids) ? false : true; if (is_valid && OB_FAIL(relation_exprs.assign(all_exprs))) { LOG_WARN("failed to assign cross stmt exprs.", K(ret)); } else { /* do nothing. */ } @@ -1046,12 +1090,19 @@ int ObTransformJoinElimination::do_eliminate_left_outer_join(ObDMLStmt *stmt, 0, /* result_flag */ ctx_->session_info_, cm))) { LOG_WARN("fail to get default cast mode", K(ret)); - } else if (OB_FAIL(ObRawExprUtils::create_cast_expr(*ctx_->expr_factory_, + } else if (is_mysql_mode() && + OB_FAIL(ObRawExprUtils::create_cast_expr(*ctx_->expr_factory_, to_expr, from_expr->get_result_type(), cast_expr, ctx_->session_info_, false, cm))) { LOG_WARN("failed to cast expr", K(ret), K(*from_expr), K(*to_expr)); + } else if (is_oracle_mode() && + OB_FAIL(ObRawExprUtils::create_cast_expr(*ctx_->expr_factory_, + to_expr, + from_expr->get_result_type(), + cast_expr, ctx_->session_info_))) { + LOG_WARN("failed to cast expr", K(ret), K(*from_expr), K(*to_expr)); } else if (OB_ISNULL(cast_expr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null expr", K(ret)); diff --git a/src/sql/rewrite/ob_transform_join_elimination.h b/src/sql/rewrite/ob_transform_join_elimination.h index 8e55795a7..a8a4cb81c 100644 --- a/src/sql/rewrite/ob_transform_join_elimination.h +++ b/src/sql/rewrite/ob_transform_join_elimination.h @@ -196,16 +196,19 @@ private: const ObIArray &output_map, ObIArray &source_exprs, ObIArray &target_exprs); - + int extract_semi_left_rel_ids(ObDMLStmt *stmt, + TableItem *table_item, + ObSqlBitSet<> &semi_left_rel_ids); int extract_child_conditions(ObDMLStmt *stmt, TableItem *source_table, ObIArray &join_conditions, - ObSqlBitSet<8, int64_t> &rel_ids); + ObSqlBitSet<8, int64_t> &right_rel_ids); - int adjust_relation_exprs(const ObSqlBitSet<8, int64_t> &rel_ids, - const ObIArray &join_conditions, - ObIArray &relation_exprs, - bool &is_valid); + int adjust_relation_exprs(const ObSqlBitSet<> &semi_left_rel_ids, + const ObSqlBitSet<8, int64_t> &right_rel_ids, + const ObIArray &join_conditions, + ObIArray &relation_exprs, + bool &is_valid); int extract_equal_join_columns(const ObIArray &join_conds, const TableItem *source_table, diff --git a/src/sql/rewrite/ob_transform_left_join_to_anti.cpp b/src/sql/rewrite/ob_transform_left_join_to_anti.cpp index 0d0317b6a..0c9251ca7 100644 --- a/src/sql/rewrite/ob_transform_left_join_to_anti.cpp +++ b/src/sql/rewrite/ob_transform_left_join_to_anti.cpp @@ -180,7 +180,8 @@ int ObTransformLeftJoinToAnti::transform_left_join_to_anti_join(ObDMLStmt *&stmt } else if (OB_FAIL(ObOptimizerUtil::remove_item(stmt->get_condition_exprs(), target_exprs))) { LOG_WARN("failed to remove condition exprs", K(ret)); } else if (is_root_table) { - if (right_table->is_joined_table() && + if ((right_table->is_joined_table() || + (lib::is_mysql_mode() && right_table->has_for_update())) && OB_FAIL(ObTransformUtils::create_view_with_table(stmt, ctx_, right_table, @@ -194,7 +195,14 @@ int ObTransformLeftJoinToAnti::transform_left_join_to_anti_join(ObDMLStmt *&stmt } else { TableItem *table = joined_table; ObDMLStmt *ref_query = NULL; - if (OB_FAIL(ObTransformUtils::create_view_with_table(stmt, + TableItem *view_table_for_update = NULL; + if (lib::is_mysql_mode() && right_table->has_for_update() && + OB_FAIL(ObTransformUtils::create_view_with_table(stmt, + ctx_, + right_table, + view_table_for_update))) { + LOG_WARN("failed to create semi view", K(ret)); + } else if (OB_FAIL(ObTransformUtils::create_view_with_table(stmt, ctx_, table, view_table))) { @@ -295,6 +303,7 @@ int ObTransformLeftJoinToAnti::trans_stmt_to_anti(ObDMLStmt *stmt, const JoinedT ObRawExpr *from_expr = NULL; ObRawExpr *to_expr = NULL; ObSysFunRawExpr *cast_expr = NULL; + ObCastMode cm; if (OB_ISNULL(col_item = stmt->get_column_item(i)) || OB_ISNULL(from_expr = col_item->expr_)) { ret = OB_ERR_UNEXPECTED; @@ -304,12 +313,23 @@ int ObTransformLeftJoinToAnti::trans_stmt_to_anti(ObDMLStmt *stmt, const JoinedT } else if (OB_FAIL(ObRawExprUtils::build_null_expr(*ctx_->expr_factory_, to_expr))) { LOG_WARN("failed to build null expr", K(ret)); - } else if (OB_FAIL(ObRawExprUtils::create_cast_expr(*ctx_->expr_factory_, - to_expr, - from_expr->get_result_type(), - cast_expr, - ctx_->session_info_))) { - LOG_WARN("failed to create cast expr", K(ret)); + } else if (OB_FAIL(ObSQLUtils::get_default_cast_mode(true,/* explicit_cast */ + 0, /* result_flag */ + ctx_->session_info_, cm))) { + LOG_WARN("fail to get default cast mode", K(ret)); + } else if (is_mysql_mode() && + OB_FAIL(ObRawExprUtils::create_cast_expr(*ctx_->expr_factory_, + to_expr, + from_expr->get_result_type(), + cast_expr, ctx_->session_info_, + false, cm))) { + LOG_WARN("failed to cast expr", K(ret), K(*from_expr), K(*to_expr)); + } else if (is_oracle_mode() && + OB_FAIL(ObRawExprUtils::create_cast_expr(*ctx_->expr_factory_, + to_expr, + from_expr->get_result_type(), + cast_expr, ctx_->session_info_))) { + LOG_WARN("failed to cast expr", K(ret), K(*from_expr), K(*to_expr)); } else if (OB_ISNULL(to_expr = cast_expr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null cast expr", K(ret)); diff --git a/src/sql/rewrite/ob_transform_post_process.cpp b/src/sql/rewrite/ob_transform_post_process.cpp index 898c3e442..0a76e260b 100644 --- a/src/sql/rewrite/ob_transform_post_process.cpp +++ b/src/sql/rewrite/ob_transform_post_process.cpp @@ -1046,12 +1046,17 @@ int ObTransformPostProcess::extract_onetime_subquery(ObRawExpr *&expr, !expr->has_flag(CNT_ALIAS); if (is_valid) { + int64_t ref_count = 0; if (OB_FAIL(is_non_correlated_exists_for_onetime(expr, - is_valid_non_correlated_exists))) { + is_valid_non_correlated_exists, + ref_count))) { LOG_WARN("failed to check non correlated exist for one time", K(ret)); - } else if (is_valid_non_correlated_exists && - OB_FAIL(onetime_list.push_back(expr))) { + } else if (!is_valid_non_correlated_exists) { + // do nothing + } else if (OB_FAIL(onetime_list.push_back(expr))) { LOG_WARN("failed to push back non-correlated exists", K(ret)); + } else if (ref_count > 1) { + is_valid = false; } } } @@ -1165,7 +1170,8 @@ int ObTransformPostProcess::create_onetime_param(ObDMLStmt *stmt, } int ObTransformPostProcess::is_non_correlated_exists_for_onetime(ObRawExpr *expr, - bool &is_non_correlated_exists_for_onetime) + bool &is_non_correlated_exists_for_onetime, + int64_t &ref_count) { int ret = OB_SUCCESS; is_non_correlated_exists_for_onetime = false; @@ -1181,6 +1187,7 @@ int ObTransformPostProcess::is_non_correlated_exists_for_onetime(ObRawExpr *expr LOG_WARN("failed to check subquery has ref assign user var", K(ret)); } else if (!has_ref_assign_user_var && query_ref_expr->get_param_count() == 0) { is_non_correlated_exists_for_onetime = true; + ref_count = query_ref_expr->get_ref_count(); } } return ret; diff --git a/src/sql/rewrite/ob_transform_post_process.h b/src/sql/rewrite/ob_transform_post_process.h index 8b177620e..bf71efdab 100644 --- a/src/sql/rewrite/ob_transform_post_process.h +++ b/src/sql/rewrite/ob_transform_post_process.h @@ -142,7 +142,8 @@ private: const int64_t stmt_level); int is_non_correlated_exists_for_onetime(ObRawExpr *expr, - bool &is_non_correlated_exists_for_onetime); + bool &is_non_correlated_exists_for_onetime, + int64_t &ref_count); DISALLOW_COPY_AND_ASSIGN(ObTransformPostProcess); }; diff --git a/src/sql/rewrite/ob_transform_rule.cpp b/src/sql/rewrite/ob_transform_rule.cpp index b4d9639ae..3220877c3 100644 --- a/src/sql/rewrite/ob_transform_rule.cpp +++ b/src/sql/rewrite/ob_transform_rule.cpp @@ -263,6 +263,8 @@ int ObTransformRule::accept_transform(common::ObIArray &parent_ ObDMLStmt *top_stmt = parent_stmts.empty() ? stmt : parent_stmts.at(0).stmt_; bool is_expected = false; bool dummy = false; + ObDMLStmt *tmp1 = NULL; + ObDMLStmt *tmp2 = NULL; cost_based_trans_tried_ = true; if (OB_ISNULL(ctx_) || OB_ISNULL(stmt) || OB_ISNULL(trans_stmt) || OB_ISNULL(top_stmt)) { ret = OB_ERR_UNEXPECTED; @@ -285,6 +287,8 @@ int ObTransformRule::accept_transform(common::ObIArray &parent_ } else if (!trans_happened) { LOG_TRACE("reject transform because the cost is increased or the query plan is unexpected", K_(stmt_cost), K(trans_stmt_cost), K(is_expected)); + } else if (OB_FAIL(adjust_transformed_stmt(parent_stmts, trans_stmt, tmp1, tmp2))) { + LOG_WARN("failed to adjust transformed stmt", K(ret)); } else if (force_accept) { LOG_TRACE("succeed force accept transform because hint/rule"); stmt = trans_stmt; @@ -408,15 +412,15 @@ int ObTransformRule::prepare_eval_cost_stmt(common::ObIArray &p } else if (OB_FAIL(ObTransformUtils::deep_copy_stmt(*ctx_->stmt_factory_, *ctx_->expr_factory_, root_stmt, copied_stmt))) { LOG_WARN("failed to deep copy stmt", K(ret)); - } else if (NULL != orig_stmt && &stmt != orig_stmt // reset stmt - && OB_FAIL(adjust_transformed_stmt(parent_stmts, orig_stmt, temp, root_stmt))) { - LOG_WARN("failed to adjust transformed stmt", K(ret)); } else if (OB_FAIL(deep_copy_temp_table(*copied_stmt, *ctx_->stmt_factory_, *ctx_->expr_factory_, old_temp_table_stmts, new_temp_table_stmts))) { LOG_WARN("failed to deep copy temp table", K(ret)); + } else if (NULL != orig_stmt && &stmt != orig_stmt // reset stmt + && OB_FAIL(adjust_transformed_stmt(parent_stmts, orig_stmt, temp, root_stmt))) { + LOG_WARN("failed to adjust transformed stmt", K(ret)); } else if (!is_trans_stmt) { /* do nothing */ } else if (OB_FAIL(stmt.get_qb_name(cur_qb_name))) { @@ -430,10 +434,10 @@ int ObTransformRule::prepare_eval_cost_stmt(common::ObIArray &p } else if (skip_adjust_qb_name() || cur_qb_name != ctx_->src_qb_name_) { ctx_->src_qb_name_ = cur_qb_name; } else if (OB_FAIL(copied_stmt->get_stmt_by_stmt_id(stmt.get_stmt_id(), copied_trans_stmt))) { - LOG_WARN("failed to get stmt by stmt id", K(ret)); + LOG_WARN("failed to get stmt by stmt id", K(ret), K(stmt.get_stmt_id()), K(*copied_stmt)); } else if(OB_ISNULL(copied_trans_stmt)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null stmt", K(ret), K(copied_trans_stmt)); + LOG_WARN("unexpect null stmt", K(ret), K(stmt.get_stmt_id()), K(*copied_stmt)); } else if (OB_FAIL(copied_trans_stmt->adjust_qb_name(ctx_->allocator_, ctx_->src_qb_name_, ctx_->src_hash_val_))) { diff --git a/src/sql/rewrite/ob_transform_temp_table.cpp b/src/sql/rewrite/ob_transform_temp_table.cpp index 8298c0b4f..abc6e6b6a 100644 --- a/src/sql/rewrite/ob_transform_temp_table.cpp +++ b/src/sql/rewrite/ob_transform_temp_table.cpp @@ -14,6 +14,7 @@ #include "ob_transform_temp_table.h" #include "lib/allocator/ob_allocator.h" +#include "lib/hash/ob_hashmap.h" #include "lib/oblog/ob_log_module.h" #include "common/ob_common_utility.h" #include "sql/resolver/expr/ob_raw_expr.h" @@ -49,6 +50,7 @@ int ObTransformTempTable::transform_one_stmt(common::ObIArray & bool is_happened = false; ObSEArray child_stmts; ObArray temp_table_infos; + hash::ObHashMap parent_map; trans_happened = false; //当前stmt是root stmt时才改写 if (parent_stmts.empty()) { @@ -60,10 +62,14 @@ int ObTransformTempTable::transform_one_stmt(common::ObIArray & trans_param_ = new(buf)TempTableTransParam; } if (OB_FAIL(ret)) { - } else if (OB_FAIL(ObTransformUtils::get_all_child_stmts(stmt, child_stmts))) { + } else if (OB_FAIL(parent_map.create(128, "TempTable"))) { + LOG_WARN("failed to init stmt map", K(ret)); + } else if (OB_FAIL(ObTransformUtils::get_all_child_stmts(stmt, child_stmts, &parent_map))) { LOG_WARN("failed to get all child stmts", K(ret)); - } else if (OB_FAIL(extract_common_subquery_as_cte(stmt, child_stmts, is_happened))) { + } else if (OB_FAIL(extract_common_subquery_as_cte(stmt, child_stmts, parent_map, is_happened))) { LOG_WARN("failed to extract common subquery as cte", K(ret)); + } else if (OB_FAIL(parent_map.destroy())) { + LOG_WARN("failed to destroy map", K(ret)); } else { trans_happened |= is_happened; LOG_TRACE("succeed to extract common subquery as cte", K(is_happened)); @@ -350,6 +356,7 @@ int ObTransformTempTable::check_stmt_has_cross_product(ObSelectStmt *stmt, bool */ int ObTransformTempTable::extract_common_subquery_as_cte(ObDMLStmt *stmt, ObIArray &stmts, + hash::ObHashMap &parent_map, bool &trans_happened) { int ret = OB_SUCCESS; @@ -369,6 +376,7 @@ int ObTransformTempTable::extract_common_subquery_as_cte(ObDMLStmt *stmt, for (int64_t i = 0; OB_SUCC(ret) && i < stmt_groups.count(); ++i) { if (OB_FAIL(inner_extract_common_subquery_as_cte(*stmt, stmt_groups.at(i).stmts_, + parent_map, trans_happened))) { LOG_WARN("failed to convert temp table", K(ret)); } @@ -391,6 +399,7 @@ int ObTransformTempTable::extract_common_subquery_as_cte(ObDMLStmt *stmt, */ int ObTransformTempTable::inner_extract_common_subquery_as_cte(ObDMLStmt &root_stmt, ObIArray &stmts, + hash::ObHashMap &parent_map, bool &trans_happened) { int ret = OB_SUCCESS; @@ -418,11 +427,19 @@ int ObTransformTempTable::inner_extract_common_subquery_as_cte(ObDMLStmt &root_s !helper.hint_force_stmt_set_.has_qb_name(stmt)) { //hint forbid，do nothing } else if (OB_FAIL(check_has_stmt(helper.stmt_, - stmt, - has_stmt))) { + stmt, + parent_map, + has_stmt))) { LOG_WARN("failed to check has stmt", K(ret)); } else if (has_stmt) { //do nothing + } else if (OB_FAIL(check_has_stmt(stmt, + helper.stmt_, + parent_map, + has_stmt))) { + LOG_WARN("failed to check has stmt", K(ret)); + } else if (has_stmt) { + // do nothing } else if (OB_FAIL(ObStmtComparer::check_stmt_containment(helper.stmt_, stmt, map_info, @@ -515,102 +532,28 @@ int ObTransformTempTable::add_materialize_stmts(const ObIArray &s int ObTransformTempTable::check_has_stmt(ObSelectStmt *left_stmt, ObSelectStmt *right_stmt, + hash::ObHashMap &parent_map, bool &has_stmt) { int ret = OB_SUCCESS; has_stmt = false; - if (OB_ISNULL(left_stmt) || OB_ISNULL(right_stmt)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null stmt", K(ret)); - } else if (left_stmt->get_current_level() == right_stmt->get_current_level() && - OB_FAIL(check_has_stmt_in_same_level(left_stmt, right_stmt, has_stmt))) { - LOG_WARN("failed to check has stmt in same level", K(ret)); - } else if (!has_stmt && - left_stmt->get_current_level() != right_stmt->get_current_level() && - OB_FAIL(check_has_stmt_in_diff_level(left_stmt, right_stmt, has_stmt))) { - LOG_WARN("failed to check has stmt in diff level", K(ret)); - } - return ret; -} - -int ObTransformTempTable::check_has_stmt_in_same_level(ObSelectStmt *left_stmt, - ObSelectStmt *right_stmt, - bool &has_stmt) -{ - int ret = OB_SUCCESS; - has_stmt = false; - ObSEArray child_stmts; - if (OB_ISNULL(left_stmt) || OB_ISNULL(right_stmt)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null stmt", K(ret)); - } else if (OB_FAIL(left_stmt->get_child_stmts(child_stmts))) { - LOG_WARN("failed to get child stmts", K(ret)); - } else if (ObOptimizerUtil::find_item(child_stmts, right_stmt)) { - has_stmt = true; - } else { - for (int64_t i = 0; !has_stmt && OB_SUCC(ret) && i < child_stmts.count(); ++i) { - ObSelectStmt *child_stmt = child_stmts.at(i); - if (OB_ISNULL(child_stmt)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null stmt", K(ret)); - } else if (child_stmt->get_current_level() != right_stmt->get_current_level()) { - //do nothing - } else if (OB_FAIL(SMART_CALL(check_has_stmt_in_same_level(child_stmts.at(i), right_stmt, has_stmt)))) { - LOG_WARN("failed to check left stmt has right stmt", K(ret)); + ObDMLStmt *current = left_stmt; + ObDMLStmt *parent = NULL; + while (OB_SUCC(ret) && current != right_stmt && NULL != current) { + uint64_t key = reinterpret_cast(current); + if (OB_FAIL(parent_map.get_refactored(key, parent))) { + if (ret == OB_HASH_NOT_EXIST) { + current = NULL; + ret = OB_SUCCESS; + } else { + LOG_WARN("failed to get value", K(ret)); } - } - } - if (OB_SUCC(ret) && !has_stmt) { - child_stmts.reuse(); - if (OB_FAIL(right_stmt->get_child_stmts(child_stmts))) { - LOG_WARN("failed to get child stmts", K(ret)); - } else if (ObOptimizerUtil::find_item(child_stmts, left_stmt)) { - has_stmt = true; } else { - for (int64_t i = 0; !has_stmt && OB_SUCC(ret) && i < child_stmts.count(); ++i) { - ObSelectStmt *child_stmt = child_stmts.at(i); - if (OB_ISNULL(child_stmt)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null stmt", K(ret)); - } else if (child_stmt->get_current_level() != left_stmt->get_current_level()) { - //do nothing - } else if (OB_FAIL(SMART_CALL(check_has_stmt_in_same_level(child_stmts.at(i), left_stmt, has_stmt)))) { - LOG_WARN("failed to check left stmt has right stmt", K(ret)); - } - } + current = parent; } } - return ret; -} - -int ObTransformTempTable::check_has_stmt_in_diff_level(ObSelectStmt *left_stmt, - ObSelectStmt *right_stmt, - bool &has_stmt) -{ - int ret = OB_SUCCESS; - has_stmt = false; - if (OB_ISNULL(left_stmt) || OB_ISNULL(right_stmt)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect null stmt", K(ret)); - } else { - ObDMLStmt *parent_stmt = right_stmt; - //check left contain right - while (!has_stmt && NULL != parent_stmt) { - if (parent_stmt == left_stmt) { - has_stmt = true; - } else { - parent_stmt = parent_stmt->get_parent_namespace_stmt(); - } - } - //check right contain left - parent_stmt = left_stmt; - while (!has_stmt && NULL != parent_stmt) { - if (parent_stmt == right_stmt) { - has_stmt = true; - } else { - parent_stmt = parent_stmt->get_parent_namespace_stmt(); - } - } + if (OB_SUCC(ret) && current == right_stmt) { + has_stmt = true; } return ret; } diff --git a/src/sql/rewrite/ob_transform_temp_table.h b/src/sql/rewrite/ob_transform_temp_table.h index f5ae737d3..b53ee16c2 100644 --- a/src/sql/rewrite/ob_transform_temp_table.h +++ b/src/sql/rewrite/ob_transform_temp_table.h @@ -121,19 +121,20 @@ public: int extract_common_subquery_as_cte(ObDMLStmt *stmt, ObIArray &stmts, + hash::ObHashMap &parent_map, bool &trans_happened); int inner_extract_common_subquery_as_cte(ObDMLStmt &root_stmt, ObIArray &stmts, + hash::ObHashMap &parent_map, bool &trans_happened); int add_materialize_stmts(const ObIArray &stms); - int check_has_stmt(ObSelectStmt *left_stmt, ObSelectStmt *right_stmt, bool &has_stmt); - - int check_has_stmt_in_same_level(ObSelectStmt *left_stmt, ObSelectStmt *right_stmt, bool &has_stmt); - - int check_has_stmt_in_diff_level(ObSelectStmt *left_stmt, ObSelectStmt *right_stmt, bool &has_stmt); + int check_has_stmt(ObSelectStmt *left_stmt, + ObSelectStmt *right_stmt, + hash::ObHashMap &parent_map, + bool &has_stmt); bool is_similar_stmt(ObSelectStmt& stmt, const ObStmtMapInfo &map_info, diff --git a/src/sql/rewrite/ob_transform_utils.cpp b/src/sql/rewrite/ob_transform_utils.cpp index 297fb4653..d11cd35c4 100644 --- a/src/sql/rewrite/ob_transform_utils.cpp +++ b/src/sql/rewrite/ob_transform_utils.cpp @@ -1841,12 +1841,12 @@ int ObTransformUtils::is_expr_not_null(ObNotNullContext &ctx, T_FUN_SYS_ROWNUM == expr->get_expr_type() || T_LEVEL == expr->get_expr_type()) { is_not_null = true; - } else if (OB_FAIL(!expr->is_const_raw_expr() && - is_general_expr_not_null(ctx, expr, is_not_null, constraints))) { + } else if (!expr->is_const_raw_expr() && + OB_FAIL(is_general_expr_not_null(ctx, expr, is_not_null, constraints))) { LOG_WARN("failed to check compound expr", K(ret)); } else if (is_not_null) { // do nothing - } else if (expr->is_static_const_expr()) { + } else if (expr->is_static_scalar_const_expr()) { if (OB_FAIL(is_const_expr_not_null(ctx, expr, is_not_null))) { LOG_WARN("failed to check calculable expr not null", K(ret)); } else if (is_not_null && !expr->is_const_raw_expr() && @@ -9482,7 +9482,8 @@ int ObTransformUtils::add_param_bool_constraint(ObTransformerCtx *ctx, } int ObTransformUtils::get_all_child_stmts(ObDMLStmt *stmt, - ObIArray &child_stmts) + ObIArray &child_stmts, + hash::ObHashMap *parent_map) { int ret = OB_SUCCESS; ObSEArray temp_stmts; @@ -9506,25 +9507,20 @@ int ObTransformUtils::get_all_child_stmts(ObDMLStmt *stmt, LOG_WARN("failed to push back stmt", K(ret)); } } - if (OB_SUCC(ret) && !temp_stmts.empty()) { + if (OB_SUCC(ret)) { if (OB_FAIL(append(child_stmts, temp_stmts))) { - LOG_WARN("failed to append stmts", K(ret)); - } else if (OB_FAIL(SMART_CALL(get_all_child_stmts(temp_stmts, - child_stmts)))) { - LOG_WARN("failed to get all child stmt", K(ret)); + LOG_WARN("failed to append temp stmts", K(ret)); } } - return ret; -} - -int ObTransformUtils::get_all_child_stmts(ObIArray &stmts, - ObIArray &child_stmts) -{ - int ret = OB_SUCCESS; - for (int64_t i = 0; OB_SUCC(ret) && i < stmts.count(); ++i) { - if (OB_FAIL(get_all_child_stmts(stmts.at(i), - child_stmts))) { - LOG_WARN("failed to get all child stmt", K(ret)); + for (int64_t i = 0; OB_SUCC(ret)&& i < temp_stmts.count(); ++i) { + uint64_t key = reinterpret_cast(temp_stmts.at(i)); + if (OB_ISNULL(temp_stmts.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("temp stmt is null", K(ret)); + } else if (parent_map != NULL && OB_FAIL(parent_map->set_refactored(key, stmt))) { + LOG_WARN("failed to add parent child relation", K(ret)); + } else if (OB_FAIL(SMART_CALL(get_all_child_stmts(temp_stmts.at(i), child_stmts, parent_map)))) { + LOG_WARN("failed to get all child stmts", K(ret)); } } return ret; diff --git a/src/sql/rewrite/ob_transform_utils.h b/src/sql/rewrite/ob_transform_utils.h index f03e76197..c57c6092d 100644 --- a/src/sql/rewrite/ob_transform_utils.h +++ b/src/sql/rewrite/ob_transform_utils.h @@ -1368,10 +1368,8 @@ public: ObRawExpr *not_null_expr); static int get_all_child_stmts(ObDMLStmt *stmt, - ObIArray &child_stmts); - - static int get_all_child_stmts(ObIArray &stmts, - ObIArray &child_stmts); + ObIArray &child_stmts, + hash::ObHashMap *parent_map = NULL); static int check_select_expr_is_const(ObSelectStmt *stmt, ObRawExpr *expr, bool &is_const); diff --git a/src/sql/rewrite/ob_transform_view_merge.cpp b/src/sql/rewrite/ob_transform_view_merge.cpp index 1c1850e23..295bb18b2 100644 --- a/src/sql/rewrite/ob_transform_view_merge.cpp +++ b/src/sql/rewrite/ob_transform_view_merge.cpp @@ -356,7 +356,8 @@ int ObTransformViewMerge::check_semi_right_table_can_be_merged(ObDMLStmt *stmt, || ref_query->is_contains_assignment() || ref_query->has_sequence() || ref_query->is_hierarchical_query() - || ref_query->has_ora_rowscn()) { + || ref_query->has_ora_rowscn() + || (lib::is_mysql_mode() && ref_query->has_for_update())) { can_be = false; } else if (OB_FAIL(ref_query->has_rownum(has_rownum))) { LOG_WARN("failed to check has rownum expr", K(ret)); diff --git a/src/sql/session/ob_basic_session_info.cpp b/src/sql/session/ob_basic_session_info.cpp index ec2c231eb..83e04735b 100644 --- a/src/sql/session/ob_basic_session_info.cpp +++ b/src/sql/session/ob_basic_session_info.cpp @@ -973,6 +973,7 @@ int ObBasicSessionInfo::init_system_variables(const bool print_info_log, const b } } } // end for + release_to_pool_ = OB_SUCC(ret); if (OB_SUCC(ret)) { if (OB_FAIL(gen_sys_var_in_pc_str())) { //将影响plan的系统变量序列化并缓存 @@ -1270,7 +1271,8 @@ int ObBasicSessionInfo::load_sys_variable(ObIAllocator &calc_buf, LOG_WARN("fail to init sys var", K(ret), K(sys_var->get_type()), K(real_val), K(name), K(value)); } else if (OB_FAIL(process_session_variable(var_id, real_val, - false /*check_timezone_valid*/))) { + false /*check_timezone_valid*/, + false /*is_update_sys_var*/))) { LOG_WARN("process system variable error", K(name), K(type), K(real_val), K(value), K(ret)); } else { variables_last_modify_time_ = ObTimeUtility::current_time(); @@ -1577,7 +1579,8 @@ int ObBasicSessionInfo::update_sys_variable(const ObSysVarClassType sys_var_id, } // 更新变量 if (OB_SUCC(ret)) { - if (OB_FAIL(process_session_variable(sys_var_id, val, false /*check_timezone_valid*/))) { + if (OB_FAIL(process_session_variable(sys_var_id, val, false /*check_timezone_valid*/, + true /*is_update_sys_var*/))) { LOG_WARN("process system variable error", K(sys_var_id), K(val), K(ret)); } else if (OB_FAIL(sys_var_inc_info_.add_sys_var_id(sys_var_id))) { LOG_WARN("add sys var id error", K(sys_var_id), K(ret)); @@ -2038,7 +2041,7 @@ ObObjType ObBasicSessionInfo::get_sys_variable_meta_type(const ObString &var_nam } while(0) OB_INLINE int ObBasicSessionInfo::process_session_variable(ObSysVarClassType var, const ObObj &val, - const bool check_timezone_valid/*true*/) + const bool check_timezone_valid/*true*/, const bool is_update_sys_var/*false*/) { int ret = OB_SUCCESS; switch (var) { @@ -2069,12 +2072,12 @@ OB_INLINE int ObBasicSessionInfo::process_session_variable(ObSysVarClassType var } case SYS_VAR_DEBUG_SYNC: { const bool is_global = false; - ret = process_session_debug_sync(val, is_global); + ret = process_session_debug_sync(val, is_global, is_update_sys_var); break; } case SYS_VAR_OB_GLOBAL_DEBUG_SYNC: { const bool is_global = true; - ret = process_session_debug_sync(val, is_global); + ret = process_session_debug_sync(val, is_global, is_update_sys_var); break; } case SYS_VAR_OB_READ_CONSISTENCY: { @@ -2847,11 +2850,11 @@ int ObBasicSessionInfo::process_session_variable_fast() // SYS_VAR_DEBUG_SYNC OZ (ObSysVarFactory::calc_sys_var_store_idx(SYS_VAR_DEBUG_SYNC, store_idx)); OV (ObSysVarFactory::is_valid_sys_var_store_idx(store_idx)); - OZ (process_session_debug_sync(sys_vars_[store_idx]->get_value(), false)); + OZ (process_session_debug_sync(sys_vars_[store_idx]->get_value(), false, false)); // SYS_VAR_OB_GLOBAL_DEBUG_SYNC OZ (ObSysVarFactory::calc_sys_var_store_idx(SYS_VAR_OB_GLOBAL_DEBUG_SYNC, store_idx)); OV (ObSysVarFactory::is_valid_sys_var_store_idx(store_idx)); - OZ (process_session_debug_sync(sys_vars_[store_idx]->get_value(), true)); + OZ (process_session_debug_sync(sys_vars_[store_idx]->get_value(), true, false)); // SYS_VAR_OB_READ_CONSISTENCY // 这个系统变量对应consistency_level_，该属性只能通过常规途径修改，所以适合加入sys_vars_cache_， // 但这样涉及到的相关修改比较大，稳妥起见保留现有的序列化操作，只在本接口里执行，保证主线程正确初始化。 @@ -2995,7 +2998,8 @@ int ObBasicSessionInfo::process_session_log_level(const ObObj &val) } int ObBasicSessionInfo::process_session_debug_sync(const ObObj &val, - const bool is_global) + const bool is_global, + const bool is_update_sys_var) { int ret = OB_SUCCESS; if (OB_SYS_TENANT_ID == tenant_id_ && GCONF.is_debug_sync_enabled()) { @@ -3009,6 +3013,12 @@ int ObBasicSessionInfo::process_session_debug_sync(const ObObj &val, } } } + } else { + if (is_update_sys_var) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, + "Non-system tenant or debug_sync is turned off, set debug_sync is"); + } } return ret; } @@ -3873,8 +3883,9 @@ OB_DEF_DESERIALIZE(ObBasicSessionInfo) } else if (OB_FAIL(deep_copy_sys_variable(*sys_var, sys_var_id, sys_var->get_value()))) { LOG_WARN("fail to update system variable", K(sys_var_id), K(sys_var->get_value()), K(ret)); } else if (OB_FAIL(process_session_variable(sys_var_id, sys_var->get_value(), - check_timezone_valid))) { - LOG_ERROR("process system variable error", K(ret), K(*sys_var)); + check_timezone_valid, + false /*is_update_sys_var*/))) { + LOG_WARN("process system variable error", K(ret), K(*sys_var)); } } } @@ -4083,7 +4094,7 @@ int ObBasicSessionInfo::load_all_sys_vars(const ObSysVariableSchema &sys_var_sch OZ (load_sys_variable(calc_buf, sys_var->get_name(), sys_var->get_data_type(), sys_var->get_value(), sys_var->get_min_val(), sys_var->get_max_val(), sys_var->get_flags(), true)); - if (OB_NOT_NULL(sys_vars_[i])) { + if (OB_NOT_NULL(sys_vars_[i]) && OB_SUCC(ret)) { if (sys_vars_[i]->is_influence_plan()) { OZ (influence_plan_var_indexs_.push_back(i)); } @@ -4094,6 +4105,7 @@ int ObBasicSessionInfo::load_all_sys_vars(const ObSysVariableSchema &sys_var_sch } } } + release_to_pool_ = OB_SUCC(ret); if (!is_deserialized_) { OZ (gen_sys_var_in_pc_str()); } diff --git a/src/sql/session/ob_basic_session_info.h b/src/sql/session/ob_basic_session_info.h index 5e6c9c794..c80278a0c 100644 --- a/src/sql/session/ob_basic_session_info.h +++ b/src/sql/session/ob_basic_session_info.h @@ -1247,7 +1247,8 @@ public: common::ActiveSessionStat &get_ash_stat() { return ash_stat_; } protected: int process_session_variable(share::ObSysVarClassType var, const common::ObObj &value, - const bool check_timezone_valid = true); + const bool check_timezone_valid = true, + const bool is_update_sys_var = false); int process_session_variable_fast(); //@brief process session log_level setting like 'all.*:info, sql.*:debug'. //int process_session_ob_binlog_row_image(const common::ObObj &value); @@ -1257,7 +1258,8 @@ protected: int process_session_time_zone_value(const common::ObObj &value, const bool check_timezone_valid); int process_session_overlap_time_value(const ObObj &value); int process_session_autocommit_value(const common::ObObj &val); - int process_session_debug_sync(const common::ObObj &val, const bool is_global); + int process_session_debug_sync(const common::ObObj &val, const bool is_global, + const bool is_update_sys_var); // session切换接口 int base_save_session(BaseSavedValue &saved_value, bool skip_cur_stmt_tables = false); int base_restore_session(BaseSavedValue &saved_value); diff --git a/src/sql/spm/ob_plan_baseline_sql_service.h b/src/sql/spm/ob_plan_baseline_sql_service.h index 46a14b86c..44d818ec3 100644 --- a/src/sql/spm/ob_plan_baseline_sql_service.h +++ b/src/sql/spm/ob_plan_baseline_sql_service.h @@ -114,6 +114,7 @@ public: ObString &output_str); private: + const static char *EMPTY_STR; bool inited_; ObMySQLProxy* mysql_proxy_; }; diff --git a/src/storage/access/ob_single_merge.cpp b/src/storage/access/ob_single_merge.cpp index 2dbf46b3b..4a696cbc8 100644 --- a/src/storage/access/ob_single_merge.cpp +++ b/src/storage/access/ob_single_merge.cpp @@ -261,7 +261,7 @@ int ObSingleMerge::inner_get_next_row(ObDatumRow &row) if (table_idx == tables_.count() - 1) { access_ctx_->defensive_check_record_.start_access_table_idx_ = table_idx; access_ctx_->defensive_check_record_.total_table_handle_cnt_ = tables_.count(); - access_ctx_->defensive_check_record_.fist_access_table_start_log_ts_ = table->get_start_scn().get_val_for_tx(); + access_ctx_->defensive_check_record_.fist_access_table_start_scn_ = table->get_start_scn(); } #endif } @@ -322,16 +322,17 @@ int ObSingleMerge::inner_get_next_row(ObDatumRow &row) #ifdef ENABLE_DEBUG_LOG if (OB_SUCC(ret)) { access_ctx_->defensive_check_record_.query_flag_ = access_ctx_->query_flag_; - if (NULL != access_ctx_->store_ctx_ - && NULL != access_ctx_->store_ctx_->mvcc_acc_ctx_.get_mem_ctx() - && NULL != access_ctx_->store_ctx_->mvcc_acc_ctx_.get_mem_ctx()->get_defensive_check_mgr()) { - (void)access_ctx_->store_ctx_ - ->mvcc_acc_ctx_ - .get_mem_ctx() - ->get_defensive_check_mgr()->put(tablet_meta.tablet_id_, - row, - *rowkey_, - access_ctx_->defensive_check_record_); + transaction::ObTransService *trx = MTL(transaction::ObTransService *); + bool trx_id_valid = (NULL != access_ctx_->store_ctx_ + && access_ctx_->store_ctx_->mvcc_acc_ctx_.tx_id_.is_valid()); + if (OB_NOT_NULL(trx) + && trx_id_valid + && NULL != trx->get_defensive_check_mgr()) { + (void)trx->get_defensive_check_mgr()->put(tablet_meta.tablet_id_, + access_ctx_->store_ctx_->mvcc_acc_ctx_.tx_id_, + row, + *rowkey_, + access_ctx_->defensive_check_record_); } } access_ctx_->defensive_check_record_.reset(); diff --git a/src/storage/access/ob_sstable_row_lock_checker.cpp b/src/storage/access/ob_sstable_row_lock_checker.cpp index 1367c10b3..b799081aa 100644 --- a/src/storage/access/ob_sstable_row_lock_checker.cpp +++ b/src/storage/access/ob_sstable_row_lock_checker.cpp @@ -117,18 +117,14 @@ int ObSSTableRowLockChecker::check_row_locked(ObStoreRowLockState &lock_state) ret = OB_SUCCESS; } } - if (OB_SUCC(ret) && + if (OB_SUCC(ret) && transaction::ObTransVersion::INVALID_TRANS_VERSION != prefetcher_.row_lock_check_version_) { if (OB_UNLIKELY(lock_state.trans_version_ != palf::SCN::min_scn() || lock_state.is_locked_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("Unexpected lock state", K(ret), K_(lock_state.trans_version), K_(lock_state.is_locked)); - } else { - // TODO: fix it - palf::SCN scn; - scn.convert_for_lsn_allocator(prefetcher_.row_lock_check_version_); - - lock_state.trans_version_ = scn; - } + LOG_WARN("Unexpected lock state", K(ret), K_(lock_state.trans_version), K_(lock_state.is_locked)); + } else if (OB_FAIL(lock_state.trans_version_.convert_for_tx(prefetcher_.row_lock_check_version_))) { + LOG_WARN("failed to convert_for_tx", K(ret), K(prefetcher_.row_lock_check_version_)); + } else {/*do nothing*/} } return ret; } diff --git a/src/storage/blocksstable/encoding/ob_encoding_util.h b/src/storage/blocksstable/encoding/ob_encoding_util.h index ec386d9ca..00d5f6a7d 100644 --- a/src/storage/blocksstable/encoding/ob_encoding_util.h +++ b/src/storage/blocksstable/encoding/ob_encoding_util.h @@ -324,14 +324,15 @@ OB_INLINE static int batch_load_number_data_to_datum( } else { for (int64_t i = 0; i < row_cap; ++i) { if (!datums[i].is_null()) { - MEMCPY(const_cast(datums[i].ptr_), cell_datas[i], - sizeof(ObNumberDesc) + sizeof(uint32_t)); - uint8_t num_len = datums[i].num_->desc_.len_; + MEMCPY(const_cast(datums[i].ptr_), cell_datas[i], sizeof(ObNumberDesc)); + const uint8_t num_len = datums[i].num_->desc_.len_; datums[i].len_ = sizeof(ObNumberDesc) + num_len * sizeof(uint32_t); - if (OB_UNLIKELY(num_len > 1)) { - int64_t copy_offset = sizeof(ObNumberDesc) + sizeof(uint32_t); - ENCODING_ADAPT_MEMCPY(const_cast(datums[i].ptr_) + copy_offset, - cell_datas[i] + copy_offset, (num_len - 1) * sizeof(uint32_t)); + if (OB_LIKELY(1 == num_len)) { + MEMCPY(const_cast(datums[i].ptr_) + sizeof(ObNumberDesc), + cell_datas[i] + sizeof(ObNumberDesc), sizeof(uint32_t)); + } else { + ENCODING_ADAPT_MEMCPY(const_cast(datums[i].ptr_) + sizeof(ObNumberDesc), + cell_datas[i] + sizeof(ObNumberDesc), num_len * sizeof(uint32_t)); } } else { datums[i].set_null(); diff --git a/src/storage/blocksstable/ob_index_block_row_struct.cpp b/src/storage/blocksstable/ob_index_block_row_struct.cpp index 31484070c..a42eb3b9f 100644 --- a/src/storage/blocksstable/ob_index_block_row_struct.cpp +++ b/src/storage/blocksstable/ob_index_block_row_struct.cpp @@ -77,8 +77,8 @@ int ObIndexBlockRowHeader::fill_micro_des_meta( } ObIndexBlockRowBuilder::ObIndexBlockRowBuilder() - : allocator_(ObModIds::OB_BLOCK_INDEX_INTERMEDIATE), - index_data_allocator_(ObModIds::OB_BLOCK_INDEX_INTERMEDIATE), + : allocator_(ObModIds::OB_BLOCK_INDEX_INTERMEDIATE, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), + index_data_allocator_(ObModIds::OB_BLOCK_INDEX_INTERMEDIATE, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), row_(), rowkey_column_count_(0), rowkey_column_types_(nullptr), @@ -227,7 +227,7 @@ int ObIndexBlockRowBuilder::set_rowkey(const ObDatumRowkey &rowkey) LOG_WARN("Rowkey column count mismatch", K(ret), K_(rowkey_column_count), K(rowkey)); } else if (OB_FAIL(dest_rowkey.assign(row_.storage_datums_, rowkey_column_count_))) { STORAGE_LOG(WARN, "Failed to assign dest rowkey", K(ret), K(rowkey_column_count_)); - } else if (OB_FAIL(rowkey.semi_copy(dest_rowkey, allocator_))) { + } else if (OB_FAIL(rowkey.semi_copy(dest_rowkey, index_data_allocator_))) { STORAGE_LOG(WARN, "Failed to semi copy dest rowkey", K(ret), K(rowkey)); } diff --git a/src/storage/blocksstable/ob_sstable.cpp b/src/storage/blocksstable/ob_sstable.cpp index 4678cc871..2ca4731d1 100644 --- a/src/storage/blocksstable/ob_sstable.cpp +++ b/src/storage/blocksstable/ob_sstable.cpp @@ -680,7 +680,9 @@ int ObSSTable::check_row_locked(ObStoreCtx &ctx, if (!meta_.is_empty()) { // skip reference upper_trans_version of empty_sstable, which may greater than real // committed transaction's version - lock_state.trans_version_.convert_for_tx(get_upper_trans_version()); + if (OB_FAIL(lock_state.trans_version_.convert_for_tx(get_upper_trans_version()))) { + LOG_WARN("Fail to convert_for_tx", K(get_upper_trans_version()), K_(meta), K(ret)); + } } } else if (NULL == (buf = allocator.alloc(sizeof(ObSSTableRowLockChecker)))) { ret = OB_ALLOCATE_MEMORY_FAILED; diff --git a/src/storage/blocksstable/ob_tmp_file_cache.cpp b/src/storage/blocksstable/ob_tmp_file_cache.cpp index 8ff0c14de..dbbfc303f 100644 --- a/src/storage/blocksstable/ob_tmp_file_cache.cpp +++ b/src/storage/blocksstable/ob_tmp_file_cache.cpp @@ -122,7 +122,8 @@ int ObTmpPageCacheValue::deep_copy(char *buf, const int64_t buf_len, ObIKVCacheV int ObTmpPageCache::prefetch( const ObTmpPageCacheKey &key, const ObTmpBlockIOInfo &info, - ObMacroBlockHandle &mb_handle) + ObMacroBlockHandle &mb_handle, + common::ObIAllocator &allocator) { int ret = OB_SUCCESS; if (OB_UNLIKELY(!key.is_valid() )) { @@ -134,7 +135,7 @@ int ObTmpPageCache::prefetch( callback.cache_ = this; callback.offset_ = info.offset_; callback.buf_size_ = info.size_; - callback.allocator_ = &allocator_; + callback.allocator_ = &allocator; callback.key_ = key; if (OB_FAIL(read_io(info, callback, mb_handle))) { if (mb_handle.get_io_handle().is_empty()) { @@ -156,7 +157,8 @@ int ObTmpPageCache::prefetch( int ObTmpPageCache::prefetch( const ObTmpBlockIOInfo &info, const common::ObIArray &page_io_infos, - ObMacroBlockHandle &mb_handle) + ObMacroBlockHandle &mb_handle, + common::ObIAllocator &allocator) { int ret = OB_SUCCESS; if (OB_UNLIKELY(page_io_infos.count() <= 0)) { @@ -167,31 +169,22 @@ int ObTmpPageCache::prefetch( callback.cache_ = this; callback.offset_ = info.offset_; callback.buf_size_ = info.size_; - callback.allocator_ = &allocator_; - void *buf = allocator_.alloc(sizeof(common::ObSEArray)); - if (NULL == buf) { - ret = OB_ALLOCATE_MEMORY_FAILED; - STORAGE_LOG(WARN, "fail to alloc a buf", K(ret), K(info)); - } else { - callback.page_io_infos_ = new (buf) common::ObSEArray(); - callback.page_io_infos_->assign(page_io_infos); - if (OB_FAIL(read_io(info, callback, mb_handle))) { - if (mb_handle.get_io_handle().is_empty()) { - // TODO: After the continuous IO has been optimized, this should - // not happen. - if (OB_FAIL(mb_handle.wait(DEFAULT_IO_WAIT_TIME_MS))) { - STORAGE_LOG(WARN, "fail to wait tmp page io", K(ret)); - } else if (OB_FAIL(read_io(info, callback, mb_handle))) { - STORAGE_LOG(WARN, "fail to read tmp page from io", K(ret)); - } - } else { + callback.allocator_ = &allocator; + if (OB_FAIL(callback.page_io_infos_.assign(page_io_infos))) { + STORAGE_LOG(WARN, "fail to assign page io infos", K(ret), K(page_io_infos.count()), K(info)); + } else if (OB_FAIL(read_io(info, callback, mb_handle))) { + if (mb_handle.get_io_handle().is_empty()) { + // TODO: After the continuous IO has been optimized, this should + // not happen. + if (OB_FAIL(mb_handle.wait(DEFAULT_IO_WAIT_TIME_MS))) { + STORAGE_LOG(WARN, "fail to wait tmp page io", K(ret)); + } else if (OB_FAIL(read_io(info, callback, mb_handle))) { STORAGE_LOG(WARN, "fail to read tmp page from io", K(ret)); } + } else { + STORAGE_LOG(WARN, "fail to read tmp page from io", K(ret)); } } - if (OB_FAIL(ret) && OB_NOT_NULL(buf)) { - allocator_.free(buf); - } } return ret; } @@ -340,6 +333,8 @@ ObTmpPageCache::ObTmpMultiPageIOCallback::ObTmpMultiPageIOCallback() ObTmpPageCache::ObTmpMultiPageIOCallback::~ObTmpMultiPageIOCallback() { + page_io_infos_.reset(); + page_io_infos_.~ObIArray(); } int ObTmpPageCache::ObTmpMultiPageIOCallback::inner_process(const bool is_success) @@ -350,17 +345,16 @@ int ObTmpPageCache::ObTmpMultiPageIOCallback::inner_process(const bool is_succes STORAGE_LOG(WARN, "Invalid tmp page cache callback, ", KP_(cache), K(ret)); } else if (is_success) { char *buf = const_cast(get_data()); - for (int32_t i = 0; OB_SUCC(ret) && i < page_io_infos_->count(); i++) { - int64_t offset = page_io_infos_->at(i).key_.get_page_id() + for (int32_t i = 0; OB_SUCC(ret) && i < page_io_infos_.count(); i++) { + int64_t offset = page_io_infos_.at(i).key_.get_page_id() * ObTmpMacroBlock::get_default_page_size() - offset_; offset += ObTmpMacroBlock::get_header_padding(); ObTmpPageCacheValue value(buf + offset); - if (OB_FAIL(process_page(page_io_infos_->at(i).key_, value))) { + if (OB_FAIL(process_page(page_io_infos_.at(i).key_, value))) { STORAGE_LOG(WARN, "fail to process tmp page cache in callback", K(ret)); } } - page_io_infos_->reset(); - allocator_->free(page_io_infos_); + page_io_infos_.reset(); } if (OB_FAIL(ret) && NULL != allocator_ && NULL != io_buf_) { allocator_->free(io_buf_); @@ -388,7 +382,11 @@ int ObTmpPageCache::ObTmpMultiPageIOCallback::inner_deep_copy(char *buf, } else { ObTmpMultiPageIOCallback *pcallback = new (buf) ObTmpMultiPageIOCallback(); *pcallback = *this; - callback = pcallback; + if (OB_FAIL(pcallback->page_io_infos_.assign(page_io_infos_))) { + STORAGE_LOG(WARN, "The tmp page io assign failed", K(ret)); + } else { + callback = pcallback; + } } return ret; } @@ -432,16 +430,9 @@ ObTmpPageCache::~ObTmpPageCache() int ObTmpPageCache::init(const char *cache_name, const int64_t priority) { int ret = OB_SUCCESS; - const int64_t mem_limit = 4 * 1024 * 1024 * 1024LL; if (OB_FAIL((common::ObKVCache::init( cache_name, priority)))) { STORAGE_LOG(WARN, "Fail to init kv cache, ", K(ret)); - } else if (OB_FAIL(allocator_.init(mem_limit, - OB_MALLOC_BIG_BLOCK_SIZE, - OB_MALLOC_BIG_BLOCK_SIZE))) { - STORAGE_LOG(WARN, "Fail to init io allocator, ", K(ret)); - } else { - allocator_.set_label(ObModIds::OB_TMP_PAGE_CACHE); } return ret; } @@ -449,7 +440,6 @@ int ObTmpPageCache::init(const char *cache_name, const int64_t priority) void ObTmpPageCache::destroy() { common::ObKVCache::destroy(); - allocator_.destroy(); } int ObTmpPageCache::put_page(const ObTmpPageCacheKey &key, const ObTmpPageCacheValue &value) diff --git a/src/storage/blocksstable/ob_tmp_file_cache.h b/src/storage/blocksstable/ob_tmp_file_cache.h index 954d5fa37..46b5af0a2 100644 --- a/src/storage/blocksstable/ob_tmp_file_cache.h +++ b/src/storage/blocksstable/ob_tmp_file_cache.h @@ -106,12 +106,14 @@ public: int prefetch( const ObTmpPageCacheKey &key, const ObTmpBlockIOInfo &info, - ObMacroBlockHandle &mb_handle); + ObMacroBlockHandle &mb_handle, + common::ObIAllocator &allocator); // multi page prefetch int prefetch( const ObTmpBlockIOInfo &info, const common::ObIArray &page_io_infos, - ObMacroBlockHandle &mb_handle); + ObMacroBlockHandle &mb_handle, + common::ObIAllocator &allocator); int get_cache_page(const ObTmpPageCacheKey &key, ObTmpPageValueHandle &handle); int get_page(const ObTmpPageCacheKey &key, ObTmpPageValueHandle &handle); int put_page(const ObTmpPageCacheKey &key, const ObTmpPageCacheValue &value); @@ -163,7 +165,7 @@ public: TO_STRING_KV(KP_(data_buf)); private: friend class ObTmpPageCache; - common::ObIArray *page_io_infos_; + common::ObArray page_io_infos_; }; private: ObTmpPageCache(); @@ -172,7 +174,6 @@ private: ObMacroBlockHandle &handle); private: - common::ObConcurrentFIFOAllocator allocator_; DISALLOW_COPY_AND_ASSIGN(ObTmpPageCache); }; diff --git a/src/storage/blocksstable/ob_tmp_file_store.cpp b/src/storage/blocksstable/ob_tmp_file_store.cpp index b1c82b0f4..90eff12d2 100644 --- a/src/storage/blocksstable/ob_tmp_file_store.cpp +++ b/src/storage/blocksstable/ob_tmp_file_store.cpp @@ -814,6 +814,7 @@ ObTmpTenantFileStore::ObTmpTenantFileStore() : page_cache_(NULL), tmp_block_manager_(), allocator_(), + io_allocator_(), tmp_mem_block_manager_(), is_inited_(false), page_cache_num_(0), @@ -850,8 +851,10 @@ int ObTmpTenantFileStore::init(const uint64_t tenant_id) if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; STORAGE_LOG(WARN, "ObTmpTenantFileStore has not been inited", K(ret)); - } else if (OB_FAIL(allocator_.init(TOTAL_LIMIT, HOLD_LIMIT, BLOCK_SIZE))) { + } else if (OB_FAIL(allocator_.init(BLOCK_SIZE, ObModIds::OB_TMP_BLOCK_MANAGER, tenant_id, TOTAL_LIMIT))) { STORAGE_LOG(WARN, "fail to init allocator", K(ret)); + } else if (OB_FAIL(io_allocator_.init(OB_MALLOC_BIG_BLOCK_SIZE, ObModIds::OB_TMP_PAGE_CACHE, tenant_id, IO_LIMIT))) { + STORAGE_LOG(WARN, "Fail to init io allocator, ", K(ret)); } else if (OB_ISNULL(page_cache_ = &ObTmpPageCache::get_instance())) { ret = OB_ERR_UNEXPECTED; STORAGE_LOG(WARN, "fail to get the page cache", K(ret)); @@ -860,7 +863,6 @@ int ObTmpTenantFileStore::init(const uint64_t tenant_id) } else if (OB_FAIL(tmp_mem_block_manager_.init(tenant_id, allocator_))) { STORAGE_LOG(WARN, "fail to init memory block manager", K(ret)); } else { - allocator_.set_label(ObModIds::OB_TMP_BLOCK_MANAGER); is_inited_ = true; } if (!is_inited_) { @@ -877,6 +879,7 @@ void ObTmpTenantFileStore::destroy() page_cache_ = NULL; } allocator_.destroy(); + io_allocator_.destroy(); is_inited_ = false; STORAGE_LOG(INFO, "cache num when destroy", K(ATOMIC_LOAD(&page_cache_num_)), K(ATOMIC_LOAD(&block_cache_num_))); @@ -1250,7 +1253,7 @@ int ObTmpTenantFileStore::read_page(ObTmpMacroBlock *block, ObTmpBlockIOInfo &io info.offset_ = p_offset + ObTmpMacroBlock::get_header_padding(); info.size_ = page_nums * ObTmpMacroBlock::get_default_page_size(); info.macro_block_id_ = block->get_macro_block_id(); - if (OB_FAIL(page_cache_->prefetch(info, *page_io_infos, mb_handle))) { + if (OB_FAIL(page_cache_->prefetch(info, *page_io_infos, mb_handle, io_allocator_))) { STORAGE_LOG(WARN, "fail to prefetch multi tmp page", K(ret)); } else { ObTmpFileIOHandle::ObIOReadHandle read_handle(mb_handle, io_info.buf_, @@ -1269,7 +1272,7 @@ int ObTmpTenantFileStore::read_page(ObTmpMacroBlock *block, ObTmpBlockIOInfo &io info.offset_ += ObTmpMacroBlock::get_header_padding(); info.size_ = ObTmpMacroBlock::get_default_page_size(); info.macro_block_id_ = block->get_macro_block_id(); - if (OB_FAIL(page_cache_->prefetch(page_io_infos->at(i).key_, info, mb_handle))) { + if (OB_FAIL(page_cache_->prefetch(page_io_infos->at(i).key_, info, mb_handle, io_allocator_))) { STORAGE_LOG(WARN, "fail to prefetch tmp page", K(ret)); } else { char *buf = io_info.buf_ + ObTmpMacroBlock::calculate_offset( diff --git a/src/storage/blocksstable/ob_tmp_file_store.h b/src/storage/blocksstable/ob_tmp_file_store.h index f6d0f6b91..b7e8df28f 100644 --- a/src/storage/blocksstable/ob_tmp_file_store.h +++ b/src/storage/blocksstable/ob_tmp_file_store.h @@ -262,6 +262,7 @@ private: int wait_write_io_finish_if_need(); private: + static const uint64_t IO_LIMIT = 4 * 1024L * 1024L * 1024L; static const uint64_t TOTAL_LIMIT = 15 * 1024L * 1024L * 1024L; static const uint64_t HOLD_LIMIT = 8 * 1024L * 1024L; static const uint64_t BLOCK_SIZE = common::OB_MALLOC_MIDDLE_BLOCK_SIZE; @@ -270,6 +271,7 @@ private: ObTmpPageCache *page_cache_; ObTmpTenantMacroBlockManager tmp_block_manager_; common::ObConcurrentFIFOAllocator allocator_; + common::ObConcurrentFIFOAllocator io_allocator_; ObTmpTenantMemBlockManager tmp_mem_block_manager_; common::SpinRWLock lock_; bool is_inited_; diff --git a/src/storage/compaction/ob_i_compaction_filter.cpp b/src/storage/compaction/ob_i_compaction_filter.cpp index b77e018ae..4f3767eef 100644 --- a/src/storage/compaction/ob_i_compaction_filter.cpp +++ b/src/storage/compaction/ob_i_compaction_filter.cpp @@ -78,10 +78,10 @@ int64_t ObICompactionFilter::ObFilterStatistics::to_string(char *buf, const int6 return pos; } -int ObTransStatusFilter::init(const int64_t filter_val, const int64_t filter_col_idx) +int ObTransStatusFilter::init(const palf::SCN &filter_val, const int64_t filter_col_idx) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(filter_val <= 0 || filter_col_idx < 0)) { + if (OB_UNLIKELY(!filter_val.is_valid() || filter_val.is_min() || filter_col_idx < 0)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(filter_val), K(filter_col_idx)); } else if (IS_INIT) { @@ -90,7 +90,7 @@ int ObTransStatusFilter::init(const int64_t filter_val, const int64_t filter_col } else { filter_val_ = filter_val; filter_col_idx_ = filter_col_idx; - max_filtered_end_scn_ = 0; + max_filtered_end_scn_.set_min(); is_inited_ = true; } return ret; @@ -112,10 +112,13 @@ int ObTransStatusFilter::filter( ret = OB_INVALID_ARGUMENT; LOG_WARN("uncommitted row in trans state table or multi version row", K(ret), K(row)); } else { - const int64_t trans_end_scn = row.storage_datums_[filter_col_idx_].get_int(); - if (trans_end_scn <= filter_val_) { + const int64_t trans_end_scn_storage_val = row.storage_datums_[filter_col_idx_].get_int(); + palf::SCN trans_end_scn; + if (OB_FAIL(trans_end_scn.convert_for_tx(trans_end_scn_storage_val))) { + LOG_WARN("failed to convert for tx", K(ret), K(trans_end_scn_storage_val)); + } else if (trans_end_scn <= filter_val_) { filter_ret = FILTER_RET_REMOVE; - max_filtered_end_scn_ = std::max(max_filtered_end_scn_, trans_end_scn); + max_filtered_end_scn_ = palf::SCN::max(max_filtered_end_scn_, trans_end_scn); LOG_DEBUG("filter row", K(ret), K(row), K(filter_val_)); } else { filter_ret = FILTER_RET_NOT_CHANGE; diff --git a/src/storage/compaction/ob_i_compaction_filter.h b/src/storage/compaction/ob_i_compaction_filter.h index fd06cbbc8..4081c0e3f 100644 --- a/src/storage/compaction/ob_i_compaction_filter.h +++ b/src/storage/compaction/ob_i_compaction_filter.h @@ -83,17 +83,21 @@ public: ObTransStatusFilter() : ObICompactionFilter(true), is_inited_(false), - filter_val_(INT64_MAX), + filter_val_(), filter_col_idx_(0), - max_filtered_end_scn_(0) {} + max_filtered_end_scn_() + { + filter_val_.set_max(); + max_filtered_end_scn_.set_min(); + } ~ObTransStatusFilter() {} - int init(const int64_t filter_val, const int64_t filter_col_idx); + int init(const palf::SCN &filter_val, const int64_t filter_col_idx); OB_INLINE virtual void reset() override { ObICompactionFilter::reset(); - filter_val_ = INT64_MAX; + filter_val_.set_max(); filter_col_idx_ = 0; - max_filtered_end_scn_ = 0; + max_filtered_end_scn_.set_min(); is_inited_ = false; } @@ -103,17 +107,14 @@ public: K_(filter_col_idx), K_(max_filtered_end_scn)); public: - // TODO(scn): change scn of int64_t type to palf::SCN - int64_t get_max_filtered_end_scn_v0() { return max_filtered_end_scn_; } - palf::SCN get_max_filtered_end_scn() { palf::SCN tmp_scn; tmp_scn.convert_for_tx(max_filtered_end_scn_); return tmp_scn; } - int64_t get_recycle_scn_v0() { return filter_val_; } - palf::SCN get_recycle_scn() { palf::SCN tmp_scn; tmp_scn.convert_for_tx(filter_val_); return tmp_scn; } + palf::SCN get_max_filtered_end_scn() { return max_filtered_end_scn_; } + palf::SCN get_recycle_scn() { return filter_val_; } private: bool is_inited_; - int64_t filter_val_; + palf::SCN filter_val_; int64_t filter_col_idx_; - int64_t max_filtered_end_scn_; + palf::SCN max_filtered_end_scn_; }; } // namespace compaction diff --git a/src/storage/compaction/ob_partition_merge_policy.cpp b/src/storage/compaction/ob_partition_merge_policy.cpp index c91b45936..4dcceece0 100644 --- a/src/storage/compaction/ob_partition_merge_policy.cpp +++ b/src/storage/compaction/ob_partition_merge_policy.cpp @@ -72,7 +72,9 @@ int ObPartitionMergePolicy::get_neighbour_freeze_info( freeze_info.reset(); freeze_info.next.freeze_scn.set_max(); if (OB_NOT_NULL(last_major)) { - freeze_info.prev.freeze_scn.convert_for_tx(last_major->get_snapshot_version()); + if (OB_FAIL(freeze_info.prev.freeze_scn.convert_for_tx(last_major->get_snapshot_version()))) { + LOG_WARN("failed to convert scn", K(ret), K(last_major)); + } } } else { LOG_WARN("Failed to get neighbour major freeze info", K(ret), K(snapshot_version)); diff --git a/src/storage/compaction/ob_partition_merger.cpp b/src/storage/compaction/ob_partition_merger.cpp index 7f6ebda60..249085313 100644 --- a/src/storage/compaction/ob_partition_merger.cpp +++ b/src/storage/compaction/ob_partition_merger.cpp @@ -1942,7 +1942,7 @@ int ObPartitionMergeDumper::generate_dump_table_name(const char *dir_name, STORAGE_LOG(WARN, "table is null", K(ret)); } else { int64_t pret = snprintf( - file_name, OB_MAX_FILE_NAME_LENGTH, "%s/%s.%s.%ld.%s.%d.%s.%ld.%s.%ld", + file_name, OB_MAX_FILE_NAME_LENGTH, "%s/%s.%s.%ld.%s.%d.%s.%lu.%s.%lu", dir_name, table->is_memtable() ? "dump_memtable" : "dump_sstable", "tablet_id", table->get_key().tablet_id_.id(), diff --git a/src/storage/compaction/ob_tablet_merge_ctx.cpp b/src/storage/compaction/ob_tablet_merge_ctx.cpp index e3b47e17f..d7e789ce3 100644 --- a/src/storage/compaction/ob_tablet_merge_ctx.cpp +++ b/src/storage/compaction/ob_tablet_merge_ctx.cpp @@ -668,7 +668,7 @@ int ObTabletMergeCtx::inner_init_for_major() } else if (get_merge_table_result.handle_.get_count() > 1 && !ObTenantTabletScheduler::check_tx_table_ready( *ls_handle_.get_ls(), - get_merge_table_result.scn_range_.end_scn_.get_val_for_tx())) { + get_merge_table_result.scn_range_.end_scn_)) { ret = OB_EAGAIN; LOG_INFO("tx table is not ready. waiting for max_decided_log_ts ...", KR(ret), "merge_scn", get_merge_table_result.scn_range_.end_scn_); @@ -728,7 +728,7 @@ int ObTabletMergeCtx::inner_init_for_minor(bool &skip_rest_operation) } } else if (!ObTenantTabletScheduler::check_tx_table_ready( *ls_handle_.get_ls(), - get_merge_table_result.scn_range_.end_scn_.get_val_for_tx())) { + get_merge_table_result.scn_range_.end_scn_)) { ret = OB_EAGAIN; LOG_INFO("tx table is not ready. waiting for max_decided_log_ts ...", KR(ret), "merge_scn", get_merge_table_result.scn_range_.end_scn_); diff --git a/src/storage/compaction/ob_tablet_merge_ctx.h b/src/storage/compaction/ob_tablet_merge_ctx.h index 298d02a7d..9708dadb7 100644 --- a/src/storage/compaction/ob_tablet_merge_ctx.h +++ b/src/storage/compaction/ob_tablet_merge_ctx.h @@ -196,7 +196,7 @@ struct ObTabletMergeCtx int64_t get_compaction_scn() const { return is_multi_version_minor_merge(param_.merge_type_) ? - scn_range_.end_scn_.get_val_for_inner_table_field() : sstable_version_range_.snapshot_version_; + scn_range_.end_scn_.get_val_for_tx() : sstable_version_range_.snapshot_version_; } // 1. init in dag diff --git a/src/storage/compaction/ob_tenant_freeze_info_mgr.cpp b/src/storage/compaction/ob_tenant_freeze_info_mgr.cpp index 3f502d10d..fec93d517 100644 --- a/src/storage/compaction/ob_tenant_freeze_info_mgr.cpp +++ b/src/storage/compaction/ob_tenant_freeze_info_mgr.cpp @@ -446,14 +446,14 @@ int ObTenantFreezeInfoMgr::get_min_reserved_snapshot( } } snapshot_version = std::max(0L, snapshot_gc_ts_ - duration * 1000L * 1000L *1000L); - snapshot_version = std::min(snapshot_version, static_cast(freeze_info.freeze_scn.get_val_for_tx())); + snapshot_version = std::min(snapshot_version, freeze_info.freeze_scn.get_val_for_tx()); for (int64_t i = 0; i < snapshots.count() && OB_SUCC(ret); ++i) { bool related = false; const ObSnapshotInfo &snapshot = snapshots.at(i); if (OB_FAIL(is_snapshot_related_to_tablet(tablet_id, snapshot, related))) { STORAGE_LOG(WARN, "fail to check snapshot relation", K(ret), K(tablet_id), K(snapshot)); } else if (related) { - snapshot_version = std::min(snapshot_version, (int64_t)snapshot.snapshot_scn_.get_val_for_tx()); + snapshot_version = std::min(snapshot_version, snapshot.snapshot_scn_.get_val_for_tx()); } } } diff --git a/src/storage/compaction/ob_tenant_freeze_info_mgr.h b/src/storage/compaction/ob_tenant_freeze_info_mgr.h index 3017761ea..a1d607997 100644 --- a/src/storage/compaction/ob_tenant_freeze_info_mgr.h +++ b/src/storage/compaction/ob_tenant_freeze_info_mgr.h @@ -52,10 +52,6 @@ public: int64_t cluster_version; FreezeInfo() : freeze_scn(), schema_version(-1), cluster_version(0) {} - FreezeInfo(const int64_t scn, const int64_t schema_ver, const int64_t cluster_ver) - : freeze_scn(), schema_version(schema_ver), cluster_version(cluster_ver) { - freeze_scn.convert_for_tx(scn); - } FreezeInfo(const palf::SCN &scn, const int64_t schema_ver, const int64_t cluster_ver) : freeze_scn(scn), schema_version(schema_ver), cluster_version(cluster_ver) {} FreezeInfo &operator =(const FreezeInfo &o) diff --git a/src/storage/compaction/ob_tenant_tablet_scheduler.cpp b/src/storage/compaction/ob_tenant_tablet_scheduler.cpp index 92f20aecd..a4ac19452 100644 --- a/src/storage/compaction/ob_tenant_tablet_scheduler.cpp +++ b/src/storage/compaction/ob_tenant_tablet_scheduler.cpp @@ -508,16 +508,16 @@ int64_t ObTenantTabletScheduler::get_frozen_version() const return frozen_version_; } -bool ObTenantTabletScheduler::check_tx_table_ready(ObLS &ls, const int64_t check_log_ts) +bool ObTenantTabletScheduler::check_tx_table_ready(ObLS &ls, const palf::SCN &check_scn) { int ret = OB_SUCCESS; bool tx_table_ready = false; palf::SCN max_decided_scn; if (OB_FAIL(ls.get_max_decided_scn(max_decided_scn))) { LOG_WARN("failed to get max decided log_ts", K(ret), "ls_id", ls.get_ls_id()); - } else if (check_log_ts <= max_decided_scn.get_val_for_lsn_allocator()) { + } else if (check_scn <= max_decided_scn) { tx_table_ready = true; - LOG_INFO("tx table ready", "sstable_end_log_ts", check_log_ts, K(max_decided_scn)); + LOG_INFO("tx table ready", "sstable_end_scn", check_scn, K(max_decided_scn)); } return tx_table_ready; diff --git a/src/storage/compaction/ob_tenant_tablet_scheduler.h b/src/storage/compaction/ob_tenant_tablet_scheduler.h index bac145548..958437b7d 100644 --- a/src/storage/compaction/ob_tenant_tablet_scheduler.h +++ b/src/storage/compaction/ob_tenant_tablet_scheduler.h @@ -120,7 +120,7 @@ public: const blocksstable::MacroBlockId ¯o_id, const int64_t prefix_len); int schedule_load_bloomfilter(const blocksstable::MacroBlockId ¯o_id); - static bool check_tx_table_ready(ObLS &ls, const int64_t check_log_ts); + static bool check_tx_table_ready(ObLS &ls, const palf::SCN &check_scn); static int check_ls_state(ObLS &ls, bool &need_merge); static int schedule_tablet_minor_merge(const share::ObLSID ls_id, ObTablet &tablet); static int schedule_tablet_major_merge( diff --git a/src/storage/compaction/ob_tx_table_merge_task.cpp b/src/storage/compaction/ob_tx_table_merge_task.cpp index abc1447aa..2b6ccc6a1 100644 --- a/src/storage/compaction/ob_tx_table_merge_task.cpp +++ b/src/storage/compaction/ob_tx_table_merge_task.cpp @@ -76,7 +76,7 @@ int ObTxTableMergeDag::init_by_param(const ObIDagInitParam *param) LOG_WARN("tx table guard is invalid", K(tmp_ret), KPC(merge_param), K(guard)); } else if (OB_TMP_FAIL(guard.get_tx_table()->get_recycle_scn(recycle_scn))) { LOG_WARN("failed to get recycle ts", K(tmp_ret), KPC(merge_param)); - } else if (OB_TMP_FAIL(compaction_filter_.init(recycle_scn.get_val_for_tx(), ObTxTable::get_filter_col_idx()))) { + } else if (OB_TMP_FAIL(compaction_filter_.init(recycle_scn, ObTxTable::get_filter_col_idx()))) { LOG_WARN("failed to get init compaction filter", K(tmp_ret), KPC(merge_param), K(recycle_scn)); } else { ctx_->compaction_filter_ = &compaction_filter_; diff --git a/src/storage/ddl/ob_ddl_merge_task.cpp b/src/storage/ddl/ob_ddl_merge_task.cpp index 503a9349c..8b2f573f7 100644 --- a/src/storage/ddl/ob_ddl_merge_task.cpp +++ b/src/storage/ddl/ob_ddl_merge_task.cpp @@ -288,7 +288,13 @@ int ObDDLTableMergeTask::init(const ObDDLTableMergeDagParam &ddl_dag_param) int ObDDLTableMergeTask::process() { int ret = OB_SUCCESS; - const int64_t MAX_DDL_SSTABLE = 128; + int64_t MAX_DDL_SSTABLE = 128; +#ifdef ERRSIM + if (0 != GCONF.errsim_max_ddl_sstable_count) { + MAX_DDL_SSTABLE = GCONF.errsim_max_ddl_sstable_count; + LOG_INFO("set max ddl sstable in errsim mode", K(MAX_DDL_SSTABLE)); + } +#endif LOG_INFO("ddl merge task start process", K(*this)); ObTabletHandle tablet_handle; ObDDLKvMgrHandle ddl_kv_mgr_handle; @@ -312,6 +318,14 @@ int ObDDLTableMergeTask::process() LOG_WARN("get ddl sstable handles failed", K(ret)); } else if (ddl_sstable_handles.get_count() >= MAX_DDL_SSTABLE || merge_param_.is_commit_) { DEBUG_SYNC(BEFORE_DDL_TABLE_MERGE_TASK); +#ifdef ERRSIM + static int64_t counter = 0; + counter++; + if (counter >= 2) { + DEBUG_SYNC(BEFORE_MIG_DDL_TABLE_MERGE_TASK); + } +#endif + ObTabletDDLParam ddl_param; ObTableHandleV2 table_handle; bool is_data_complete = false; diff --git a/src/storage/ddl/ob_ddl_redo_log_writer.cpp b/src/storage/ddl/ob_ddl_redo_log_writer.cpp index f8180f743..bf05f7e41 100644 --- a/src/storage/ddl/ob_ddl_redo_log_writer.cpp +++ b/src/storage/ddl/ob_ddl_redo_log_writer.cpp @@ -25,6 +25,7 @@ #include "storage/tx/ob_ts_mgr.h" #include "storage/ddl/ob_tablet_ddl_kv_mgr.h" #include "storage/blocksstable/ob_logic_macro_id.h" +#include "observer/ob_server_event_history_table_operator.h" using namespace oceanbase::common; using namespace oceanbase::storage; @@ -1020,6 +1021,14 @@ int ObDDLSSTableRedoWriter::write_prepare_log(const ObITable::TableKey &table_ke { int ret = OB_SUCCESS; +#ifdef ERRSIM + SERVER_EVENT_SYNC_ADD("storage_ddl", "before_write_prepare_log", + "table_key", table_key, + "table_id", table_id, + "execution_id", execution_id, + "ddl_task_id", ddl_task_id); + DEBUG_SYNC(BEFORE_DDL_WRITE_PREPARE_LOG); +#endif prepare_scn.reset(); ObLS *ls = nullptr; ObDDLPrepareLog log; diff --git a/src/storage/ddl/ob_ddl_struct.cpp b/src/storage/ddl/ob_ddl_struct.cpp index 9c5296012..6b7d1535c 100644 --- a/src/storage/ddl/ob_ddl_struct.cpp +++ b/src/storage/ddl/ob_ddl_struct.cpp @@ -212,6 +212,12 @@ int ObDDLKV::set_macro_block(const ObDDLMacroBlock ¯o_block) int ret = OB_SUCCESS; const int64_t MAX_DDL_BLOCK_COUNT = 10L * 1024L * 1024L * 1024L / OB_SERVER_BLOCK_MGR.get_macro_block_size(); int64_t freeze_block_count = MAX_DDL_BLOCK_COUNT; +#ifdef ERRSIM + if (0 != GCONF.errsim_max_ddl_block_count) { + freeze_block_count = GCONF.errsim_max_ddl_block_count; + LOG_INFO("ddl set macro block count", K(freeze_block_count)); + } +#endif if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("ddl kv is not init", K(ret)); diff --git a/src/storage/high_availability/ob_ls_migration.cpp b/src/storage/high_availability/ob_ls_migration.cpp index 30cf24ff0..c986953df 100644 --- a/src/storage/high_availability/ob_ls_migration.cpp +++ b/src/storage/high_availability/ob_ls_migration.cpp @@ -1084,8 +1084,8 @@ int ObStartMigrationTask::choose_src_() palf::SCN local_clog_checkpoint_scn = palf::SCN::min_scn(); if (OB_FAIL(get_local_ls_checkpoint_scn_(local_clog_checkpoint_scn))) { LOG_WARN("failed to get local ls checkpoint ts", K(ret)); - } else if (OB_FAIL(src_provider.init(tenant_id, storage_rpc_))) { - LOG_WARN("failed to init src provider", K(ret), K(tenant_id)); + } else if (OB_FAIL(src_provider.init(tenant_id, ctx_->arg_.type_, storage_rpc_))) { + LOG_WARN("failed to init src provider", K(ret), K(tenant_id), "type", ctx_->arg_.type_); } else if (OB_FAIL(src_provider.choose_ob_src(ls_id, local_clog_checkpoint_scn, src_info))) { LOG_WARN("failed to choose ob src", K(ret), K(tenant_id), K(ls_id), K(local_clog_checkpoint_scn)); } else if (OB_FAIL(fetch_ls_info_(tenant_id, ls_id, src_info.src_addr_, ls_info))) { diff --git a/src/storage/high_availability/ob_storage_ha_src_provider.cpp b/src/storage/high_availability/ob_storage_ha_src_provider.cpp index a22d01868..7585f7a45 100644 --- a/src/storage/high_availability/ob_storage_ha_src_provider.cpp +++ b/src/storage/high_availability/ob_storage_ha_src_provider.cpp @@ -18,13 +18,18 @@ namespace oceanbase { namespace storage { -ObStorageHASrcProvider::ObStorageHASrcProvider() : is_inited_(false), tenant_id_(OB_INVALID_ID), storage_rpc_(nullptr) +ObStorageHASrcProvider::ObStorageHASrcProvider() + : is_inited_(false), + tenant_id_(OB_INVALID_ID), + type_(ObMigrationOpType::MAX_LS_OP), + storage_rpc_(nullptr) {} ObStorageHASrcProvider::~ObStorageHASrcProvider() {} -int ObStorageHASrcProvider::init(const uint64_t tenant_id, storage::ObStorageRpc *storage_rpc) +int ObStorageHASrcProvider::init(const uint64_t tenant_id, const ObMigrationOpType::TYPE &type, + storage::ObStorageRpc *storage_rpc) { int ret = OB_SUCCESS; if (is_inited_) { @@ -35,6 +40,7 @@ int ObStorageHASrcProvider::init(const uint64_t tenant_id, storage::ObStorageRpc LOG_WARN("get invalid argument", K(ret), K(tenant_id), K(ls_id), KP(storage_rpc)); } else { tenant_id_ = tenant_id; + type_ = type; storage_rpc_ = storage_rpc; is_inited_ = true; } @@ -62,8 +68,27 @@ int ObStorageHASrcProvider::choose_ob_src(const share::ObLSID &ls_id, const palf } else { src_info.src_addr_ = chosen_src_addr; src_info.cluster_id_ = GCONF.cluster_id; - SERVER_EVENT_ADD( - "storage_ha", "choose_src", "tenant_id", tenant_id_, "ls_id", ls_id.id(), "src_addr", chosen_src_addr); +#ifdef ERRSIM + if (ObMigrationOpType::ADD_LS_OP == type_ || ObMigrationOpType::MIGRATE_LS_OP == type_) { + const ObString &errsim_server = GCONF.errsim_migration_src_server_addr.str(); + if (!errsim_server.empty()) { + common::ObAddr tmp_errsim_addr; + if (OB_FAIL(tmp_errsim_addr.parse_from_string(errsim_server))) { + LOG_WARN("failed to parse from string", K(ret), K(errsim_server)); + } else { + src_info.src_addr_ = tmp_errsim_addr; + src_info.cluster_id_ = GCONF.cluster_id; + LOG_INFO("storage ha choose errsim src", K(tmp_errsim_addr)); + } + } + } +#endif + SERVER_EVENT_ADD("storage_ha", "choose_src", + "tenant_id", tenant_id_, + "ls_id", ls_id.id(), + "src_addr", src_info.src_addr_, + "op_type", ObMigrationOpType::get_str(type_)); + } return ret; } diff --git a/src/storage/high_availability/ob_storage_ha_src_provider.h b/src/storage/high_availability/ob_storage_ha_src_provider.h index 2065595bd..196ddc7a3 100644 --- a/src/storage/high_availability/ob_storage_ha_src_provider.h +++ b/src/storage/high_availability/ob_storage_ha_src_provider.h @@ -24,7 +24,7 @@ class ObStorageHASrcProvider { public: ObStorageHASrcProvider(); virtual ~ObStorageHASrcProvider(); - int init(const uint64_t tenant_id, storage::ObStorageRpc *storage_rpc); + int init(const uint64_t tenant_id, const ObMigrationOpType::TYPE &type, storage::ObStorageRpc *storage_rpc); int choose_ob_src(const share::ObLSID &ls_id, const palf::SCN &local_clog_checkpoint_scn, ObStorageHASrcInfo &src_info); @@ -41,6 +41,7 @@ private: private: bool is_inited_; uint64_t tenant_id_; + ObMigrationOpType::TYPE type_; storage::ObStorageRpc *storage_rpc_; DISALLOW_COPY_AND_ASSIGN(ObStorageHASrcProvider); }; diff --git a/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp b/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp index be2d3a805..08d92debf 100644 --- a/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp +++ b/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp @@ -534,6 +534,15 @@ int ObStorageHATabletsBuilder::build_copy_tablet_sstable_info_arg_( ObTablet *tablet = nullptr; arg.reset(); +#ifdef ERRSIM + const int64_t errsim_tablet_id = GCONF.errsim_migration_tablet_id; + if (errsim_tablet_id == tablet_id.id()) { + SERVER_EVENT_SYNC_ADD("storage_ha", "before_copy_ddl_sstable", + "tablet_id", tablet_id); + DEBUG_SYNC(BEFORE_COPY_DDL_SSTABLE); + } +#endif + if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("storage ha tablets builder do not init", K(ret)); @@ -704,6 +713,14 @@ int ObStorageHATabletsBuilder::get_need_copy_ddl_sstable_range_( need_copy_scn_range.start_scn_ = ddl_start_scn; need_copy_scn_range.end_scn_ = ddl_checkpoint_scn; } +#ifdef ERRSIM + LOG_INFO("ddl checkpoint pushed", K(ddl_checkpoint_pushed), K(ddl_sstable_array), K(ddl_start_log_ts), K(ddl_checkpoint_ts)); + SERVER_EVENT_SYNC_ADD("storage_ha", "get_need_copy_ddl_sstable_range", + "tablet_id", tablet->get_tablet_meta().tablet_id_, + "dest_ddl_checkpoint_pushed", ddl_checkpoint_pushed, + "start_log_ts", need_copy_log_ts_range.start_log_ts_, + "end_log_ts", need_copy_log_ts_range.end_log_ts_); +#endif } else { ret = OB_ERR_UNEXPECTED; LOG_WARN("checkpoint ts should be greater than start ts", @@ -724,7 +741,10 @@ int ObStorageHATabletsBuilder::get_ddl_sstable_max_start_scn_( if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("storage ha tables builder do not init", K(ret)); - } else if (ddl_sstable_array.count() > 0 && OB_FAIL(ddl_sstable_array.get_all_tables(sstables))) { + } else if (ddl_sstable_array.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("ddl sstable should not be empty", K(ret)); + } else if (OB_FAIL(ddl_sstable_array.get_all_tables(sstables))) { LOG_WARN("failed to get all tables", K(ret), K(param_)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < sstables.count(); ++i) { diff --git a/src/storage/lob/ob_lob_data_reader.cpp b/src/storage/lob/ob_lob_data_reader.cpp index c64391c01..91043cdb2 100644 --- a/src/storage/lob/ob_lob_data_reader.cpp +++ b/src/storage/lob/ob_lob_data_reader.cpp @@ -23,7 +23,8 @@ namespace storage { ObLobDataReader::ObLobDataReader() - : is_inited_(false), tablet_id_(), access_ctx_(nullptr), allocator_(ObModIds::OB_LOB_READER) + : is_inited_(false), tablet_id_(), access_ctx_(nullptr), + allocator_(ObModIds::OB_LOB_READER, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()) { } diff --git a/src/storage/ls/ob_freezer.cpp b/src/storage/ls/ob_freezer.cpp index 1d5c613b8..cd8fdb3e6 100644 --- a/src/storage/ls/ob_freezer.cpp +++ b/src/storage/ls/ob_freezer.cpp @@ -867,7 +867,7 @@ int ObFreezer::get_newest_snapshot_version(const ObTabletID &tablet_id, } else if (OB_FAIL(ls_tablet_svr_->get_tablet(tablet_id, handle, ObTabletCommon::NO_CHECK_GET_TABLET_TIMEOUT_US))) { TRANS_LOG(WARN, "[Freezer] fail to get tablet", K(ret), K_(ls_id), K(tablet_id)); - } else if (OB_FAIL(snapshot_version.convert_for_lsn_allocator(handle.get_obj()->get_snapshot_version()))) { + } else if (OB_FAIL(snapshot_version.convert_for_tx(handle.get_obj()->get_snapshot_version()))) { TRANS_LOG(WARN, "[Freezer] fail to convert from ts", K(ret), K_(ls_id), K(tablet_id)); } else { TRANS_LOG(TRACE, "[Freezer] get_snapshot_version", K(ret), K_(ls_id), K(tablet_id), K(snapshot_version)); diff --git a/src/storage/ls/ob_ls.cpp b/src/storage/ls/ob_ls.cpp index 5b9b64a65..f220637e0 100644 --- a/src/storage/ls/ob_ls.cpp +++ b/src/storage/ls/ob_ls.cpp @@ -329,7 +329,7 @@ int ObLS::remove_ls() if (OB_TMP_FAIL(log_handler_.unregister_rebuild_cb())) { LOG_WARN("unregister rebuild cb failed", K(ret), K(ls_meta_)); } - if (OB_FAIL(logservice->remove_ls(ls_meta_.ls_id_))) { + if (OB_FAIL(logservice->remove_ls(ls_meta_.ls_id_, log_handler_, restore_handler_))) { LOG_ERROR("remove log stream from logservice failed", K(ret), K(ls_meta_.ls_id_)); } } diff --git a/src/storage/ls/ob_ls_sync_tablet_seq_handler.cpp b/src/storage/ls/ob_ls_sync_tablet_seq_handler.cpp index 3ef6da8cf..e1885e206 100644 --- a/src/storage/ls/ob_ls_sync_tablet_seq_handler.cpp +++ b/src/storage/ls/ob_ls_sync_tablet_seq_handler.cpp @@ -17,10 +17,12 @@ #include "logservice/ob_log_base_header.h" #include "logservice/palf/scn.h" #include "lib/oblog/ob_log_module.h" +#include "share/ob_tablet_autoincrement_service.h" namespace oceanbase { using namespace palf; +using namespace share; namespace storage { @@ -56,7 +58,7 @@ int ObLSSyncTabletSeqHandler::replay(const void *buffer, ObSyncTabletSeqLog log; int64_t tmp_pos = 0; const char *log_buf = static_cast(buffer); - ObTabletHandle tablet_handle; + ObTabletAutoincSeqRpcHandler &autoinc_seq_handler = ObTabletAutoincSeqRpcHandler::get_instance(); if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("ObLSSyncTabletSeqHandler not inited", K(ret)); @@ -64,17 +66,10 @@ int ObLSSyncTabletSeqHandler::replay(const void *buffer, LOG_WARN("log base header deserialize error", K(ret)); } else if (OB_FAIL(log.deserialize(log_buf, nbytes, tmp_pos))) { LOG_WARN("ObSyncTabletSeqLog deserialize error", K(ret)); - } else if (OB_FAIL(ls_->replay_get_tablet(log.get_tablet_id(), scn, tablet_handle))) { - if (OB_TABLET_NOT_EXIST == ret) { - LOG_INFO("tablet may be deleted, skip this log", K(ret), "tablet_id", log.get_tablet_id(), K(scn)); - ret = OB_SUCCESS; - } else if (OB_EAGAIN == ret) { - // retry replay again - } else { - LOG_WARN("fail to replay get tablet, retry again", K(ret), K(log), K(scn)); - ret = OB_EAGAIN; - } - } else if (OB_FAIL(tablet_handle.get_obj()->update_tablet_autoinc_seq(log.get_autoinc_seq(), scn))) { + } else if (OB_FAIL(autoinc_seq_handler.replay_update_tablet_autoinc_seq(ls_, + log.get_tablet_id(), + log.get_autoinc_seq(), + scn))) { LOG_WARN("failed to update tablet auto inc seq", K(ret), K(log)); } return ret; diff --git a/src/storage/ls/ob_ls_tablet_service.cpp b/src/storage/ls/ob_ls_tablet_service.cpp index 2d3c5a27e..828b6e63a 100644 --- a/src/storage/ls/ob_ls_tablet_service.cpp +++ b/src/storage/ls/ob_ls_tablet_service.cpp @@ -3266,6 +3266,39 @@ int ObLSTabletService::insert_lob_col( return ret; } +int ObLSTabletService::check_lob_tablet_valid(ObTabletHandle &data_tablet) +{ + int ret = OB_SUCCESS; + bool is_valid_aux_lob_table = false; + ObTabletBindingInfo ddl_data; + if (OB_FAIL(data_tablet.get_obj()->get_ddl_data(ddl_data))) { + LOG_WARN("failed to get ddl data from tablet", K(ret), K(data_tablet)); + } else { + is_valid_aux_lob_table = ddl_data.lob_meta_tablet_id_.is_valid() && ddl_data.lob_piece_tablet_id_.is_valid(); + if (!is_valid_aux_lob_table) { + ObTabletTxMultiSourceDataUnit tx_data; + if (OB_FAIL(data_tablet.get_obj()->get_tx_data(tx_data))) { + LOG_WARN("fail to get tx data", K(ret), K(data_tablet)); + } else if (tx_data.is_in_tx()) { + ret = OB_SCHEMA_EAGAIN; + LOG_WARN("do retry for data tablet is in tx, maybe wait for lob tablet finish", K(ret), K(ddl_data)); + } else { + // maybe has committed, refresh binding info and check once + if (OB_FAIL(data_tablet.get_obj()->get_ddl_data(ddl_data))) { + LOG_WARN("failed to get ddl data from tablet", K(ret), K(data_tablet)); + } else { + is_valid_aux_lob_table = ddl_data.lob_meta_tablet_id_.is_valid() && ddl_data.lob_piece_tablet_id_.is_valid(); + if (!is_valid_aux_lob_table) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("aux lob table must valid when lob column exist", K(ret), K(ddl_data)); + } + } + } + } + } + return ret; +} + int ObLSTabletService::insert_lob_tablet_row( ObTabletHandle &data_tablet, ObDMLRunningCtx &run_ctx, @@ -3274,7 +3307,7 @@ int ObLSTabletService::insert_lob_tablet_row( int ret = OB_SUCCESS; int64_t col_cnt = run_ctx.col_descs_->count(); ObLobManager *lob_mngr = MTL(ObLobManager*); - bool check_ddl_data = false; + bool check_lob = false; if (OB_ISNULL(lob_mngr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("[STORAGE_LOB]failed to get lob manager handle.", K(ret)); @@ -3282,27 +3315,20 @@ int ObLSTabletService::insert_lob_tablet_row( ret = OB_ERR_UNEXPECTED; LOG_WARN("[STORAGE_LOB]column count invalid", K(ret), K(col_cnt), K(row.row_val_.count_)); } else { - bool is_valid_aux_lob_table = false; for (int64_t i = 0; OB_SUCC(ret) && i < col_cnt; ++i) { const ObColDesc &column = run_ctx.col_descs_->at(i); - if (column.col_type_.is_lob_v2()) { - if (!check_ddl_data) { - ObTabletBindingInfo ddl_data; - if (OB_FAIL(data_tablet.get_obj()->get_ddl_data(ddl_data))) { - LOG_WARN("failed to get ddl data from tablet", K(ret), K(data_tablet)); + ObObj &obj = row.row_val_.get_cell(i); + if (!column.col_type_.is_lob_v2() || obj.is_null() || obj.is_nop_value()) { + // do nothing + } else { + if (!check_lob) { + if (OB_FAIL(check_lob_tablet_valid(data_tablet))) { + LOG_WARN("failed to check_lob_tablet_valid", K(ret), K(data_tablet)); } else { - check_ddl_data = true; - const common::ObTabletID &data_tablet_id = data_tablet.get_obj()->tablet_meta_.tablet_id_; - const common::ObTabletID &lob_meta_tablet_id = ddl_data.lob_meta_tablet_id_; - const common::ObTabletID &lob_piece_tablet_id = ddl_data.lob_piece_tablet_id_; - is_valid_aux_lob_table = lob_meta_tablet_id.is_valid() && lob_piece_tablet_id.is_valid(); + check_lob = true; } } - ObObj &obj = row.row_val_.get_cell(i); if (OB_FAIL(ret)) { - } else if (!is_valid_aux_lob_table) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("aux lob table must valid when lob column exist", K(ret), K(data_tablet)); } else if(OB_FAIL(insert_lob_col(run_ctx, column, obj, nullptr, nullptr))) { LOG_WARN("[STORAGE_LOB]failed to insert lob col.", K(ret), K(row), K(i)); } @@ -3541,8 +3567,7 @@ int ObLSTabletService::process_lob_row( ObStoreRow &new_row) { int ret = OB_SUCCESS; - bool is_valid_aux_lob_table = false; - bool check_ddl_data = false; + bool check_lob = false; if (OB_UNLIKELY(old_row.row_val_.get_count() != new_row.row_val_.get_count())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("[STORAGE_LOB]invalid args", K(old_row), K(new_row), K(ret)); @@ -3565,20 +3590,12 @@ int ObLSTabletService::process_lob_row( ObLobCommon *lob_common = nullptr; ObLobAccessParam lob_param; lob_param.update_len_ = new_obj.get_string_len(); - if (!check_ddl_data) { + if (!check_lob) { ObTabletBindingInfo ddl_data; - if (OB_FAIL(tablet_handle.get_obj()->get_ddl_data(ddl_data))) { - LOG_WARN("failed to get ddl data from tablet", K(ret), K(tablet_handle)); + if (OB_FAIL(check_lob_tablet_valid(tablet_handle))) { + LOG_WARN("failed to check_lob_tablet_valid", K(ret), K(tablet_handle)); } else { - check_ddl_data = true; - const common::ObTabletID &data_tablet_id = tablet_handle.get_obj()->tablet_meta_.tablet_id_; - const common::ObTabletID &lob_meta_tablet_id = ddl_data.lob_meta_tablet_id_; - const common::ObTabletID &lob_piece_tablet_id = ddl_data.lob_piece_tablet_id_; - is_valid_aux_lob_table = lob_meta_tablet_id.is_valid() && lob_piece_tablet_id.is_valid(); - if (!is_valid_aux_lob_table) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("aux lob table must valid when lob column exist", K(ret), K(lob_meta_tablet_id), K(lob_piece_tablet_id)); - } + check_lob = true; } } if (OB_FAIL(ret)) { @@ -4708,9 +4725,12 @@ void ObLSTabletService::dump_diag_info_for_old_row_loss( } #ifdef ENABLE_DEBUG_LOG // print single row check info - if (NULL != store_ctx.mvcc_acc_ctx_.get_mem_ctx() - && NULL != store_ctx.mvcc_acc_ctx_.get_mem_ctx()->get_defensive_check_mgr()) { - (void)store_ctx.mvcc_acc_ctx_.get_mem_ctx()->get_defensive_check_mgr()->dump(access_param.iter_param_.tablet_id_); + if (store_ctx.mvcc_acc_ctx_.tx_id_.is_valid()) { + transaction::ObTransService *trx = MTL(transaction::ObTransService *); + if (OB_NOT_NULL(trx) + && NULL != trx->get_defensive_check_mgr()) { + (void)trx->get_defensive_check_mgr()->dump(store_ctx.mvcc_acc_ctx_.tx_id_); + } } #endif } diff --git a/src/storage/ls/ob_ls_tablet_service.h b/src/storage/ls/ob_ls_tablet_service.h index 99533a9f4..d747130ac 100644 --- a/src/storage/ls/ob_ls_tablet_service.h +++ b/src/storage/ls/ob_ls_tablet_service.h @@ -556,6 +556,8 @@ private: ObObj &obj, ObLobAccessParam *del_param, ObLobCommon *lob_common); + static int check_lob_tablet_valid( + ObTabletHandle &data_tablet); static int insert_lob_tablet_row( ObTabletHandle &data_tablet, ObDMLRunningCtx &run_ctx, diff --git a/src/storage/memtable/mvcc/ob_multi_version_iterator.cpp b/src/storage/memtable/mvcc/ob_multi_version_iterator.cpp index 0a5e2a534..89effa010 100644 --- a/src/storage/memtable/mvcc/ob_multi_version_iterator.cpp +++ b/src/storage/memtable/mvcc/ob_multi_version_iterator.cpp @@ -92,9 +92,10 @@ int ObMultiVersionValueIterator::init_multi_version_iter() max_committed_trans_version_ = (NULL != version_iter_) ? version_iter_->trans_version_.get_val_for_tx() : -1; // NB: It will assign -1 to cur_trans_version_, while it will not // cause any wrong logic, but take care of it - cur_trans_version_.convert_for_tx(max_committed_trans_version_); multi_version_iter_ = iter; - if (max_committed_trans_version_ <= version_range_.multi_version_start_) { + if (OB_FAIL(cur_trans_version_.convert_for_tx(max_committed_trans_version_))) { + TRANS_LOG(ERROR, "failed to convert scn", K(ret), K_(max_committed_trans_version)); + } else if (max_committed_trans_version_ <= version_range_.multi_version_start_) { //如果多版本的开始版本大于等于当前以提交的最大版本 //则只迭代出所有trans node compact结果 } else { diff --git a/src/storage/memtable/mvcc/ob_mvcc_row.cpp b/src/storage/memtable/mvcc/ob_mvcc_row.cpp index 21d27cbc9..c29fd8516 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_row.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_row.cpp @@ -26,6 +26,7 @@ #include "storage/tx/ob_trans_part_ctx.h" #include "storage/memtable/ob_memtable_context.h" #include "storage/tx/ob_trans_ctx.h" +#include "storage/tx/ob_trans_event.h" #include "storage/memtable/mvcc/ob_mvcc_trans_ctx.h" #include "storage/blocksstable/ob_datum_row.h" @@ -992,6 +993,14 @@ int ObMvccRow::mvcc_write_(ObIMemtableCtx &ctx, res.tx_node_ = &writer_node; total_trans_node_cnt_++; } + if (ctx.is_can_elr() + && NULL != writer_node.prev_ + && writer_node.prev_->is_elr()) { + ObMemtableCtx &mt_ctx = static_cast(ctx); + if (NULL != mt_ctx.get_trans_ctx()) { + TX_STAT_READ_ELR_ROW_COUNT_INC(mt_ctx.get_trans_ctx()->get_tenant_id()); + } + } } } diff --git a/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.h b/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.h index 980c25f5b..eedb8406b 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.h +++ b/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.h @@ -57,7 +57,7 @@ struct RedoDataNode uint32_t modify_count_; uint32_t acc_checksum_; int64_t version_; - int32_t flag_; + int32_t flag_; // currently, unused int64_t seq_no_; ObITransCallback *callback_; common::ObTabletID tablet_id_; diff --git a/src/storage/memtable/ob_memtable_context.cpp b/src/storage/memtable/ob_memtable_context.cpp index 1852e559a..245ff992f 100644 --- a/src/storage/memtable/ob_memtable_context.cpp +++ b/src/storage/memtable/ob_memtable_context.cpp @@ -28,7 +28,6 @@ #include "storage/tablelock/ob_table_lock_callback.h" #include "storage/tablelock/ob_table_lock_common.h" #include "storage/tx/ob_trans_deadlock_adapter.h" -#include "storage/tx/ob_defensive_check_mgr.h" namespace oceanbase { @@ -64,9 +63,6 @@ ObMemtableCtx::ObMemtableCtx() is_master_(true), read_elr_data_(false), lock_mem_ctx_(ctx_cb_allocator_), -#ifdef ENABLE_DEBUG_LOG - defensive_check_mgr_(NULL), -#endif is_inited_(false) { } @@ -93,17 +89,6 @@ int ObMemtableCtx::init(const uint64_t tenant_id) TRANS_LOG(ERROR, "ctx_allocator_ init error", K(ret)); } else if (OB_FAIL(reset_log_generator_())) { TRANS_LOG(ERROR, "fail to reset log generator", K(ret)); -#ifdef ENABLE_DEBUG_LOG - } else if (!GCONF.enable_defensive_check()) { - // do nothing - } else if (NULL == (defensive_check_mgr_ = op_alloc(ObDefensiveCheckMgr))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - TRANS_LOG(ERROR, "memory alloc failed", K(ret), KP(defensive_check_mgr_)); - } else if (OB_FAIL(defensive_check_mgr_->init(lib::ObMemAttr(tenant_id, "MemtableCtx")))) { - TRANS_LOG(ERROR, "defensive check mgr init failed", K(ret), KP(defensive_check_mgr_)); - op_free(defensive_check_mgr_); - defensive_check_mgr_ = NULL; -#endif } else { // do nothing } @@ -173,12 +158,6 @@ void ObMemtableCtx::reset() tx_table_guard_.reset(); //FIXME: ObIMemtableCtx don't have resetfunction, //thus ObIMvccCtx::reset is called, so resource_link_is not reset -#ifdef ENABLE_DEBUG_LOG - if (NULL != defensive_check_mgr_) { - op_free(defensive_check_mgr_); - defensive_check_mgr_ = NULL; - } -#endif ObIMemtableCtx::reset(); } } diff --git a/src/storage/memtable/ob_memtable_context.h b/src/storage/memtable/ob_memtable_context.h index b9eb3ee2a..f1e9be3d6 100644 --- a/src/storage/memtable/ob_memtable_context.h +++ b/src/storage/memtable/ob_memtable_context.h @@ -485,9 +485,6 @@ public: // for deadlock detect. void set_table_lock_killed() { lock_mem_ctx_.set_killed(); } bool is_table_lock_killed() const { return lock_mem_ctx_.is_killed(); } -#ifdef ENABLE_DEBUG_LOG - transaction::ObDefensiveCheckMgr *get_defensive_check_mgr() { return defensive_check_mgr_; } -#endif private: int do_trans_end( const bool commit, @@ -560,9 +557,6 @@ private: common::ObArray conflict_trans_ids_; // table lock mem ctx. transaction::tablelock::ObLockMemCtx lock_mem_ctx_; -#ifdef ENABLE_DEBUG_LOG - transaction::ObDefensiveCheckMgr *defensive_check_mgr_; -#endif bool is_inited_; }; diff --git a/src/storage/memtable/ob_memtable_interface.h b/src/storage/memtable/ob_memtable_interface.h index ce3494641..9313924a8 100644 --- a/src/storage/memtable/ob_memtable_interface.h +++ b/src/storage/memtable/ob_memtable_interface.h @@ -210,9 +210,8 @@ public: virtual int on_memtable_flushed() { return common::OB_SUCCESS; } virtual bool can_be_minor_merged() { return false; } void set_snapshot_version(const palf::SCN snapshot_version) { snapshot_version_ = snapshot_version; } - // TODO: remove get_snapshot_version() and use get_snapshot_version_scn() instead of it virtual int64_t get_snapshot_version() const override { return snapshot_version_.get_val_for_tx(); } - virtual palf::SCN get_snapshot_version_scn() const override { return snapshot_version_; } + virtual palf::SCN get_snapshot_version_scn() const { return snapshot_version_; } virtual int64_t get_upper_trans_version() const override { return OB_NOT_SUPPORTED; } virtual int64_t get_max_merged_trans_version() const override diff --git a/src/storage/memtable/ob_memtable_mutator.cpp b/src/storage/memtable/ob_memtable_mutator.cpp index 87154ed59..fb55657ca 100644 --- a/src/storage/memtable/ob_memtable_mutator.cpp +++ b/src/storage/memtable/ob_memtable_mutator.cpp @@ -39,7 +39,7 @@ ObMemtableMutatorMeta::ObMemtableMutatorMeta(): data_crc_(0), data_size_(0), row_count_(0), - have_rollback_to_savepoint_(1) + unused_(0) { } @@ -159,8 +159,8 @@ int64_t ObMemtableMutatorMeta::to_string(char *buffer, const int64_t length) con { int64_t pos = 0; common::databuff_printf(buffer, length, pos, - "%p data_crc=%x meta_size=%d data_size=%d row_count=%d have_rollback_to_savepoint=%d", - this, data_crc_, meta_size_, data_size_, row_count_, have_rollback_to_savepoint_); + "%p data_crc=%x meta_size=%d data_size=%d row_count=%d", + this, data_crc_, meta_size_, data_size_, row_count_); return pos; } @@ -169,23 +169,6 @@ bool ObMemtableMutatorMeta::is_row_start() const return ObTransRowFlag::is_row_start(flags_); } -int ObMemtableMutatorMeta::set_savepoint(const uint32_t flag) { - int ret = OB_SUCCESS; - - if (flag != 0 && flag != 1) { - TRANS_LOG(WARN, "invalid argument", K(flag)); - ret = OB_INVALID_ARGUMENT; - } else { - have_rollback_to_savepoint_ = flag; - } - - return ret; -} - -uint32_t ObMemtableMutatorMeta::get_savepoint() const { - return have_rollback_to_savepoint_; -} - //only meta_crc is newly generated here, other information will keep unchanged void ObMemtableMutatorMeta::generate_new_header() { @@ -863,7 +846,6 @@ int ObMutatorWriter::append_row_kv( const ObMemtableKey *mtk = &redo.key_; uint64_t cluster_version = 0; bool is_with_head = true; - uint32 have_rollback_to_savepoint = redo.flag_; if (OB_ISNULL(redo.callback_)) { is_with_head = false; } else if (OB_FAIL(redo.callback_->get_cluster_version(cluster_version))) { @@ -918,9 +900,6 @@ int ObMutatorWriter::append_row_kv( } } else if (OB_FAIL(meta_.inc_row_count())) { TRANS_LOG(WARN, "meta inc_row_count failed", K(ret)); - } else if (have_rollback_to_savepoint == 1 && - OB_FAIL(meta_.set_savepoint(have_rollback_to_savepoint))) { - TRANS_LOG(WARN, "set savepoint flag failed", K(ret)); } else { buf_.get_position() = tmp_pos; } @@ -938,7 +917,6 @@ int ObMutatorWriter::append_row( const bool is_with_head) { int ret = OB_SUCCESS; - uint32 have_rollback_to_savepoint = row.get_savepoint_flag(); ObMutatorRowHeader row_header; row_header.mutator_type_ = MutatorType::MUTATOR_ROW; //TODO replace pkey with tablet_id for clog_encrypt_info @@ -964,9 +942,6 @@ int ObMutatorWriter::append_row( } } else if (OB_FAIL(meta_.inc_row_count())) { TRANS_LOG(WARN, "meta inc_row_count failed", K(ret)); - } else if (have_rollback_to_savepoint == 1 && - OB_FAIL(meta_.set_savepoint(have_rollback_to_savepoint))) { - TRANS_LOG(WARN, "set savepoint flag failed", K(ret)); } else { buf_.get_position() = tmp_pos; } diff --git a/src/storage/memtable/ob_memtable_mutator.h b/src/storage/memtable/ob_memtable_mutator.h index aca11815c..a86abac22 100644 --- a/src/storage/memtable/ob_memtable_mutator.h +++ b/src/storage/memtable/ob_memtable_mutator.h @@ -66,8 +66,6 @@ public: int64_t get_meta_size() const { return meta_size_; } int64_t get_data_size() const { return data_size_; } bool is_row_start() const; - int set_savepoint(const uint32_t is_savepoint); - uint32_t get_savepoint() const; public: int64_t get_serialize_size() const; @@ -87,7 +85,7 @@ private: uint32_t data_crc_; uint32_t data_size_; uint32_t row_count_; - uint32_t have_rollback_to_savepoint_; + uint32_t unused_; DISALLOW_COPY_AND_ASSIGN(ObMemtableMutatorMeta); }; @@ -190,7 +188,7 @@ public: share::ObEncryptMeta &final_encrypt_meta, share::ObCLogEncryptStatMap &encrypt_stat_map, const bool is_big_row = false); - uint32_t get_savepoint_flag() const { return flag_; }; + uint32_t get_flag() const { return flag_; }; TO_STRING_KV(K_(row_size), K_(table_id), @@ -212,7 +210,7 @@ public: ObRowData old_row_; uint32_t acc_checksum_; int64_t version_; - int32_t flag_; // savepoint flag + int32_t flag_; // currently, unused uint8_t rowid_version_; }; diff --git a/src/storage/memtable/ob_redo_log_generator.cpp b/src/storage/memtable/ob_redo_log_generator.cpp index d5fcfe2c1..08baee135 100644 --- a/src/storage/memtable/ob_redo_log_generator.cpp +++ b/src/storage/memtable/ob_redo_log_generator.cpp @@ -74,7 +74,6 @@ int ObRedoLogGenerator::fill_redo_log(char *buf, } else { helper.reset(); ObMutatorWriter mmw; - mmw.get_meta().set_savepoint(0); mmw.set_buffer(buf, buf_len - buf_pos); RedoDataNode redo; TableLockRedoDataNode table_lock_redo; diff --git a/src/storage/ob_i_memtable_mgr.cpp b/src/storage/ob_i_memtable_mgr.cpp index 4217343a5..c3d278cd8 100644 --- a/src/storage/ob_i_memtable_mgr.cpp +++ b/src/storage/ob_i_memtable_mgr.cpp @@ -109,7 +109,7 @@ int ObIMemtableMgr::get_newest_snapshot_version(palf::SCN &snapshot_version) return ret; } -int ObIMemtableMgr::release_memtables(const int64_t log_ts) +int ObIMemtableMgr::release_memtables(const palf::SCN &scn) { SpinWLockGuard lock_guard(lock_); int ret = OB_SUCCESS; @@ -117,9 +117,9 @@ int ObIMemtableMgr::release_memtables(const int64_t log_ts) if (IS_NOT_INIT) { ret = OB_NOT_INIT; STORAGE_LOG(WARN, "not inited", K(ret)); - } else if (OB_UNLIKELY(log_ts < 0)) { + } else if (OB_UNLIKELY(!scn.is_valid())) { ret = OB_INVALID_ARGUMENT; - STORAGE_LOG(WARN, "invalid log ts", K(ret), K(log_ts)); + STORAGE_LOG(WARN, "invalid log ts", K(ret), K(scn)); } else { for (int64_t i = memtable_head_; OB_SUCC(ret) && i < memtable_tail_; ++i) { // memtable that cannot be released will block memtables behind it @@ -132,13 +132,13 @@ int ObIMemtableMgr::release_memtables(const int64_t log_ts) && memtable->is_empty() && !memtable->get_is_force_freeze()) { break; - } else if (memtable->get_end_scn().get_val_for_tx() <= log_ts + } else if (memtable->get_end_scn() <= scn && memtable->can_be_minor_merged()) { if (OB_FAIL(release_head_memtable_(memtable))) { STORAGE_LOG(WARN, "fail to release memtable", K(ret), KPC(memtable)); break; } else { - STORAGE_LOG(INFO, "succeed to release memtable", K(ret), K(i), K(log_ts)); + STORAGE_LOG(INFO, "succeed to release memtable", K(ret), K(i), K(scn)); } } else { break; diff --git a/src/storage/ob_i_memtable_mgr.h b/src/storage/ob_i_memtable_mgr.h index c61f233e3..620c83b94 100644 --- a/src/storage/ob_i_memtable_mgr.h +++ b/src/storage/ob_i_memtable_mgr.h @@ -94,7 +94,7 @@ public: memtable::ObMemtable *&memtable, const memtable::MultiSourceDataUnitType type) const; - int release_memtables(const int64_t log_ts); + int release_memtables(const palf::SCN &scn); // force release all memtables // WARNING: this will release all the ref of memtable, make sure you will not use it again. int release_memtables(); diff --git a/src/storage/ob_i_table.h b/src/storage/ob_i_table.h index b13d48765..410fbada0 100644 --- a/src/storage/ob_i_table.h +++ b/src/storage/ob_i_table.h @@ -208,14 +208,6 @@ public: virtual OB_INLINE share::ObScnRange &get_scn_range() { return key_.scn_range_; } virtual OB_INLINE bool is_trans_state_deterministic() { return get_upper_trans_version() < INT64_MAX; } virtual int64_t get_snapshot_version() const { return key_.get_snapshot_version(); } - // TODO: remove it - virtual palf::SCN get_snapshot_version_scn() const - { - palf::SCN scn; - // TODO qianchen, ret_code for convert_for_tx. - scn.convert_for_tx(key_.get_snapshot_version()); - return scn; - } virtual int64_t get_upper_trans_version() const { return get_snapshot_version(); } virtual int64_t get_max_merged_trans_version() const { return get_snapshot_version(); } virtual int get_frozen_schema_version(int64_t &schema_version) const = 0; diff --git a/src/storage/tablelock/ob_lock_memtable_mgr.cpp b/src/storage/tablelock/ob_lock_memtable_mgr.cpp index 65847c8d4..2052d9562 100644 --- a/src/storage/tablelock/ob_lock_memtable_mgr.cpp +++ b/src/storage/tablelock/ob_lock_memtable_mgr.cpp @@ -90,8 +90,9 @@ int ObLockMemtableMgr::create_memtable(const palf::SCN clog_checkpoint_scn, table_key.table_type_ = ObITable::LOCK_MEMTABLE; table_key.tablet_id_ = LS_LOCK_TABLET; - table_key.scn_range_.start_scn_.convert_for_gts(1); // fake - table_key.scn_range_.end_scn_.convert_for_gts(2); // fake + + table_key.scn_range_.start_scn_ = palf::SCN::base_scn();//fake + table_key.scn_range_.end_scn_ = palf::SCN::plus(table_key.scn_range_.start_scn_, 1);//fake if (get_memtable_count_() > 0) { ret = OB_ERR_UNEXPECTED; diff --git a/src/storage/tablet/ob_tablet.cpp b/src/storage/tablet/ob_tablet.cpp index 914ca4c3c..ce242464a 100644 --- a/src/storage/tablet/ob_tablet.cpp +++ b/src/storage/tablet/ob_tablet.cpp @@ -1632,7 +1632,7 @@ int ObTablet::release_memtables(const palf::SCN scn) LOG_WARN("not inited", K(ret), K_(is_inited)); } else if (OB_FAIL(get_memtable_mgr(memtable_mgr))) { LOG_WARN("failed to get memtable mgr", K(ret)); - } else if (OB_FAIL(memtable_mgr->release_memtables(scn.get_val_for_tx()))) { + } else if (OB_FAIL(memtable_mgr->release_memtables(scn))) { LOG_WARN("failed to release memtables", K(ret), K(scn)); } @@ -2111,9 +2111,7 @@ int ObTablet::write_sync_tablet_seq_log(ObTabletAutoincSeq &autoinc_seq, ret = OB_TIMEOUT; LOG_WARN("submit sync tablet seq log timeout", K(ret)); } else if (cb->is_failed()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("submit sync tablet seq log failed", K(ret)); - } else if (OB_FAIL(cb->get_ret_code())) { + ret = cb->get_ret_code(); LOG_WARN("submit sync tablet seq log failed", K(ret)); } else { int64_t wait_time = ObTimeUtility::fast_current_time() - start_time; diff --git a/src/storage/tablet/ob_tablet_create_delete_helper.cpp b/src/storage/tablet/ob_tablet_create_delete_helper.cpp index be3bc254b..4f80b03fc 100644 --- a/src/storage/tablet/ob_tablet_create_delete_helper.cpp +++ b/src/storage/tablet/ob_tablet_create_delete_helper.cpp @@ -914,7 +914,7 @@ int ObTabletCreateDeleteHelper::do_abort_create_tablet( } else if (OB_UNLIKELY(!trans_flags.for_replay_ && tx_data.tx_scn_ != palf::SCN::max_scn() && trans_flags.scn_ <= tx_data.tx_scn_)) { - // If tx log ts equals ObScnRange::MAX_TS, it means redo callback has not been called. + // If tx log ts equals SCN::max_scn(), it means redo callback has not been called. // Thus, we should handle this situation ret = OB_ERR_UNEXPECTED; LOG_WARN("log ts is smaller than tx log ts", K(ret), K(tablet_id), K(trans_flags), K(tx_data)); diff --git a/src/storage/tx/ob_defensive_check_mgr.cpp b/src/storage/tx/ob_defensive_check_mgr.cpp index 207331c0e..9efa40a4b 100644 --- a/src/storage/tx/ob_defensive_check_mgr.cpp +++ b/src/storage/tx/ob_defensive_check_mgr.cpp @@ -27,12 +27,14 @@ void SingleRowDefensiveRecord::reset() generate_ts_ = 0; rowkey_.reset(); row_.reset(); + tablet_id_.reset(); allocator_.reset(); } int SingleRowDefensiveRecord::deep_copy(const blocksstable::ObDatumRow &row, const blocksstable::ObDatumRowkey &rowkey, - const ObDefensiveCheckRecordExtend &extend_info) + const ObDefensiveCheckRecordExtend &extend_info, + const ObTabletID &tablet_id) { int ret = OB_SUCCESS; @@ -44,6 +46,7 @@ int SingleRowDefensiveRecord::deep_copy(const blocksstable::ObDatumRow &row, TRANS_LOG(WARN, "datum row deep copy error", K(ret), K(row)); } else { extend_info_ = extend_info; + tablet_id_ = tablet_id; generate_ts_ = ObTimeUtility::current_time(); } @@ -52,7 +55,7 @@ int SingleRowDefensiveRecord::deep_copy(const blocksstable::ObDatumRow &row, void ObSingleTabletDefensiveCheckInfo::reset() { - tablet_id_.reset(); + tx_id_.reset(); for (int64_t i = 0; i < record_arr_.count(); ++i) { if (NULL != record_arr_.at(i)) { op_free(record_arr_.at(i)); @@ -61,15 +64,15 @@ void ObSingleTabletDefensiveCheckInfo::reset() } } -int ObSingleTabletDefensiveCheckInfo::init(const ObTabletID &tablet_id) +int ObSingleTabletDefensiveCheckInfo::init(const ObTransID &tx_id) { int ret = OB_SUCCESS; - if (!tablet_id.is_valid()) { + if (!tx_id.is_valid()) { ret = OB_INVALID_ARGUMENT; - TRANS_LOG(WARN, "invalid argument", K(ret), K(tablet_id)); + TRANS_LOG(WARN, "invalid argument", K(ret), K(tx_id)); } else { - tablet_id_ = tablet_id; + tx_id_ = tx_id; } return ret; @@ -117,6 +120,7 @@ void ObDefensiveCheckMgr::reset() } int ObDefensiveCheckMgr::put(const ObTabletID &tablet_id, + const ObTransID &tx_id, const blocksstable::ObDatumRow &row, const blocksstable::ObDatumRowkey &rowkey, const ObDefensiveCheckRecordExtend &extend_info) @@ -130,27 +134,31 @@ int ObDefensiveCheckMgr::put(const ObTabletID &tablet_id, SingleRowDefensiveRecord *r = NULL; if (NULL == (r = op_alloc(SingleRowDefensiveRecord))) { ret = OB_ALLOCATE_MEMORY_FAILED; - TRANS_LOG(WARN, "alloc memory fail", K(ret), K(tablet_id)); - } else if (OB_FAIL(r->deep_copy(row, rowkey, extend_info))) { - TRANS_LOG(WARN, "Defensive reocrd deep copy error", K(ret), K(row), K(rowkey)); + TRANS_LOG(WARN, "alloc memory fail", K(ret), K(tablet_id), K(tx_id)); + } else if (OB_FAIL(r->deep_copy(row, rowkey, extend_info, tablet_id))) { + TRANS_LOG(WARN, "Defensive reocrd deep copy error", K(ret), + K(row), + K(rowkey), + K(extend_info), + K(tablet_id)); } else { Guard spin_guard(lock_); ObSingleTabletDefensiveCheckInfo *info = NULL; - if (OB_FAIL(map_.get(tablet_id, info))) { + if (OB_FAIL(map_.get(tx_id, info))) { if (OB_ENTRY_NOT_EXIST != ret) { - TRANS_LOG(WARN, "map get error", K(ret), K(tablet_id)); + TRANS_LOG(WARN, "map get error", K(ret), K(tx_id), K(tablet_id)); } else if (NULL == (info = op_alloc(ObSingleTabletDefensiveCheckInfo)) - || OB_FAIL(info->init(tablet_id))) { + || OB_FAIL(info->init(tx_id))) { ret = OB_ALLOCATE_MEMORY_FAILED; - TRANS_LOG(WARN, "Defensive info alloc or init error", K(tablet_id)); + TRANS_LOG(WARN, "Defensive info alloc or init error", K(tx_id), K(tablet_id)); } else if (OB_FAIL(info->add_record(r))) { TRANS_LOG(WARN, "add defensive record error", K(ret)); - } else if (OB_FAIL(map_.insert_and_get(tablet_id, info, NULL))) { - TRANS_LOG(WARN, "defensive check info insert map error", K(ret), K(tablet_id), KP(info)); + } else if (OB_FAIL(map_.insert_and_get(tx_id, info, NULL))) { + TRANS_LOG(WARN, "defensive check info insert map error", K(ret), K(tx_id), K(tablet_id), KP(info)); } else { - TRANS_LOG(DEBUG, "add record", K(tablet_id), K(row), K(rowkey), K(extend_info)); + TRANS_LOG(DEBUG, "add record", K(tx_id), K(tablet_id), K(row), K(rowkey), K(extend_info)); // do nothing } if (OB_FAIL(ret)) { @@ -165,7 +173,7 @@ int ObDefensiveCheckMgr::put(const ObTabletID &tablet_id, } else if (OB_FAIL(info->add_record(r))) { TRANS_LOG(WARN, "add defensive record error", K(ret)); } else { - TRANS_LOG(DEBUG, "add record", K(tablet_id), K(row), K(rowkey), K(extend_info)); + TRANS_LOG(DEBUG, "add record", K(tx_id), K(tablet_id), K(row), K(rowkey), K(extend_info)); // do nothing } if (OB_SUCC(ret) && NULL != info) { @@ -181,7 +189,7 @@ int ObDefensiveCheckMgr::put(const ObTabletID &tablet_id, return ret; } -void ObDefensiveCheckMgr::dump(const ObTabletID &tablet_id) +void ObDefensiveCheckMgr::del(const ObTransID &tx_id) { int ret = OB_SUCCESS; @@ -193,23 +201,43 @@ void ObDefensiveCheckMgr::dump(const ObTabletID &tablet_id) Guard spin_guard(lock_); ObSingleTabletDefensiveCheckInfo *info = NULL; - if (OB_FAIL(map_.get(tablet_id, info))) { + if (OB_FAIL(map_.get(tx_id, info))) { if (OB_ENTRY_NOT_EXIST != ret) { - TRANS_LOG(WARN, "map get error", K(ret), K(tablet_id)); + TRANS_LOG(WARN, "map get error", K(ret), K(tx_id)); + } + } else { + (void)map_.del(tx_id, info); + (void)map_.revert(info); + } + } + UNUSED(ret); +} + +void ObDefensiveCheckMgr::dump(const ObTransID &tx_id) +{ + int ret = OB_SUCCESS; + + if (!is_inited_) { + ret = OB_NOT_INIT; + TRANS_LOG(WARN, "ObDefensiveCheckMgr not init ", K(ret)); + } else { + + Guard spin_guard(lock_); + + ObSingleTabletDefensiveCheckInfo *info = NULL; + if (OB_FAIL(map_.get(tx_id, info))) { + if (OB_ENTRY_NOT_EXIST != ret) { + TRANS_LOG(WARN, "map get error", K(ret), K(tx_id)); } } else { const int64_t max_count = std::min(max_record_cnt_, info->get_record_arr().count()); TRANS_LOG(INFO, "###################### start to print defensice check info##########################", - K(max_count), "tablet_id", info->get_tablet_id()); + K(max_count), K(tx_id), K(max_count)); for (int64_t i = 0; i < max_count; ++i) { - TRANS_LOG(INFO, "print single row defensive check info", - "rowkey", info->get_record_arr().at(i)->rowkey_, - "row", info->get_record_arr().at(i)->row_, - "extend_info", info->get_record_arr().at(i)->extend_info_); + TRANS_LOG(INFO, "print single row defensive check info", "record", *(info->get_record_arr().at(i))); } TRANS_LOG(INFO, "##################### print defensice check info finished #########################", - K(max_count), - "tablet_id", info->get_tablet_id()); + K(max_count), K(tx_id), K(max_count)); map_.revert(info); } diff --git a/src/storage/tx/ob_defensive_check_mgr.h b/src/storage/tx/ob_defensive_check_mgr.h index 167ab0a44..aac49a988 100644 --- a/src/storage/tx/ob_defensive_check_mgr.h +++ b/src/storage/tx/ob_defensive_check_mgr.h @@ -13,6 +13,7 @@ #ifndef OB_DEFENSIVE_CHECK_MGR_H_ #define OB_DEFENSIVE_CHECK_MGR_H_ #include "storage/blocksstable/ob_fuse_row_cache.h" +#include "logservice/palf/scn.h" namespace oceanbase { @@ -25,7 +26,7 @@ struct ObDefensiveCheckRecordExtend ~ObDefensiveCheckRecordExtend() { reset(); } void reset() { - fist_access_table_start_log_ts_ = 0; + fist_access_table_start_scn_.set_min(); total_table_handle_cnt_ = 0; start_access_table_idx_ = INT64_MAX; end_access_table_idx_ = INT64_MAX; @@ -33,7 +34,7 @@ struct ObDefensiveCheckRecordExtend is_all_data_from_memtable_ = false; query_flag_.reset(); } - TO_STRING_KV(K_(fist_access_table_start_log_ts), + TO_STRING_KV(K_(fist_access_table_start_scn), K_(total_table_handle_cnt), K_(start_access_table_idx), K_(end_access_table_idx), @@ -41,7 +42,7 @@ struct ObDefensiveCheckRecordExtend K_(is_all_data_from_memtable), K_(query_flag)); public: - int64_t fist_access_table_start_log_ts_; + palf::SCN fist_access_table_start_scn_; int64_t total_table_handle_cnt_; int64_t start_access_table_idx_; int64_t end_access_table_idx_; @@ -59,14 +60,16 @@ public: void destroy() { reset(); } int deep_copy(const blocksstable::ObDatumRow &row, const blocksstable::ObDatumRowkey &rowkey, - const ObDefensiveCheckRecordExtend &extend_info); + const ObDefensiveCheckRecordExtend &extend_info, + const ObTabletID &tablet_id); - TO_STRING_KV(K_(row), K_(generate_ts), K_(rowkey), K_(extend_info)); + TO_STRING_KV(K_(row), K_(generate_ts), K_(rowkey), K_(tablet_id), K_(extend_info)); blocksstable::ObDatumRow row_; int64_t generate_ts_; ObArenaAllocator allocator_; blocksstable::ObDatumRowkey rowkey_; + ObTabletID tablet_id_; ObDefensiveCheckRecordExtend extend_info_; }; @@ -77,15 +80,15 @@ class ObSingleTabletDefensiveCheckInfo : public ObTransHashLink ObTxDefensiveCheckInfoMap; + common::SpinRWLock, 2 << 16 /*bucket_num*/> ObTxDefensiveCheckInfoMap; class ObDefensiveCheckMgr { @@ -123,10 +126,12 @@ public: void reset(); void destroy() { reset(); } int put(const ObTabletID &tablet_id, + const ObTransID &tx_id, const blocksstable::ObDatumRow &row, const blocksstable::ObDatumRowkey &rowkey, const ObDefensiveCheckRecordExtend &extend_info); - void dump(const ObTabletID &tablet_id); + void del(const ObTransID &tx_id); + void dump(const ObTransID &tx_id); private: static int64_t max_record_cnt_; typedef ObSmallSpinLockGuard Guard; diff --git a/src/storage/tx/ob_id_service.cpp b/src/storage/tx/ob_id_service.cpp index d6157dff0..0d3feb7ec 100644 --- a/src/storage/tx/ob_id_service.cpp +++ b/src/storage/tx/ob_id_service.cpp @@ -137,8 +137,9 @@ int ObIDService::submit_log_(const int64_t last_id, const int64_t limited_id) base_ts = ATOMIC_LOAD(&last_id_); } } - base_scn.convert_for_gts(base_ts); - if (OB_FAIL(cb_.serialize_ls_log(ls_log, service_type_))) { + if (OB_FAIL(base_scn.convert_for_gts(base_ts))) { + TRANS_LOG(ERROR, "failed to convert scn", KR(ret), K(base_ts)); + } else if (OB_FAIL(cb_.serialize_ls_log(ls_log, service_type_))) { TRANS_LOG(WARN, "serialize ls log error", KR(ret), K(cb_)); } else { cb_.set_srv_type(service_type_); diff --git a/src/storage/tx/ob_trans_ctx_mgr_v4.cpp b/src/storage/tx/ob_trans_ctx_mgr_v4.cpp index 6814bf188..bdcda7ebf 100644 --- a/src/storage/tx/ob_trans_ctx_mgr_v4.cpp +++ b/src/storage/tx/ob_trans_ctx_mgr_v4.cpp @@ -199,7 +199,7 @@ int ObLSTxCtxMgr::init(const int64_t tenant_id, void ObLSTxCtxMgr::destroy() { - WLockGuard guard(rwlock_); + WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); if (IS_INIT) { ls_log_writer_.destroy(); is_inited_ = false; @@ -249,7 +249,6 @@ int ObLSTxCtxMgr::process_callback_(ObIArray &cb_array) cons void ObLSTxCtxMgr::print_all_tx_ctx(const int64_t max_print, const bool verbose) { - RLockGuard guard(rwlock_); print_all_tx_ctx_(max_print, verbose); } @@ -510,7 +509,6 @@ int ObLSTxCtxMgr::get_tx_ctx_(const ObTransID &tx_id, const bool for_replay, ObP int ObLSTxCtxMgr::iterator_tx_id_in_one_bucket(ObTxIDIterator& iter, int bucket_pos) { int ret = OB_SUCCESS; - RLockGuard guard(rwlock_); IteratorTxIDFunctor fn(iter); if (OB_FAIL(ls_tx_ctx_map_.for_each_in_one_bucket(fn, bucket_pos))) { @@ -524,7 +522,6 @@ int ObLSTxCtxMgr::iterator_tx_id_in_one_bucket(ObTxIDIterator& iter, int bucket_ int ObLSTxCtxMgr::iterator_tx_id(ObTxIDIterator& iter) { int ret = OB_SUCCESS; - RLockGuard guard(rwlock_); IteratorTxIDFunctor fn(iter); if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { @@ -588,7 +585,7 @@ int ObLSTxCtxMgr::replay_start_working_log(const ObTxStartWorkingLog &log, palf: { int ret = OB_SUCCESS; UNUSED(log); - WLockGuard guard(rwlock_); + WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); ReplayTxStartWorkingLogFunctor fn(start_working_ts); if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { TRANS_LOG(WARN, "[LsTxCtxMgr Role Change] replay start working log failed", KR(ret), K(ls_id_)); @@ -601,7 +598,7 @@ int ObLSTxCtxMgr::replay_start_working_log(const ObTxStartWorkingLog &log, palf: int ObLSTxCtxMgr::on_start_working_log_cb_succ(palf::SCN start_working_ts) { int ret = OB_SUCCESS; - WLockGuard guard(rwlock_); + WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); StateHelper state_helper(ls_id_, state_); if (State::T_PENDING == state_ || State::T_BLOCKED_PENDING == state_) { SwitchToLeaderFunctor fn(start_working_ts); @@ -640,7 +637,7 @@ int ObLSTxCtxMgr::on_start_working_log_cb_succ(palf::SCN start_working_ts) int ObLSTxCtxMgr::on_start_working_log_cb_fail() { int ret = OB_SUCCESS; - WLockGuard guard(rwlock_); + WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); StateHelper state_helper(ls_id_, state_); if (OB_FAIL(state_helper.switch_state(Ops::SWL_CB_FAIL))) { TRANS_LOG(WARN, "switch state fail", KR(ret), K(tenant_id_), K(ls_id_)); @@ -666,7 +663,7 @@ int ObLSTxCtxMgr::switch_to_follower_forcedly() ObTimeGuard timeguard("ObLSTxCtxMgr::switch_to_follower_forcedly"); ObSEArray cb_array; { - WLockGuard guard(rwlock_); + WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); StateHelper state_helper(ls_id_, state_); if (IS_NOT_INIT) { TRANS_LOG(ERROR, "ObLSTxCtxMgr not inited", K(ls_id_)); @@ -739,7 +736,7 @@ int ObLSTxCtxMgr::try_wait_gts_and_inc_max_commit_ts_() int ObLSTxCtxMgr::switch_to_leader() { int ret = OB_SUCCESS; - WLockGuard guard(rwlock_); + WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); StateHelper state_helper(ls_id_, state_); if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -781,7 +778,7 @@ int ObLSTxCtxMgr::switch_to_follower_gracefully() ObSEArray cb_array; { - WLockGuard guard(rwlock_); + WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); timeguard.click(); if (OB_FAIL(ret)) { @@ -836,7 +833,7 @@ int ObLSTxCtxMgr::switch_to_follower_gracefully() int ObLSTxCtxMgr::resume_leader() { int ret = OB_SUCCESS; - WLockGuard guard(rwlock_); + WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); StateHelper state_helper(ls_id_, state_); if (IS_NOT_INIT) { ret = OB_NOT_INIT; @@ -886,7 +883,7 @@ int ObLSTxCtxMgr::stop(const bool graceful) const KillTransArg arg(graceful); ObTimeGuard timeguard("ctxmgr stop"); { - WLockGuard guard(rwlock_); + WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); if (OB_FAIL(ls_log_writer_.stop())) { TRANS_LOG(WARN, "ls_log_writer_ stop error", KR(ret)); } else { @@ -926,7 +923,7 @@ int ObLSTxCtxMgr::kill_all_tx(const bool graceful, bool &is_all_tx_cleaned_up) ObSEArray cb_array; const KillTransArg arg(graceful); { - WLockGuard guard(rwlock_); + WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); KillTxCtxFunctor fn(arg, cb_array); if (OB_FAIL(ls_retain_ctx_mgr_.force_gc_retain_ctx())) { TRANS_LOG(WARN, "force gc retain ctx mgr", K(ret)); @@ -947,7 +944,7 @@ int ObLSTxCtxMgr::block(bool &is_all_tx_cleaned_up) { int ret = OB_SUCCESS; StateHelper state_helper(ls_id_, state_); - WLockGuard guard(rwlock_); + WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); if (OB_FAIL(state_helper.switch_state(Ops::BLOCK))) { TRANS_LOG(WARN, "switch state error", KR(ret), "manager", *this); @@ -961,7 +958,7 @@ int ObLSTxCtxMgr::online() { int ret = OB_SUCCESS; StateHelper state_helper(ls_id_, state_); - WLockGuard guard(rwlock_); + WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); if (OB_FAIL(state_helper.switch_state(Ops::ONLINE))) { TRANS_LOG(WARN, "switch state error", KR(ret), "manager", *this); @@ -974,7 +971,6 @@ int ObLSTxCtxMgr::online() int ObLSTxCtxMgr::get_ls_min_uncommit_tx_prepare_version(palf::SCN &min_prepare_version) { int ret = OB_SUCCESS; - RLockGuard guard(rwlock_); if (ATOMIC_LOAD(&total_tx_ctx_count_) > 0 || ls_tx_ctx_map_.count() > 0) { IterateMinPrepareVersionFunctor fn; @@ -1297,7 +1293,6 @@ int ObLSTxCtxMgr::on_tx_ctx_table_flushed() int ObLSTxCtxMgr::get_min_start_scn(palf::SCN &min_start_scn) { int ret = OB_SUCCESS; - RLockGuard guard(rwlock_); GetMinStartSCNFunctor fn; if (OB_FAIL(ls_tx_ctx_map_.for_each(fn))) { @@ -1335,7 +1330,7 @@ palf::SCN ObLSTxCtxMgr::get_aggre_rec_scn_() int ObLSTxCtxMgr::refresh_aggre_rec_scn() { int ret = OB_SUCCESS; - WLockGuard guard(rwlock_); + WLockGuardWithRetryInterval guard(rwlock_, TRY_THRESOLD_US, RETRY_INTERVAL_US); if (!prev_aggre_rec_scn_.is_valid()) { // We should remember the rec_log_ts before the tx ctx table is successfully diff --git a/src/storage/tx/ob_trans_ctx_mgr_v4.h b/src/storage/tx/ob_trans_ctx_mgr_v4.h index 269261aa7..129a964c8 100644 --- a/src/storage/tx/ob_trans_ctx_mgr_v4.h +++ b/src/storage/tx/ob_trans_ctx_mgr_v4.h @@ -154,6 +154,7 @@ public: typedef common::RWLock RWLock; typedef RWLock::RLockGuard RLockGuard; typedef RWLock::WLockGuard WLockGuard; + typedef RWLock::WLockGuardWithRetryInterval WLockGuardWithRetryInterval; ObLSTxCtxMgr() : tx_log_adapter_(&log_adapter_def_), rwlock_(ObLatchIds::DEFAULT_SPIN_RWLOCK), @@ -492,6 +493,8 @@ private: private: static const int64_t OB_TRANS_STATISTICS_INTERVAL = 60 * 1000 * 1000; static const int64_t OB_PARTITION_AUDIT_LOCAL_STORAGE_COUNT = 4; + static const int64_t TRY_THRESOLD_US = 1 * 1000 *1000; + static const int64_t RETRY_INTERVAL_US = 10 *1000; private: int process_callback_(ObIArray &cb_array) const; diff --git a/src/storage/tx/ob_trans_event.cpp b/src/storage/tx/ob_trans_event.cpp index e07185b87..641341a03 100644 --- a/src/storage/tx/ob_trans_event.cpp +++ b/src/storage/tx/ob_trans_event.cpp @@ -129,6 +129,22 @@ void ObTransStatistic::add_trans_total_used_time(const uint64_t tenant_id, const //trans_total_used_time_stat_.add(value); } +void ObTransStatistic::add_elr_enable_trans_count(const uint64_t tenant_id, const int64_t value) +{ + common::ObTenantStatEstGuard guard(tenant_id); + EVENT_ADD(TRANS_ELR_ENABLE_COUNT, value); +} +void ObTransStatistic::add_elr_unable_trans_count(const uint64_t tenant_id, const int64_t value) +{ + common::ObTenantStatEstGuard guard(tenant_id); + EVENT_ADD(TRANS_ELR_UNABLE_COUNT, value); +} +void ObTransStatistic::add_read_elr_row_count(const uint64_t tenant_id, const int64_t value) +{ + common::ObTenantStatEstGuard guard(tenant_id); + EVENT_ADD(READ_ELR_ROW_COUNT, value); +} + void ObTransStatistic::add_local_stmt_count(const uint64_t tenant_id, const int64_t value) { UNUSED(value); diff --git a/src/storage/tx/ob_trans_event.h b/src/storage/tx/ob_trans_event.h index daba4caa8..9d4ea970b 100644 --- a/src/storage/tx/ob_trans_event.h +++ b/src/storage/tx/ob_trans_event.h @@ -61,6 +61,11 @@ public: void add_commit_trans_count(const uint64_t tenant_id, const int64_t value); // count the number of aborted transactions void add_rollback_trans_count(const uint64_t tenant_id, const int64_t value); + // count the number of elr enable transactions + void add_elr_enable_trans_count(const uint64_t tenant_id, const int64_t value); + // count the number of elr unable transactions + void add_elr_unable_trans_count(const uint64_t tenant_id, const int64_t value); + void add_read_elr_row_count(const uint64_t tenant_id, const int64_t value); // count the number of timeout transactions: count when commit the transaction(end_trans) void add_trans_timeout_count(const uint64_t tenant_id, const int64_t value); // count how many transactions are started, via start_trans @@ -284,6 +289,9 @@ private: #define TX_STAT_DIST_INC ObTransStatistic::get_instance().add_dist_count(tenant_id_, 1); #define TX_STAT_LOCAL_INC ObTransStatistic::get_instance().add_local_count(tenant_id_, 1); #define TX_STAT_READONLY_INC ObTransStatistic::get_instance().add_readonly_count(tenant_id_, 1); +#define TX_STAT_ELR_ENABLE_TRANS_INC ObTransStatistic::get_instance().add_elr_enable_trans_count(MTL_ID(), 1); +#define TX_STAT_ELR_UNABLE_TRANS_INC ObTransStatistic::get_instance().add_elr_unable_trans_count(MTL_ID(), 1); +#define TX_STAT_READ_ELR_ROW_COUNT_INC transaction::ObTransStatistic::get_instance().add_read_elr_row_count(MTL_ID(), 1); // TODO: following events is not used, do clean up // count the interval time between statements diff --git a/src/storage/tx/ob_trans_part_ctx.cpp b/src/storage/tx/ob_trans_part_ctx.cpp index d157ee185..15e434ed8 100644 --- a/src/storage/tx/ob_trans_part_ctx.cpp +++ b/src/storage/tx/ob_trans_part_ctx.cpp @@ -184,6 +184,12 @@ void ObPartTransCtx::destroy() if (OB_LIKELY(is_inited_)) { TRANS_LOG(DEBUG, "part_ctx_destroy", K(*this)); +#ifdef ENABLE_DEBUG_LOG + if (NULL != trans_service_->get_defensive_check_mgr()) { + trans_service_->get_defensive_check_mgr()->del(trans_id_); + } +#endif + // Defensive Check 1 : earse ctx id descriptor mt_ctx_.reset(); @@ -467,24 +473,9 @@ int ObPartTransCtx::handle_timeout(const int64_t delay) TRANS_LOG(INFO, "handle timeout", K(ret), - KP(this), - K_(ls_id), - K(trans_id_), + K(*this), K(tx_expired), K(commit_expired), - K(upstream_state_), - K(exec_info_.state_), - K(start_replay_ts_), - K(start_working_log_ts_), - K(rec_log_ts_), - K(prev_rec_log_ts_), - K(is_exiting_), - K(part_trans_action_), - K(is_incomplete_replay_ctx_), - "is_logging", - is_logging_(), - "ctx_ref", - get_ref(), K(delay)); if (busy_cbs_.get_size() > 0) { TRANS_LOG(INFO, "trx is waiting log_cb", K(busy_cbs_.get_size()), KPC(busy_cbs_.get_first()), @@ -2140,7 +2131,9 @@ int ObPartTransCtx::submit_redo_log_(ObTxLogBlock &log_block, log_cb = NULL; helper.reset(); - if (OB_FAIL(prepare_log_cb_(!NEED_FINAL_CB, log_cb))) { + if (OB_FAIL(exec_info_.redo_lsns_.reserve(exec_info_.redo_lsns_.count() + 1))) { + TRANS_LOG(WARN, "reserve memory for redo lsn failed", K(ret)); + } else if (OB_FAIL(prepare_log_cb_(!NEED_FINAL_CB, log_cb))) { if (OB_UNLIKELY(OB_TX_NOLOGCB != ret)) { TRANS_LOG(WARN, "get log cb failed", KR(ret), K(*this)); } @@ -4980,7 +4973,9 @@ int ObPartTransCtx::submit_multi_data_source_(ObTxLogBlock &log_block) ObTxMDSRange range; while (OB_SUCC(ret)) { log.reset(); - if (OB_FAIL(prepare_log_cb_(!NEED_FINAL_CB, log_cb))) { + if (OB_FAIL(exec_info_.redo_lsns_.reserve(exec_info_.redo_lsns_.count() + 1))) { + TRANS_LOG(WARN, "reserve memory for redo lsn failed", K(ret)); + } else if (OB_FAIL(prepare_log_cb_(!NEED_FINAL_CB, log_cb))) { if (OB_UNLIKELY(OB_TX_NOLOGCB != ret)) { TRANS_LOG(WARN, "get log cb failed", KR(ret), K(*this)); } @@ -5002,9 +4997,8 @@ int ObPartTransCtx::submit_multi_data_source_(ObTxLogBlock &log_block) // enough to hold multi source data, if not, take it as an error. TRANS_LOG(WARN, "add new log failed", KR(ret), K(*this)); } else if (need_pre_replay_barrier - && OB_FAIL( - log_block.rewrite_barrier_log_block(trans_id_.get_id(), - logservice::ObReplayBarrierType::PRE_BARRIER))) { + && OB_FAIL(log_block.rewrite_barrier_log_block( + trans_id_.get_id(), logservice::ObReplayBarrierType::PRE_BARRIER))) { TRANS_LOG(WARN, "rewrite multi data source log barrier failed", K(ret)); return_log_cb_(log_cb); @@ -5638,7 +5632,9 @@ int ObPartTransCtx::rollback_to_savepoint_(const int64_t from_scn, return ret; } -int ObPartTransCtx::submit_rollback_to_log_(const int64_t from_scn, const int64_t to_scn, ObTxData *tx_data) +int ObPartTransCtx::submit_rollback_to_log_(const int64_t from_scn, + const int64_t to_scn, + ObTxData *tx_data) { int ret = OB_SUCCESS; ObTxLogBlock log_block; @@ -5646,11 +5642,12 @@ int ObPartTransCtx::submit_rollback_to_log_(const int64_t from_scn, const int64_ ObTxRollbackToLog log(from_scn, to_scn); ObTxLogCb *log_cb = NULL; - ObTxLogBlockHeader - log_block_header(cluster_id_, exec_info_.next_log_entry_no_, trans_id_); + ObTxLogBlockHeader log_block_header(cluster_id_, exec_info_.next_log_entry_no_, trans_id_); if (OB_FAIL(log_block.init(replay_hint, log_block_header))) { TRANS_LOG(WARN, "init log block fail", K(ret), KPC(this)); + } else if (OB_FAIL(exec_info_.redo_lsns_.reserve(exec_info_.redo_lsns_.count() + 1))) { + TRANS_LOG(WARN, "reserve memory for redo lsn failed", K(ret)); } else if (OB_FAIL(get_log_cb_(!NEED_FINAL_CB, log_cb))) { TRANS_LOG(WARN, "get log_cb fail", K(ret), KPC(this)); } else if (OB_FAIL(log_block.add_new_log(log))) { @@ -5660,11 +5657,9 @@ int ObPartTransCtx::submit_rollback_to_log_(const int64_t from_scn, const int64_ TRANS_LOG(ERROR, "cb arg array is empty", K(ret), K(log_block)); return_log_cb_(log_cb); log_cb = NULL; - } else if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter() - ->submit_log(log_block.get_buf(), - log_block.get_size(), palf::SCN::min_scn(), - log_cb, false /*nonblock on EAGAIN*/ - ))) { + } else if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->submit_log( + log_block.get_buf(), log_block.get_size(), palf::SCN::min_scn(), log_cb, false /*nonblock on EAGAIN*/ + ))) { TRANS_LOG(WARN, "submit log fail", K(ret), K(log_block), KPC(this)); return_log_cb_(log_cb); } else if (OB_FAIL(acquire_ctx_ref())) { @@ -5674,12 +5669,9 @@ int ObPartTransCtx::submit_rollback_to_log_(const int64_t from_scn, const int64_ } else { log_cb->set_tx_data(tx_data); } - REC_TRANS_TRACE_EXT(tlog_, submit_rollback_log, - OB_ID(ret), ret, - OB_ID(from), from_scn, - OB_ID(to), to_scn); - TRANS_LOG(INFO, "RollbackToLog submit", K(ret), K(from_scn), K(to_scn), - KP(log_cb), KPC(this)); + REC_TRANS_TRACE_EXT(tlog_, submit_rollback_log, OB_ID(ret), ret, OB_ID(from), from_scn, OB_ID(to), + to_scn); + TRANS_LOG(INFO, "RollbackToLog submit", K(ret), K(from_scn), K(to_scn), KP(log_cb), KPC(this)); return ret; } diff --git a/src/storage/tx/ob_trans_service.cpp b/src/storage/tx/ob_trans_service.cpp index 824aaaa4f..3edcea740 100644 --- a/src/storage/tx/ob_trans_service.cpp +++ b/src/storage/tx/ob_trans_service.cpp @@ -61,6 +61,9 @@ ObTransService::ObTransService() server_tracer_(NULL), input_queue_count_(0), output_queue_count_(0), +#ifdef ENABLE_DEBUG_LOG + defensive_check_mgr_(NULL), +#endif tx_desc_mgr_(*this) { check_env_(); @@ -159,6 +162,28 @@ int ObTransService::init(const ObAddr &self, is_inited_ = true; TRANS_LOG(INFO, "transaction service inited success", KP(this)); } + if (OB_SUCC(ret)) { +#ifdef ENABLE_DEBUG_LOG + void *p = NULL; + if (!GCONF.enable_defensive_check()) { + // do nothing + } else if (NULL == (p = ob_malloc(sizeof(ObDefensiveCheckMgr), + lib::ObMemAttr(tenant_id, "ObDefenCheckMgr")))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + TRANS_LOG(WARN, "memory alloc failed", KR(ret)); + } else { + defensive_check_mgr_ = new(p) ObDefensiveCheckMgr(); + if (OB_FAIL(defensive_check_mgr_->init(lib::ObMemAttr(tenant_id, "ObDefenCheckMgr")))) { + TRANS_LOG(ERROR, "defensive check mgr init failed", K(ret), KP(defensive_check_mgr_)); + defensive_check_mgr_->destroy(); + ob_free(defensive_check_mgr_); + defensive_check_mgr_ = NULL; + } else { + // do nothing + } + } +#endif + } return ret; } @@ -267,6 +292,13 @@ void ObTransService::destroy() tx_ctx_mgr_.destroy(); tx_desc_mgr_.destroy(); dup_table_rpc_->destroy(); +#ifdef ENABLE_DEBUG_LOG + if (NULL != defensive_check_mgr_) { + defensive_check_mgr_->destroy(); + ob_free(defensive_check_mgr_); + defensive_check_mgr_ = NULL; + } +#endif is_inited_ = false; TRANS_LOG(INFO, "transaction service destroyed", KPC(this)); } diff --git a/src/storage/tx/ob_trans_service.h b/src/storage/tx/ob_trans_service.h index 055797803..f34847cb3 100644 --- a/src/storage/tx/ob_trans_service.h +++ b/src/storage/tx/ob_trans_service.h @@ -198,6 +198,9 @@ public: const char *buf, const int64_t buf_len); ObTxELRUtil &get_tx_elr_util() { return elr_util_; } +#ifdef ENABLE_DEBUG_LOG + transaction::ObDefensiveCheckMgr *get_defensive_check_mgr() { return defensive_check_mgr_; } +#endif private: void check_env_(); bool can_create_ctx_(const int64_t trx_start_ts, const common::ObTsWindows &changing_leader_windows); @@ -266,6 +269,9 @@ private: // account task qeuue's inqueue and dequeue uint32_t input_queue_count_; uint32_t output_queue_count_; +#ifdef ENABLE_DEBUG_LOG + transaction::ObDefensiveCheckMgr *defensive_check_mgr_; +#endif // txDesc's manager ObTxDescMgr tx_desc_mgr_; diff --git a/src/storage/tx/ob_trans_service_v4.cpp b/src/storage/tx/ob_trans_service_v4.cpp index af920292b..41674972a 100644 --- a/src/storage/tx/ob_trans_service_v4.cpp +++ b/src/storage/tx/ob_trans_service_v4.cpp @@ -1108,7 +1108,6 @@ int ObTransService::create_tx_ctx_(const share::ObLSID &ls_id, { int ret = OB_SUCCESS; bool existed = false; - int64_t epoch = 0; ObTxCreateArg arg(tx.can_elr_, /* can_elr */ false, /* for_replay */ tx.tenant_id_, @@ -1379,7 +1378,6 @@ int ObTransService::acquire_local_snapshot_(const share::ObLSID &ls_id, palf::SCN &snapshot) { int ret = OB_SUCCESS; - int64_t epoch = 0; bool leader = false; palf::SCN snapshot0; ObLSTxCtxMgr *ls_tx_ctx_mgr = NULL; @@ -1487,19 +1485,14 @@ int ObTransService::batch_post_tx_msg_(ObTxRollbackSPMsg &msg, { int ret = OB_SUCCESS; int last_ret = OB_SUCCESS; - const ObTxDesc *tx_ptr = msg.tx_ptr_; ARRAY_FOREACH_NORET(list, idx) { auto &p = list.at(idx); msg.receiver_ = p.left_; msg.epoch_ = p.right_; - if (msg.epoch_ > 0) { - msg.tx_ptr_ = NULL; - } if (OB_FAIL(rpc_->post_msg(p.left_, msg))) { TRANS_LOG(WARN, "post msg falied", K(ret), K(msg), K(p)); last_ret = ret; } - msg.tx_ptr_ = tx_ptr; } return last_ret; } @@ -1684,22 +1677,45 @@ int ObTransService::handle_trans_abort_request(ObTxAbortMsg &abort_req, ObTransR return ret; } +int ObTransService::create_tx_ctx_(ObTxRollbackSPMsg &msg, ObPartTransCtx *&ctx) +{ + int ret = OB_SUCCESS; + bool existed = false; + ObTxCreateArg arg(msg.can_elr_, /* can_elr */ + false, /* for_replay */ + msg.tenant_id_, + msg.tx_id_, + msg.receiver_, + msg.cluster_id_, + msg.cluster_version_, + msg.session_id_, /*session_id*/ + msg.tx_addr_, + msg.tx_expire_ts_, + this); + ret = tx_ctx_mgr_.create_tx_ctx(arg, existed, ctx); + if (OB_FAIL(ret)) { + TRANS_LOG(WARN, "create tx ctx fail", K(ret), K(msg), K(arg)); + ctx = NULL; + } + TRANS_LOG(TRACE, "create tx ctx for savepoint rollback", K(ret), K(msg), K(arg)); + return ret; +} + int ObTransService::handle_sp_rollback_request(ObTxRollbackSPMsg &msg, obrpc::ObTxRpcRollbackSPResult &result) { int ret = OB_SUCCESS; int64_t ctx_born_epoch = -1; + ObFunction create_tx_ctx_func = [this, &msg](ObPartTransCtx *&ctx) -> int { + return this->create_tx_ctx_(msg, ctx); + }; ret = ls_rollback_to_savepoint_(msg.tx_id_, msg.receiver_, msg.epoch_, msg.op_sn_, msg.savepoint_, ctx_born_epoch, - msg.tx_ptr_); - if (OB_NOT_NULL(msg.tx_ptr_)) { - ob_free((void*)msg.tx_ptr_); - msg.tx_ptr_ = NULL; - } + create_tx_ctx_func); result.status_ = ret; result.addr_ = self_; result.born_epoch_ = ctx_born_epoch; @@ -1778,6 +1794,10 @@ int ObTransService::handle_tx_batch_req(int msg_type, } else if (!leader) { \ ret = OB_NOT_MASTER; \ TRANS_LOG(WARN, "ls not master", K(ret), K(msg)); \ + } else if (ctx->is_exiting()) { \ + ret = OB_TRANS_CTX_NOT_EXIST; \ + TRANS_LOG(INFO, "tx context is exiting",K(ret),K(msg)); \ + handle_orphan_2pc_msg_(msg, false); \ } else if (OB_FAIL(ctx->msg_handler__(msg))) { \ TRANS_LOG(WARN, "handle 2pc request fail", K(ret), K(msg)); \ } \ diff --git a/src/storage/tx/ob_trans_service_v4.h b/src/storage/tx/ob_trans_service_v4.h index 91f21a39c..5ea334249 100644 --- a/src/storage/tx/ob_trans_service_v4.h +++ b/src/storage/tx/ob_trans_service_v4.h @@ -173,7 +173,7 @@ int rollback_savepoint_slowpath_(ObTxDesc &tx, int create_tx_ctx_(const share::ObLSID &ls_id, const ObTxDesc &tx, ObPartTransCtx *&ctx); - +int create_tx_ctx_(ObTxRollbackSPMsg &msg, ObPartTransCtx *&ctx); int create_tx_ctx_(const share::ObLSID &ls_id, ObLS *ls, const ObTxDesc &tx, @@ -281,7 +281,7 @@ int ls_rollback_to_savepoint_(const ObTransID &tx_id, const int64_t op_sn, const int64_t savepoint, int64_t &ctx_born_epoch, - const ObTxDesc *tx, + ObFunction &func, int64_t expire_ts = -1); int sync_rollback_savepoint__(ObTxDesc &tx, ObTxRollbackSPMsg &msg, diff --git a/src/storage/tx/ob_tx_api.cpp b/src/storage/tx/ob_tx_api.cpp index f41d76d99..28713d874 100644 --- a/src/storage/tx/ob_tx_api.cpp +++ b/src/storage/tx/ob_tx_api.cpp @@ -1279,13 +1279,16 @@ int ObTransService::rollback_savepoint_(ObTxDesc &tx, slowpath = false; ObTxPart &p = parts[0]; int64_t born_epoch = 0; + ObFunction create_tx_ctx_func = [this, &p, &tx](ObPartTransCtx *&ctx) -> int { + return this->create_tx_ctx_(p.id_, tx, ctx); + }; if (OB_FAIL(ls_rollback_to_savepoint_(tx.tx_id_, p.id_, p.epoch_, tx.op_sn_, savepoint, born_epoch, - &tx, + create_tx_ctx_func, expire_ts))) { if (OB_NOT_MASTER == ret) { slowpath = true; @@ -1339,7 +1342,7 @@ int ObTransService::ls_rollback_to_savepoint_(const ObTransID &tx_id, const int64_t op_sn, const int64_t savepoint, int64_t &ctx_born_epoch, - const ObTxDesc *tx, + ObFunction &func, int64_t expire_ts) { int ret = OB_SUCCESS; @@ -1348,8 +1351,8 @@ int ObTransService::ls_rollback_to_savepoint_(const ObTransID &tx_id, if (OB_FAIL(get_tx_ctx_(ls, tx_id, ctx))) { if (OB_NOT_MASTER == ret) { } else if (OB_TRANS_CTX_NOT_EXIST == ret && verify_epoch <= 0) { - if (OB_FAIL(create_tx_ctx_(ls, *tx, ctx))) { - TRANS_LOG(WARN, "create tx ctx fail", K(ret), K(ls), KPC(tx)); + if (OB_FAIL(func(ctx))) { + TRANS_LOG(WARN, "create tx ctx fail", K(ret), K(ls), K(tx_id)); } } else { TRANS_LOG(WARN, "get transaction context error", K(ret), K(tx_id), K(ls)); @@ -1400,32 +1403,12 @@ inline int ObTransService::rollback_savepoint_slowpath_(ObTxDesc &tx, msg.tx_id_ = tx.tx_id_; msg.savepoint_ = savepoint; msg.op_sn_ = tx.op_sn_; + msg.can_elr_ = tx.can_elr_; + msg.session_id_ = tx.sess_id_; + msg.tx_addr_ = tx.addr_; + msg.tx_expire_ts_ = tx.get_expire_ts(); msg.epoch_ = -1; msg.request_id_ = tx.op_sn_; - // prepare msg.tx_ptr_ if required - // TODO(yunxing.cyx) : in 4.1 rework here, won't serialize txDesc - ObTxDesc *tmp_tx_desc = NULL; - ARRAY_FOREACH_NORET(parts, i) { - if (parts[i].epoch_ <= 0) { - int64_t len = tx.get_serialize_size() + sizeof(ObTxDesc); - char *buf = (char*)ob_malloc(len); - int64_t pos = sizeof(ObTxDesc); - if (OB_FAIL(tx.serialize(buf, len, pos))) { - TRANS_LOG(WARN, "serialize tx fail", KR(ret), K(tx)); - ob_free(buf); - } else { - tmp_tx_desc = new(buf)ObTxDesc(); - pos = sizeof(ObTxDesc); - if (OB_FAIL(tmp_tx_desc->deserialize(buf, len, pos))) { - TRANS_LOG(WARN, "deserialize tx fail", KR(ret)); - } else { - tmp_tx_desc->parts_.reset(); - msg.tx_ptr_ = tmp_tx_desc; - } - } - break; - } - } int64_t start_ts = ObTimeUtility::current_time(); int retries = 0; if (OB_SUCC(ret)) { @@ -1446,11 +1429,6 @@ inline int ObTransService::rollback_savepoint_slowpath_(ObTxDesc &tx, // clear interrupt flag tx.flags_.INTERRUPTED_ = false; } - if (OB_NOT_NULL(tmp_tx_desc)) { - msg.tx_ptr_ = NULL; - tmp_tx_desc->~ObTxDesc(); - ob_free(tmp_tx_desc); - } auto elapsed_us = ObTimeUtility::current_time() - start_ts; TRANS_LOG(INFO, "rollback savepoint slowpath", K(ret), K_(tx.tx_id), K(start_ts), K(retries), diff --git a/src/storage/tx/ob_tx_data_define.h b/src/storage/tx/ob_tx_data_define.h index 6e2ccc791..f2bc63788 100644 --- a/src/storage/tx/ob_tx_data_define.h +++ b/src/storage/tx/ob_tx_data_define.h @@ -198,7 +198,7 @@ public: ObTxCCCtx(transaction::ObTxState state, palf::SCN prepare_version) : state_(state), prepare_version_(prepare_version) {} // For Tx Data Table - ObTxCCCtx() : state_(transaction::ObTxState::MAX), prepare_version_(palf::SCN::invalid_scn()) {} + ObTxCCCtx() : state_(transaction::ObTxState::MAX), prepare_version_() {} TO_STRING_KV(K_(state), K_(prepare_version)); public: transaction::ObTxState state_; diff --git a/src/storage/tx/ob_tx_elr_util.cpp b/src/storage/tx/ob_tx_elr_util.cpp index 16eee2815..0055c5ddf 100644 --- a/src/storage/tx/ob_tx_elr_util.cpp +++ b/src/storage/tx/ob_tx_elr_util.cpp @@ -13,6 +13,7 @@ #include "ob_tx_elr_util.h" #include "common/ob_clock_generator.h" #include "observer/omt/ob_tenant_config_mgr.h" +#include "ob_trans_event.h" namespace oceanbase { @@ -24,6 +25,7 @@ int ObTxELRUtil::check_and_update_tx_elr_info(ObTxDesc &tx, const bool can_elr) int ret = OB_SUCCESS; if (can_elr && can_tenant_elr_) { // tenant config enable elr tx.set_can_elr(true); + TX_STAT_ELR_ENABLE_TRANS_INC(MTL_ID()); } else { refresh_elr_tenant_config_(); } diff --git a/src/storage/tx/ob_tx_msg.cpp b/src/storage/tx/ob_tx_msg.cpp index efedff76f..3dff62735 100644 --- a/src/storage/tx/ob_tx_msg.cpp +++ b/src/storage/tx/ob_tx_msg.cpp @@ -53,56 +53,7 @@ OB_SERIALIZE_MEMBER_INHERIT(Ob2pcPrepareRedoReqMsg, ObTxMsg, xid_, upstream_, ap OB_SERIALIZE_MEMBER_INHERIT(Ob2pcPrepareRedoRespMsg, ObTxMsg); OB_SERIALIZE_MEMBER_INHERIT(Ob2pcPrepareVersionReqMsg, ObTxMsg); OB_SERIALIZE_MEMBER_INHERIT(Ob2pcPrepareVersionRespMsg, ObTxMsg, prepare_version_, prepare_info_array_); - -OB_DEF_SERIALIZE_SIZE(ObTxRollbackSPMsg) -{ - int len = 0; - len += ObTxMsg::get_serialize_size(); - LST_DO_CODE(OB_UNIS_ADD_LEN, savepoint_, op_sn_, branch_id_); - if (OB_NOT_NULL(tx_ptr_)) { - OB_UNIS_ADD_LEN(true); - OB_UNIS_ADD_LEN(*tx_ptr_); - } else { - OB_UNIS_ADD_LEN(false); - } - return len; -} - -OB_DEF_SERIALIZE(ObTxRollbackSPMsg) -{ - int ret = ObTxMsg::serialize(buf, buf_len, pos); - if (OB_SUCC(ret)) { - LST_DO_CODE(OB_UNIS_ENCODE, savepoint_, op_sn_, branch_id_); - if (OB_NOT_NULL(tx_ptr_)) { - OB_UNIS_ENCODE(true); - OB_UNIS_ENCODE(*tx_ptr_); - } else { - OB_UNIS_ENCODE(false); - } - } - return ret; -} - -OB_DEF_DESERIALIZE(ObTxRollbackSPMsg) -{ - int ret = ObTxMsg::deserialize(buf, data_len, pos); - if (OB_SUCC(ret)) { - LST_DO_CODE(OB_UNIS_DECODE, savepoint_, op_sn_, branch_id_); - bool has_tx_ptr = false; - OB_UNIS_DECODE(has_tx_ptr); - if (has_tx_ptr) { - void *buffer = ob_malloc(sizeof(ObTxDesc)); - if (OB_ISNULL(buffer)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - } else { - ObTxDesc *tmp = new(buffer)ObTxDesc(); - OB_UNIS_DECODE(*tmp); - tx_ptr_ = tmp; - } - } - } - return ret; -} +OB_SERIALIZE_MEMBER_INHERIT(ObTxRollbackSPMsg, ObTxMsg, savepoint_, op_sn_, can_elr_, session_id_, tx_addr_, tx_expire_ts_); bool ObTxMsg::is_valid() const { @@ -208,7 +159,9 @@ bool ObTxRollbackSPMsg::is_valid() const { bool ret = false; if (ObTxMsg::is_valid() && type_ == ROLLBACK_SAVEPOINT - && savepoint_ > -1 && op_sn_ > -1) { + && savepoint_ > -1 && op_sn_ > -1 + && session_id_ > 0 && tx_addr_.is_valid() + && tx_expire_ts_ > 0) { ret = true; } return ret; diff --git a/src/storage/tx/ob_tx_msg.h b/src/storage/tx/ob_tx_msg.h index 9defc19bc..4af616194 100644 --- a/src/storage/tx/ob_tx_msg.h +++ b/src/storage/tx/ob_tx_msg.h @@ -229,26 +229,21 @@ namespace transaction ObTxMsg(ROLLBACK_SAVEPOINT), savepoint_(-1), op_sn_(-1), - //todo:后续branch_id使用方式确定后，需要相应修改 - branch_id_(-1), - tx_ptr_(NULL) + can_elr_(false), + session_id_(0), + tx_addr_(), + tx_expire_ts_(-1) {} - ~ObTxRollbackSPMsg() { - if (OB_NOT_NULL(tx_ptr_)) { - tx_ptr_->~ObTxDesc(); - ob_free((void*)tx_ptr_); - tx_ptr_ = NULL; - } - } + ~ObTxRollbackSPMsg() {} int64_t savepoint_; int64_t op_sn_; - //todo:后期设计中操作编号是否等于branch_id - int64_t branch_id_; - const ObTxDesc *tx_ptr_; + bool can_elr_; + uint32_t session_id_; + ObAddr tx_addr_; + int64_t tx_expire_ts_; bool is_valid() const; INHERIT_TO_STRING_KV("txMsg", ObTxMsg, - K_(savepoint), K_(op_sn), K_(branch_id), - KP_(tx_ptr)); + K_(savepoint), K_(op_sn), K_(can_elr), K_(session_id), K_(tx_addr), K_(tx_expire_ts)); OB_UNIS_VERSION(1); }; diff --git a/src/storage/tx/ob_xa_ctx.cpp b/src/storage/tx/ob_xa_ctx.cpp index 4d97c456b..b8f7a7521 100644 --- a/src/storage/tx/ob_xa_ctx.cpp +++ b/src/storage/tx/ob_xa_ctx.cpp @@ -51,7 +51,6 @@ void ObXACtx::reset() is_exiting_ = false; trans_id_.reset(); is_executing_ = false; - has_submited_ = false; is_xa_end_trans_ = false; is_xa_readonly_ = false; xa_trans_state_ = ObXATransState::UNKNOWN; @@ -143,7 +142,7 @@ int ObXACtx::handle_timeout(const int64_t delay) } else if (is_terminated_) { ret = OB_TRANS_IS_EXITING; TRANS_LOG(WARN, "xa trans has terminated", K(ret)); - } else if (has_submited_) { + } else if (ObXATransState::has_submitted(xa_trans_state_)) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "xa trans has entered commit phase, unexpected", K(ret), K(*this)); } else { @@ -2037,7 +2036,6 @@ int ObXACtx::one_phase_end_trans_(const bool is_rollback, const int64_t timeout_ } else { xa_trans_state_ = ObXATransState::COMMITTING; } - has_submited_ = true; } if (OB_FAIL(ret)) { @@ -2117,6 +2115,8 @@ int ObXACtx::try_heartbeat() const int64_t now = ObTimeUtility::current_time(); if (original_sche_addr_ != GCTX.self_addr()) { // temproray scheduler, do nothing + } else if (OB_ISNULL(xa_branch_info_) && xa_trans_state_ > ObXATransState::IDLE) { + // do nothing } else if (OB_ISNULL(xa_branch_info_)) { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "unexpected branch count", K(ret), K(*this)); @@ -2252,7 +2252,7 @@ int ObXACtx::check_for_execution_(const ObXATransID &xid, const bool is_new_bran } else { ret = OB_TRANS_IS_EXITING; } - } else if (has_submited_) { + } else if (ObXATransState::has_submitted(xa_trans_state_)) { ret = OB_TRANS_XA_PROTO; TRANS_LOG(WARN, "xa trans has entered into commit phase", K(ret), K(*this)); } else if (is_tightly_coupled_) { @@ -2422,7 +2422,6 @@ int ObXACtx::drive_prepare_(const ObXATransID &xid, const int64_t timeout_us) const bool is_readonly = false; // TODO, get a sche ctx (may be newly created), and use sche ctx to drive commit // first update coordinator into inner table - has_submited_ = true; if (OB_FAIL(MTL(ObTransService*)->prepare_tx(*tx_desc_, timeout_us, end_trans_cb_))) { if (OB_LIKELY(!is_exiting_)) { is_exiting_ = true; diff --git a/src/storage/tx/ob_xa_ctx.h b/src/storage/tx/ob_xa_ctx.h index 56b5ba5f2..371dc8426 100644 --- a/src/storage/tx/ob_xa_ctx.h +++ b/src/storage/tx/ob_xa_ctx.h @@ -157,7 +157,7 @@ public: K_(is_xa_readonly), K_(xa_trans_state), K_(is_xa_one_phase), K_(xa_branch_count), K_(xa_ref_count), K_(lock_grant), K_(is_tightly_coupled), K_(lock_xid), K_(is_terminated), - K_(has_submited), K_(executing_xid), "uref", get_uref(), + K_(executing_xid), "uref", get_uref(), K_(has_tx_level_temp_table)); private: int register_timeout_task_(const int64_t interval_us); @@ -255,8 +255,6 @@ private: bool is_executing_; ObITransTimer *timer_; ObXATimeoutTask timeout_task_; - //has submited to scheduler, TODO, maintains it - bool has_submited_; uint64_t tenant_id_; //========================================================================================= // receive xa commit or xa rollback diff --git a/src/storage/tx/ob_xa_define.h b/src/storage/tx/ob_xa_define.h index 10a3c5670..94fb13f2c 100644 --- a/src/storage/tx/ob_xa_define.h +++ b/src/storage/tx/ob_xa_define.h @@ -55,6 +55,13 @@ public: { return state == PREPARED; } + static bool has_submitted(const int32_t state) + { + return COMMITTING == state + || ROLLBACKING == state + || ROLLBACKED == state + || COMMITTED == state; + } static bool can_convert(const int32_t src_state, const int32_t dst_state); static const char* to_string(int32_t state) { const char* state_str = NULL; diff --git a/src/storage/tx_storage/ob_ls_map.cpp b/src/storage/tx_storage/ob_ls_map.cpp index fcd164c8d..23feb3df0 100644 --- a/src/storage/tx_storage/ob_ls_map.cpp +++ b/src/storage/tx_storage/ob_ls_map.cpp @@ -122,6 +122,11 @@ void ObLSMap::reset() ls_buckets_ = NULL; } if (OB_NOT_NULL(buckets_lock_)) { + for (int64_t i = 0; i < BUCKETS_CNT; ++i) { + if (OB_NOT_NULL(buckets_lock_ + i)) { + (buckets_lock_ + i)->~ObQSyncLock(); + } + } ob_free(buckets_lock_); buckets_lock_ = nullptr; } @@ -176,6 +181,11 @@ int ObLSMap::init(const int64_t tenant_id, ObIAllocator *ls_allocator) return ret; } +void ObLSMap::destroy() +{ + reset(); +} + int ObLSMap::add_ls( ObLS &ls) { diff --git a/src/storage/tx_storage/ob_ls_map.h b/src/storage/tx_storage/ob_ls_map.h index b8699811e..9f4ca8bb3 100644 --- a/src/storage/tx_storage/ob_ls_map.h +++ b/src/storage/tx_storage/ob_ls_map.h @@ -41,9 +41,10 @@ public: { reset(); } - ~ObLSMap() { reset(); } + ~ObLSMap() { destroy(); } void reset(); int init(const int64_t tenant_id, common::ObIAllocator *ls_allocator); + void destroy(); // allow_multi_true is used during replay int add_ls(ObLS &ls); int del_ls(const share::ObLSID &ls_id); diff --git a/src/storage/tx_table/ob_tx_ctx_memtable.cpp b/src/storage/tx_table/ob_tx_ctx_memtable.cpp index b62f43896..ca14f79c3 100644 --- a/src/storage/tx_table/ob_tx_ctx_memtable.cpp +++ b/src/storage/tx_table/ob_tx_ctx_memtable.cpp @@ -265,23 +265,26 @@ int ObTxCtxMemtable::flush(palf::SCN recycle_scn, bool need_freeze) { int ret = OB_SUCCESS; ObSpinLockGuard guard(flush_lock_); - + if (need_freeze) { SCN rec_scn = get_rec_scn(); if (rec_scn >= recycle_scn) { TRANS_LOG(INFO, "no need to freeze", K(rec_scn), K(recycle_scn)); } else if (is_active_memtable()) { - int64_t cur_ts = common::ObClockGenerator::getClock(); + int64_t cur_time_us = ObTimeUtility::current_time(); ObScnRange scn_range; - scn_range.start_scn_.convert_for_gts(1); - scn_range.end_scn_.convert_for_gts(cur_ts); - set_scn_range(scn_range); - set_snapshot_version(scn_range.end_scn_); - ATOMIC_STORE(&is_frozen_, true); + scn_range.start_scn_.set_base(); + if (OB_FAIL(scn_range.end_scn_.convert_from_ts(cur_time_us))) { + TRANS_LOG(WARN, "failed to convert_from_ts", K(ret), K(cur_time_us)); + } else { + set_scn_range(scn_range); + set_snapshot_version(scn_range.end_scn_); + ATOMIC_STORE(&is_frozen_, true); + } } } - if (is_frozen_memtable()) { + if (OB_SUCC(ret) && is_frozen_memtable()) { compaction::ObTabletMergeDagParam param; param.ls_id_ = ls_id_; param.tablet_id_ = LS_TX_CTX_TABLET; diff --git a/src/storage/tx_table/ob_tx_ctx_memtable_mgr.cpp b/src/storage/tx_table/ob_tx_ctx_memtable_mgr.cpp index a12e1ba31..22196fb47 100644 --- a/src/storage/tx_table/ob_tx_ctx_memtable_mgr.cpp +++ b/src/storage/tx_table/ob_tx_ctx_memtable_mgr.cpp @@ -79,7 +79,7 @@ int ObTxCtxMemtableMgr::create_memtable(const SCN last_replay_scn, table_key.table_type_ = ObITable::TX_CTX_MEMTABLE; table_key.tablet_id_ = ObTabletID(ObTabletID::LS_TX_CTX_TABLET_ID); table_key.scn_range_.start_scn_ = palf::SCN::base_scn(); - table_key.scn_range_.end_scn_.convert_for_gts(2); + table_key.scn_range_.end_scn_ = palf::SCN::plus(table_key.scn_range_.start_scn_, 1); // TODO: Donot use pool to create the only memtable if (get_memtable_count_() > 0) { diff --git a/src/storage/tx_table/ob_tx_data_memtable.cpp b/src/storage/tx_table/ob_tx_data_memtable.cpp index aff167904..7ac503ba7 100644 --- a/src/storage/tx_table/ob_tx_data_memtable.cpp +++ b/src/storage/tx_table/ob_tx_data_memtable.cpp @@ -424,9 +424,9 @@ bool ObTxDataMemtable::ready_for_flush() set_snapshot_version(min_tx_scn_); bool_ret = true; } else { - int64_t freeze_ts = key_.scn_range_.end_scn_.get_val_for_inner_table_field(); + const palf::SCN &freeze_scn = key_.scn_range_.end_scn_; STORAGE_LOG(INFO, "tx data metmable is not ready for flush", - K(max_consequent_callbacked_scn), K(freeze_ts)); + K(max_consequent_callbacked_scn), K(freeze_scn)); } return bool_ret; diff --git a/src/storage/tx_table/ob_tx_data_memtable_mgr.cpp b/src/storage/tx_table/ob_tx_data_memtable_mgr.cpp index 56e047e49..02801d082 100644 --- a/src/storage/tx_table/ob_tx_data_memtable_mgr.cpp +++ b/src/storage/tx_table/ob_tx_data_memtable_mgr.cpp @@ -87,6 +87,18 @@ void ObTxDataMemtableMgr::destroy() is_inited_ = false; } +int ObTxDataMemtableMgr::offline() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(release_memtables())) { + STORAGE_LOG(WARN, "release tx data memtables failed", KR(ret)); + } else { + memtable_head_ = 0; + memtable_tail_ = 0; + } + return ret; +} + int ObTxDataMemtableMgr::release_head_memtable_(memtable::ObIMemtable *imemtable, const bool force) { diff --git a/src/storage/tx_table/ob_tx_data_memtable_mgr.h b/src/storage/tx_table/ob_tx_data_memtable_mgr.h index 1feee9a4d..cdee14343 100644 --- a/src/storage/tx_table/ob_tx_data_memtable_mgr.h +++ b/src/storage/tx_table/ob_tx_data_memtable_mgr.h @@ -48,6 +48,10 @@ public: // ObTxDataMemtableMgr ObFreezer *freezer, ObTenantMetaMemMgr *t3m) override; virtual void destroy() override; + + int offline(); + + /** * @brief This function do the following operations: * 1. check some parameters which is required by freeze; diff --git a/src/storage/tx_table/ob_tx_data_table.cpp b/src/storage/tx_table/ob_tx_data_table.cpp index 1dace2349..c2ddd95cd 100644 --- a/src/storage/tx_table/ob_tx_data_table.cpp +++ b/src/storage/tx_table/ob_tx_data_table.cpp @@ -202,12 +202,14 @@ int ObTxDataTable::offline() if (IS_NOT_INIT) { ret = OB_NOT_INIT; STORAGE_LOG(WARN, "tx data table is not inited", KR(ret), KPC(this)); - } else if (get_memtable_mgr_()->release_memtables()) { + } else if (get_memtable_mgr_()->offline()) { STORAGE_LOG(WARN, "release memtables failed", KR(ret)); } else if (OB_FAIL(clean_memtables_cache_())) { STORAGE_LOG(WARN, "clean memtables cache failed", KR(ret), KPC(this)); } else { last_update_ts_ = 0; + min_start_scn_in_ctx_.set_min(); + calc_upper_trans_version_cache_.reset(); } return ret; } diff --git a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_repeat.result b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_repeat.result index 4b9adf9ce..599bcd63f 100644 --- a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_repeat.result +++ b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_repeat.result @@ -80,9 +80,5 @@ select repeat(1.414, 2); +------------------+ select repeat("abc", 200000000); -+--------------------------+ -| repeat("abc", 200000000) | -+--------------------------+ -| NULL | -+--------------------------+ +ERROR HY000: Result of repeat() was larger than max_allowed_packet (4194304) - truncated diff --git a/tools/deploy/mysql_test/test_suite/static_engine/t/expr_repeat.test b/tools/deploy/mysql_test/test_suite/static_engine/t/expr_repeat.test index 584ea553e..7d7d04bb1 100644 --- a/tools/deploy/mysql_test/test_suite/static_engine/t/expr_repeat.test +++ b/tools/deploy/mysql_test/test_suite/static_engine/t/expr_repeat.test @@ -30,6 +30,7 @@ select repeat("abc", NULL); select repeat(1.414, 1); select repeat(1.414, 2); +--error 1301 select repeat("abc", 200000000); --enable_warnings diff --git a/unittest/logservice/mock_logservice_container/mock_election.h b/unittest/logservice/mock_logservice_container/mock_election.h index 5932eb476..077f6ae04 100644 --- a/unittest/logservice/mock_logservice_container/mock_election.h +++ b/unittest/logservice/mock_logservice_container/mock_election.h @@ -43,6 +43,12 @@ public: UNUSED(new_member_list); return ret; } + int revoke(const RoleChangeReason &reason) + { + UNUSED(reason); + int ret = OB_SUCCESS; + return ret; + } int set_priority(ElectionPriority *) override final { return OB_SUCCESS; } int reset_priority() override final { return OB_SUCCESS; } // 获取选举当前的角色 diff --git a/unittest/storage/test_compaction_policy.cpp b/unittest/storage/test_compaction_policy.cpp index 6030192da..73c544a25 100644 --- a/unittest/storage/test_compaction_policy.cpp +++ b/unittest/storage/test_compaction_policy.cpp @@ -751,7 +751,9 @@ TEST_F(TestCompactionPolicy, check_minor_merge_basic) common::ObArray freeze_info; common::ObArray snapshots; - ASSERT_EQ(OB_SUCCESS, freeze_info.push_back(ObTenantFreezeInfoMgr::FreezeInfo(1, 1, 0))); + palf::SCN scn; + scn.convert_for_tx(1); + ASSERT_EQ(OB_SUCCESS, freeze_info.push_back(ObTenantFreezeInfoMgr::FreezeInfo(scn, 1, 0))); ret = TestCompactionPolicy::prepare_freeze_info(500, freeze_info, snapshots); ASSERT_EQ(OB_SUCCESS, ret); @@ -784,9 +786,13 @@ TEST_F(TestCompactionPolicy, check_no_need_minor_merge) common::ObArray freeze_info; common::ObArray snapshots; - ASSERT_EQ(OB_SUCCESS, freeze_info.push_back(ObTenantFreezeInfoMgr::FreezeInfo(1, 1, 0))); - ASSERT_EQ(OB_SUCCESS, freeze_info.push_back(ObTenantFreezeInfoMgr::FreezeInfo(320, 1, 0))); - ASSERT_EQ(OB_SUCCESS, freeze_info.push_back(ObTenantFreezeInfoMgr::FreezeInfo(400, 1, 0))); + palf::SCN scn; + scn.convert_for_tx(1); + ASSERT_EQ(OB_SUCCESS, freeze_info.push_back(ObTenantFreezeInfoMgr::FreezeInfo(scn, 1, 0))); + scn.convert_for_tx(320); + ASSERT_EQ(OB_SUCCESS, freeze_info.push_back(ObTenantFreezeInfoMgr::FreezeInfo(scn, 1, 0))); + scn.convert_for_tx(400); + ASSERT_EQ(OB_SUCCESS, freeze_info.push_back(ObTenantFreezeInfoMgr::FreezeInfo(scn, 1, 0))); ret = TestCompactionPolicy::prepare_freeze_info(500, freeze_info, snapshots); ASSERT_EQ(OB_SUCCESS, ret); @@ -819,8 +825,11 @@ TEST_F(TestCompactionPolicy, check_major_merge_basic) common::ObArray freeze_info; common::ObArray snapshots; - ASSERT_EQ(OB_SUCCESS, freeze_info.push_back(ObTenantFreezeInfoMgr::FreezeInfo(1, 1, 0))); - ASSERT_EQ(OB_SUCCESS, freeze_info.push_back(ObTenantFreezeInfoMgr::FreezeInfo(340, 1, 0))); + palf::SCN scn; + scn.convert_for_tx(1); + ASSERT_EQ(OB_SUCCESS, freeze_info.push_back(ObTenantFreezeInfoMgr::FreezeInfo(scn, 1, 0))); + scn.convert_for_tx(340); + ASSERT_EQ(OB_SUCCESS, freeze_info.push_back(ObTenantFreezeInfoMgr::FreezeInfo(scn, 1, 0))); ret = TestCompactionPolicy::prepare_freeze_info(500, freeze_info, snapshots); ASSERT_EQ(OB_SUCCESS, ret); @@ -854,8 +863,11 @@ TEST_F(TestCompactionPolicy, check_no_need_major_merge) common::ObArray freeze_info; common::ObArray snapshots; - ASSERT_EQ(OB_SUCCESS, freeze_info.push_back(ObTenantFreezeInfoMgr::FreezeInfo(1, 1, 0))); - ASSERT_EQ(OB_SUCCESS, freeze_info.push_back(ObTenantFreezeInfoMgr::FreezeInfo(340, 1, 0))); + palf::SCN scn; + scn.convert_for_tx(1); + ASSERT_EQ(OB_SUCCESS, freeze_info.push_back(ObTenantFreezeInfoMgr::FreezeInfo(scn, 1, 0))); + scn.convert_for_tx(340); + ASSERT_EQ(OB_SUCCESS, freeze_info.push_back(ObTenantFreezeInfoMgr::FreezeInfo(scn, 1, 0))); ret = TestCompactionPolicy::prepare_freeze_info(500, freeze_info, snapshots); ASSERT_EQ(OB_SUCCESS, ret); diff --git a/unittest/storage/test_io_manager.cpp b/unittest/storage/test_io_manager.cpp index 0fc4bb7a3..48883882b 100644 --- a/unittest/storage/test_io_manager.cpp +++ b/unittest/storage/test_io_manager.cpp @@ -283,6 +283,7 @@ TEST_F(TestIOStruct, IOAllocator) TEST_F(TestIOStruct, IORequest) { ObTenantIOManager tenant_io_mgr; + tenant_io_mgr.inc_ref(); ASSERT_SUCC(tenant_io_mgr.io_allocator_.init(TEST_TENANT_ID, IO_MEMORY_LIMIT)); ObRefHolder holder(&tenant_io_mgr); ObIOFd fd; diff --git a/unittest/storage/tx/test_ob_tx_msg.cpp b/unittest/storage/tx/test_ob_tx_msg.cpp index f195f94ce..4b2dc6568 100644 --- a/unittest/storage/tx/test_ob_tx_msg.cpp +++ b/unittest/storage/tx/test_ob_tx_msg.cpp @@ -121,8 +121,10 @@ public: msg.request_id_ = op_sn_; msg.savepoint_ = 1; msg.op_sn_ = op_sn_; - msg.branch_id_ = 1; - msg.tx_ptr_ = tx; + msg.can_elr_ = true; + msg.session_id_ = 202; + msg.tx_addr_ = ObAddr(ObAddr::VER::IPV4, "127.1.1.2", 8919); + msg.tx_expire_ts_ = 120000; } void build_tx_keepalive_msg(ObTxKeepaliveMsg &msg) { @@ -432,11 +434,10 @@ TEST_F(TestObTxMsg, trans_rollback_sp_msg) EXPECT_EQ(msg.cluster_id_, msg1.cluster_id_); EXPECT_EQ(msg.savepoint_, msg1.savepoint_); EXPECT_EQ(msg.op_sn_, msg1.op_sn_); - EXPECT_EQ(msg.branch_id_, msg1.branch_id_); - EXPECT_EQ(msg.tx_ptr_->parts_[0].id_, msg1.tx_ptr_->parts_[0].id_); - if (OB_NOT_NULL(msg.tx_ptr_)) { - msg.tx_ptr_ = NULL; - } + EXPECT_EQ(msg.can_elr_, msg1.can_elr_); + EXPECT_EQ(msg.session_id_, msg1.session_id_); + EXPECT_EQ(msg.tx_addr_, msg1.tx_addr_); + EXPECT_EQ(msg.tx_expire_ts_, msg1.tx_expire_ts_); } TEST_F(TestObTxMsg, trans_keepalive_msg)