hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

remove unused code

Browse Source
This commit is contained in:
nroskill
2023-05-22 03:41:34 +00:00
committed by ob-robot
parent f8699503fa
commit b6cb96f6ce
123 changed files with 28 additions and 21432 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
deps/easy/src/io/easy_io_struct.h vendored
View File

@ -44,11 +44,7 @@ EASY_CPP_START
extern easy_atomic_t easy_debug_uuid;
#endif
///// define
#ifdef PERF_MODE
#define EASY_MAX_THREAD_CNT 128
#else
#define EASY_MAX_THREAD_CNT 64
#endif
#define EASY_IOTH_DOING_REQ_CNT 65536
#define EASY_CONN_DOING_REQ_CNT 65536
#define EASY_WARN_LOG_INTERVAL 100

1003
deps/easy/src/packet/easy_tbnet.h vendored
View File

File diff suppressed because it is too large Load Diff

1
deps/oblib/src/lib/CMakeLists.txt vendored
View File

@ -105,7 +105,6 @@ ob_set_subtarget(oblib_lib common_mixed
net/ob_addr.cpp
net/ob_net_util.cpp
number/ob_number_v2.cpp
objectpool/ob_concurrency_objpool.cpp
objectpool/ob_pool.ipp
objectpool/ob_server_object_pool.cpp
profile/ob_atomic_event.cpp

7
deps/oblib/src/lib/alloc/ob_malloc_sample_struct.h vendored
View File

@ -97,7 +97,6 @@ inline bool ObMallocSampleLimiter::try_acquire(int64_t alloc_bytes)
return ret;
}
#ifndef PERF_MODE
inline bool ObMallocSampleLimiter::malloc_sample_allowed(const int64_t size, const ObMemAttr &attr)
{
bool ret = false;
@ -114,12 +113,6 @@ inline bool ObMallocSampleLimiter::malloc_sample_allowed(const int64_t size, con
}
return ret;
}
#else
inline bool ObMallocSampleLimiter::malloc_sample_allowed(const int64_t size, const ObMemAttr &attr)
{
return false;
}
#endif
inline void ObMallocSampleLimiter::set_interval(int32_t max_interval, int32_t min_interval)
{

56
deps/oblib/src/lib/allocator/ob_adapter_allocator.h vendored
View File

@ -1,56 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OB_ADAPTER_ALLOCATOR_H_
#define OB_ADAPTER_ALLOCATOR_H_
namespace oceanbase
{
namespace common
{
class AdapterAllocator
{
public:
AdapterAllocator() : buffer_(NULL), pos_(0), cap_(0)
{
}
~AdapterAllocator()
{
}
inline void init(char *buffer, const int64_t cap)
{
buffer_ = buffer;
pos_ = 0;
cap_ = cap;
}
inline void *alloc(const int64_t size)
{
void *ret = NULL;
if (pos_ + size <= cap_) {
ret = static_cast<void *>(buffer_ + pos_);
pos_ += size;
}
return ret;
}
inline void free(void *buf)
{
UNUSED(buf);
}
private:
char *buffer_;
int64_t pos_;
int64_t cap_;
};
}
}
#endif

2
deps/oblib/src/lib/allocator/ob_concurrent_fifo_allocator.cpp vendored
View File

@ -38,7 +38,7 @@ int ObConcurrentFIFOAllocator::init(const int64_t total_limit,
const int64_t cache_page_count = lib::is_mini_mode() ? 0 : get_cpu_count() * STORAGE_SIZE_TIMES;
if (OB_FAIL(inner_allocator_.init(
page_size,
ObModIds::OB_CON_FIFO_ALLOC,
"ConFifoAlloc",
OB_SERVER_TENANT_ID,
cache_page_count,
total_limit))) {

617
deps/oblib/src/lib/allocator/ob_mod_define.h vendored
View File

File diff suppressed because it is too large Load Diff

101
deps/oblib/src/lib/container/ob_easy_array.h vendored
View File

@ -1,101 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef _OB_EASY_ARRAY_H
#define _OB_EASY_ARRAY_H
#include "lib/container/ob_array.h"
#include "common/object/ob_object.h"
namespace oceanbase
{
namespace common
{
template<class T>
class EasyArray
{
public:
EasyArray();
EasyArray(T element);
//Overload () to support the use of EasyArray("name")("value")("info")
EasyArray &operator()(T element);
int at(int32_t index, T &element) const;
const T &at(int32_t index) const;
int64_t count() const; //Number of column names
inline int get_exec_status() const
{
return exec_status_;
}
private:
ObArray<T> array_;
int exec_status_;
};
template<class T>
EasyArray<T>::EasyArray() : exec_status_(OB_SUCCESS)
{
}
template<class T>
EasyArray<T>::EasyArray(T element)
{
exec_status_ = OB_SUCCESS;
if (OB_SUCCESS == exec_status_) {
exec_status_ = array_.push_back(element);
if (OB_SUCCESS != exec_status_) {
_OB_LOG(WARN, "push back fail:exec_status_[%d]", exec_status_);
}
}
}
template<class T>
EasyArray<T> &EasyArray<T>::operator()(T element)
{
if (OB_SUCCESS == exec_status_) {
exec_status_ = array_.push_back(element);
if (OB_SUCCESS != exec_status_) {
_OB_LOG(WARN, "push back fail:exec_status_[%d]", exec_status_);
}
}
return *this;
}
template<class T>
int EasyArray<T>::at(int32_t index, T &element) const
{
int ret = OB_SUCCESS;
ret = array_.at(index, element);
return ret;
}
template<class T>
const T &EasyArray<T>::at(int32_t index) const
{
return array_.at(index);
}
template<class T>
int64_t EasyArray<T>::count() const
{
return array_.count();
}
}
}
#endif /* _OB_EASY_ARRAY_H */

71
deps/oblib/src/lib/container/ob_pointer_array.h vendored
View File

@ -1,71 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_COMMON_POINTER_ARRAY_H_
#define OCEANBASE_COMMON_POINTER_ARRAY_H_
#include <stdlib.h>
#include <stdint.h>
#include <stdio.h>
#include <pthread.h>
#include <new>
#include <typeinfo>
#include "lib/ob_define.h"
#include "lib/allocator/page_arena.h"
#include "lib/allocator/ob_malloc.h"
#include "lib/allocator/ob_mod_define.h"
namespace oceanbase
{
namespace common
{
template <typename T, int64_t SIZE>
class ObPointerArray
{
public:
ObPointerArray()
{
memset(array_, 0, sizeof(array_));
};
~ObPointerArray()
{
for (int64_t i = 0; i < SIZE; i++) {
if (NULL != array_[i]) {
array_[i]->~T();
array_[i] = NULL;
}
}
};
public:
T *operator [](const int64_t index)
{
T *ret = NULL;
if (0 <= index
&& SIZE > index) {
if (NULL == (ret = array_[index])) {
char *buffer = allocator_.alloc(sizeof(T));
if (NULL != buffer) {
array_[index] = new(buffer) T();
ret = array_[index];
}
}
}
return ret;
};
private:
T *array_[SIZE];
PageArena<char> allocator_;
};
}
}
#endif // OCEANBASE_COMMON_POINTER_ARRAY_H_

1161
deps/oblib/src/lib/container/ob_seg_array.h vendored
View File

File diff suppressed because it is too large Load Diff

23
deps/oblib/src/lib/cpu/ob_endian.h vendored
View File

@ -1,23 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_COMMON_ENDIAN_H
#define OCEANBASE_COMMON_ENDIAN_H
#include <endian.h>
#if (__BYTE_ORDER == __LITTLE_ENDIAN)
#define OB_LITTLE_ENDIAN
#elif (__BYTE_ORDER == __BIG_ENDIAN)
#define OB_BIG_ENDIAN
#endif
#endif

51
deps/oblib/src/lib/lock/ob_fcond.h vendored
View File

@ -1,51 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCENABASE_LIB_LOCK_OB_FCOND_H
#define OCENABASE_LIB_LOCK_OB_FCOND_H
#include "lib/thread_local/ob_tsi_utils.h"
namespace oceanbase
{
namespace common
{
struct ObFCond
{
public:
ObFCond(): n_waiters_(0) {}
~ObFCond() {}
void signal()
{
(void)ATOMIC_FAA(&futex_.val(), 1);
if (ATOMIC_LOAD(&n_waiters_) > 0) {
futex_.wake(INT32_MAX);
}
}
uint32_t get_seq()
{
return ATOMIC_LOAD(&futex_.val());
}
int wait(uint32_t cmp, int64_t timeout)
{
int err = 0;
(void)ATOMIC_FAA(&n_waiters_, 1);
err = futex_.wait(cmp, timeout);
(void)ATOMIC_FAA(&n_waiters_, -1);
return err;
}
private:
lib::ObFutex futex_;
uint32_t n_waiters_;
};
}; // end namespace lock
}; // end namespace oceanbase
#endif //OCENABASE_LIB_LOCK_OB_FCOND_H

31
deps/oblib/src/lib/metrics/ob_gauge.h vendored
View File

@ -1,31 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef _OB_GAUGE_H
#define _OB_GAUGE_H 1
#include <stdint.h>
namespace oceanbase
{
namespace common
{
// A gauge just returns a value.
class ObGauge
{
public:
virtual ~ObGauge() {};
virtual int64_t get_value() const = 0;
};
} // end namespace common
} // end namespace oceanbase
#endif /* _OB_GAUGE_H */

838
deps/oblib/src/lib/objectpool/ob_concurrency_objpool.cpp vendored
View File

@ -1,838 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "lib/objectpool/ob_concurrency_objpool.h"
namespace oceanbase
{
namespace common
{
volatile int64_t ObObjFreeListList::once_ = 0;
volatile ObObjFreeListList *ObObjFreeListList::instance_ = NULL;
const int64_t ObObjFreeListList::MAX_NUM_FREELIST = 1024;
pthread_key_t ObObjFreeListList::objpool_key_;
const int64_t ObObjFreeList::THREAD_CACHE_TOTAL_SIZE = 4 * 1024 * 1024;
const int64_t ObObjFreeList::THREAD_CACHE_LOW_MARK = 4;
static const uint8_t ITEM_MAGIC = 0xFF;
static int64_t upper_round(const int64_t value, const int64_t align)
{
return (value + align - 1L) & ~(align - 1L);
}
static ObThreadCache *&get_thread_cache(const int64_t thread_cache_idx)
{
static TLOCAL(ByteBuf<sizeof(ObThreadCache *) * ObObjFreeListList::MAX_NUM_FREELIST>, g_thread_caches_buf);
static TLOCAL(ObThreadCache **, g_thread_caches);
if (OB_ISNULL(g_thread_caches)) {
g_thread_caches = reinterpret_cast<ObThreadCache **>(&g_thread_caches_buf[0]);
if (0 != pthread_setspecific(ObObjFreeListList::objpool_key_, (void *)(g_thread_caches))) {
OB_LOG_RET(ERROR, OB_ERR_SYS, "failed to pthread_setspecific");
}
}
return g_thread_caches[thread_cache_idx];
}
void thread_shutdown_cleanup(void *arg)
{
ObThreadCache **thread_caches = reinterpret_cast<ObThreadCache **>(arg);
if (NULL != thread_caches) {
int64_t freelist_count = ObObjFreeListList::get_freelists().get_freelist_count();
for (int64_t i = 0; i < freelist_count && i < ObObjFreeListList::MAX_NUM_FREELIST; ++i) {
if (NULL != thread_caches[i]) {
thread_caches[i]->f_->destroy_cur_thread_cache();
}
}
}
}
ObChunkInfo *&ObObjFreeList::get_chunk_info(void *item)
{
return *reinterpret_cast<ObChunkInfo **>(reinterpret_cast<char *>(item) + type_size_ - sizeof(void *));
}
ObObjFreeList::ObObjFreeList()
: freelists_(NULL), tc_allocator_(NULL), is_inited_(false),
only_global_(false), label_(nullptr),
thread_cache_idx_(0), name_(NULL), type_size_(0), type_size_base_(0),
alignment_(0), obj_count_base_(0), obj_count_per_chunk_(0),
chunk_byte_size_(0), chunk_count_high_mark_(0), chunk_count_low_mark_(0),
used_(0), allocated_(0), allocated_base_(0), used_base_(0),
nr_thread_cache_(0), nr_orphaned_thread_cache_(0), clock_(1)
{
}
#ifdef DOUBLE_FREE_CHECK
int64_t ObObjFreeList::get_chunk_item_magic_idx(
void *item, ObChunkInfo **chunk_info, const bool do_check /* = false */)
{
int64_t idx = 0;
if (NULL == *chunk_info) {
*chunk_info = get_chunk_info(item);
}
idx = (reinterpret_cast<int64_t>(item) - reinterpret_cast<int64_t>((*chunk_info)->head_)) / type_size_;
if (do_check && (idx >= obj_count_per_chunk_
|| (0 != (reinterpret_cast<int64_t>(item) - reinterpret_cast<int64_t>((*chunk_info)->head_)) % type_size_))) {
_OB_LOG_RET(ERROR, OB_ERROR, "Invalid address:%p, chunk_addr:%p, type_size:%ld, obj_count:%ld, idx:%ld",
item, (*chunk_info)->head_, type_size_, obj_count_per_chunk_, idx);
}
return idx;
}
void ObObjFreeList::set_chunk_item_magic(ObChunkInfo *chunk_info, void *item)
{
int64_t idx = 0;
idx = get_chunk_item_magic_idx(item, &chunk_info);
if (0 != chunk_info->item_magic_[idx]) {
OB_LOG_RET(ERROR, OB_ERROR, "the chunk is free, but the magic is not zero", "magic", chunk_info->item_magic_[idx]);
}
chunk_info->item_magic_[idx] = ITEM_MAGIC;
}
void ObObjFreeList::clear_chunk_item_magic(ObChunkInfo *chunk_info, void *item)
{
int64_t idx = 0;
idx = get_chunk_item_magic_idx(item, &chunk_info, true);
if (ITEM_MAGIC != chunk_info->item_magic_[idx]) {
OB_LOG_RET(ERROR, OB_ERROR, "the chunk is used, but without the right magic",
"actual_magic", chunk_info->item_magic_[idx], "expected_magic", ITEM_MAGIC);
}
chunk_info->item_magic_[idx] = 0;
}
#else
void ObObjFreeList::set_chunk_item_magic(ObChunkInfo *chunk_info, void *item)
{
UNUSED(chunk_info);
UNUSED(item);
}
void ObObjFreeList::clear_chunk_item_magic(ObChunkInfo *chunk_info, void *item)
{
UNUSED(chunk_info);
UNUSED(item);
}
#endif
ObChunkInfo *ObObjFreeList::chunk_create(ObThreadCache *thread_cache)
{
void *chunk_addr = NULL;
void *curr = NULL;
void *next = NULL;
ObChunkInfo *chunk_info = NULL;
ObMemAttr attr(OB_SERVER_TENANT_ID, label_);
SET_USE_500(attr);
if (NULL == (chunk_addr = ob_malloc_align(alignment_, chunk_byte_size_, attr))) {
OB_LOG_RET(ERROR, OB_ALLOCATE_MEMORY_FAILED, "failed to allocate chunk", K_(chunk_byte_size));
} else {
chunk_info = new (reinterpret_cast<char *>(chunk_addr) + type_size_ * obj_count_per_chunk_) ObChunkInfo();
#ifdef DOUBLE_FREE_CHECK
memset(chunk_info->item_magic_, 0, chunk_byte_size_ - type_size_ * obj_count_per_chunk_ - sizeof(ObChunkInfo));
#endif
chunk_info->tid_ = pthread_self();
chunk_info->head_ = chunk_addr;
chunk_info->type_size_ = type_size_;
chunk_info->obj_count_ = obj_count_per_chunk_;
chunk_info->length_ = chunk_byte_size_;
chunk_info->allocated_ = 0;
chunk_info->thread_cache_ = thread_cache;
chunk_info->link_ = Link<ObChunkInfo>();
curr = chunk_info->head_;
chunk_info->inner_free_list_ = curr;
for (int64_t i = 1; i < obj_count_per_chunk_; i++) {
next = reinterpret_cast<void *>(reinterpret_cast<char *>(curr) + type_size_);
*reinterpret_cast<void **>(curr) = next;
curr = next;
}
*reinterpret_cast<void **>(curr) = NULL;
(void)ATOMIC_FAA(&allocated_, obj_count_per_chunk_);
(void)ATOMIC_FAA(&ObObjFreeListList::get_freelists().mem_total_, chunk_byte_size_);
thread_cache->nr_chunks_++;
thread_cache->nr_total_ += obj_count_per_chunk_;
(void)ATOMIC_FAA(&thread_cache->nr_free_, obj_count_per_chunk_);
}
return chunk_info;
}
void ObObjFreeList::chunk_delete(ObThreadCache *thread_cache, ObChunkInfo *chunk_info)
{
void *chunk_addr = chunk_info->head_;
if (OB_UNLIKELY(0 != chunk_info->allocated_)) {
OB_LOG_RET(ERROR, OB_ERROR, "the chunk allocated size isn't 0 when it deleting", K(chunk_info->allocated_));
}
thread_cache->nr_chunks_--;
thread_cache->nr_total_ -= obj_count_per_chunk_;
(void)ATOMIC_FAA(&thread_cache->nr_free_, -obj_count_per_chunk_);
ob_free_align(chunk_addr);
(void)ATOMIC_FAA(&allocated_, -obj_count_per_chunk_);
(void)ATOMIC_FAA(&ObObjFreeListList::get_freelists().mem_total_, -chunk_byte_size_);
}
void *ObObjFreeList::alloc_from_chunk(ObChunkInfo *chunk_info)
{
void *item = NULL;
if (NULL != (item = chunk_info->inner_free_list_)) {
chunk_info->inner_free_list_ = *reinterpret_cast<void **>(item);
get_chunk_info(item) = chunk_info;
chunk_info->allocated_++;
}
return item;
}
void ObObjFreeList::free_to_chunk(ObThreadCache *thread_cache, void *item)
{
ObChunkInfo *chunk_info = NULL;
chunk_info = get_chunk_info(item);
chunk_info->allocated_--;
*reinterpret_cast<void **>(item) = chunk_info->inner_free_list_;
chunk_info->inner_free_list_ = item;
if (0 == chunk_info->allocated_) {
if (chunk_info == thread_cache->active_chunk_) {
thread_cache->active_chunk_ = NULL;
} else if (1 == obj_count_per_chunk_) {
// The following chunk's conditional is necessary because if the
// chunk only contains one object, then it never gets inserted
// into the non-full chunk list.
thread_cache->full_chunk_list_.remove(chunk_info);
} else {
thread_cache->non_full_chunk_list_.remove(chunk_info);
}
// keep enough object thread cache, if one object is in one chunk, we free this chunk
if (thread_cache->nr_chunks_ > chunk_count_high_mark_
|| thread_cache->nr_chunks_ * chunk_byte_size_ > THREAD_CACHE_TOTAL_SIZE) {
chunk_delete(thread_cache, chunk_info);
// flush more large object
if (1 == obj_count_per_chunk_) {
// must meet both size and number of cached objects constrains
while (thread_cache->nr_chunks_ * chunk_byte_size_ > THREAD_CACHE_TOTAL_SIZE
|| thread_cache->nr_chunks_ > chunk_count_low_mark_) {
// scanning from tail until meet conditions
chunk_info = thread_cache->non_full_chunk_list_.tail_;
if (NULL != chunk_info) {
thread_cache->non_full_chunk_list_.remove(chunk_info);
chunk_delete(thread_cache, chunk_info);
} else {
break;
}
}
}
} else {
if (NULL == thread_cache->active_chunk_) {
thread_cache->active_chunk_ = chunk_info;
} else {
thread_cache->non_full_chunk_list_.push(chunk_info);
}
}
} else if (obj_count_per_chunk_ - 1 == chunk_info->allocated_
&& chunk_info != thread_cache->active_chunk_) {
thread_cache->full_chunk_list_.remove(chunk_info);
thread_cache->non_full_chunk_list_.push(chunk_info);
}
}
void *ObObjFreeList::alloc_from_cache(ObThreadCache *thread_cache)
{
void *ret = NULL;
if (NULL != thread_cache->active_chunk_
&& NULL != (ret = alloc_from_chunk(thread_cache->active_chunk_))) {
// do nothing
} else {
if (NULL != thread_cache->active_chunk_) {
thread_cache->full_chunk_list_.push(thread_cache->active_chunk_);
thread_cache->active_chunk_ = NULL;
}
thread_cache->active_chunk_ = thread_cache->non_full_chunk_list_.pop();
if (NULL != thread_cache->active_chunk_
&& NULL != (ret = alloc_from_chunk(thread_cache->active_chunk_))) {
// do nothing
}
}
return ret;
}
void *ObObjFreeList::privatize_public_freelist(ObThreadCache *thread_cache, const bool alloc)
{
void *ret = NULL;
ObFreeObject *item = NULL;
if (NULL != thread_cache->outer_free_list_.head()
&& NULL != (item = reinterpret_cast<ObFreeObject *>(thread_cache->outer_free_list_.popall()))) {
SLL<ObFreeObject> org_list;
SLL<ObFreeObject> invert_list;
org_list.head_ = item;
if (alloc) {
ret = org_list.pop();
}
while (NULL != (item = org_list.pop())) {
invert_list.push(item);
}
while (NULL != (item = invert_list.pop())) {
free_to_chunk(thread_cache, item);
}
}
return ret;
}
void *ObObjFreeList::alloc_from_public_freelist(ObThreadCache *thread_cache)
{
return privatize_public_freelist(thread_cache, true);
}
void *ObObjFreeList::alloc_from_orphaned_thread_cache(ObThreadCache *thread_cache)
{
int ret = OB_SUCCESS;
void *item = NULL;
ObThreadCache *orphaned_thread_cache = NULL;
if (nr_orphaned_thread_cache_ > 0) {
if (OB_FAIL(mutex_acquire(&lock_))) {
OB_LOG(WARN, "failed to lock of freelist", KCSTRING(name_));
} else {
if (nr_orphaned_thread_cache_ > 0) {
orphaned_thread_cache = orphaned_thread_cache_list_.pop();
nr_orphaned_thread_cache_--;
}
mutex_release(&lock_);
if (NULL != orphaned_thread_cache) {
privatize_thread_cache(thread_cache, orphaned_thread_cache);
tc_allocator_->free_void(orphaned_thread_cache);
item = alloc_from_cache(thread_cache);
}
}
}
return item;
}
void *ObObjFreeList::alloc_from_new_chunk(ObThreadCache *thread_cache)
{
void *ret = NULL;
ObChunkInfo *chunk_info = NULL;
if (NULL != (chunk_info = chunk_create(thread_cache))) {
if (NULL != thread_cache->active_chunk_) {
thread_cache->full_chunk_list_.push(thread_cache->active_chunk_);
thread_cache->active_chunk_ = NULL;
}
thread_cache->active_chunk_ = chunk_info;
if (NULL != (ret = alloc_from_chunk(thread_cache->active_chunk_))) {
// do nothing
}
}
return ret;
}
int ObObjFreeList::init(const char *name, const int64_t obj_size,
const int64_t obj_count, const int64_t alignment /* = 16 */,
const ObMemCacheType cache_type /* = OP_RECLAIM */)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(is_inited_)) {
ret = OB_INIT_TWICE;
OB_LOG(WARN, "init twice");
} else if (OB_ISNULL(name) || OB_UNLIKELY(obj_size <= 0) || OB_UNLIKELY(obj_count <= 0)
|| OB_UNLIKELY(obj_count < 0) || OB_UNLIKELY(alignment <= 0)
|| OB_UNLIKELY(alignment > OB_MALLOC_NORMAL_BLOCK_SIZE)) {
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "invalid argument", KCSTRING(name), K(obj_size), K(obj_count), K(alignment));
} else if (OB_FAIL(obj_free_list_.init(name, 0))) {
OB_LOG(WARN, "failed to initialize atomic free list", KCSTRING(name));
} else if (OB_FAIL(mutex_init(&lock_))) {
OB_LOG(WARN, "failed to init mutex");
} else {
alignment_ = alignment;
obj_count_base_ = (OP_GLOBAL == cache_type) ? 0 : obj_count;
type_size_base_ = obj_size;
only_global_ = (OP_GLOBAL == cache_type);
name_ = name;
// Make sure we align *all* the objects in the allocation,
// not just the first one, each chunk with an extra
// pointer to store the pointer of chunk info
if (OB_UNLIKELY(only_global_)) {
type_size_ = upper_round(obj_size, alignment);
} else {
type_size_ = upper_round(obj_size + sizeof(void *), alignment);
}
const int64_t ob_malloc_header_size = 32; // reserved size for ob_malloc header
int64_t meta_size = only_global_ ? ob_malloc_header_size : sizeof(ObChunkInfo) + ob_malloc_header_size;
int64_t real_type_size = type_size_;
#ifdef DOUBLE_FREE_CHECK
// enlarge type_size to hold a item_magic
real_type_size = type_size_ + sizeof(uint8_t);
#endif
// find fit chunk size
// object_size < 128K, one chunk with multiply object; object_size > 128K, one chunk with one object
int64_t min_chunk_size = upper_round(real_type_size + meta_size, alignment);
int64_t low_mark_chunk_size = upper_round(real_type_size * THREAD_CACHE_LOW_MARK + meta_size, alignment);
if (low_mark_chunk_size <= OB_MALLOC_MIDDLE_BLOCK_SIZE * THREAD_CACHE_LOW_MARK * 2) {
chunk_byte_size_ = upper_round(low_mark_chunk_size, OB_MALLOC_NORMAL_BLOCK_SIZE);
obj_count_per_chunk_ = (chunk_byte_size_ - meta_size) / real_type_size;
} else {
OB_LOG(INFO, "big object pool to initialize", K(obj_size));
obj_count_per_chunk_ = 1;
if (min_chunk_size > OB_MALLOC_BIG_BLOCK_SIZE) {
chunk_byte_size_ = min_chunk_size;
} else {
chunk_byte_size_ = upper_round(min_chunk_size, OB_MALLOC_NORMAL_BLOCK_SIZE);
}
}
chunk_byte_size_ -= ob_malloc_header_size;
int64_t low_mark_obj_count = std::min(obj_count, std::max(obj_count / 4, THREAD_CACHE_LOW_MARK));
chunk_count_low_mark_ = (low_mark_obj_count + obj_count_per_chunk_ - 1) / obj_count_per_chunk_;
chunk_count_high_mark_ = (obj_count_base_ + obj_count_per_chunk_ - 1) / obj_count_per_chunk_;
used_ = 0;
allocated_ = 0;
allocated_base_ = 0;
used_base_ = 0;
nr_thread_cache_ = 0;
nr_orphaned_thread_cache_ = 0;
is_inited_ = true;
}
return ret;
}
void *ObObjFreeList::global_alloc()
{
void *chunk_addr = NULL;
void *item = NULL;
void *next = NULL;
bool break_loop = false;
do {
if (obj_free_list_.empty()) {
ObMemAttr attr(OB_SERVER_TENANT_ID, label_);
SET_USE_500(attr);
if (NULL == (chunk_addr = ob_malloc_align(alignment_, chunk_byte_size_, attr))) {
OB_LOG_RET(ERROR, OB_ALLOCATE_MEMORY_FAILED, "failed to allocate chunk", K_(chunk_byte_size));
break_loop = true;
} else {
(void)ATOMIC_FAA(&allocated_, obj_count_per_chunk_);
(void)ATOMIC_FAA(&ObObjFreeListList::get_freelists().mem_total_, chunk_byte_size_);
item = chunk_addr;
break_loop = true;
// free each of the new elements
for (int64_t i = 1; i < obj_count_per_chunk_; i++) {
next = reinterpret_cast<void *>(reinterpret_cast<char *>(item) + i * type_size_);
obj_free_list_.push(next);
}
}
} else if (NULL != (item = obj_free_list_.pop())) {
break_loop = true;
}
} while (!break_loop);
return item;
}
void ObObjFreeList::global_free(void *item)
{
if (OB_LIKELY(NULL != item)) {
obj_free_list_.push(item);
}
}
ObThreadCache *ObObjFreeList::init_thread_cache()
{
int ret = OB_SUCCESS;
ObThreadCache *thread_cache = NULL;
if (NULL == (thread_cache = reinterpret_cast<ObThreadCache *>(tc_allocator_->alloc_void()))) {
OB_LOG(ERROR, "failed to allocate memory for thread cache");
} else {
new (thread_cache) ObThreadCache();
thread_cache->inner_free_list_ = SLL<ObFreeObject>();
thread_cache->f_ = this;
thread_cache->active_chunk_ = NULL;
thread_cache->full_chunk_list_ = DLL<ObChunkInfo>();
thread_cache->non_full_chunk_list_ = Queue<ObChunkInfo>();
thread_cache->clock_ = UINT64_MAX;
// this lock will only be accessed when initializing
// thread cache, so it won't damage performance
if (OB_FAIL(mutex_acquire(&lock_))) {
OB_LOG(WARN, "failed to lock of freelist", KCSTRING(name_));
} else if (OB_FAIL(thread_cache->outer_free_list_.init(name_, 0))) {
mutex_release(&lock_);
OB_LOG(WARN, "failed to init outer freelist", KCSTRING(name_));
} else {
get_thread_cache(thread_cache_idx_) = thread_cache;
thread_cache_list_.push(thread_cache);
nr_thread_cache_++;
mutex_release(&lock_);
}
}
return thread_cache;
}
int ObObjFreeList::destroy_cur_thread_cache()
{
int ret = OB_SUCCESS;
ObChunkInfo *chunk_info = NULL;
ObThreadCache *cur_thread_cache = NULL;
void *item = NULL;
// unregister thread cache
if (!OB_ISNULL(cur_thread_cache = get_thread_cache(thread_cache_idx_))) {
if (OB_FAIL(mutex_acquire(&lock_))) {
OB_LOG(WARN, "failed to lock of freelist", KCSTRING(name_));
} else {
thread_cache_list_.remove(cur_thread_cache);
nr_thread_cache_--;
mutex_release(&lock_);
if (only_global_) {
while (NULL != (item = cur_thread_cache->inner_free_list_.pop())) {
obj_free_list_.push(item);
cur_thread_cache->nr_free_--;
}
tc_allocator_->free_void(cur_thread_cache);
} else {
if (NULL != cur_thread_cache->outer_free_list_.head()) {
privatize_public_freelist(cur_thread_cache, false);
}
if (NULL != (chunk_info = cur_thread_cache->active_chunk_)) {
if (0 == chunk_info->allocated_) {
chunk_delete(cur_thread_cache, chunk_info);
cur_thread_cache->active_chunk_ = NULL;
}
}
if (0 == cur_thread_cache->nr_chunks_) {
tc_allocator_->free_void(cur_thread_cache);
} else {
if (OB_FAIL(mutex_acquire(&lock_))) {
OB_LOG(WARN, "failed to lock of freelist", KCSTRING(name_));
} else {
orphaned_thread_cache_list_.push(cur_thread_cache);
nr_orphaned_thread_cache_++;
mutex_release(&lock_);
}
}
}
}
}
return ret;
}
void ObObjFreeList::privatize_thread_cache(ObThreadCache *cur_thread_cache, ObThreadCache *src_thread_cache)
{
ObChunkInfo *chunk_info = NULL;
// modify the owner thread cache of the chunks in source thread cache
if (NULL != (chunk_info = src_thread_cache->active_chunk_)) {
ATOMIC_STORE(&chunk_info->thread_cache_, cur_thread_cache);
}
chunk_info = src_thread_cache->full_chunk_list_.head_;
while (NULL != chunk_info) {
ATOMIC_STORE(&chunk_info->thread_cache_, cur_thread_cache);
chunk_info = src_thread_cache->full_chunk_list_.next(chunk_info);
}
chunk_info = src_thread_cache->non_full_chunk_list_.head_;
while (NULL != chunk_info) {
ATOMIC_STORE(&chunk_info->thread_cache_, cur_thread_cache);
chunk_info = src_thread_cache->non_full_chunk_list_.next(chunk_info);
}
// after modifying the owner of chunks, wait until no other threads use the old owner thread cache
rcu_sync();
// nobody uses the src thread cache, current thread cache can private it
if (NULL != src_thread_cache->outer_free_list_.head()) {
privatize_public_freelist(src_thread_cache, false);
}
if (NULL != (chunk_info = src_thread_cache->active_chunk_)) {
if (0 == chunk_info->allocated_) {
chunk_delete(src_thread_cache, chunk_info);
src_thread_cache->active_chunk_ = NULL;
} else if (chunk_info->allocated_ == obj_count_per_chunk_) {
cur_thread_cache->full_chunk_list_.push(chunk_info);
} else {
cur_thread_cache->non_full_chunk_list_.push(chunk_info);
}
}
while (NULL != (chunk_info = src_thread_cache->full_chunk_list_.pop())) {
cur_thread_cache->full_chunk_list_.push(chunk_info);
}
while (NULL != (chunk_info = src_thread_cache->non_full_chunk_list_.pop())) {
cur_thread_cache->non_full_chunk_list_.push(chunk_info);
}
// merge the stat to current thread cache
cur_thread_cache->nr_chunks_ += src_thread_cache->nr_chunks_;
cur_thread_cache->nr_total_ += src_thread_cache->nr_total_;
(void)ATOMIC_FAA(&cur_thread_cache->nr_malloc_, src_thread_cache->nr_malloc_);
(void)ATOMIC_FAA(&cur_thread_cache->nr_free_, src_thread_cache->nr_free_);
}
void *ObObjFreeList::reclaim_alloc(ObThreadCache *thread_cache)
{
void *ret = NULL;
if (OB_LIKELY(NULL != thread_cache)){
// get from local chunk freelist first
if (NULL != (ret = alloc_from_cache(thread_cache))) {
// do nothing
}
// fetch memory from outer_free_list
else if (NULL != (ret = alloc_from_public_freelist(thread_cache))) {
// do nothing
}
// fetch memory from orphaned thread cache
else if (NULL != (ret = alloc_from_orphaned_thread_cache(thread_cache))) {
// do nothing
}
// add new chunk
else if (NULL != (ret = alloc_from_new_chunk(thread_cache))) {
// do nothing
}
if (NULL != ret) {
(void)ATOMIC_FAA(&thread_cache->nr_free_, -1);
(void)ATOMIC_FAA(&thread_cache->nr_malloc_, 1);
set_chunk_item_magic(NULL, ret);
}
}
return ret;
}
void *ObObjFreeList::alloc()
{
void *ret = NULL;
ObThreadCache *thread_cache = NULL;
// no thread cache, create it
if (OB_ISNULL(thread_cache = get_thread_cache(thread_cache_idx_))) {
if (OB_ISNULL(thread_cache = init_thread_cache())) {
OB_LOG_RET(ERROR, OB_ERROR, "failed to init object free list thread cache");
ret = NULL; // allocate failed
}
}
if (only_global_) {
if (OB_NOT_NULL(ret = global_alloc())) {
thread_cache->nr_malloc_++;
}
} else {
ret = reclaim_alloc(thread_cache);
}
return ret;
}
void ObObjFreeList::reclaim_free(ObThreadCache *cur_thread_cache, void *item)
{
ObChunkInfo *chunk_info = NULL;
ObThreadCache *thread_cache = NULL;
chunk_info = get_chunk_info(item);
clear_chunk_item_magic(chunk_info, item);
rcu_read_lock(cur_thread_cache);
thread_cache = ATOMIC_LOAD(&chunk_info->thread_cache_);
(void)ATOMIC_FAA(&thread_cache->nr_free_, 1);
(void)ATOMIC_FAA(&thread_cache->nr_malloc_, -1);
if (thread_cache == cur_thread_cache) {
thread_cache->f_->free_to_chunk(thread_cache, item);
if (NULL != thread_cache->outer_free_list_.head()) {
privatize_public_freelist(thread_cache, false);
}
} else {
thread_cache->outer_free_list_.push(item);
}
rcu_read_unlock(cur_thread_cache);
}
void ObObjFreeList::free(void *item)
{
ObThreadCache *thread_cache = NULL;
if (OB_LIKELY(NULL != item)) {
// no thread cache, create it
if (OB_ISNULL(thread_cache = get_thread_cache(thread_cache_idx_))) {
if (OB_ISNULL(thread_cache = init_thread_cache())) {
OB_LOG_RET(ERROR, OB_ERROR, "failed to init object free list thread cache");
}
}
if (OB_LIKELY(NULL != thread_cache)) {
if (only_global_) {
global_free(item);
thread_cache->nr_malloc_--;
} else {
reclaim_free(thread_cache, item);
}
}
}
}
void ObObjFreeList::update_used()
{
int64_t used = 0;
mutex_acquire(&lock_);
ObThreadCache *next_thread_cache = thread_cache_list_.head_;
while (NULL != next_thread_cache) {
used += next_thread_cache->nr_malloc_;
next_thread_cache = thread_cache_list_.next(next_thread_cache);
}
mutex_release(&lock_);
used_ = used;
}
ObObjFreeListList::ObObjFreeListList()
: tc_meta_allocator_(ObModIds::OB_CONCURRENCY_OBJ_POOL)
{
mem_total_ = 0;
nr_freelists_ = 0;
is_inited_ = false;
}
int ObObjFreeListList::create_freelist(
ObObjFreeList *&freelist,
const char *name,
const int64_t obj_size,
const int64_t obj_count,
const int64_t alignment /* = 16 */,
const ObMemCacheType cache_type /* = OP_RECLAIM */,
const bool is_meta /* = false */,
const lib::ObLabel &label)
{
int ret = OB_SUCCESS;
freelist = NULL;
if (OB_ISNULL(name) || OB_UNLIKELY(obj_size <= 0)
|| OB_UNLIKELY(obj_count < 0) || OB_UNLIKELY(alignment <= 0)) {
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "invalid argument", KP(name), K(obj_size), K(obj_count), K(alignment));
} else if (!is_meta && OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
OB_LOG(WARN, "global object freelist list is not initialized success");
} else if (nr_freelists_ >= ObObjFreeListList::MAX_NUM_FREELIST) {
ret = OB_SIZE_OVERFLOW;
OB_LOG(ERROR, "can't allocate object freelist, freelist is full",
"max_freelist_num", ObObjFreeListList::MAX_NUM_FREELIST, K(ret));
} else {
if (OB_UNLIKELY(is_meta)) {
freelist = new (std::nothrow) ObObjFreeList();
} else {
freelist = fl_allocator_.alloc();
}
if (OB_ISNULL(freelist)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
OB_LOG(ERROR, "failed to allocate memory for freelist", K(ret));
} else {
freelist->thread_cache_idx_ = get_next_free_slot();
if (freelist->thread_cache_idx_ >= ObObjFreeListList::MAX_NUM_FREELIST) {
ret = OB_SIZE_OVERFLOW;
OB_LOG(ERROR, "can't allocate object freelist, freelist is full",
"max_freelist_num", ObObjFreeListList::MAX_NUM_FREELIST, K(ret));
if (OB_UNLIKELY(is_meta)) {
delete freelist;
} else {
fl_allocator_.free(freelist);
}
freelist = NULL;
} else {
freelist->label_ = label;
freelist->freelists_ = &freelists_;
freelists_.push(freelist);
if (OB_UNLIKELY(is_meta)) {
freelist->tc_allocator_ = &tc_meta_allocator_;
} else {
freelist->tc_allocator_ = &tc_allocator_;
}
if (OB_FAIL(freelist->init(name, obj_size, obj_count, alignment, cache_type))) {
OB_LOG(WARN, "failed to initialize object freelist",
KCSTRING(name), K(obj_size), K(obj_count), K(alignment));
}
}
}
}
return ret;
}
int ObObjFreeListList::get_info(ObVector<ObObjFreeList *> &flls)
{
int ret = OB_SUCCESS;
ObAtomicSLL<ObObjFreeList> &fll = ObObjFreeListList::get_freelists().freelists_;
ObObjFreeList *fl = fll.head();
while (OB_SUCC(ret) && NULL != fl) {
fl->update_used();
if (OB_FAIL(flls.push_back(fl))) {
OB_LOG(WARN, "failed to push back fl", K(ret));
} else {
fl = fll.next(fl);
}
}
return ret;
}
void ObObjFreeListList::dump()
{
char buf[8000];
buf[sizeof(buf) - 1] = '\0';
to_string(buf, sizeof(buf) - 1);
_OB_LOG(INFO, "[MEMORY] dump object freelist statistic:%s", buf);
}
int64_t ObObjFreeListList::to_string(char *buf, const int64_t len) const
{
int ret = OB_SUCCESS;
ObAtomicSLL<ObObjFreeList> &fll = ObObjFreeListList::get_freelists().freelists_;
ObObjFreeList *fl = fll.head();
int64_t pos = 0;
databuff_printf(buf, len, pos, "\n[MEMORY] allocated | in-use | type size | cache type | free list name\n");
databuff_printf(buf, len, pos, "[MEMORY] --------------------|--------------------|------------|------------|----------------------------------\n");
while (NULL != fl && OB_SUCC(ret)) {
fl->update_used();
ret = databuff_printf(buf, len, pos, "[MEMORY] %'18ld | %'18ld | %'10ld | %10s | %s\n",
fl->allocated_ * fl->type_size_,
fl->used_ * fl->type_size_, fl->type_size_,
fl->only_global_ ? (fl->obj_count_base_ > 0 ? "tc" : "global") : "reclaim",
fl->name_ ? fl->name_ : "<unknown>");
fl = fll.next(fl);
}
return pos;
}
} // end of namespace common
} // end of namespace oceanbase

798
deps/oblib/src/lib/objectpool/ob_concurrency_objpool.h vendored
View File

@ -14,371 +14,15 @@
#define OB_LIB_CONCURRENCY_OBJPOOL_H_
#include <typeinfo>
#if __cplusplus >= 201103L
#include <type_traits>
#endif
#include "lib/allocator/page_arena.h"
#include "lib/utility/ob_print_utils.h"
#include "lib/list/ob_atomic_list.h"
#include "lib/list/ob_intrusive_list.h"
#include "lib/lock/ob_mutex.h"
#include "lib/container/ob_vector.h"
#include "lib/allocator/ob_slice_alloc.h"
#include "lib/cpu/ob_cpu_topology.h"
DEFINE_HAS_MEMBER(OP_LOCAL_NUM);
DEFINE_HAS_MEMBER(OP_LABEL);
namespace oceanbase
{
namespace common
{
struct ObThreadCache;
class ObObjFreeList;
class ObObjFreeListList;
class ObFixedMemAllocator;
extern void thread_shutdown_cleanup(void *);
enum ObMemCacheType
{
OP_GLOBAL,
OP_RECLAIM
};
struct ObChunkInfo
{
ObChunkInfo()
{
memset(this, 0, sizeof(ObChunkInfo));
}
~ObChunkInfo() { }
pthread_t tid_;
int64_t type_size_;
int64_t obj_count_;
int64_t allocated_;
int64_t length_;
// inner free list will only be
// accessed by creator-thread
void *inner_free_list_;
void *head_;
ObThreadCache *thread_cache_;
LINK(ObChunkInfo, link_);
#ifdef DOUBLE_FREE_CHECK
// magic code for each item,
// it's used to check double-free issue.
uint8_t item_magic_[0];
#endif
};
struct ObFreeObject
{
SLINK(ObFreeObject, link_);
};
struct ObThreadCache
{
ObThreadCache()
{
memset(this, 0, sizeof(ObThreadCache));
clock_ = UINT64_MAX;
}
~ObThreadCache() { }
TO_STRING_KV(K_(f), K_(nr_total), K_(nr_malloc), K_(nr_free), K_(nr_chunks), K_(clock));
// the address must be aligned with 16 bytes if it uses cas 128
//
// outer free list will be accessed by:
// - creator-thread, as a producer-thread
// - consumer-thread
ObAtomicList outer_free_list_;
// inner free list for tc alloc
SLL<ObFreeObject> inner_free_list_;
ObObjFreeList *f_;
// using for memory reclaim algorithm
int64_t nr_total_;
int64_t nr_free_;
int64_t nr_malloc_;
int64_t nr_chunks_;
ObChunkInfo *active_chunk_;
Queue<ObChunkInfo> non_full_chunk_list_;
DLL<ObChunkInfo> full_chunk_list_;
LINK(ObThreadCache, link_);
uint64_t clock_;
};
class ObObjFreeList
{
friend class ObObjFreeListList;
friend struct ObThreadCache;
public:
ObObjFreeList();
~ObObjFreeList() { (void)mutex_destroy(&lock_); }
// alignment must be a power of 2
int init(const char *name, const int64_t type_size, const int64_t obj_count,
const int64_t alignment = 16, const ObMemCacheType cache_type = OP_RECLAIM);
void *alloc();
void free(void *item);
const char *get_name() const { return name_; }
int64_t get_allocated() const { return allocated_; }
int64_t get_allocated_base() const { return allocated_base_; }
int64_t get_type_size() const { return type_size_; }
int64_t get_used() const { return used_; }
int64_t get_used_base() const { return used_base_; }
int64_t get_chunk_count() const { return chunk_count_high_mark_; }
int64_t get_chunk_byte_size() const { return chunk_byte_size_; }
int destroy_cur_thread_cache();
TO_STRING_KV(KCSTRING_(name), K_(is_inited), K_(only_global), K_(label), K_(thread_cache_idx),
K_(type_size), K_(type_size_base), K_(alignment), K_(obj_count_base),
K_(obj_count_per_chunk), K_(chunk_byte_size), K_(chunk_count_high_mark),
K_(chunk_count_low_mark), K_(used), K_(allocated), K_(allocated_base),
K_(used_base), K_(nr_thread_cache), K_(nr_orphaned_thread_cache), K_(clock));
SLINK(ObObjFreeList, link_);
private:
void show_info(const char *file, const int line,
const char *tag, ObThreadCache *thread_cache);
ObChunkInfo *&get_chunk_info(void *item);
#ifdef DOUBLE_FREE_CHECK
int64_t get_chunk_item_magic_idx(void *item, ObChunkInfo **chunk_info, const bool do_check = false);
#endif
void set_chunk_item_magic(ObChunkInfo *chunk_info, void *item);
void clear_chunk_item_magic(ObChunkInfo *chunk_info, void *item);
ObChunkInfo *chunk_create(ObThreadCache *thread_cache);
void chunk_delete(ObThreadCache *thread_cache, ObChunkInfo *chunk_info);
void *alloc_from_chunk(ObChunkInfo *chunk_info);
void free_to_chunk(ObThreadCache *thread_cache, void *item);
void *alloc_from_cache(ObThreadCache *thread_cache);
void *privatize_public_freelist(ObThreadCache *thread_cache, const bool alloc);
void *alloc_from_public_freelist(ObThreadCache *thread_cache);
void *alloc_from_orphaned_thread_cache(ObThreadCache *thread_cache);
void *alloc_from_new_chunk(ObThreadCache *thread_cache);
void *global_alloc();
void global_free(void *item);
void *reclaim_alloc(ObThreadCache *thread_cache);
void reclaim_free(ObThreadCache *cur_thread_cache, void *item);
ObThreadCache *init_thread_cache();
void privatize_thread_cache(ObThreadCache *cur_thread_cache, ObThreadCache *src_thread_cache);
void update_used();
uint64_t inc_clock() { return ATOMIC_AAF(&clock_, 1); }
uint64_t get_clock() { return ATOMIC_LOAD(&clock_); }
void rcu_read_lock(ObThreadCache *thread_cache)
{
WEAK_BARRIER();
ATOMIC_STORE(&thread_cache->clock_, get_clock());
}
void rcu_read_unlock(ObThreadCache *thread_cache)
{
WEAK_BARRIER();
ATOMIC_STORE(&thread_cache->clock_, UINT64_MAX);
}
void rcu_sync()
{
uint64_t clock = inc_clock();
while(!is_quiescent(clock)) {
PAUSE();
}
}
bool is_quiescent(uint64_t cur_clock)
{
bool ret = true;
mutex_acquire(&lock_);
ObThreadCache *next_thread_cache = thread_cache_list_.head_;
while (NULL != next_thread_cache) {
if (ATOMIC_LOAD(&next_thread_cache->clock_) < cur_clock) {
ret = false;
break;
}
next_thread_cache = thread_cache_list_.next(next_thread_cache);
}
mutex_release(&lock_);
return ret;
}
private:
static const int64_t THREAD_CACHE_TOTAL_SIZE;
static const int64_t THREAD_CACHE_LOW_MARK;
ObAtomicList obj_free_list_; // global object freelist for global alloc
ObAtomicSLL<ObObjFreeList> *freelists_; // global free list list
ObFixedMemAllocator *tc_allocator_;
bool is_inited_;
bool only_global_;
lib::ObLabel label_;
int64_t thread_cache_idx_;
const char *name_;
// actual size of each object, each object in one chunk, each chunk includes
// an extra void* to store instance of ObChunkInfo.
int64_t type_size_;
// user specified size of object
int64_t type_size_base_;
int64_t alignment_;
// user specified number of elements
int64_t obj_count_base_;
// number of elements of each chunk
int64_t obj_count_per_chunk_;
// byte size of one block allocated from ob_malloc()
int64_t chunk_byte_size_;
// memory block count, blocks are allocated from ob_malloc()
int64_t chunk_count_high_mark_;
int64_t chunk_count_low_mark_;
int64_t used_;
int64_t allocated_;
int64_t allocated_base_;
int64_t used_base_;
// number of thread cache in one object freelist
int64_t nr_thread_cache_;
DLL<ObThreadCache> thread_cache_list_;
int64_t nr_orphaned_thread_cache_;
DLL<ObThreadCache> orphaned_thread_cache_list_;
::ObMutex0 lock_;
uint64_t clock_;
DISALLOW_COPY_AND_ASSIGN(ObObjFreeList);
};
// Allocator for fixed size memory blocks.
class ObFixedMemAllocator
{
public:
ObFixedMemAllocator() { fl_ = NULL; }
virtual ~ObFixedMemAllocator() { }
// Creates a new global allocator.
// @param name identification tag used for mem tracking .
// @param obj_size size of memory blocks to be allocated.
// @param obj_count number of units to be allocated if free pool is
// empty.
// @param alignment of objects must be a power of 2.
// initialize the parameters of the allocator.
int init(ObObjFreeListList &fll, const char *name, const int64_t obj_size,
const int64_t obj_count, const int64_t alignment,
const ObMemCacheType cache_type = OP_RECLAIM, const bool is_meta = false,
const lib::ObLabel &label = common::ObModIds::OB_CONCURRENCY_OBJ_POOL);
int init(const char *name, const int64_t obj_size,
const int64_t obj_count, const int64_t alignment,
const ObMemCacheType cache_type = OP_RECLAIM, const bool is_meta = false,
const lib::ObLabel &label = common::ObModIds::OB_CONCURRENCY_OBJ_POOL);
// Allocate a block of memory (size specified during construction
// of Allocator.
virtual void *alloc_void()
{
void *ret = NULL;
if (OB_LIKELY(NULL != fl_)) {
ret = fl_->alloc();
}
return ret;
}
// Deallocate a block of memory allocated by the Allocator.
virtual void free_void(void *ptr)
{
if (OB_LIKELY(NULL != fl_) && OB_LIKELY(NULL != ptr)) {
fl_->free(ptr);
ptr = NULL;
}
}
protected:
ObObjFreeList *fl_;
DISALLOW_COPY_AND_ASSIGN(ObFixedMemAllocator);
};
// use std::is_default_constructible for c++11
#if __cplusplus < 201103L
template <class T = void> struct __is_default_constructible__;
template <> struct __is_default_constructible__<void>
{
protected:
// Put base typedefs here to avoid pollution
struct twoc { char a, b; };
template <bool> struct test { typedef char type; };
public:
static bool const value = false;
};
template <> struct __is_default_constructible__<>::test<true> { typedef twoc type; };
template <class T> struct __is_default_constructible__ : __is_default_constructible__<>
{
private:
template <class U> static typename test<!!sizeof(::new U())>::type sfinae(U*);
template <class U> static char sfinae(...);
public:
static bool const value = sizeof(sfinae<T>(0)) > 1;
};
#else
template <class T>
using __is_default_constructible__ = std::is_default_constructible<T>;
#endif
template <class T>
struct ObClassConstructor
{
template <class Type, bool Cond = true>
struct Constructor
{
Type *operator ()(void *ptr)
{
return new (ptr) Type();
}
};
template <class Type>
struct Constructor<Type, false>
{
Type *operator ()(void *ptr)
{
return reinterpret_cast<Type *>(ptr);
}
};
T *operator ()(void *ptr)
{
Constructor<T, __is_default_constructible__<T>::value> construct;
return construct(ptr);
}
};
#define RND16(x) (((x) + 15) & ~15)
template<class T>
class ObFixedClassAllocator
@ -431,379 +75,6 @@ private:
ObSliceAlloc allocator_;
};
// Allocator for Class objects. It uses a prototype object to do
// fast initialization. Prototype of the template class is created
// when the fast allocator is created. This is instantiated with
// default (no argument) constructor. Constructor is not called for
// the allocated objects. Instead, the prototype is just memory
// copied onto the new objects. This is done for performance reasons.
template<class T>
class ObClassAllocator : public ObFixedMemAllocator
{
public:
ObClassAllocator() { fl_ = NULL; }
virtual ~ObClassAllocator() { }
// Create a new global class specific ClassAllocator.
// @param name some identifying name, used for mem tracking purposes.
// @param obj_count number of units to be allocated if free pool is
// empty.
// @param alignment of objects must be a power of 2.
// @param is_meta is allocator for meta
static ObClassAllocator<T> *get(const int64_t obj_count = 64,
const ObMemCacheType cache_type = OP_RECLAIM,
const lib::ObLabel &label = common::ObModIds::OB_CONCURRENCY_OBJ_POOL,
const int64_t alignment = 16,
const bool is_meta = false)
{
ObClassAllocator<T> *instance = NULL;
while (OB_UNLIKELY(once_ < 2)) {
if (ATOMIC_BCAS(&once_, 0, 1)) {
instance = new (std::nothrow) ObClassAllocator<T>();
if (OB_LIKELY(NULL != instance)) {
if (common::OB_SUCCESS != instance->init(typeid(T).name(), RND16(sizeof(T)),
obj_count, RND16(alignment), cache_type, is_meta, label)) {
_OB_LOG_RET(ERROR, common::OB_ERR_UNEXPECTED, "failed to init class allocator %s", typeid(T).name());
delete instance;
instance = NULL;
ATOMIC_BCAS(&once_, 1, 0);
} else {
instance_ = instance;
(void)ATOMIC_BCAS(&once_, 1, 2);
}
} else {
(void)ATOMIC_BCAS(&once_, 1, 0);
}
}
}
return (ObClassAllocator<T> *)instance_;
}
// Allocates objects of the templated type.
virtual T *alloc()
{
T *ret = NULL;
void *ptr = NULL;
if (OB_LIKELY(NULL != fl_) && OB_LIKELY(NULL != (ptr = fl_->alloc()))) {
ObClassConstructor<T> construct;
ret = construct(ptr);
}
return ret;
}
// Deallocates objects of the templated type.
// @param ptr pointer to be freed.
virtual void free(T *ptr)
{
if (OB_LIKELY(NULL != fl_) && OB_LIKELY(NULL != ptr)) {
ptr->~T();
fl_->free(ptr);
ptr = NULL;
}
}
// Deallocate objects of the templated type via the inherited
// interface using void pointers.
// @param ptr pointer to be freed.
virtual void free_void(void *ptr) { free(reinterpret_cast<T *>(ptr)); }
protected:
static volatile int64_t once_; // for creating singleton instance
static volatile ObClassAllocator<T> *instance_;
private:
DISALLOW_COPY_AND_ASSIGN(ObClassAllocator);
};
// Allocator for meta of ObObjFreeList, ObChunkInfo, ObThreadCache. The other
// objects need created object pools with these three objects. But these three
// objects are also allocated form their object pools. The object pool is
// implied with these three objects. So the object pools of these three objects
// need an especial allocator to allocate themself.
template<class T>
class ObMetaAllocator : public ObFixedMemAllocator
{
public:
// Create a new class specific ClassAllocator.
// @param mod allocate mode.
explicit ObMetaAllocator(const lib::ObLabel &label)
{
free_list_ = NULL;
allocator_.set_label(label);
(void)mutex_init(&allocator_lock_);
}
virtual ~ObMetaAllocator() { (void)mutex_destroy(&allocator_lock_); }
// Allocates objects of the templated type.
T *alloc()
{
int ret = OB_SUCCESS;
T *obj = NULL;
void *ptr = NULL;
if (OB_SUCC(mutex_acquire(&allocator_lock_))) {
if (OB_LIKELY(NULL != free_list_)) {
ptr = free_list_;
free_list_ = *(reinterpret_cast<void **>(ptr));
} else {
ptr = allocator_.alloc_aligned(sizeof(T), 16);
}
mutex_release(&allocator_lock_);
}
if (OB_LIKELY(NULL != ptr)) {
obj = new (ptr) T();
}
return obj;
}
// Deallocates objects of the templated type.
// @param ptr pointer to be freed.
void free(T *ptr)
{
int ret = OB_SUCCESS;
if (OB_LIKELY(NULL != ptr)) {
ptr->~T();
if (OB_SUCC(mutex_acquire(&allocator_lock_))) {
*(reinterpret_cast<void **>(ptr)) = free_list_;
free_list_ = ptr;
mutex_release(&allocator_lock_);
}
}
}
// Allocate objects of the templated type via the inherited interface
// using void pointers.
virtual void *alloc_void() { return reinterpret_cast<void *>(alloc()); }
// Deallocate objects of the templated type via the inherited
// interface using void pointers.
// @param ptr pointer to be freed.
virtual void free_void(void *ptr) { free(reinterpret_cast<T *>(ptr)); }
private:
void *free_list_;
PageArena<char> allocator_;
::ObMutex0 allocator_lock_;
DISALLOW_COPY_AND_ASSIGN(ObMetaAllocator);
};
class ObObjFreeListList
{
friend class ObObjFreeList;
public:
static const int64_t MAX_NUM_FREELIST;
static pthread_key_t objpool_key_;
~ObObjFreeListList() { }
int create_freelist(ObObjFreeList *&free_list,
const char *name,
const int64_t obj_size,
const int64_t obj_count,
const int64_t alignment = 16,
const ObMemCacheType cache_type = OP_RECLAIM,
const bool is_meta = false,
const lib::ObLabel &label = common::ObModIds::OB_CONCURRENCY_OBJ_POOL);
static ObObjFreeListList &get_freelists()
{
ObObjFreeListList *instance = NULL;
while (OB_UNLIKELY(once_ < 2)) {
if (ATOMIC_BCAS(&once_, 0, 1)) {
instance = new (std::nothrow) ObObjFreeListList();
if (OB_LIKELY(NULL != instance)) {
if (common::OB_SUCCESS != instance->init()) {
_OB_LOG_RET(ERROR, common::OB_ERR_UNEXPECTED, "failed to init object freelist list");
delete instance;
instance = NULL;
(void)ATOMIC_BCAS(&once_, 1, 0);
} else {
instance_ = instance;
(void)ATOMIC_BCAS(&once_, 1, 2);
}
} else {
(void)ATOMIC_BCAS(&once_, 1, 0);
}
}
}
return (ObObjFreeListList &)*instance_;
}
int64_t get_next_free_slot() { return ATOMIC_FAA(&nr_freelists_, 1); }
int64_t get_freelist_count() { return ATOMIC_LOAD(&nr_freelists_); }
int get_info(ObVector<ObObjFreeList *> &flls);
void dump();
int64_t to_string(char *buf, const int64_t len) const;
private:
int init()
{
int ret = OB_SUCCESS;
if (OB_FAIL(fl_allocator_.init(*this, "ObjFreeList", sizeof(ObObjFreeList), 64, 16, OP_GLOBAL, true))) {
OB_LOG(WARN, "failed to init ObjFreeList allocator", K(ret));
} else if (OB_FAIL(tc_allocator_.init(*this, "ObThreadCache", sizeof(ObThreadCache), 64, 16, OP_RECLAIM, true))) {
OB_LOG(WARN, "failed to init ObThreadCache allocator", K(ret));
} else if (0 != pthread_key_create(&objpool_key_, thread_shutdown_cleanup)) {
OB_LOG(WARN, "failed to init pthread key", K(ret));
} else {
is_inited_ = true;
}
return ret;
}
private:
static volatile int64_t once_; // for creating singleton instance
ObObjFreeListList();
static volatile ObObjFreeListList *instance_;
ObAtomicSLL<ObObjFreeList> freelists_; // global free list list
volatile int64_t nr_freelists_;
volatile int64_t mem_total_;
bool is_inited_;
ObMetaAllocator<ObThreadCache> tc_meta_allocator_;
ObClassAllocator<ObObjFreeList> fl_allocator_;
ObClassAllocator<ObThreadCache> tc_allocator_;
DISALLOW_COPY_AND_ASSIGN(ObObjFreeListList);
};
inline int ObFixedMemAllocator::init(
ObObjFreeListList &fll,
const char *name,
const int64_t obj_size,
const int64_t obj_count,
const int64_t alignment,
const ObMemCacheType cache_type /* = OP_RECLAIM */,
const bool is_meta /* = false */,
const lib::ObLabel &label/* = common::ObModIds::OB_CONCURRENCY_OBJ_POOL */)
{
int ret = OB_SUCCESS;
if (OB_FAIL(fll.create_freelist(fl_, name, obj_size, obj_count,
alignment, cache_type, is_meta, label))) {
// do nothing
} else if (OB_ISNULL(fl_)) {
ret = OB_ERR_UNEXPECTED;
}
return ret;
}
inline int ObFixedMemAllocator::init(
const char *name, const int64_t obj_size,
const int64_t obj_count, const int64_t alignment,
const ObMemCacheType cache_type /* = OP_RECLAIM */,
const bool is_meta /* = false */,
const lib::ObLabel &label/* = common::ObModIds::OB_CONCURRENCY_OBJ_POOL */)
{
return init(ObObjFreeListList::get_freelists(), name, obj_size, obj_count,
alignment, cache_type, is_meta, label);
}
struct ObThreadFreeList
{
ObThreadFreeList() : reserved_size_(64), allocated_(0), freelist_(NULL) { }
~ObThreadFreeList() { }
int64_t reserved_size_;
int64_t allocated_;
void *freelist_;
};
template<class T>
volatile int64_t ObClassAllocator<T>::once_ = 0;
template<class T>
volatile ObClassAllocator<T> *ObClassAllocator<T>::instance_ = NULL;
template <class T>
struct OPNum
{
template <class Type, bool Cond = true>
struct GetOPLocalNum
{
static const int64_t v = Type::OP_LOCAL_NUM;
};
template <class Type>
struct GetOPLocalNum<Type, false>
{
static const int64_t v = sizeof(Type) < 16 * 1024 ? 32 : 8;
};
static const int64_t LOCAL_NUM = GetOPLocalNum<T, HAS_MEMBER(T, OP_LOCAL_NUM)>::v;
template <class Type, bool Cond = true>
struct GetOPLabel
{
static constexpr const char *v = Type::OP_LABEL;
};
template <class Type>
struct GetOPLabel<Type, false>
{
static constexpr const char *v = common::ObModIds::OB_CONCURRENCY_OBJ_POOL;
};
static constexpr const char *LABEL = GetOPLabel<T, HAS_MEMBER(T, OP_LABEL)>::v;
};
template <class T>
template <class Type>
constexpr const char *OPNum<T>::GetOPLabel<Type, false>::v;
template <class T>
constexpr const char *OPNum<T>::LABEL;
template <class Type, bool Cond = true>
struct Constructor
{
Type *operator ()(void *ptr)
{
return new (ptr) Type();
}
};
template <class Type>
struct Constructor<Type, false>
{
Type *operator ()(void *ptr)
{
return reinterpret_cast<Type *>(ptr);
}
};
template<class T>
static T *call_constructor(T *ptr)
{
Constructor<T, __is_default_constructible__<T>::value> construct;
return construct(ptr);
}
template<class T>
inline void call_destructor(T *ptr) {
ptr->~T();
}
// object pool user interface
// Warning: 1. the cached memory can't be freed even through the thread is dead.
// 2. double free is very terriable.
// 3. because object pool uses singleton, please only use one of global,
// tc or reclaim interfaces for each object type in the whole procject.
// Note:
// op_alloc and op_reclaim_alloc call the default constructor if it exist,
// else it just reinterpret_cast ptr.
//
// op_alloc_args and op_reclaim_args call the constructor with args.
// It uses placement new to construct instance, if args is null and there isn't public
// default constructor, compiler isn't happy.
//
// op_alloc_args uses global object freelist, save memory but performance is poor.
// op_reclaim_alloc_args uses thread local object free list and with memory reclaim,
// performace is good and object waste less memory.
#define op_alloc_args(type, args...) \
({ \
type *ret = NULL; \
@ -842,77 +113,8 @@ inline void call_destructor(T *ptr) {
} \
})
// global pool allocator interface
#define op_g_alloc_args(type, args...) \
({ \
type *ret = NULL; \
common::ObClassAllocator<type> *instance = \
common::ObClassAllocator<type>::get(common::OPNum<type>::LOCAL_NUM, common::OP_GLOBAL, \
common::OPNum<type>::LABEL); \
if (OB_LIKELY(NULL != instance)) { \
void *tmp = instance->alloc_void(); \
if (OB_LIKELY(NULL != tmp)) { \
ret = new (tmp) type(args); \
} \
} \
ret; \
})
#define op_g_alloc(type) \
({ \
OLD_STATIC_ASSERT((std::is_default_constructible<type>::value), "type is not default constructible"); \
type *ret = NULL; \
common::ObClassAllocator<type> *instance = \
common::ObClassAllocator<type>::get(common::OPNum<type>::LOCAL_NUM, common::OP_GLOBAL, \
common::OPNum<type>::LABEL); \
if (OB_LIKELY(NULL != instance)) { \
ret = instance->alloc(); \
} \
ret; \
})
#define op_g_free(ptr) \
({ \
common::ObClassAllocator<__typeof__(*ptr)> *instance = \
common::ObClassAllocator<__typeof__(*ptr)>::get(common::OPNum<__typeof__(*ptr)>::LOCAL_NUM, \
common::OP_GLOBAL, \
common::OPNum<__typeof__(*ptr)>::LABEL); \
if (OB_LIKELY(NULL != instance)) { \
instance->free(ptr); \
} \
})
#define op_reclaim_alloc_old(type) \
({ \
OLD_STATIC_ASSERT((std::is_default_constructible<type>::value), "type is not default constructible"); \
type *ret = NULL; \
common::ObClassAllocator<type> *instance = \
common::ObClassAllocator<type>::get(common::OPNum<type>::LOCAL_NUM, common::OP_RECLAIM, \
common::OPNum<type>::LABEL); \
if (OB_LIKELY(NULL != instance)) { \
ret = instance->alloc(); \
} \
ret; \
})
#define op_reclaim_free_old(ptr) \
({ \
common::ObClassAllocator<__typeof__(*ptr)> *instance = \
common::ObClassAllocator<__typeof__(*ptr)>::get(common::OPNum<__typeof__(*ptr)>::LOCAL_NUM, \
common::OP_RECLAIM, \
common::OPNum<__typeof__(*ptr)>::LABEL); \
if (OB_LIKELY(NULL != instance)) { \
instance->free(ptr); \
} \
})
#ifndef PERF_MODE
#define op_reclaim_alloc(type) op_alloc(type)
#define op_reclaim_free(ptr) op_free(ptr)
#else
#define op_reclaim_alloc(type) op_reclaim_alloc_old(type)
#define op_reclaim_free(ptr) op_reclaim_free_old(ptr)
#endif
} // end of namespace common
} // end of namespace oceanbase

192
deps/oblib/src/lib/ptr/ob_ptr.h vendored
View File

@ -1,192 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OB_LIB_PTR_H_
#define OB_LIB_PTR_H_
#include "lib/ob_define.h"
#include "lib/atomic/ob_atomic.h"
namespace oceanbase
{
namespace common
{
class ObRefCount
{
public:
ObRefCount() : ref_count_(0) {}
virtual ~ObRefCount() {}
virtual void free() = 0;
inline void inc_ref() { ATOMIC_FAA(&ref_count_, 1); }
inline void dec_ref()
{
if (1 == ATOMIC_FAA(&ref_count_, -1)) {
free();
}
}
volatile int64_t ref_count_;
};
class ObForceVFPTToTop
{
public:
ObForceVFPTToTop() { }
virtual ~ObForceVFPTToTop() { }
};
// class ObRefCountObj
// prototypical class for reference counting
class ObRefCountObj : public ObForceVFPTToTop
{
public:
ObRefCountObj() : refcount_(0) { }
virtual ~ObRefCountObj() { }
ObRefCountObj(const ObRefCountObj &s) : refcount_(0)
{
UNUSED(s);
}
ObRefCountObj &operator=(const ObRefCountObj &s)
{
UNUSED(s);
return (*this);
}
int64_t refcount_inc();
int64_t refcount_dec();
int64_t refcount() const;
virtual void free() { delete this; }
volatile int64_t refcount_;
};
// Increment the reference count, returning the new count.
inline int64_t ObRefCountObj::refcount_inc()
{
return ATOMIC_FAA(&refcount_, 1) + 1;
}
// Decrement the reference count, returning the new count.
inline int64_t ObRefCountObj::refcount_dec()
{
return ATOMIC_FAA(&refcount_, -1) - 1;
}
inline int64_t ObRefCountObj::refcount() const
{
return refcount_;
}
// class ObPtr
template<class T>
class ObPtr
{
public:
explicit ObPtr(T *p = NULL);
ObPtr(const ObPtr<T> &);
~ObPtr();
void release();
ObPtr<T> &operator=(const ObPtr<T>&);
ObPtr<T> &operator=(T*);
operator T *() const { return (ptr_); }
T *operator->() const { return (ptr_); }
T &operator*() const { return (*ptr_); }
bool operator==(const T *p) { return (ptr_ == p); }
bool operator==(const ObPtr<T> &p) { return (ptr_ == p.ptr_); }
bool operator!=(const T *p) { return (ptr_ != p); }
bool operator!=(const ObPtr<T> &p) { return (ptr_ != p.ptr_); }
ObRefCountObj *get_ptr() { return reinterpret_cast<ObRefCountObj *>(ptr_); }
T *ptr_;
};
template<typename T>
ObPtr<T> make_ptr(T *p)
{
return ObPtr<T>(p);
}
template<class T>
inline ObPtr<T>::ObPtr(T *ptr /* = NULL */) : ptr_(ptr)
{
if (NULL != ptr_) {
get_ptr()->refcount_inc();
}
}
template<class T>
inline ObPtr<T>::ObPtr(const ObPtr<T> &src) : ptr_(src.ptr_)
{
if (NULL != ptr_) {
get_ptr()->refcount_inc();
}
}
template<class T>
inline ObPtr<T>::~ObPtr()
{
if ((NULL != ptr_) && 0 == get_ptr()->refcount_dec()) {
get_ptr()->free();
}
return;
}
template<class T>
inline ObPtr<T>& ObPtr<T>::operator = (T *p)
{
T *temp_ptr = ptr_;
if (ptr_ != p) {
ptr_ = p;
if (NULL != ptr_) {
get_ptr()->refcount_inc();
}
if ((NULL != temp_ptr) && 0 == (reinterpret_cast<ObRefCountObj *>(temp_ptr))->refcount_dec()) {
(reinterpret_cast<ObRefCountObj *>(temp_ptr))->free();
}
}
return (*this);
}
template<class T>
inline void ObPtr<T>::release()
{
if (NULL != ptr_) {
if (0 == (reinterpret_cast<ObRefCountObj *>(ptr_))->refcount_dec()) {
(reinterpret_cast<ObRefCountObj *>(ptr_))->free();
}
ptr_ = NULL;
}
}
template<class T>
inline ObPtr<T>& ObPtr<T>::operator = (const ObPtr<T> &src)
{
return (operator =(src.ptr_));
}
} // end of namespace common
} // end of namespace oceanbase
#endif // OB_LIB_PTR_H_

2
deps/oblib/src/lib/thread/thread_define.h vendored
View File

@ -23,7 +23,7 @@ TG_DEF(TEST7, test7, "", TG_STATIC, QUEUE_THREAD, ThreadCountPair(10 ,10), 10)
TG_DEF(TEST8, test8, "", TG_STATIC, REENTRANT_THREAD_POOL, ThreadCountPair(1 ,1))
// other
TG_DEF(MEMORY_DUMP, memDump, "", TG_STATIC, THREAD_POOL, ThreadCountPair(1, 1))
TG_DEF(SchemaRefTask, SchemaRefTask, "", TG_STATIC, DEDUP_QUEUE, ThreadCountPair(1, 1), 1024, 1024, 1L << 30, 512L << 20, common::OB_MALLOC_BIG_BLOCK_SIZE, common::ObModIds::OB_SCHEMA_DEDUP_QUEUE)
TG_DEF(SchemaRefTask, SchemaRefTask, "", TG_STATIC, DEDUP_QUEUE, ThreadCountPair(1, 1), 1024, 1024, 1L << 30, 512L << 20, common::OB_MALLOC_BIG_BLOCK_SIZE, "SchemaDedupQueu")
TG_DEF(CONFIG_MGR, ConfigMgr, "", TG_STATIC, TIMER, 1024)
TG_DEF(ReqMemEvict, ReqMemEvict, "", TG_DYNAMIC, TIMER)
TG_DEF(IO_TUNING, IO_TUNING, "", TG_STATIC, THREAD_POOL, ThreadCountPair(1, 1))

42
deps/oblib/src/rpc/obmysql/ob_mysql_handler.cpp vendored
View File

@ -115,41 +115,6 @@ void do_wakeup_request(easy_request_t *r)
do_wakeup(r);
}
#ifdef PERF_MODE
int do_layer_perf_response(easy_request_t *r)
{
//response
int ret = OB_SUCCESS;
void *buf = easy_pool_alloc(r->ms->pool, OB_MULTI_RESPONSE_BUF_SIZE + sizeof(easy_buf_t));
easy_buf_t *b = reinterpret_cast<easy_buf_t*>(buf);
init_easy_buf(b, reinterpret_cast<char*>(b + 1), r, OB_MULTI_RESPONSE_BUF_SIZE);
OMPKOK okp;
ObMySQLCapabilityFlags cap_flags(163553933);
okp.set_seq(1);
okp.set_capability(cap_flags);
int64_t seri_size = 0;
if (OB_FAIL(okp.encode(b->last, b->end - b->pos, seri_size))) {
LOG_ERROR("okp.serialize", K(ret));
} else {
b->last += seri_size;
//easy_request_addbuf(r, b);
r->opacket = b;
}
return ret;
}
bool do_rpc_layer_perf(easy_request_t *r)
{
bool hit = false;
const oceanbase::obmysql::ObMySQLRawPacket &pkt
= reinterpret_cast<const oceanbase::obmysql::ObMySQLRawPacket &>(*((ObPacket*)(r->ipacket)));
if (pkt.get_cmd() == ObMySQLCmd::COM_QUERY && pkt.get_clen() > 9 && memcmp(pkt.get_cdata(), "[PM_EASY]", 9) == 0) {
hit = true;
}
return hit;
}
#endif
int ObMySQLHandler::process(easy_request_t *r)
{
int eret = EASY_OK;
@ -187,13 +152,6 @@ int ObMySQLHandler::process(easy_request_t *r)
eret = EASY_OK;
LOG_INFO("MySQL SSL Request", "sessid", sessid,
KCSTRING(easy_connection_str(r->ms->c)));
#ifdef PERF_MODE
} else if (do_rpc_layer_perf(r)) {
if (do_layer_perf_response(r) != OB_SUCCESS) {
ob_abort();
}
eret = EASY_OK;
#endif
} else if (OB_ISNULL(buf = easy_alloc(r->ms->pool, sizeof (ObRequest)))) {
RPC_LOG_RET(WARN, common::OB_ALLOCATE_MEMORY_FAILED, "alloc easy memory fail", K(sessid));
eret = EASY_BREAK;

4
deps/oblib/src/rpc/obmysql/ob_mysql_request_utils.h vendored
View File

@ -53,7 +53,7 @@ public:
READ_BODY,
READ_COMPLETE
};
ObMysqlPktContext() : arena_(common::ObModIds::LIB_MULTI_PACKETS) { reset(); }
ObMysqlPktContext() : arena_("LibMultiPackets") { reset(); }
~ObMysqlPktContext() {}
void reset()
{
@ -137,7 +137,7 @@ private:
class ObProto20PktContext
{
public:
ObProto20PktContext() : arena_(common::ObModIds::LIB_MULTI_PACKETS){ reset(); }
ObProto20PktContext() : arena_("LibMultiPackets"){ reset(); }
~ObProto20PktContext() { }
void reset()
{

31
deps/oblib/src/rpc/obmysql/packet/ompk_ping.h vendored
View File

@ -1,31 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef _OMPK_PING_H_
#define _OMPK_PING_H_
#include "rpc/obmysql/ob_mysql_packet.h"
namespace oceanbase
{
namespace obmysql
{
class OMPKPing
: public ObMySQLRawPacket
{
}; // end of class OMPKPing
} // end of namespace obmysql
} // end of namespace oceanbase
#endif /* _OMPK_PING_H_ */

4
deps/oblib/src/rpc/obrpc/ob_listener.h vendored
View File

@ -27,11 +27,7 @@ namespace obrpc
#define NEGOTIATION_PACKET_HEADER_MAGIC_EASY (0x1234567877668833)
#define MAX_PROTOCOL_TYPE_SIZE (5)
#ifdef PERF_MODE
#define OB_LISTENER_MAX_THREAD_CNT 128
#else
#define OB_LISTENER_MAX_THREAD_CNT 64
#endif
inline struct sockaddr_in* make_unix_sockaddr(struct sockaddr_in *sin, in_addr_t ip, int port) {
if (NULL != sin) {

4
deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.h vendored
View File

@ -183,11 +183,7 @@ public:
if (get_proxy_group_id(proxy) == ObPocServerHandleContext::OBCG_ELECTION) {
src_tenant_id = OB_SERVER_TENANT_ID;
}
#ifndef PERF_MODE
const int init_alloc_sz = 0;
#else
const int init_alloc_sz = 400<<10;
#endif
auto &set = obrpc::ObRpcPacketSet::instance();
const char* pcode_label = set.name_of_idx(set.idx_of_pcode(pcode));
if (NULL == (pool = ObRpcMemPool::create(src_tenant_id, pcode_label, init_alloc_sz))) {

4
deps/oblib/src/rpc/obrpc/ob_poc_rpc_server.cpp vendored
View File

@ -46,11 +46,7 @@ int ObPocServerHandleContext::create(int64_t resp_id, const char* buf, int64_t s
int ret = OB_SUCCESS;
ObPocServerHandleContext* ctx = NULL;
ObRpcPacket tmp_pkt;
#ifndef PERF_MODE
const int64_t alloc_payload_sz = sz;
#else
const int64_t alloc_payload_sz = 0;
#endif
if (OB_FAIL(tmp_pkt.decode(buf, sz))) {
RPC_LOG(ERROR, "decode packet fail", K(ret));
} else {

4
deps/oblib/src/rpc/obrpc/ob_rpc_endec.h vendored
View File

@ -46,11 +46,7 @@ template <typename T>
ObRpcPacket pkt;
const int64_t header_sz = pkt.get_header_size();
const int64_t payload_sz = calc_extra_payload_size() + common::serialization::encoded_length(args);
#ifdef PERF_MODE
const int64_t reserve_bytes_for_pnio = 200;
#else
const int64_t reserve_bytes_for_pnio = 0;
#endif
char* header_buf = (char*)pool.alloc(reserve_bytes_for_pnio + header_sz + payload_sz) + reserve_bytes_for_pnio;
char* payload_buf = header_buf + header_sz;
int64_t pos = 0;

44
deps/oblib/src/rpc/pnio/interface/group.c vendored
View File

@ -1,11 +1,7 @@
#define MAX_PN_LISTEN 256
#define MAX_PN_GRP (1<<17)
#define MAX_PN_PER_GRP 64
#ifdef PERF_MODE
#define CHUNK_SIZE ((1<<21) - (17<<10))
#else
#define CHUNK_SIZE (1<<14) - 128
#endif
typedef struct pn_listen_t
{
@ -276,28 +272,6 @@ typedef struct pn_client_slice_t
pn_client_req_t req_;
} pn_client_slice_t;
#ifdef PERF_MODE
static void pn_pktc_flush_cb(pktc_req_t* r)
{
pn_client_req_t* pn_req = structof(r, pn_client_req_t, req);
pn_client_slice_t* slice = structof(pn_req, pn_client_slice_t, req_);
if (0 == --slice->ref_) {
pn_pktc_cb_t* pn_cb = &slice->cb_;
PNIO_DELAY_WARN(STAT_TIME_GUARD(eloop_client_cb_count, eloop_client_cb_time));
pn_cb->client_cb(pn_cb->arg, (&pn_cb->cb)->errcode, NULL, 0);
}
}
static void pn_pktc_resp_cb(pktc_cb_t* cb, const char* resp, int64_t sz)
{
pn_pktc_cb_t* pn_cb = structof(cb, pn_pktc_cb_t, cb);
pn_client_slice_t* slice = structof(pn_cb, pn_client_slice_t, cb_);
if (0 == ++slice->ref_) {
PNIO_DELAY_WARN(STAT_TIME_GUARD(eloop_client_cb_count, eloop_client_cb_time));
pn_cb->client_cb(pn_cb->arg, cb->errcode, resp, sz);
}
}
#else
static void pn_pktc_flush_cb(pktc_req_t* r)
{
pn_client_req_t* pn_req = structof(r, pn_client_req_t, req);
@ -319,19 +293,9 @@ static void pn_pktc_resp_cb(pktc_cb_t* cb, const char* resp, int64_t sz)
pn_cb->client_cb(pn_cb->arg, cb->errcode, resp, sz);
cfifo_free(pn_cb);
}
#endif
static pktc_req_t* pn_create_pktc_req(pn_t* pn, uint64_t pkt_id, addr_t dest, const char* req, int64_t req_sz, int16_t categ_id, int64_t expire_us, client_cb_t client_cb, void* arg)
{
#ifdef PERF_MODE
pn_client_slice_t* slice = ((typeof(slice))req) - 1;
if (unlikely(NULL == slice)) {
return NULL;
}
pn_client_req_t* pn_req = &slice->req_;
struct pn_pktc_cb_t* pn_cb = &slice->cb_;
slice->ref_ = 0;
#else
pn_client_req_t* pn_req = (typeof(pn_req))cfifo_alloc(&pn->client_req_alloc, sizeof(*pn_req) + req_sz);
if (unlikely(NULL == pn_req)) {
return NULL;
@ -341,7 +305,6 @@ static pktc_req_t* pn_create_pktc_req(pn_t* pn, uint64_t pkt_id, addr_t dest, co
cfifo_free(pn_req);
return NULL;
}
#endif
pktc_cb_t* cb = &pn_cb->cb;
pktc_req_t* r = &pn_req->req;
pn_cb->client_cb = client_cb;
@ -421,11 +384,7 @@ typedef struct pn_resp_ctx_t
void* req_handle;
uint64_t sock_id;
uint64_t pkt_id;
#ifdef PERF_MODE
char reserve[1<<10];
#else
char reserve[sizeof(pkts_req_t)];
#endif
} pn_resp_ctx_t;
static pn_resp_ctx_t* create_resp_ctx(pn_t* pn, void* req_handle, uint64_t sock_id, uint64_t pkt_id)
@ -484,9 +443,6 @@ static void pn_pkts_flush_cb_error_func(pkts_req_t* req)
PN_API int pn_resp(uint64_t req_id, const char* buf, int64_t sz, int64_t resp_expired_abs_us)
{
pn_resp_ctx_t* ctx = (typeof(ctx))req_id;
#ifdef PERF_MODE
ref_free(ctx->req_handle);
#endif
pn_resp_t* resp = NULL;
if (sizeof(pn_resp_t) + sz <= sizeof(ctx->reserve)) {
resp = (typeof(resp))(ctx->reserve);

4
deps/oblib/src/rpc/pnio/nio/packet_server.c vendored
View File

@ -16,11 +16,7 @@ static int pkts_sk_read(void** b, pkts_sk_t* s, int64_t sz, int64_t* avail_bytes
static int pkts_sk_handle_msg(pkts_sk_t* s, pkts_msg_t* msg) {
pkts_t* pkts = structof(s->fty, pkts_t, sf);
#ifdef PERF_MODE
int ret = pkts->on_req(pkts, ib_ref(&s->ib), msg->payload, msg->sz, s->id);
#else
int ret = pkts->on_req(pkts, s->ib.b, msg->payload, msg->sz, s->id);
#endif
ib_consumed(&s->ib, msg->sz);
return ret;
}

3
deps/oblib/src/rpc/pnio/nio/pktc_resp.h vendored
View File

@ -4,8 +4,6 @@ static void pktc_do_cb(pktc_t* io, pktc_cb_t* cb, pktc_msg_t* m) {
}
static void pktc_do_cb_exception(pktc_t* io, pktc_cb_t* cb) {
#ifdef PERF_MODE
#else
pktc_req_t* req = cb->req;
if (req) {
// pktc_flush_cb hasn't be executed
@ -21,7 +19,6 @@ static void pktc_do_cb_exception(pktc_t* io, pktc_cb_t* cb) {
pktc_flush_cb(io, req);
}
}
#endif
cb->resp_cb(cb, NULL, 0);
}

2
deps/oblib/unittest/common/test_segmented_buffer.cpp vendored
View File

@ -22,7 +22,7 @@ using namespace oceanbase::lib;
TEST(TestObSegmentedBuffer, base)
{
const int block_size = 100;
ObMemAttr attr(OB_SERVER_TENANT_ID, ObModIds::OB_MEM_META,
ObMemAttr attr(OB_SERVER_TENANT_ID, "MemMeta",
ObCtxIds::DEFAULT_CTX_ID);
ObSegmentedBufffer sb(block_size, attr);
char buf[block_size * 10];

37
deps/oblib/unittest/lib/thread/test_thread_pool.h vendored
View File

@ -1,37 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#include <iostream>
#include "lib/thread/thread_pool.h"
using namespace oceanbase::lib;
using namespace std;
TEST(TestThreadPool, Submit)
{
class : public ThreadPool
{
void run1() override()
{
cout << "ok" << endl;
}
} tp;
tp.init();
tp.start();
}
int main(int argc, char *argv[])
{
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

182
src/libtable/src/ob_htable.cpp
View File

@ -1,182 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX CLIENT
#include "ob_htable.h"
using namespace oceanbase::common;
using namespace oceanbase::table;
using namespace oceanbase::table::hbase;
int ObHTable::put(const ObHPut &put)
{
int ret = OB_SUCCESS;
int64_t N = put.get_column_count();
ObHKVTable::Entities entities;
for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i)
{
ObHCell cell;
put.get_column(i, cell);
ObHKVTable::Key key;
key.rowkey_ = put.get_row();
// @todo ignore cell.get_column_family();
key.column_qualifier_ = cell.get_column_qualifier();
key.version_ = cell.get_timestamp();
ObHKVTable::Value value;
value.set_varbinary(cell.get_value());
OHKVTable::Entity entity;
entity.set_key(key);
entity.set_value(value);
entities.push_back(entity);
} // end for
ret = hkv_table_->multi_put(entities);
return ret;
}
int ObHTable::multi_put(const ObIArray<ObHPut> &puts)
{
int ret = OB_SUCCESS;
const int64_t N = puts.count();
for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i)
{
} // end for
return ret;
}
int ObHTable::del(const ObHDelete &del)
{
int ret = OB_SUCCESS;
return ret;
}
int ObHTable::multi_del(const ObIArray<ObHDelete> &deletes);
int ObHTable::mutate_row(const ObHRowMutations &row_mutations);
// with timestamp range or no timestamp
int ObHTable::get_by_mscan_with_time_range(const ObHGet &get)
{
int ret = OB_SUCCESS;
ObTableQuery query;
query.add_select_column(ObHKVTable::CQ_CNAME_STR);
query.add_select_column(ObHKVTable::VERSION_CNAME_STR);
query.add_select_column(ObHKVTable::VALUE_CNAME_STR);
query.set_scan_order(ObQueryFlag::Forward);
ObHTableFilter htable_filter;
htable_filter.set_max_versions(scan.get_max_versions());
htable_filter.set_limit(scan.get_limit());
const int64_t N = get.get_column_count();
for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i)
{
htable_filter.add_column(scan.get_qualifier(i));
}
query.set_htable_filter(htable_filter);
ObObj pk_objs_start[3];
pk_objs_start[0].set_varbinary(get.get_row());
pk_objs_start[1].set_min_value();
pk_objs_start[2].set_min_value();
ObObj pk_objs_end[3];
pk_objs_end[0].set_varbinary(get.get_row());
pk_objs_start[1].set_max_value();
pk_objs_start[2].set_max_value();
ObNewRange range;
range.start_key_.assign(pk_objs_start, 3);
range.end_key_.assign(pk_objs_end, 3);
range.border_flag_.set_inclusive_start();
range.border_flag_.unset_inclusive_end();
query.add_scan_range(range);
ObTableEntityIterator *iter = nullptr;
ret = hkv_table_->execute_query(query, iter);
return ret;
}
// with specific timestamp
int ObHTable::get_by_mget(const ObHGet &get)
{
int ret = OB_SUCCESS;
int64_t N = get.get_column_count();
ObHKVTable::Keys keys;
for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i)
{
ObHCell cell;
get.get_column(i, cell);
ObHKVTable::Key key;
key.rowkey_ = put.get_row();
// @todo ignore cell.get_column_family();
key.column_qualifier_ = cell.get_column_qualifier();
key.version_ = cell.get_timestamp();
keys.push_back(entity);
} // end for
ObHKVTable::Values values;
ret = hkv_table_->multi_get(keys, values);
return ret;
}
int ObHTable::multi_get(const ObIArray<ObHGet> &gets);
int ObHTable::scan(const ObHScan &scan)
{
int ret = OB_SUCCESS;
ObTableQuery query;
query.add_select_column(ObHKVTable::ROWKEY_CNAME_STR);
query.add_select_column(ObHKVTable::CQ_CNAME_STR);
query.add_select_column(ObHKVTable::VERSION_CNAME_STR);
query.add_select_column(ObHKVTable::VALUE_CNAME_STR);
query.set_scan_order(scan.reversed() ? ObQueryFlag::Reversed : ObQueryFlag::Forward);
ObHTableFilter htable_filter;
htable_filter.set_max_versions(scan.get_max_versions());
htable_filter.set_limit(scan.get_limit());
const int64_t N = scan.get_column_count();
for (int64_t i = 0; OB_SUCCESS == ret && i < N; ++i)
{
htable_filter.add_column(scan.get_qualifier(i));
}
query.set_htable_filter(htable_filter);
ObObj pk_objs_start[3];
pk_objs_start[0].set_varbinary(scan.get_start_row());
pk_objs_start[1].set_min_value();
pk_objs_start[2].set_min_value();
ObObj pk_objs_end[3];
pk_objs_end[0].set_varbinary(scan.get_stop_row());
pk_objs_start[1].set_max_value();
pk_objs_start[2].set_max_value();
ObNewRange range;
range.start_key_.assign(pk_objs_start, 3);
range.end_key_.assign(pk_objs_end, 3);
if (scan.include_start_row()) {
range.border_flag_.set_inclusive_start();
} else {
range.border_flag_.unset_inclusive_start();
}
if (scan.include_stop_row()) {
range.border_flag_.set_inclusive_end();
} else {
range.border_flag_.unset_inclusive_end();
}
query.add_scan_range(range);
query.set_batch(scan.get_batch());
ObTableEntityIterator *iter = nullptr;
ret = hkv_table_->execute_query(query, iter);
return ret;
}

241
src/libtable/src/ob_htable.h
View File

@ -1,241 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef _OB_HTABLE_H
#define _OB_HTABLE_H 1
#include "ob_hkv_table.h"
namespace oceanbase
{
namespace table
{
/// HBase Interface
namespace hbase
{
class ObHMutation
{
public:
ObHMutation();
virtual ~ObHMutation();
const ObString &get_row() const;
private:
// types and constants
private:
// disallow copy
DISALLOW_COPY_AND_ASSIGN(ObHMutation);
// function members
protected:
// data members
ObString rowkey_;
int64_t ts_;
};
class ObHCell
{
public:
const ObString &get_row();
const ObString &get_column_family();
const ObString &get_column_qualifier();
int64_t get_timestamp();
const ObString &get_value();
};
// http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/Put.html
class ObHPut: public ObHMutation
{
public:
ObHPut(const ObString &row);
virtual ~ObHPut();
int add_column(const ObString &family, const ObString &qualifier, int64_t ts, const ObString &value);
int64_t get_column_count() const;
int get_column(int64_t i, ObHCell &cell);
private:
// types and constants
private:
// disallow copy
DISALLOW_COPY_AND_ASSIGN(ObHPut);
// function members
private:
// data members
};
// http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/Delete.html
class ObHDelete: public ObHMutation
{
public:
ObHDelete(const ObString &row);
virtual ~ObHDelete();
/// Delete the latest version of the specified column.
int add_column(const ObString &family, const ObString &qualifier);
/// Delete the specified version of the specified column.
int add_column(const ObString &family, const ObString &qualifier, int64_t timestamp);
/// Delete all versions of the specified column.
int add_columns(const ObString &family, const ObString &qualifier);
/// Delete all versions of the specified column with a timestamp less than or equal to the specified timestamp.
int add_columns(const ObString &family, const ObString &qualifier, int64_t timestamp);
private:
// types and constants
private:
// disallow copy
DISALLOW_COPY_AND_ASSIGN(ObHDelete);
// function members
private:
// data members
};
class ObHRowMutation
{
public:
ObHRowMutation(const ObString &row);
virtual ~ObHRowMutation();
int add(const ObHMutation &mutation);
private:
// types and constants
private:
// disallow copy
DISALLOW_COPY_AND_ASSIGN(ObHRowMutation);
// function members
private:
// data members
};
class ObHQuery
{
public:
ObHQuery();
virtual ~ObHQuery();
private:
// types and constants
private:
// disallow copy
DISALLOW_COPY_AND_ASSIGN(ObHQuery);
// function members
private:
// data members
};
class ObHGet: public ObHQuery
{
public:
ObHGet(const ObString &row);
virtual ~ObHGet();
/// Get the column from the specific family with the specified qualifier.
int add_column(const ObString &family, const ObString &qualifier);
/// Get all columns from the specified family.
int add_family(const ObString &family);
/// Get versions of columns with the specified timestamp.
int set_timestamp(int64_t timestamp);
/// Get versions of columns only within the specified timestamp range, [minStamp, maxStamp).
int set_time_range(int64_t min_stamp, int64_t max_stamp);
/// Get up to the specified number of versions of each column.
int read_versions(int32_t versions);
private:
// types and constants
private:
// disallow copy
DISALLOW_COPY_AND_ASSIGN(ObHGet);
// function members
private:
// data members
};
/// http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/Table.html
class ObHScan: public ObHQuery
{
public:
ObHScan();
virtual ~ObHScan();
/// Set the start row of the scan.
int with_start_row(const ObString &row, bool inclusive);
/// Set the stop row of the scan.
int with_stop_row(const ObString &row, bool inclusive);
/// Get the column from the specified family with the specified qualifier.
int add_column(const ObString &family, const ObString &qualifier);
/// Get all columns from the specified family.
int add_family(const ObString &family);
/// Get versions of columns with the specified timestamp.
int set_timestamp(int64_t timestamp);
/// Get versions of columns only within the specified timestamp range, [minStamp, maxStamp).
int set_time_range(int64_t min_stamp, int64_t max_stamp);
/// Get up to the specified number of versions of each column.
int read_versions(int32_t versions);
int read_all_versions();
/// Set the maximum number of cells to return for each call to next().
int set_batch(int32_t batch_size);
/// Set the limit of rows for this scan.
/// We will terminate the scan if the number of returned rows reaches this value.
/// @param limit - the limit of rows for this scan
int set_limit(int32_t limit);
/// Set the maximum result size.
/// The default is -1; this means that no specific maximum result size will be set for this scan.
/// @param max_result_size - The maximum result size in bytes.
int set_max_result_size(int64_t max_result_size);
/// Set the maximum number of values to return per row per Column Family
/// @param limit - the maximum number of values returned / row / CF
int set_max_results_per_column_family(int32_t limit);
/// Set offset for the row per Column Family.
/// @param offset - is the number of kvs that will be skipped.
int set_row_offset_per_column_family(int32_t offset);
/// Set whether this scan is a reversed one
int set_reversed(bool reversed);
private:
// types and constants
private:
// disallow copy
DISALLOW_COPY_AND_ASSIGN(ObHScan);
// function members
private:
// data members
common::ObArenaAllocator alloc_;
ObString start_row_;
ObString stop_row_;
bool include_start_row_;
bool include_stop_row_;
ObArray<ObString> qualifiers_;
int64_t timestamp_;
};
/// A HBase Table for demo
class ObHTable
{
public:
ObHTable();
virtual ~ObHTable();
int put(const ObHPut &put);
int multi_put(const ObIArray<ObHPut> &puts);
int del(const ObHDelete &del);
int multi_del(const ObIArray<ObHDelete> &deletes);
/// Performs multiple mutations atomically on a single row.
int mutate_row(const ObHRowMutations &row_mutations);
int get(const ObHGet &get);
int multi_get(const ObIArray<ObHGet> &gets);
int scan(const ObHScan &scan);
private:
// types and constants
private:
// disallow copy
DISALLOW_COPY_AND_ASSIGN(ObHTable);
// function members
private:
// data members
ObHKVTable *hkv_table_;
};
} // end namespace hbase
} // end namespace table
} // end namespace oceanbase
#endif /* _OB_HTABLE_H */

86
src/libtable/src/ob_kv_table.h
View File

@ -1,86 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef _OB_KV_TABLE_H
#define _OB_KV_TABLE_H 1
namespace oceanbase
{
namespace table
{
/** A Key-Value Table for HBase
* 1. provide kv inerface like memcached
* 2. the schema of each kvtable is 'create table kv (K varbinary(1024), V varbinary(1024), primary key(K)) partition by key(K) partitions 16;'
*/
class ObKVTable
{
public:
/// Key type for ObKVTable
typedef ObString Key;
/// Value type for ObKVTable
typedef ObObj Value;
/// Entity of ObKVTable
class Entity: public ObITableEntity
{
public:
Entity();
~Entity();
virtual void reset();
virtual int set_rowkey(const ObRowkey &rowkey) override;
virtual int set_rowkey(const ObITableEntity &other) override;
virtual int set_rowkey_value(int64_t idx, const ObObj &value) override;
virtual int add_rowkey_value(const ObObj &value) override;
virtual int64_t get_rowkey_size() const override { return 1; }
virtual int get_rowkey_value(int64_t idx, ObObj &value) const override;
virtual int64_t hash_rowkey() const override;
virtual int get_property(const ObString &prop_name, ObObj &prop_value) const override;
virtual int set_property(const ObString &prop_name, const ObObj &prop_value) override;
virtual int get_properties(ObIArray<std::pair<ObString, ObObj> > &properties) const override;
virtual int get_properties_names(ObIArray<ObString> &properties) const override;
virtual int get_properties_values(ObIArray<ObObj> &values) const override;
const Key &key() const { return key_; }
const Value &value() const { return value_; }
void set_key(const Key &k) { key_ = k;}
void set_value(const Value &v) { value_ = v;}
TO_STRING_KV(K_(key), K_(value));
private:
Key key_;
Value value_;
};
public:
int init(ObTableServiceClient &client, ObTable *tbl);
void destroy();
int get(const Key &key, Value &value);
int multi_get(const IKeys &keys, IValues &values);
int put(const Key &key, const Value &value);
int multi_put(const IKeys &keys, const IValues &values);
int multi_put(const IEntities &entities);
int remove(const Key &key);
int multi_remove(const IKeys &keys);
private:
ObKVTable();
virtual ~ObKVTable();
private:
// disallow copy
DISALLOW_COPY_AND_ASSIGN(ObKVTable);
};
} // end namespace table
} // end namespace oceanbase
#endif /* _OB_KV_TABLE_H */

20
src/logservice/archiveservice/ob_archive_compensate_engine.cpp
View File

@ -1,20 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "ob_archive_support_engine.h"
namespace oceanbase
{
namespace archive
{
} // namespace archive
} // namespace oceanbase

57
src/logservice/archiveservice/ob_archive_compensate_engine.h
View File

@ -1,57 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_ARCHIVE_OB_ARCHIVE_COMPENSATE_ENGINE_H_
#define OCEANBASE_ARCHIVE_OB_ARCHIVE_COMPENSATE_ENGINE_H_
#include "logservice/log_define.h"
#include "common/ob_queue_thread.h" // ObCond
namespace oceanbase
{
namespace archive
{
/*
* 为满足归档熟虑压缩率, 归档数据需要凑够一定块大小才会压缩并归档出去,
* 对于冷的日志流, 可能较长时间无法写入足够多的数据进行压缩以及归档;
*
* 另外某些存储介质不支持修改(worm), 需要整个日志文件写满之后一次性归档出去,
* 这同样可能需要很长时间才能写满一个日志文件(64M大小)
*
* 为兼顾压缩效率/worm以及归档时效性, 需要为日志文件周期性补占位日志, 来达到一定时间
* 内可以凑够足够大小数据进行压缩归档以及一定时间内可以写满日志文件满足worm归档需求
* */
class ObArchiveCompensateEngine : public share::ObThreadPool
{
//TODO
public:
ObArchiveCompensateEngine();
~ObArchiveCompensateEngine();
public:
int start();
void stop();
void wait();
private:
void run1();
void do_thread_task_();
private:
bool inited_;
bool need_force_check_;
common::ObCond cond_;
};
} // namespace archive
} // namespace oceanbase
#endif /* OCEANBASE_ARCHIVE_OB_ARCHIVE_COMPENSATE_ENGINE_H_ */

283
src/logservice/libobcdc/src/ob_seq_thread.h
View File

@ -1,283 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*
* Thread class with ObConcurrentSeqQueue, support for concurrent push for task specified seq,
* support for concurrent sequential consumption of tasks
*
* Assume two tasks, task numbers: M and N, 0 <= M < N.
* If the Nth task has been assigned a thread to process, ensure that the Mth task must have
* been assigned a thread to ensure no starvation
*/
#ifndef OCEANBASE_OB_SEQ_QUEUE_THREAD_H__
#define OCEANBASE_OB_SEQ_QUEUE_THREAD_H__
#include "lib/ob_define.h" // RETRY_FUNC
#include "lib/allocator/ob_malloc.h" // ObMemAttr
#include "share/ob_errno.h" // KR
#include "ob_concurrent_seq_queue.h" // ObConcurrentSeqQueue
namespace oceanbase
{
namespace common
{
// MAX_THREAD_NUM: Specifies the maximum number of threads supported
// ModuleClass: The module type of the thread class used, to distinguish between different modules for easy debugging
template <int MAX_THREAD_NUM = 256, typename ModuleClass = void>
class ObSeqThread
{
enum { DATA_OP_TIMEOUT = 1 * 1000 * 1000 };
typedef ObConcurrentSeqQueue QueueType;
public:
ObSeqThread();
virtual ~ObSeqThread();
public:
virtual int handle(void *task, const int64_t task_seq, const int64_t thread_index, volatile bool &stop_flag) = 0;
public:
int push(void *task, const int64_t task_seq, const int64_t timeout);
int start();
void stop();
void mark_stop_flag() { stop_flag_ = true; }
bool is_stoped() const { return ATOMIC_LOAD(&stop_flag_); }
int64_t get_thread_num() const { return thread_num_; }
int64_t get_task_num() const { return queue_.size(); }
public:
int init(const int64_t thread_num,
const int64_t queue_size,
const ObMemAttr &memattr);
void destroy();
public:
void run();
private:
static void *thread_func_(void *arg);
int next_task_(const int64_t task_seq, void *&task);
int64_t next_seq_();
private:
bool inited_;
int64_t thread_num_;
int64_t thread_counter_;
volatile bool stop_flag_ CACHE_ALIGNED;
int64_t task_seq_ CACHE_ALIGNED;
QueueType queue_;
pthread_t tids_[MAX_THREAD_NUM];
private:
DISALLOW_COPY_AND_ASSIGN(ObSeqThread);
};
template <int MAX_THREAD_NUM, typename ModuleClass>
ObSeqThread<MAX_THREAD_NUM, ModuleClass>::ObSeqThread() :
inited_(false),
thread_num_(0),
thread_counter_(0),
stop_flag_(true),
task_seq_(0),
queue_()
{
(void)memset(tids_, 0, sizeof(tids_));
}
template <int MAX_THREAD_NUM, typename ModuleClass>
ObSeqThread<MAX_THREAD_NUM, ModuleClass>::~ObSeqThread()
{
destroy();
}
template <int MAX_THREAD_NUM, typename ModuleClass>
int ObSeqThread<MAX_THREAD_NUM, ModuleClass>::init(const int64_t thread_num,
const int64_t queue_size,
const ObMemAttr &memattr)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(inited_)) {
LIB_LOG(ERROR, "ObSeqThread has been initialized", K(inited_));
ret = OB_INIT_TWICE;
} else if (OB_UNLIKELY(thread_num <= 0)
|| OB_UNLIKELY(thread_num > MAX_THREAD_NUM)
|| OB_UNLIKELY(queue_size <= 0)) {
LIB_LOG(ERROR, "invalid arguments", K(thread_num), K(MAX_THREAD_NUM), K(queue_size));
ret = OB_INVALID_ARGUMENT;
} else if (OB_FAIL(queue_.init(queue_size, memattr))) {
LIB_LOG(ERROR, "initialize queue fail", KR(ret), K(queue_size));
} else {
thread_num_ = thread_num;
thread_counter_ = 0;
stop_flag_ = true;
task_seq_ = 0;
(void)memset(tids_, 0, sizeof(tids_));
inited_ = true;
}
return ret;
}
template <int MAX_THREAD_NUM, typename ModuleClass>
void ObSeqThread<MAX_THREAD_NUM, ModuleClass>::destroy()
{
stop();
inited_ = false;
thread_num_ = 0;
thread_counter_ = 0;
stop_flag_ = true;
task_seq_ = 0;
queue_.destroy();
(void)memset(tids_, 0, sizeof(tids_));
}
template <int MAX_THREAD_NUM, typename ModuleClass>
int ObSeqThread<MAX_THREAD_NUM, ModuleClass>::start()
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(! inited_)) {
LIB_LOG(ERROR, "ObSeqThread has not been initialized");
ret = OB_NOT_INIT;
} else if (stop_flag_) {
stop_flag_ = false;
for (int64_t index = 0; OB_SUCC(ret) && index < thread_num_; index++) {
int pthread_ret = 0;
if (0 != (pthread_ret = pthread_create(tids_ + index, NULL, thread_func_, this))) {
ret = OB_ERR_UNEXPECTED;
LIB_LOG(ERROR, "pthread_create fail", K(pthread_ret), KERRNOMSG(pthread_ret), K(index));
}
}
}
return ret;
}
template <int MAX_THREAD_NUM, typename ModuleClass>
void ObSeqThread<MAX_THREAD_NUM, ModuleClass>::stop()
{
if (inited_) {
stop_flag_ = true;
for (int64_t index = 0; index < thread_num_; index++) {
if (0 != tids_[index]) {
int pthread_ret = pthread_join(tids_[index], NULL);
if (0 != pthread_ret) {
LIB_LOG_RET(ERROR, common::OB_ERR_SYS, "pthread_join fail", "thread_id", tids_[index], K(pthread_ret));
} else {
// do nothing
}
}
}
(void)memset(tids_, 0, sizeof(tids_));
}
}
template <int MAX_THREAD_NUM, typename ModuleClass>
void *ObSeqThread<MAX_THREAD_NUM, ModuleClass>::thread_func_(void *arg)
{
if (NULL != arg) {
ObSeqThread<MAX_THREAD_NUM, ModuleClass> *td = static_cast<ObSeqThread<MAX_THREAD_NUM, ModuleClass> *>(arg);
td->run();
}
return NULL;
}
template <int MAX_THREAD_NUM, typename ModuleClass>
void ObSeqThread<MAX_THREAD_NUM, ModuleClass>::run()
{
int ret = OB_SUCCESS;
int64_t thread_index = ATOMIC_FAA(&thread_counter_, 1);
if (OB_UNLIKELY(! inited_)) {
LIB_LOG(ERROR, "ObSeqThread not initialized", K(thread_index));
ret = OB_NOT_INIT;
} else {
while (! stop_flag_ && OB_SUCCESS == ret) {
void *task = NULL;
// Get the next sequence number to be consumed
int64_t task_seq = next_seq_();
if (OB_FAIL(next_task_(task_seq, task))) {
if (OB_IN_STOP_STATE != ret) {
LIB_LOG(ERROR, "next_task_ fail", KR(ret), K(task_seq));
}
} else if (OB_FAIL(handle(task, task_seq, thread_index, stop_flag_))) {
if (OB_IN_STOP_STATE != ret) {
LIB_LOG(ERROR, "handle task fail", KR(ret), "task", (int64_t)task, K(task_seq),
K(thread_index));
}
} else {
// do nothing
}
}
}
// NOTE: One thread exits, others exit at the same time
stop_flag_ = true;
}
template <int MAX_THREAD_NUM, typename ModuleClass>
int64_t ObSeqThread<MAX_THREAD_NUM, ModuleClass>::next_seq_()
{
return ATOMIC_FAA(&task_seq_, 1);
}
template <int MAX_THREAD_NUM, typename ModuleClass>
int ObSeqThread<MAX_THREAD_NUM, ModuleClass>::next_task_(const int64_t task_seq, void *&task)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(! inited_)) {
LIB_LOG(ERROR, "ObSeqThread not initialized");
ret = OB_NOT_INIT;
} else {
RETRY_FUNC(stop_flag_, queue_, pop, task, task_seq, DATA_OP_TIMEOUT);
}
return ret;
}
template <int MAX_THREAD_NUM, typename ModuleClass>
int ObSeqThread<MAX_THREAD_NUM, ModuleClass>::push(void *task, const int64_t task_seq, const int64_t timeout)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(! inited_)) {
LIB_LOG(ERROR, "ObSeqThread not initialized");
ret = OB_NOT_INIT;
} else if (OB_ISNULL(task) || OB_UNLIKELY(task_seq < 0)) {
LIB_LOG(ERROR, "invalid argument", K(task), K(task_seq));
ret = OB_INVALID_ARGUMENT;
} else if (OB_UNLIKELY(stop_flag_)) {
ret = OB_IN_STOP_STATE;
} else {
ret = queue_.push(task, task_seq, timeout);
}
return ret;
}
} // namespace common
} // namespace oceanbase
#endif /* OCEANBASE_MULTI_FIXED_QUEUE_THREAD_H__ */

23
src/logservice/logrpc/ob_log_rpc_processor.cpp
View File

@ -1,23 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "ob_log_rpc_processor.h"
#include "logservice/ob_log_service.h"
#include "logservice/palf/palf_env.h"
namespace oceanbase
{
namespace logservice
{
} // end namespace logservice
} // end namespace oceanbase

1
src/observer/CMakeLists.txt
View File

@ -44,7 +44,6 @@ ob_set_subtarget(ob_server common_mixed
report/ob_tablet_table_updater.cpp
report/ob_tenant_meta_checker.cpp
report/ob_server_meta_table_checker.cpp
layer_perf/ob_layer_perf.cpp
)
ob_set_subtarget(ob_server dbms_job

148
src/observer/layer_perf/ob_layer_perf.cpp
View File

@ -1,148 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SERVER
#include "observer/layer_perf/ob_layer_perf.h"
#include "lib/ob_errno.h"
#include "lib/string/ob_string.h"
namespace oceanbase
{
namespace observer
{
#ifdef PERF_MODE
thread_local ObLS *ObLayerPerf::ls_ = nullptr;
int ObLayerPerf::process(bool &hit)
{
int ret = OB_SUCCESS;
hit = false;
common::ObString &sql = query_->sql_;
if (sql.prefix_match("[PM_")) {
hit = true;
bool need_sync_resp = true;
r_ = query_->get_ob_request()->get_ez_req();
LOG_INFO("ObLayerPerf::process", K(sql));
if (sql.prefix_match("[PM_WORKER]")) {
// do nothing
} else if (sql.prefix_match("[PM_CLOG]")) {
// clog layer perf
need_sync_resp = false;
if (OB_FAIL(do_clog_layer_perf())) {
LOG_ERROR("do_clog_layer_perf", K(ret));
}
}
if (OB_SUCC(ret)) {
if (need_sync_resp) {
ret = do_query_response();
} else {
// async response
query_->packet_sender_.disable_response();
}
}
}
return ret;
}
int ObLayerPerf::do_response()
{
//response
int ret = OB_SUCCESS;
void *buf = easy_pool_alloc(r_->ms->pool, obmysql::OB_MULTI_RESPONSE_BUF_SIZE + sizeof(easy_buf_t));
easy_buf_t *b = reinterpret_cast<easy_buf_t*>(buf);
init_easy_buf(b, reinterpret_cast<char*>(b + 1), r_, obmysql::OB_MULTI_RESPONSE_BUF_SIZE);
obmysql::OMPKOK okp;
obmysql::ObMySQLCapabilityFlags cap_flags(163553933);
okp.set_seq(1);
okp.set_capability(cap_flags);
int64_t seri_size = 0;
if (OB_FAIL(okp.encode(b->last, b->end - b->pos, seri_size))) {
LOG_ERROR("okp.serialize", K(ret));
} else {
b->last += seri_size;
//easy_request_addbuf(r_, b);
r_->opacket = b;
easy_request_wakeup(r_);
}
return ret;
}
int ObLayerPerf::do_query_response()
{
obmysql::OMPKOK okp;
okp.set_capability(query_->get_conn()->cap_flags_);
query_->response_packet(okp, NULL);
return query_->flush_buffer(true);
}
int ObLayerPerf::do_clog_layer_perf()
{
int ret = OB_SUCCESS;
ObLS *ls = nullptr;
if (ObLayerPerf::ls_ != nullptr) {
ls = ObLayerPerf::ls_;
} else {
ObLSService *ls_svr = MTL(ObLSService*);
common::ObSharedGuard<ObLSIterator> iter;
if (OB_FAIL(ls_svr->get_ls_iter(iter, ObLSGetMod::LOG_MOD))) {
} else {
while (OB_SUCC(ret)) {
ls = nullptr;
if (OB_FAIL(iter->get_next(ls))) {
if (OB_ITER_END != ret) {
LOG_WARN("iter ls get next failed", K(ret));
}
} else if (OB_ISNULL(ls)) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("ls is NULL", K(ret));
} else if (ls->get_ls_id().id() > ObLSID::MIN_USER_LS_ID) {
// find first user ls
break;
}
}
}
if (OB_SUCC(ret) && ls == nullptr) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("not found user ls");
}
if (OB_SUCC(ret)) {
ObLayerPerf::ls_ = ls;
}
}
if (OB_SUCC(ret)) {
char *buf = (char*)"clog layer perf test";
PerfLogCb *cb = nullptr;
palf::LSN lsn;
share::SCN zero, ts_ns;
LOG_INFO("perf layer append", KP(r_), KP(buf));
if (nullptr == (cb = static_cast<PerfLogCb*>(ob_malloc(sizeof(PerfLogCb), "PerfLogCb")))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("allo mem", K(ret));
} else if (FALSE_IT(new (cb) PerfLogCb(r_))) {
} else if (OB_FAIL(ls->get_log_handler()->append(buf, strlen(buf), zero, true, cb, lsn, ts_ns))) {
LOG_ERROR("append fail", K(ret));
}
}
return ret;
}
#endif
} // end observer
} // end oceanbase

67
src/observer/layer_perf/ob_layer_perf.h
View File

@ -1,67 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "observer/mysql/obmp_query.h"
#include "rpc/obmysql/packet/ompk_ok.h"
namespace oceanbase
{
namespace observer
{
#ifdef PERF_MODE
class ObLayerPerf
{
public:
ObLayerPerf(ObMPQuery *query) :query_(query),
r_(nullptr)
{}
int process(bool &hit);
// use mpquery do response
int do_query_response();
// use easy request do response
int do_response();
void set_ez_req(easy_request_t *r) {
r_ = r;
}
public:
static thread_local ObLS* ls_;
private:
int do_clog_layer_perf();
ObMPQuery *query_;
easy_request_t *r_;
};
class PerfLogCb : public logservice::AppendCb
{
public:
explicit PerfLogCb(easy_request_t *r) : layer_perf_(nullptr) {
layer_perf_.set_ez_req(r);
}
~PerfLogCb() {}
int on_success() override {
int ret = layer_perf_.do_response();
this->~PerfLogCb();
ob_free(this);
return ret;
}
int on_failure() override {
this->~PerfLogCb();
ob_free(this);
return OB_ERR_UNEXPECTED;
}
private:
ObLayerPerf layer_perf_;
};
#endif
} // end observer
} // end oceanbase

17
src/observer/mysql/obmp_query.cpp
View File

@ -56,9 +56,6 @@
#include "sql/monitor/ob_security_audit_utils.h"
#include "observer/mysql/obmp_utils.h"
#include "lib/ash/ob_active_session_guard.h"
#ifdef PERF_MODE
#include "observer/layer_perf/ob_layer_perf.h"
#endif
#include "lib/trace/ob_trace.h"
using namespace oceanbase::rpc;
@ -90,17 +87,6 @@ ObMPQuery::~ObMPQuery()
int ObMPQuery::process()
{
int ret = OB_SUCCESS;
#ifdef PERF_MODE
// use for ob layer benchmark
//
bool layer_perf_hit = false;
ObLayerPerf layer_perf(this);
ret = layer_perf.process(layer_perf_hit);
if (layer_perf_hit) {
return ret;
}
#endif
int tmp_ret = OB_SUCCESS;
ObSQLSessionInfo *sess = NULL;
uint32_t sessid = 0;
@ -394,8 +380,7 @@ int ObMPQuery::process()
// THIS_WORKER.need_retry()是指是否扔回队列重试，包括大查询被扔回队列的情况。
session.check_and_reset_retry_info(*cur_trace_id, THIS_WORKER.need_retry());
session.set_last_trace_id(ObCurTraceId::get_trace_id());
int tmp_ret = OB_SUCCESS;
tmp_ret = record_flt_trace(session);
IGNORE_RETURN record_flt_trace(session);
}
if (OB_UNLIKELY(NULL != GCTX.cgroup_ctrl_) && GCTX.cgroup_ctrl_->is_valid()) {

3
src/observer/mysql/obmp_query.h
View File

@ -42,9 +42,6 @@ namespace observer
{
class ObMPQuery : public ObMPBase
{
#ifdef PERF_MODE
friend class ObLayerPerf;
#endif
public:
static const obmysql::ObMySQLCmd COM = obmysql::COM_QUERY;

1
src/observer/ob_signal_handle.cpp
View File

@ -178,7 +178,6 @@ int ObSignalHandle::deal_signals(int signum)
ob_print_mod_memory_usage();
//GARL_PRINT();
PC_REPORT();
ObObjFreeListList::get_freelists().dump();
ObTenantMemoryPrinter::get_instance().print_tenant_usage();
break;
}

108
src/observer/table_load/ob_table_load_queue.h
View File

@ -1,108 +0,0 @@
// Copyright (c) 2022-present Oceanbase Inc. All Rights Reserved.
// Author:
// suzhi.yt <>
#pragma once
#include "lib/allocator/ob_malloc.h"
#include "lib/oblog/ob_log_module.h"
#include "lib/utility/ob_utility.h"
#include "share/ob_errno.h"
namespace oceanbase
{
namespace observer
{
template<class T>
class ObTableLoadQueue
{
public:
ObTableLoadQueue();
~ObTableLoadQueue();
int init(uint64_t tenant_id, uint64_t capacity);
int push(T *obj, int64_t timeout_us = 0);
int pop(T *&obj, int64_t timeout_us = 0);
int push_nowait(T *obj);
int pop_nowait(T *&obj);
int64_t size() const;
private:
T **queue_;
uint64_t capacity_;
uint64_t push_;
uint64_t pop_;
int64_t size_;
mutable pthread_mutex_t mutex_;
pthread_cond_t push_cond_;
pthread_cond_t pop_cond_;
bool is_inited_;
};
template <class T>
ObTableLoadQueue<T>::ObTableLoadQueue()
: queue_(nullptr), capacity_(0), push_(0), pop_(0), size_(0), is_inited_(false)
{
OB_ASSERT(0 == pthread_mutex_init(&mutex_, nullptr));
OB_ASSERT(0 == pthread_cond_init(&push_cond_, nullptr));
OB_ASSERT(0 == pthread_cond_init(&pop_cond_, nullptr));
}
template<class T>
ObTableLoadQueue<T>::~ObTableLoadQueue()
{
if (nullptr != queue_) {
common::ob_free(queue_);
queue_ = nullptr;
}
OB_ASSERT(0 == pthread_mutex_destroy(&mutex_));
OB_ASSERT(0 == pthread_cond_destroy(&push_cond_));
OB_ASSERT(0 == pthread_cond_destroy(&pop_cond_));
}
template<class T>
int ObTableLoadQueue<T>::init(uint64_t tenant_id, uint64_t capacity)
{
int ret = common::OB_SUCCESS;
return ret;
}
template<class T>
int64_t ObTableLoadQueue<T>::size() const
{
int64_t size = 0;
pthread_mutex_lock(&mutex_);
size = size_;
pthread_mutex_unlock(&mutex_);
return size;
}
template<class T>
int ObTableLoadQueue<T>::push(T *obj, int64_t timeout_us)
{
int ret = common::OB_SUCCESS;
return ret;
}
template<class T>
int ObTableLoadQueue<T>::pop(T *&obj, int64_t timeout_us)
{
int ret = common::OB_SUCCESS;
return ret;
}
template<class T>
int ObTableLoadQueue<T>::push_nowait(T *obj)
{
int ret = common::OB_SUCCESS;
return ret;
}
template<class T>
int ObTableLoadQueue<T>::pop_nowait(T *&obj)
{
int ret = common::OB_SUCCESS;
return ret;
}
} // namespace observer
} // namespace oceanbase

111
src/observer/virtual_table/ob_all_concurrency_object_pool.cpp
View File

@ -11,120 +11,13 @@
*/
#include "ob_all_concurrency_object_pool.h"
#include "common/object/ob_object.h"
#include "share/config/ob_server_config.h"
#include "lib/allocator/ob_mem_leak_checker.h"
#include "lib/objectpool/ob_concurrency_objpool.h"
#include "observer/ob_server_utils.h"
using namespace oceanbase::lib;
using namespace oceanbase::common;
using namespace oceanbase::observer;
ObAllConcurrencyObjectPool::ObAllConcurrencyObjectPool()
: ObVirtualTableScannerIterator(),
addr_(NULL),
flls_(NULL, ObModIds::OB_VT_ALL_CONCUR_OBJ_POOL),
fl_(NULL),
idx_(0)
{
int ret = OB_SUCCESS;
if (OB_FAIL(ObObjFreeListList::get_freelists().get_info(flls_))) {
SERVER_LOG(ERROR, "get freelists failed", K(ret));
}
}
ObAllConcurrencyObjectPool::~ObAllConcurrencyObjectPool()
{
reset();
}
void ObAllConcurrencyObjectPool::reset()
{
addr_ = NULL;
fl_ = NULL;
idx_ = 0;
flls_.reset();
}
int ObAllConcurrencyObjectPool::inner_get_next_row(ObNewRow *& row)
{
int ret = OB_SUCCESS;
if (idx_ >= flls_.size()) {
ret = OB_ITER_END;
} else if (NULL == allocator_){
ret = OB_NOT_INIT;
SERVER_LOG(WARN, "allocator is not init", K(ret));
} else {
const int64_t col_count = output_column_ids_.count();
ObObj *cells = NULL;
if (OB_ISNULL(cells = cur_row_.cells_)) {
ret = OB_ERR_UNEXPECTED;
SERVER_LOG(WARN, "cur row cell is NULL", K(ret));
} else if (cur_row_.count_ < output_column_ids_.count()) {
ret = OB_ERR_UNEXPECTED;
SERVER_LOG(WARN, "cur_row count is invalid", K(ret));
} else {
fl_ = flls_.at(idx_);
if (OB_ISNULL(fl_)) {
ret = OB_ERR_UNEXPECTED;
SERVER_LOG(WARN, "fl_ is null", K(ret));
} else {
// ip
ObString ipstr;
for (int64_t i = 0; OB_SUCC(ret) && i < output_column_ids_.count(); i++) {
uint64_t col_id = output_column_ids_.at(i);
switch(col_id) {
case 16: {
if (OB_FAIL(ObServerUtils::get_server_ip(allocator_, ipstr))) {
SERVER_LOG(ERROR, "get server ip failed", K(ret));
} else {
cells[i].set_varchar(ipstr);
cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
}
} break;
case 17: {
const int32_t port = addr_->get_port();
cells[i].set_int(port);
} break;
case 18: {
cells[i].set_varchar(fl_->get_name());
cells[i].set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset()));
} break;
case 19: {
cells[i].set_int((fl_->get_allocated() - fl_->get_allocated_base()) * fl_->get_type_size());
} break;
case 20: {
cells[i].set_int((fl_->get_used() - fl_->get_used_base()) * fl_->get_type_size());
} break;
case 21: {
cells[i].set_int(fl_->get_used() - fl_->get_used_base());
} break;
case 22: {
cells[i].set_int(fl_->get_type_size());
} break;
case 23: {
cells[i].set_int(fl_->get_chunk_count());
} break;
case 24: {
cells[i].set_int(fl_->get_chunk_byte_size());
} break;
default: {
ret = OB_ERR_UNEXPECTED;
SERVER_LOG(WARN,
"invalid column id",
K(ret),
K(i));
break;
}
}
}
}
if (OB_SUCC(ret)) {
row = &cur_row_;
++idx_;
}
}
}
return ret;
UNUSED(row);
return OB_ITER_END;
}

15
src/observer/virtual_table/ob_all_concurrency_object_pool.h
View File

@ -13,9 +13,6 @@
#ifndef SRC_OBSERVER_VIRTUAL_TABLE_OB_ALL_CONCURRENCY_OBJECT_POOL_H_
#define SRC_OBSERVER_VIRTUAL_TABLE_OB_ALL_CONCURRENCY_OBJECT_POOL_H_
#include "lib/objectpool/ob_concurrency_objpool.h"
#include "lib/container/ob_vector.h"
#include "lib/net/ob_addr.h"
#include "share/ob_virtual_table_scanner_iterator.h"
namespace oceanbase
@ -25,17 +22,11 @@ namespace observer
class ObAllConcurrencyObjectPool : public common::ObVirtualTableScannerIterator
{
public:
ObAllConcurrencyObjectPool();
virtual ~ObAllConcurrencyObjectPool();
virtual int inner_get_next_row(common::ObNewRow *&row);
virtual void reset();
inline void set_addr(common::ObAddr &addr) { addr_ = &addr; }
ObAllConcurrencyObjectPool() {}
virtual ~ObAllConcurrencyObjectPool() {}
virtual int inner_get_next_row(common::ObNewRow *&row) override;
private:
DISALLOW_COPY_AND_ASSIGN(ObAllConcurrencyObjectPool);
common::ObAddr *addr_;
common::ObVector<common::ObObjFreeList *> flls_;
common::ObObjFreeList *fl_;
int64_t idx_;
};
}
}

68
src/observer/virtual_table/ob_tenant_show_restore_preview.h
View File

@ -1,68 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_OBSERVER_VIRTUAL_TABLE_OB_TENANT_SHOW_RESTORE_PREVIEW_H_
#define OCEANBASE_OBSERVER_VIRTUAL_TABLE_OB_TENANT_SHOW_RESTORE_PREVIEW_H_
#include "share/ob_virtual_table_iterator.h"
#include "share/backup/ob_backup_struct.h"
#include "lib/container/ob_array.h"
namespace oceanbase
{
namespace observer
{
class ObTenantShowRestorePreview : public common::ObVirtualTableIterator
{
enum BackupType
{
BACKUP_TYPE_SET = 0,
BACKUP_TYPE_PIECE = 1,
BACKUP_TYPE_MAX = 2,
};
enum RestorePreviewColumn
{
BACKUP_TYPE = common::OB_APP_MIN_COLUMN_ID,
BACKUP_ID = common::OB_APP_MIN_COLUMN_ID + 1,
COPY_ID = common::OB_APP_MIN_COLUMN_ID + 2,
PREVIEW_PATH = common::OB_APP_MIN_COLUMN_ID + 3,
FILE_STATUS = common::OB_APP_MIN_COLUMN_ID + 4,
};
public:
ObTenantShowRestorePreview();
virtual ~ObTenantShowRestorePreview();
int init();
virtual int inner_get_next_row(common::ObNewRow *&row);
virtual void reset();
private:
int inner_get_next_row();
int get_backup_type(BackupType &type);
int get_backup_id(int64_t &backup_id);
int get_copy_id(int64_t &copy_id);
int get_backup_path(common::ObString &str);
int get_file_status(common::ObString &str);
private:
bool is_inited_;
int64_t idx_; // index in array
int64_t total_cnt_;
common::ObArray<share::ObBackupSetPath> backup_set_list_;
common::ObArray<share::ObBackupPiecePath> backup_piece_list_;
private:
DISALLOW_COPY_AND_ASSIGN(ObTenantShowRestorePreview);
};
} // end namespace observer
} // end namespace oceanbase
#endif

2
src/observer/virtual_table/ob_virtual_table_iterator_factory.cpp
View File

@ -1339,8 +1339,6 @@ int ObVTIterCreator::create_vt_iter(ObVTableScanParam &params,
case OB_ALL_VIRTUAL_CONCURRENCY_OBJECT_POOL_TID: {
ObAllConcurrencyObjectPool *object_pool = NULL;
if (OB_SUCC(NEW_VIRTUAL_TABLE(ObAllConcurrencyObjectPool, object_pool))) {
object_pool->set_allocator(&allocator);
object_pool->set_addr(addr_);
vt_iter = static_cast<ObVirtualTableIterator *>(object_pool);
}
break;

162
src/observer/virtual_table/ob_virtual_trace_log.cpp
View File

@ -1,162 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SERVER
#include "observer/virtual_table/ob_virtual_trace_log.h"
#include "lib/string/ob_sql_string.h"
#include "lib/trace/ob_trace_event.h"
using namespace oceanbase::common;
namespace oceanbase
{
namespace observer
{
ObVirtualTraceLog::ObVirtualTraceLog()
: ObVirtualTableScannerIterator()
{
}
ObVirtualTraceLog::~ObVirtualTraceLog()
{
reset();
}
void ObVirtualTraceLog::reset()
{
ObVirtualTableScannerIterator::reset();
}
int ObVirtualTraceLog::inner_get_next_row(common::ObNewRow *&row)
{
int ret = OB_SUCCESS;
if (!start_to_read_) {
if (OB_FAIL(fill_scanner())) {
LOG_WARN( "fail to fill scanner", K(ret));
} else {
scanner_it_ = scanner_.begin();
start_to_read_ = true;
}
}
if (OB_SUCCESS == ret && start_to_read_) {
if (OB_FAIL(scanner_it_.get_next_row(cur_row_))) {
if (OB_ITER_END != ret) {
LOG_WARN( "fail to get next row", K(ret));
}
} else {
row = &cur_row_;
}
}
return ret;
}
int ObVirtualTraceLog::fill_scanner()
{
int ret = OB_SUCCESS;
ObObj *cells = NULL;
if (OB_ISNULL(allocator_)) {
ret = OB_NOT_INIT;
LOG_WARN( "allocator is NULL", K(ret));
} else if (OB_ISNULL(session_)) {
ret = OB_NOT_INIT;
LOG_WARN( "session_ is NULL", K(ret));
} else if (OB_FAIL(fill_trace_buf())) {
LOG_WARN("fail fill trace buf", K(ret));
}
return ret;
}
int ObVirtualTraceLog::fill_trace_buf()
{
int ret = OB_SUCCESS;
ObObj *cells = NULL;
if (OB_ISNULL(cells = cur_row_.cells_)) {
ret = OB_ERR_UNEXPECTED;
SERVER_LOG(ERROR, "cur row cell is NULL", K(ret));
}
if (OB_SUCC(ret)) {
ObTraceEventRecorder *trace_buf = session_->get_trace_buf();
if (NULL != trace_buf) {
int64_t N = trace_buf->count();
char buf1[512];
char buf2[32];
int64_t pos = 0;
int64_t prev_ts = 0;
ObString str;
for (int64_t i = 0; OB_SUCC(ret) && i < N; ++i) {
const ObTraceEvent &ev = trace_buf->get_event(i);
if (ev.id_ <= OB_ID(__PAIR_NAME_BEGIN__)) {
continue;
}
int cell_idx = 0;
for (int j = 0; OB_SUCC(ret) && j < output_column_ids_.count(); ++j) {
int64_t col_id = output_column_ids_.at(j);
switch(col_id) {
case TITLE: {
cells[cell_idx].set_varchar(ObString::make_string(name::get_description(ev.id_)));
cells[cell_idx].set_collation_type(
ObCharset::get_default_collation(ObCharset::get_default_charset()));
break;
};
case KEY_VALUE: {
if (ev.yson_end_pos_ > ev.yson_beg_pos_) {
pos = 0;
(void)::oceanbase::yson::databuff_print_elements(buf1, 512, pos,
trace_buf->get_buffer() + ev.yson_beg_pos_, ev.yson_end_pos_-ev.yson_beg_pos_);
str.assign_ptr(const_cast<const char*>(buf1), static_cast<int32_t>(pos));
cells[cell_idx].set_varchar(str);
} else {
// no value
cells[cell_idx].set_varchar("");
}
cells[cell_idx].set_collation_type(
ObCharset::get_default_collation(ObCharset::get_default_charset()));
break;
};
case TIME: {
pos = 0;
if (prev_ts == 0) {
prev_ts = ev.timestamp_;
}
pos = snprintf(buf2, 32, "%ld", ev.timestamp_-prev_ts);
prev_ts = ev.timestamp_;
str.assign_ptr(const_cast<const char*>(buf2), static_cast<int32_t>(pos));
cells[cell_idx].set_varchar(str);
cells[cell_idx].set_collation_type(
ObCharset::get_default_collation(ObCharset::get_default_charset()));
break;
};
default: {
ret = OB_ERR_UNEXPECTED;
SERVER_LOG(ERROR, "invalid column id", K(ret), K(cell_idx), K(j),
K(output_column_ids_), K(col_id));
break;
}
}
if (OB_SUCC(ret)) {
cell_idx++;
}
}
if (OB_SUCCESS == ret && OB_FAIL(scanner_.add_row(cur_row_))) {
LOG_WARN( "fail to add row", K(ret), K(cur_row_));
}
} // end for
} else {
session_->clear_trace_buf(); //避免下次打开trace log后show时会被show出来
}
}
return ret;
}
}
}

49
src/observer/virtual_table/ob_virtual_trace_log.h
View File

@ -1,49 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_OBSERVER_VIRTUAL_TABLE_OB_VIRTUAL_TRACE_LOG_
#define OCEANBASE_OBSERVER_VIRTUAL_TABLE_OB_VIRTUAL_TRACE_LOG_
#include "share/ob_virtual_table_scanner_iterator.h"
#include "sql/session/ob_sql_session_info.h"
namespace oceanbase
{
namespace sql
{
class ObSQLSessionInfo;
}
namespace observer
{
class ObVirtualTraceLog : public common::ObVirtualTableScannerIterator
{
public:
ObVirtualTraceLog();
virtual ~ObVirtualTraceLog();
virtual int inner_get_next_row(common::ObNewRow *&row);
virtual void reset();
private:
DISALLOW_COPY_AND_ASSIGN(ObVirtualTraceLog);
enum TRACE_COLUMN
{
TITLE = common::OB_APP_MIN_COLUMN_ID,
KEY_VALUE,
TIME
};
int fill_scanner();
int fill_trace_buf();
};
}// end namespace observer
}// end namespace oceanbase
#endif

64
src/share/ob_build_index_struct.h
View File

@ -1,64 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_SHARE_OB_BUILD_INDEX_STRUCT_H_
#define OCEANBASE_SHARE_OB_BUILD_INDEX_STRUCT_H_
#include "share/ob_define.h"
namespace oceanbase
{
namespace share
{
struct ObBuildIndexAppendLocalDataParam
{
ObBuildIndexAppendLocalDataParam()
: execution_id_(-1), task_id_(common::OB_INVALID_ID),
index_id_(common::OB_INVALID_ID), schema_version_(common::OB_INVALID_VERSION),
task_cnt_(0)
{}
virtual ~ObBuildIndexAppendLocalDataParam() {}
bool is_valid() const
{
return 0 <= execution_id_ && common::OB_INVALID_ID != task_id_
&& common::OB_INVALID_ID != index_id_ && common::OB_INVALID_VERSION != schema_version_
&& 0 != task_cnt_ && task_id_ < task_cnt_;
}
TO_STRING_KV(K_(execution_id), K_(task_id), K_(index_id), K_(schema_version), K_(task_cnt));
int64_t execution_id_;
uint64_t task_id_;
uint64_t index_id_;
int64_t schema_version_;
uint64_t task_cnt_;
};
struct ObBuildIndexAppendSSTableParam
{
ObBuildIndexAppendSSTableParam()
: index_id_(common::OB_INVALID_ID), schema_version_(common::OB_INVALID_VERSION), execution_id_(-1)
{}
virtual ~ObBuildIndexAppendSSTableParam() {}
bool is_valid() const
{
return common::OB_INVALID_ID != index_id_ && common::OB_INVALID_VERSION != schema_version_
&& 0 <= execution_id_;
}
TO_STRING_KV(K_(index_id), K_(schema_version), K_(execution_id));
uint64_t index_id_;
int64_t schema_version_;
int64_t execution_id_;
};
} // end namespace share
} // end namespace oceanbase
#endif // OCEANBASE_SHARE_OB_BUILD_INDEX_STRUCT_H_

362
src/share/ob_cluster_info_proxy.cpp
View File

@ -1,362 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SHARE
#include "ob_cluster_info_proxy.h"
#include "share/ob_core_table_proxy.h"
#include "share/ob_dml_sql_splicer.h"
#include "share/ob_encryption_util.h"
#include "share/config/ob_server_config.h"
#include "share/inner_table/ob_inner_table_schema_constants.h"
#include "lib/mysqlclient/ob_isql_client.h"
#include "lib/utility/ob_print_utils.h"
#include "lib/string/ob_sql_string.h"
#include "common/ob_timeout_ctx.h"
#include "rootserver/ob_root_utils.h"
#include "observer/ob_server_struct.h"
using namespace oceanbase;
using namespace oceanbase::common;
namespace oceanbase
{
namespace share
{
const char* ObClusterInfoProxy::OB_ALL_CLUSTER_INFO_TNAME = "__all_cluster";
const char* ObClusterInfoProxy::CLUSTER_ROLE = "cluster_role";
const char* ObClusterInfoProxy::SWITCHOVER_STATUS = "switchover_status";
const char* ObClusterInfoProxy::SWITCHOVER_TIMESTAMP = "switchover_timestamp";
const char* ObClusterInfoProxy::CLUSTER_STATUS = "cluster_status";
const char* ObClusterInfoProxy::ENCRYPTION_KEY = "eeeeffff";
const char* ObClusterInfoProxy::PROTECTION_MODE = "protection_mode";
const char* ObClusterInfoProxy::VERSION = "version";
const char* ObClusterInfoProxy::PROTECTION_LEVEL = "protection_level";
const char* ObClusterInfo::PERSISTENT_SWITCHOVER_STATUS_ARRAY[] =
{
"SWITCHOVER_INVALID",
"SWITCHOVER_NORMAL",
"SWITCHOVER_SWITCHING",
"FAILOVER_FLASHBACK",
"FAILOVER_CLEANUP",
"FAILOVER_FLASHBACK_INNER",
"FAILOVER_FLASHBACK_USER",
"FAILOVER_FLASHBACK_INNER",
"FAILOVER_FLASHBACK_USER",
"FAILOVER_FLASHBACK_USER",
"DISCONNECT",
};
const char* ObClusterInfo::IN_MEMORY_SWITCHOVER_STATUS_ARRAY[] =
{
"INVALID",
"NOT ALLOWED",
"TO STANDBY",
"TO PRIMARY",
"SWITCHOVER SWITCHING",
"FAILOVER FLASHBACK",
"FAILOVER CLEANUP",
"DISCONNECT",
};
const char* ObClusterInfo::CLUSTER_STATUS_ARRAY[]=
{
"VALID",
"DISABLE",
};
int ObClusterInfo::str_to_in_memory_switchover_status(const ObString &status_str, InMemorySwitchOverStatus &status)
{
int ret = OB_SUCCESS;
status = I_MAX_STATUS;
if (I_MAX_STATUS != ARRAYSIZEOF(IN_MEMORY_SWITCHOVER_STATUS_ARRAY)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get invalid array size", K(ret), K(I_MAX_STATUS), "array_size",
ARRAYSIZEOF(IN_MEMORY_SWITCHOVER_STATUS_ARRAY));
} else {
for (int64_t i = 0; i < ARRAYSIZEOF(IN_MEMORY_SWITCHOVER_STATUS_ARRAY) && OB_SUCC(ret); i++) {
if (STRLEN(IN_MEMORY_SWITCHOVER_STATUS_ARRAY[i]) == status_str.length()
&& 0 == STRNCASECMP(IN_MEMORY_SWITCHOVER_STATUS_ARRAY[i], status_str.ptr(), status_str.length())) {
status = static_cast<InMemorySwitchOverStatus>(i);
break;
}
}
}
if (OB_SUCC(ret) && I_MAX_STATUS == status) {
ret = OB_ENTRY_NOT_EXIST;
LOG_WARN("fail to find cluster status", K(status_str));
}
return ret;
}
const char* ObClusterInfo::in_memory_switchover_status_to_str(const InMemorySwitchOverStatus &status)
{
STATIC_ASSERT(ARRAYSIZEOF(IN_MEMORY_SWITCHOVER_STATUS_ARRAY) == I_MAX_STATUS,
"type string array size mismatch with enum InMemorySwitchOverStatus count");
const char* str = "UNKNOWN";
if (status <= I_INVALID || status >= I_MAX_STATUS) {
LOG_WARN_RET(OB_ERR_UNEXPECTED, "invalid in-memory switchover status", K(status));
} else {
str = IN_MEMORY_SWITCHOVER_STATUS_ARRAY[status];
}
return str;
}
const char* ObClusterInfo::persistent_switchover_status_to_str(const PersistentSwitchOverStatus &status)
{
STATIC_ASSERT(ARRAYSIZEOF(PERSISTENT_SWITCHOVER_STATUS_ARRAY) == P_MAX_STATUS,
"type string array size mismatch with enum PersistentSwitchOverStatus count");
const char* str = "UNKNOWN";
if (status < P_SWITCHOVER_INVALID || status >= P_MAX_STATUS) {
LOG_WARN_RET(OB_ERR_UNEXPECTED, "invalid persistent switchover status", K(status));
} else {
str = PERSISTENT_SWITCHOVER_STATUS_ARRAY[status];
}
return str;
}
bool ObClusterInfo::is_in_failover(const PersistentSwitchOverStatus &status)
{
return P_FAILOVER_CLEANUP == status
|| is_failover_flashback(status);
}
void ObClusterInfo::reset()
{
cluster_id_ = -1;
cluster_role_ = INVALID_CLUSTER_ROLE;
switchover_status_ = ObClusterInfo::P_SWITCHOVER_INVALID;
switch_timestamp_ = 0;
is_sync_ = false;
cluster_status_ = INVALID_CLUSTER_STATUS;
protection_mode_ = common::MAXIMUM_PERFORMANCE_MODE;
version_ = OB_INVALID_VERSION;
protection_level_ = common::MAXIMUM_PERFORMANCE_LEVEL;
}
int ObClusterInfo::assign(const ObClusterInfo &other)
{
int ret = OB_SUCCESS;
cluster_role_ = other.cluster_role_;
cluster_id_ = other.cluster_id_;
switchover_status_ = other.switchover_status_;
switch_timestamp_ = other.switch_timestamp_;
cluster_status_ = other.cluster_status_;
is_sync_ = other.is_sync_;
protection_mode_ = other.protection_mode_;
version_ = other.version_;
protection_level_ = other.protection_level_;
return ret;
}
ObClusterInfo::ObClusterInfo(const ObClusterInfo &other)
{
int ret = OB_SUCCESS;
if (OB_FAIL(assign(other))) {
LOG_WARN("fail to assign", K(ret));
}
}
ObClusterInfo& ObClusterInfo::operator= (const ObClusterInfo &other)
{
int ret = OB_SUCCESS;
if (this != &other) {
if (OB_FAIL(assign(other))) {
LOG_WARN("fail to assign", K(ret));
}
}
return *this;
}
bool ObClusterInfo::is_valid() const
{
bool bret = false;
if (INVALID_CLUSTER_ROLE != cluster_role_
&& -1 != cluster_id_) {
bret = true;
}
if (bret) {
bret = OB_INVALID_VERSION != version_;
}
return bret;
}
bool ObClusterInfo::operator !=(const ObClusterInfo &other) const
{
return cluster_id_ != other.cluster_id_
|| cluster_role_ != other.cluster_role_
|| switchover_status_ != other.switchover_status_
|| switch_timestamp_ != other.switch_timestamp_
|| cluster_status_ != other.cluster_status_
|| is_sync_ != other.is_sync_
|| protection_mode_ != other.protection_mode_
|| version_ != other.version_
|| protection_level_ != other.protection_level_;
}
int64_t ObClusterInfo::generate_switch_timestamp(const int64_t switch_timestamp)
{
return switch_timestamp & (~SWITCH_TIMSTAMP_MASK);
}
int ObClusterInfo::inc_switch_timestamp()
{
int ret = OB_SUCCESS;
int64_t switch_times = switch_timestamp_ & SWITCH_TIMSTAMP_MASK;
if (switch_times >= MAX_CHANGE_TIMES) {
ret = OB_OP_NOT_ALLOW;
LOG_WARN("switch retry too much times", KR(ret), K(switch_times));
} else {
++ switch_timestamp_;
}
LOG_INFO("inc switch timestamp", K(switch_timestamp_), K(switch_times));
return ret;
}
int64_t ObClusterInfo::get_pure_switch_timestamp(const int64_t switch_timestamp)
{
return switch_timestamp & (~SWITCH_TIMSTAMP_MASK);
}
bool ObClusterInfo::is_less_than(const int64_t switch_timestamp) const
{
return switch_timestamp_ < switch_timestamp;
}
OB_SERIALIZE_MEMBER(ObClusterInfo, cluster_role_, switchover_status_,
is_sync_, cluster_id_, switch_timestamp_, cluster_status_,
protection_mode_, version_, protection_level_);
//////////////////////////////////////////////////
//////////////////////////////////////////////////
ObClusterInfoProxy::ObClusterInfoProxy()
{}
ObClusterInfoProxy::~ObClusterInfoProxy()
{
}
// get the gmt_create value of specific row of __all_core_table as the cluster creation time
// table_name='__all_global_stat', column_name='snapshot_gc_scn'
int ObClusterInfoProxy::load_cluster_create_timestamp(ObISQLClient &sql_proxy,
int64_t &cluster_create_ts)
{
int ret = OB_SUCCESS;
static const char* TABLE_NAME = "__all_global_stat";
static const char* COLUMN_NAME = "snapshot_gc_scn";
ObCoreTableProxy core_table(TABLE_NAME, sql_proxy, OB_SYS_TENANT_ID);
if (OB_FAIL(core_table.load_gmt_create(COLUMN_NAME, cluster_create_ts))) {
LOG_WARN("load gmt_create column of core table fail", KR(ret), K(TABLE_NAME), K(COLUMN_NAME));
}
LOG_INFO("load cluster create timestamp finish", KR(ret), K(cluster_create_ts), K(TABLE_NAME),
K(COLUMN_NAME));
return ret;
}
int ObClusterInfoProxy::load(ObISQLClient &sql_proxy, ObClusterInfo &cluster_info,
const bool for_update)
{
int ret = OB_SUCCESS;
ObCoreTableProxy core_table(OB_ALL_CLUSTER_INFO_TNAME, sql_proxy, OB_SYS_TENANT_ID);
int64_t cluster_role = -1;
int64_t switchover_status = -1;
int64_t passwd_length = 0;
int64_t switch_timestamp = 0;
int64_t cluster_status = 0;
int64_t protection_mode = 0;
int64_t protection_level = 0;
ObTimeoutCtx ctx;
if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) {
LOG_WARN("fail to get timeout ctx", K(ret), K(ctx));
} else if (for_update && OB_FAIL(core_table.load_for_update())) {
LOG_WARN("failed to load for update", KR(ret), K(for_update));
} else if (!for_update && OB_FAIL(core_table.load())) {
LOG_WARN("fail to load core table", K(ret));
} else if (OB_FAIL(core_table.next())) {
if (OB_ITER_END == ret) {
LOG_WARN("get empty cluster info, maybe in bootstrap", K(ret));
ret = OB_SUCCESS;
} else {
LOG_WARN("fail to next", K(ret));
}
} else if (OB_FAIL(core_table.get_int(CLUSTER_ROLE, cluster_role))) {
LOG_WARN("fail to get int", KR(ret));
} else if (OB_FAIL(core_table.get_int(SWITCHOVER_STATUS, switchover_status))) {
LOG_WARN("fail to get int", KR(ret));
} else if (OB_FAIL(core_table.get_int(SWITCHOVER_TIMESTAMP, switch_timestamp))) {
LOG_WARN("fail to get int", KR(ret));
} else if (OB_FAIL(core_table.get_int(CLUSTER_STATUS, cluster_status))) {
LOG_WARN("fail to get int", KR(ret));
} else if (OB_FAIL(core_table.get_int(PROTECTION_MODE, protection_mode))) {
LOG_WARN("failed to get int", KR(ret));
} else if (OB_FAIL(core_table.get_int(VERSION, cluster_info.version_))) {
LOG_WARN("failed to get int", KR(ret));
} else if (OB_FAIL(core_table.get_int(PROTECTION_LEVEL, protection_level))) {
LOG_WARN("failed to get int", KR(ret));
} else {
cluster_info.cluster_role_ = static_cast<ObClusterRole>(cluster_role);
cluster_info.switchover_status_ = static_cast<ObClusterInfo::PersistentSwitchOverStatus>(switchover_status);
cluster_info.set_switch_timestamp(switch_timestamp);
cluster_info.cluster_id_ = GCTX.config_->cluster_id;
cluster_info.cluster_status_ = static_cast<ObClusterStatus>(cluster_status);
cluster_info.protection_mode_ = static_cast<ObProtectionMode>(protection_mode);
cluster_info.protection_level_ = static_cast<ObProtectionLevel>(protection_level);
}
if (OB_SUCC(ret)) {
if (OB_ITER_END != (ret = core_table.next())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get invalid next", K(ret));
} else {
ret = OB_SUCCESS;
}
}
LOG_INFO("load all cluster finish", K(ret), K(cluster_info));
return ret;
}
int ObClusterInfoProxy::update(ObISQLClient &sql_proxy, const ObClusterInfo &cluster_info)
{
int ret = OB_SUCCESS;
LOG_INFO("start to update cluster info", K(cluster_info));
ObDMLSqlSplicer dml;
int64_t affected_rows = 0;
ObArray<ObCoreTableProxy::UpdateCell> cells;
ObCoreTableProxy kv(OB_ALL_CLUSTER_INFO_TNAME, sql_proxy, OB_SYS_TENANT_ID);
ObTimeoutCtx ctx;
if (OB_FAIL(ret)) {
} else if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) {
LOG_WARN("fail to get timeout ctx", K(ret), K(ctx));
} else if (OB_FAIL(dml.add_column(CLUSTER_ROLE, cluster_info.cluster_role_))
|| OB_FAIL(dml.add_column(SWITCHOVER_STATUS, cluster_info.switchover_status_))
|| OB_FAIL(dml.add_column(SWITCHOVER_TIMESTAMP, cluster_info.get_switch_timestamp()))
|| OB_FAIL(dml.add_column(CLUSTER_STATUS, cluster_info.cluster_status_))
|| OB_FAIL(dml.add_column(PROTECTION_MODE, cluster_info.protection_mode_))
|| OB_FAIL(dml.add_column(VERSION, cluster_info.version_))
|| OB_FAIL(dml.add_column(PROTECTION_LEVEL, cluster_info.protection_level_))) {
LOG_WARN("fail to add column", KR(ret), K(cluster_info));
} else if (OB_FAIL(kv.load_for_update())) {
LOG_WARN("fail to load for update", K(ret));
} else if (OB_FAIL(dml.splice_core_cells(kv, cells))) {
LOG_WARN("fail to splice core cells", K(ret));
} else if (OB_FAIL(kv.replace_row(cells, affected_rows))) {
LOG_WARN("fail to replace row", K(ret), K(cluster_info));
}
return ret;
}
}
}

233
src/share/ob_cluster_info_proxy.h
View File

@ -1,233 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_SHARE_OB_ALL_CLUSTER_PROXY_H_
#define OCEANBASE_SHARE_OB_ALL_CLUSTER_PROXY_H_
#include "share/ob_zone_info.h"
#include "share/ob_web_service_root_addr.h"
#include "lib/string/ob_fixed_length_string.h"
#include "share/ob_cluster_role.h" // ObClusterRole
namespace oceanbase
{
namespace share
{
#define SWITCH_TIMSTAMP_MASK 0x3FF
class ObClusterInfo
{
OB_UNIS_VERSION(1);
public:
static const int64_t MAX_PASSWD_LENGTH = common::OB_MAX_PASSWORD_LENGTH;
typedef common::ObFixedLengthString<common::OB_MAX_USERNAME_LENGTH> UserNameString;
typedef common::ObFixedLengthString<MAX_PASSWD_LENGTH> PassWdString;
enum PersistentSwitchOverStatus //Persistent switchover state maintained by RS
{
P_SWITCHOVER_INVALID = 0,
P_SWITCHOVER_NORMAL = 1,
P_SWITCHOVER_SWITCHING = 2,
P_FAILOVER_FLASHBACK = 3,//no use any more
P_FAILOVER_CLEANUP = 4,
P_FAILOVER_FLASHBACK_INNER = 5,
P_FAILOVER_FLASHBACK_USER = 6,
P_FAILOVER_FLASHBACK_INNER_PRE = 7,
P_FAILOVER_FLASHBACK_CLEANUP = 8,
P_FAILOVER_FLASHBACK_USER_PRE = 9,
P_DISCONNECT = 10,
P_MAX_STATUS
};
/* The process of FAILOVER FLASHBACK
1. Construct all tenant’s failover_scn
2. Keep the SCN needed by flashback to avoid recycling clog before replaying it
3. Prepare partitions to do flashback for tenant's inner table
a) Write cutdata clog
b) Wait flashback info dump finished
c) Wait the restore status of replicas are in cutdata status
4. Do flashback tenant's inner table partitions, send cutdata task and wait cutdata task finished
5. Do Preparations to do user table flashback
6. Prepare partitions to do flashback for tenant's user table
a) Write cutdata clog
b) Wait flashback info dump finished
c) Wait the restore status of replicas are in cutdata status
7. Do flashback tenant's user table partitions, send cutdata task and wait cutdata task finished
The following table shows what will do in each stage of FAILOVER FLASHBACK
+-------------------------------------+--------------+
| status | process |
+-------------------------------------+--------------+
| P_FAILOVER_FLASHBACK | 1-2 |
+-------------------------------------+--------------+
| P_FAILOVER_FLASHBACK_INNER_PRE | 3 |
+-------------------------------------+--------------+
| P_FAILOVER_FLASHBACK_INNER | 4 |
+-------------------------------------+--------------+
| P_FAILOVER_FLASHBACK_CLEANUP | 5 |
+-------------------------------------+--------------+
| P_FAILOVER_FLASHBACK_USER_PRE | 6 |
+-------------------------------------+--------------+
| P_FAILOVER_FLASHBACK_USER | 7 |
+-------------------------------------+--------------+
*/
//Switchover status of external display
enum InMemorySwitchOverStatus
{
// invalid status
I_INVALID = 0,
// can not switchover to primary or standby
I_NOT_ALLOW,
// only in primary cluster, the cluster can switchover to standby
I_TO_STANDBY,
// only in standby cluster, the cluster can switchover to primary
I_TO_PRIMARY,
// the intermediate status of primary cluster switchover to standby,
// can not provide write service.
I_SWITCHOVER_SWITCHING,
// the intermediate status of standby cluster failover to primary cluster.
// cluster is flashback to a consistent state, need to retry failover until success.
// the status can not provide write service.
I_FAILOVER_FLASHBACK,
// the intermediate status of standby cluster failover to primary cluster.
// cluster is doing cleanup, remove unused schema.
// the status can not do DDL
I_FAILOVER_CLEANUP,
I_DISCONNECT,
I_MAX_STATUS
};
ObClusterInfo() : cluster_id_(-1), cluster_role_(common::INVALID_CLUSTER_ROLE),
switchover_status_(P_SWITCHOVER_INVALID), cluster_status_(common::INVALID_CLUSTER_STATUS),
switch_timestamp_(0), is_sync_(false),
protection_mode_(common::INVALID_PROTECTION_MODE), version_(common::OB_INVALID_VERSION),
protection_level_(common::INVALID_PROTECTION_LEVEL) {}
ObClusterInfo(const common::ObClusterStatus cluster_status,
const common::ObClusterRole cluster_role,
const PersistentSwitchOverStatus switchover_status,
const int64_t cluster_id,
const int64_t version)
{
reset();
cluster_status_ = cluster_status;
cluster_role_ = cluster_role;
switchover_status_ = switchover_status;
cluster_id_ = cluster_id;
version_ = version;
}
~ObClusterInfo() {}
ObClusterInfo(const ObClusterInfo &other);
ObClusterInfo &operator =(const ObClusterInfo &other);
void reset();
int assign(const ObClusterInfo &other);
bool operator !=(const ObClusterInfo &other) const;
bool is_valid() const;
static int str_to_in_memory_switchover_status(const common::ObString &status_str,
InMemorySwitchOverStatus &status);
static const char* in_memory_switchover_status_to_str(const InMemorySwitchOverStatus &status);
static const char* persistent_switchover_status_to_str(const PersistentSwitchOverStatus &status);
static bool is_in_failover(const PersistentSwitchOverStatus &status);
static int64_t generate_switch_timestamp(const int64_t switch_timestamp);
static int64_t get_pure_switch_timestamp(const int64_t switch_timestamp);
static bool is_failover_flashback(const PersistentSwitchOverStatus switchover_status)
{
return is_failover_flashback_inner(switchover_status)
|| is_failover_flashback_user(switchover_status);
}
static bool is_failover_flashback_inner(const PersistentSwitchOverStatus switchover_status)
{
return P_FAILOVER_FLASHBACK_INNER_PRE == switchover_status
|| P_FAILOVER_FLASHBACK_INNER == switchover_status;
}
static bool is_failover_flashback_user(const PersistentSwitchOverStatus switchover_status)
{
return P_FAILOVER_FLASHBACK_USER_PRE == switchover_status
|| P_FAILOVER_FLASHBACK_USER == switchover_status
|| P_FAILOVER_FLASHBACK_CLEANUP == switchover_status;
}
int inc_switch_timestamp();
int64_t get_switch_timestamp() const { return switch_timestamp_; }
int64_t atomic_get_switch_timestamp() const { return ATOMIC_LOAD(&switch_timestamp_); }
void atomic_set_switch_timestamp(const int64_t switchover_epoch)
{ ATOMIC_SET(&switch_timestamp_, switchover_epoch); }
void set_switch_timestamp(const int64_t switchover_epoch)
{ switch_timestamp_ = switchover_epoch; }
bool is_less_than(const int64_t switch_timestamp) const;
int64_t atomic_get_version() const { return ATOMIC_LOAD(&version_); }
void set_cluster_info_version(const int64_t version)
{
version_ = version;
}
TO_STRING_KV(K_(cluster_id), K_(cluster_role),
"switchover_status", persistent_switchover_status_to_str(switchover_status_),
K_(cluster_status), K_(switch_timestamp), K_(is_sync),
K_(protection_mode), K_(version), K_(protection_level));
public: // TODO public -> private
static const char* IN_MEMORY_SWITCHOVER_STATUS_ARRAY[];
static const char* PERSISTENT_SWITCHOVER_STATUS_ARRAY[];
static const char* CLUSTER_STATUS_ARRAY[];
static const int64_t MAX_CHANGE_TIMES = 1000;
int64_t cluster_id_;
common::ObClusterRole cluster_role_;
PersistentSwitchOverStatus switchover_status_;
common::ObClusterStatus cluster_status_;
private:
//It can avoid the backoff of switching state, and can switch 1000 times at most in one switching process;
//The last 10 bits are used to indicate the number of state changes in a switching process; each state change will inc_switch_timestamp
int64_t switch_timestamp_;
public:
bool is_sync_;//the cluster is sync with primary
public:
common::ObProtectionMode protection_mode_;
int64_t version_;//Mark the change of each variable in cluster info to avoid state fallback
common::ObProtectionLevel protection_level_;
};
class ObClusterInfoProxy
{
public:
ObClusterInfoProxy();
virtual ~ObClusterInfoProxy();
// the cluster's create timestamp
static int load_cluster_create_timestamp(common::ObISQLClient &sql_proxy,
int64_t &cluster_create_ts);
static int load(common::ObISQLClient &sql_proxy, ObClusterInfo &cluster_info,
const bool for_update = false);
static int update(common::ObISQLClient &sql_proxy, const ObClusterInfo &cluster_info);
private:
static const char* OB_ALL_CLUSTER_INFO_TNAME;
static const char* CLUSTER_ROLE;
static const char* SWITCHOVER_STATUS;
static const char* SWITCHOVER_TIMESTAMP;
static const char* CLUSTER_STATUS;
static const char* ENCRYPTION_KEY;
static const char* PROTECTION_MODE;
static const char* VERSION;
static const char* PROTECTION_LEVEL;
};
} //end share
} //end oceanbase
#endif

139
src/share/ob_cluster_switchover_info.cpp
View File

@ -1,139 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "ob_cluster_switchover_info.h"
namespace oceanbase
{
namespace share
{
const char *switchover_info_to_str(const ObClusterSwitchoverInfo so_info)
{
const char *cstr = "";
switch (so_info) {
case SWITCHOVER_INFO_NONE:
cstr = "";
break;
case SWITCHOVER_INFO_CHECK_PRIMARY_CLUSTER_SERVER_STATUS:
cstr = "CHECK PRIMARY CLUSTER SERVER STATUS";
break;
case SWITCHOVER_INFO_CHECK_STANDBY_CLUSTER_SERVER_STATUS:
cstr = "CHECK STANDBY CLUSTER SERVER STATUS";
break;
case SWITCHOVER_INFO_CHECK_PRIMARY_CLUSTER_REBALANCE_TASK_STATUS:
cstr = "CHECK PRIMARY CLUSTER REBALANCE TASK STATUS";
break;
case SWITCHOVER_INFO_CHECK_STANDBY_CLUSTER_REBALANCE_TASK_STATUS:
cstr = "CHECK STANDBY CLUSTER REBALANCE TASK STATUS";
break;
case SWITCHOVER_INFO_CHECK_MERGE_STATUS:
cstr = "CHECK MERGE STATUS";
break;
case SWITCHOVER_INFO_NONE_SYNCED_STANDBY_CLUSTER:
cstr = "NONE SYNCED STANDBY CLUSTER";
break;
case SWITCHOVER_INFO_HAS_SYNCED_STANDBY_CLUSTER:
cstr = "SYNCED STANDBY CLUSTERS: ";
break;
case SWITCHOVER_INFO_CHECK_SYS_SCHEMA_SYNC_STATUS:
cstr = "CHECK SYS SCHEMA SYNC STATUS";
break;
case SWITCHOVER_INFO_CHECK_USER_SCHEMA_SYNC_STATUS:
cstr = "CHECK USER SCHEMA SYNC STATUS";
break;
case SWITCHOVER_INFO_CHECK_FREEZE_INFO_SYNC_STATUS:
cstr = "CHECK FREEZE INFO SYNC STATUS";
break;
case SWITCHOVER_INFO_CHECK_ENOUGH_REPLICA:
cstr = "CHECK ENOUGH REPLICA";
break;
case SWITCHOVER_INFO_CHECK_REDO_LOG_SYNC_STATUS:
cstr = "CHECK REDO LOG SYNC STATUS";
break;
case SWITCHOVER_INFO_INNER_ERROR:
cstr = "INNER ERROR";
break;
case SWITCHOVER_INFO_CLUSTER_CAN_NOT_ACCESS:
cstr = "CAN NOT ACCESS CLUSTERS: ";
break;
case SWITCHOVER_INFO_NO_CLUSTER_IN_SWITCHING:
cstr = "NO CLUSTER IN SWITCHING";
break;
case SWITCHOVER_INFO_PRIMARY_CLUSTER_NOT_IN_SYNC_MODE:
cstr = "PRIMARY CLUSTER NOT IN SYNC MODE";
break;
case SWITCHOVER_INFO_STANDBY_CLUSTER_NOT_IN_SYNC_MODE:
cstr = "STANDBY CLUSTER NOT IN SYNC MODE";
break;
case SWITCHOVER_INFO_PRIMARY_CLUSTER_IN_PRE_MAXIMUM_PROTECTION_MODE:
cstr = "IN PRE MAXIMUM PROTECTION MODE";
break;
case SWITCHOVER_INFO_PRIMARY_CLUSTER_HAS_REPLICA_IN_RESTORE:
cstr = "PRIMARY CLUSTER HAS REPLICA IN RESTORE";
break;
case SWITCHOVER_INFO_STANDBY_CLUSTER_HAS_REPLICA_IN_RESTORE:
cstr = "STANDBY CLUSTER HAS REPLICA IN RESTORE";
break;
case SWITCHOVER_INFO_PRIMARY_CLUSTER_DOING_BACKUP:
cstr = "PRIMARY CLUSTER DOING BACKUP";
break;
case SWITCHOVER_INFO_EXIST_OTHER_PRIMARY_CLUSTER:
cstr = "CHECK OTHER PRIMARY CLUSTER";
break;
case SWITCHOVER_INFO_CLUSTER_IS_DISABLED:
cstr = "CLUSTER IS DISABLED";
break;
case SWITCHOVER_INFO_CLUSTER_INFO_NOT_SYNC:
cstr = "CLUSTER INFO NOT SYNC";
break;
case SWITCHOVER_INFO_FAILOVER_INFO_NOT_SYNC:
cstr = "FAILOVER INFO NOT SYNC";
break;
default:
cstr = "";
break;
}
return cstr;
}
OB_SERIALIZE_MEMBER(ObClusterSwitchoverInfoWrap, info_,
synced_cluster_id_array_, can_not_access_cluster_);
}
}

155
src/share/ob_cluster_switchover_info.h
View File

@ -1,155 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_SHARE_OB_CLUSTER_SWITCHOVER_INFO_H_
#define OCEANBASE_SHARE_OB_CLUSTER_SWITCHOVER_INFO_H_
#include "lib/container/ob_se_array.h" // ObSEArray
namespace oceanbase
{
namespace share
{
/**
* SWITCHOVER_INFO column of V$OB_CLUSTER view
*
* Show switchover information, which is detailed descriptions of the SWITCHOVER_STATUS column
* 1. When show NOT ALLOWED, why is it not allowed to switchover
* 2. When show TO STANDBY, which standby cluster can switchover to primary cluster
* 3. When in other status, this column value is empty
*/
enum ObClusterSwitchoverInfo
{
// Empty
SWITCHOVER_INFO_NONE = 0,
// Need to check the status of servers on primary/standby cluster, there may be some server offline
// Reason of SWITCHOVER_STATUS = 'NOT ALLOWED'
SWITCHOVER_INFO_CHECK_PRIMARY_CLUSTER_SERVER_STATUS = 1, // Both displayed on primary and standby cluster
SWITCHOVER_INFO_CHECK_STANDBY_CLUSTER_SERVER_STATUS = 2, // Displayed on standby cluster
// Obsolete: Need to check whether load balancing task is completed on primary/standby cluster
// Reason of SWITCHOVER_STATUS = 'NOT ALLOWED'
SWITCHOVER_INFO_CHECK_PRIMARY_CLUSTER_REBALANCE_TASK_STATUS = 3, // Both displayed on primary and standby cluster
SWITCHOVER_INFO_CHECK_STANDBY_CLUSTER_REBALANCE_TASK_STATUS = 4, // Displayed on standby cluster
// Need to check whether merge status is ERROR on primary/standby cluster
// Reason of SWITCHOVER_STATUS = 'NOT ALLOWED'
SWITCHOVER_INFO_CHECK_MERGE_STATUS = 5, // Both displayed on primary and standby cluster
// No synchronized standby cluster
// Reason of SWITCHOVER_STATUS = 'NOT ALLOWED'
SWITCHOVER_INFO_NONE_SYNCED_STANDBY_CLUSTER = 6, // Displayed on primary cluster
// List of standby clusters which synchronized with primary
// SWITCHOVER_STATUS = 'TO STANDBY', Display the list of standby clusters which can switchover
// Example:
// "SYNCED STANDBY CLUSTERS: cluster1, cluster2"
SWITCHOVER_INFO_HAS_SYNCED_STANDBY_CLUSTER = 7, // Displayed on primary cluster
// Need to check whether SYS schema of standby clusteris synchronized
// Reason of SWITCHOVER_STATUS = 'NOT ALLOWED'
SWITCHOVER_INFO_CHECK_SYS_SCHEMA_SYNC_STATUS = 8, // Displayed on standby cluster
// Need to check whether USER schema of standby cluster is synchronized
// Reason of SWITCHOVER_STATUS = 'NOT ALLOWED'
SWITCHOVER_INFO_CHECK_USER_SCHEMA_SYNC_STATUS = 9, // Displayed on standby cluster
// Need to check whether FREEZE INFO of standby cluster is synchronized
// Reason of SWITCHOVER_STATUS = 'NOT ALLOWED'
SWITCHOVER_INFO_CHECK_FREEZE_INFO_SYNC_STATUS = 10, // Displayed on standby cluster
// Obsolete: Need to check whether the number of replicas on standby cluster is enough
// Reason of SWITCHOVER_STATUS = 'NOT ALLOWED'
SWITCHOVER_INFO_CHECK_ENOUGH_REPLICA = 11, // Displayed on standby cluster
// Need to check whether REDO LOG of standby cluster is synchronized
// Reason of SWITCHOVER_STATUS = 'NOT ALLOWED'
SWITCHOVER_INFO_CHECK_REDO_LOG_SYNC_STATUS = 12, // Displayed on standby cluster
// INNER ERROR
SWITCHOVER_INFO_INNER_ERROR = 13, // Both displayed on primary and standby cluster
// There are inaccessible server, switchover requires all clusters to be online
// Reason of SWITCHOVER_STATUS = 'NOT ALLOWED'
SWITCHOVER_INFO_CLUSTER_CAN_NOT_ACCESS = 14, // Both displayed on primary and standby cluster
// Currently there are no clusters in SWITCHING state
// Typical scenario: When primary and standby databases are normal, the standby cluster
// displays "NOT ALLOWED", and switchover_info will display this status
// Reason of SWITCHOVER_STATUS = 'NOT ALLOWED'
SWITCHOVER_INFO_NO_CLUSTER_IN_SWITCHING = 15, // Displayed on standby cluster
// Displayed on primary cluster, primary cluster is in maximum protection mode, but
// redo_transport_options is not SYNC, if do switchover, the new primary and standby cluster
// cannot be maximum protection mode as there is no SYNC redo_transport_options standby cluster
SWITCHOVER_INFO_PRIMARY_CLUSTER_NOT_IN_SYNC_MODE = 16,
// Displayed on standby cluster, standby cluster is in maximum protection mode, but
// redo_transport_options is not SYNC, this standby cluster cannot switchover to primary cluster,
// need choose other SYNC redo_transport_options standby cluster to do switchover
SWITCHOVER_INFO_STANDBY_CLUSTER_NOT_IN_SYNC_MODE = 17,
// Changing protection mode, cannot switchover
SWITCHOVER_INFO_PRIMARY_CLUSTER_IN_PRE_MAXIMUM_PROTECTION_MODE = 18,
// There is replica in restore status, cannot switchover
SWITCHOVER_INFO_PRIMARY_CLUSTER_HAS_REPLICA_IN_RESTORE = 19,
// There is replica in restore status, cannot switchover
SWITCHOVER_INFO_STANDBY_CLUSTER_HAS_REPLICA_IN_RESTORE = 20,
// Doing backup, cannot switchover
SWITCHOVER_INFO_PRIMARY_CLUSTER_DOING_BACKUP = 21,
// There is other primary cluster
SWITCHOVER_INFO_EXIST_OTHER_PRIMARY_CLUSTER = 22,
//cluster is disabled
SWITCHOVER_INFO_CLUSTER_IS_DISABLED = 23,
//cluster_info not sync
SWITCHOVER_INFO_CLUSTER_INFO_NOT_SYNC = 24,
//failover_info not sync
SWITCHOVER_INFO_FAILOVER_INFO_NOT_SYNC = 25,
SWITCHOVER_INFO_MAX
};
const char *switchover_info_to_str(const ObClusterSwitchoverInfo so_info);
struct ObClusterSwitchoverInfoWrap
{
OB_UNIS_VERSION(1);
public:
static const int64_t DEFAULT_SYNCED_CLUSTER_COUNT = 5;
typedef common::ObSEArray<int64_t, DEFAULT_SYNCED_CLUSTER_COUNT> ClusterIdArray;
ObClusterSwitchoverInfo info_;
ClusterIdArray synced_cluster_id_array_;
ClusterIdArray can_not_access_cluster_;
void reset()
{
info_ = SWITCHOVER_INFO_NONE;
synced_cluster_id_array_.reset();
can_not_access_cluster_.reset();
}
TO_STRING_KV(K_(info), "info_str", switchover_info_to_str(info_),
K_(synced_cluster_id_array), K_(can_not_access_cluster));
};
} //end share
} //end oceanbase
#endif

93
src/share/ob_cluster_sync_status.cpp
View File

@ -1,93 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SHARE
#include "share/ob_cluster_sync_status.h" //class ObClusterSyncStatusHelp
namespace oceanbase
{
using namespace common;
namespace share
{
const char *ObClusterSyncStatusHelp::cluster_sync_status_strs[] = {
"NOT AVAILABLE",
"CLUSTER VERSION NOT MATCH",
"SYS SCHEMA NOT SYNC",
"FAILED CHECK PARTITION LOG",
"PARTITION LOG NOT SYNC",
"FAILED CHECK ENOUGH MEMBER",
"FAILED CHECK SYS SCHEMA",
"REPLICA NOT ENOUGH",
"FAILED CHECK FAILOVER INFO",
"FAILOVER INFO NOT LATEST",
"FAILED CHECK ALL CLUSTER INFO",
"ALL CLUSTER INFO NOT SYNC",
"FAILED CHECK RESTORE REPLICA",
"REPLICA IN RESTORE",
"CHECK USER SCHEMA SYNC STATUS",
"CHECK FREEZE INFO SYNC STATUS",
"FAILED CHECK MERGE STATUS",
"CHECK MERGE STATUS",
"FAILED CHECK META VALID",
"FAILED CHECK USER TENANT SCHEMA",
"FAILED CHECK FREEZE INFO",
"FAILED CHECK ALL SCHEMA EFFECTIVE",
"CHECK ALL SCHEMA EFFECTIVE",
"TENANT NOT MATCH",
"OK",
"CLUSTER IS DISABLED",
};
const char * ObClusterSyncStatusHelp::cluster_sync_status_to_str(
const share::ObClusterSyncStatus sync_status,
const int64_t last_hb_ts)
{
STATIC_ASSERT(ARRAYSIZEOF(cluster_sync_status_strs) == SYNC_STATUS_MAX,
"type string array size mismatch with enum ObClusterSyncStatus count");
const char *sync_status_str = "";
if (cluster_sync_status_is_valid(last_hb_ts)) {
if (OB_UNLIKELY(sync_status >= share::ObClusterSyncStatus::SYNC_STATUS_MAX)
|| OB_UNLIKELY(sync_status < share::ObClusterSyncStatus::NOT_AVAILABLE)) {
LOG_ERROR_RET(OB_ERR_UNEXPECTED, "fatal error, unknown cluster sync status", K(sync_status));
} else {
sync_status_str = cluster_sync_status_strs[sync_status];
}
} else {
sync_status_str = cluster_sync_status_strs[share::ObClusterSyncStatus::NOT_AVAILABLE];
}
return sync_status_str;
}
bool ObClusterSyncStatusHelp::cluster_sync_status_is_valid(const int64_t last_ts)
{
const int64_t SYNCSTATUS_TIMEOUT_US = 30L * 1000 * 1000; //30s
bool bret = false;
if (common::OB_INVALID_TIMESTAMP == last_ts
|| (ObTimeUtility::current_time() - last_ts) > SYNCSTATUS_TIMEOUT_US) {
// More than 30s, status is invalid
bret = false;
} else {
bret = true;
}
return bret;
}
} // end namespace share
} // end oceanbase

42
src/share/ob_get_primary_standby_service.h
View File

@ -1,42 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_SHARE_OB_GET_PRIMARY_STANDBY_SERVICE_H_
#define OCEANBASE_SHARE_OB_GET_PRIMARY_STANDBY_SERVICE_H_
#include "share/ob_primary_standby_service.h"
#define OB_PRIMARY_STANDBY_SERVICE_TYPE ObPrimaryStandbyService
namespace oceanbase
{
namespace standby
{
class ObPrimaryStandbyServiceGetter
{
public:
static ObPrimaryStandbyService &get_instance()
{
static OB_PRIMARY_STANDBY_SERVICE_TYPE primary_standby_service;
return primary_standby_service;
}
};
#define OB_PRIMARY_STANDBY_SERVICE (oceanbase::standby::ObPrimaryStandbyServiceGetter::get_instance())
} // end namespace standby
} // end namespace oceanbase
#endif // OCEANBASE_SHARE_OB_GET_PRIMARY_STANDBY_SERVICE_H_

1
src/share/ob_thread_mgr.cpp
View File

@ -15,7 +15,6 @@
#include "storage/tx/ob_trans_service.h"
#include "storage/tx/ob_ts_worker.h"
#include "storage/tx_storage/ob_ls_freeze_thread.h"
#include "storage/tx_storage/ob_ls_cb_queue_thread.h"
#include "rootserver/ob_index_builder.h"
#include "observer/ob_srv_deliver.h"
#include "logservice/palf/log_io_task_cb_thread_pool.h"

35
src/share/scheduler/ob_dag_type.cpp
View File

@ -1,35 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "share/scheduler/ob_dag_type.h"
namespace oceanbase
{
namespace share
{
const TypeBasicSetting OB_DAG_TYPE[] = {
#define DAG_TYPE_SETTING_DEF(def, default_score) \
{default_score},
#include "share/scheduler/ob_dag_type.h"
#undef DAG_TYPE_SETTING_DEF
};
const common::ObString ObDagTypeStr[ObDagTypeIds::TYPE_SETTING_END] = {
"MINOR_MERGE",
"MAJOR_MERGE",
"MINI_MERGE"
};
} // namespace share
} // namespace oceanbase

96
src/share/scheduler/ob_dag_type.h
View File

@ -1,96 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifdef DAG_TYPE_SETTING_DEF
DAG_TYPE_SETTING_DEF(MINOR_MERGE, 2)
DAG_TYPE_SETTING_DEF(MAJOR_MERGE, 3)
DAG_TYPE_SETTING_DEF(MINI_MERGE, 5)
DAG_TYPE_SETTING_DEF(TYPE_SETTING_END, 0)
#endif
#ifndef SRC_SHARE_SCHEDULER_OB_DAG_TYPE_H_
#define SRC_SHARE_SCHEDULER_OB_DAG_TYPE_H_
#include "lib/ob_define.h"
#include "lib/string/ob_string.h"
#include "lib/container/ob_vector.h"
#include "lib/allocator/ob_mod_define.h"
namespace oceanbase
{
namespace share
{
struct ObDagTypeIds
{
enum ObDagTypeIdEnum
{
#define DAG_TYPE_SETTING_DEF(def, default_score) def,
#include "share/scheduler/ob_dag_type.h"
#undef DAG_TYPE_SETTING_DEF
};
};
struct TypeBasicSetting
{
int64_t default_score_;
static const int64_t DEFAULT_SCORE = 2;
TypeBasicSetting(int64_t default_score = DEFAULT_SCORE) :
default_score_(default_score)
{
}
virtual ~TypeBasicSetting()
{
}
;
};
struct ObTenantTypeSetting;
struct ObTenantSetting
{
typedef common::ObVector<ObTenantTypeSetting> TenantTypeSettingVec;
const static int64_t NOT_SET = -999;
int64_t tenant_id_;
int32_t max_thread_num_;
TenantTypeSettingVec type_settings_;
ObTenantSetting(
int64_t tenant_id = -1,
int32_t max_thread_num = NOT_SET)
: tenant_id_(tenant_id),
max_thread_num_(max_thread_num),
type_settings_(0, NULL, common::ObModIds::OB_SCHEDULER)
{
}
VIRTUAL_TO_STRING_KV(K_(tenant_id), K_(max_thread_num));
};
// type setting in tenant
struct ObTenantTypeSetting
{
int64_t type_id_;
int64_t score_;
int64_t up_limit_;
ObTenantTypeSetting(
int64_t type_id = -1,
int64_t score = ObTenantSetting::NOT_SET,
int64_t uplimit = ObTenantSetting::NOT_SET)
: type_id_(type_id),
score_(score),
up_limit_(uplimit)
{
}
};
extern const TypeBasicSetting OB_DAG_TYPE[];
extern const common::ObString ObDagTypeStr[];
} // namespace share
} // namespace oceanbase
#endif

146
src/share/schema/ob_zone_sql_service.cpp
View File

@ -1,146 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SHARE_SCHEMA
#include "ob_zone_sql_service.h"
#include "lib/oblog/ob_log.h"
#include "lib/oblog/ob_log_module.h"
#include "lib/string/ob_sql_string.h"
#include "share/ob_dml_sql_splicer.h"
#include "share/inner_table/ob_inner_table_schema_constants.h"
#include "share/schema/ob_schema_struct.h"
#include "share/schema/ob_schema_service.h"
namespace oceanbase
{
using namespace common;
namespace share
{
namespace schema
{
int ObZoneSqlService::alter_zone(
const int64_t new_schema_version,
common::ObISQLClient &sql_client,
const ObString *ddl_stmt_str)
{
int ret = OB_SUCCESS;
if (OB_SUCC(ret)) {
ObSchemaOperation alter_zone_op;
alter_zone_op.tenant_id_ = OB_SYS_TENANT_ID;
alter_zone_op.database_id_ = 0;
alter_zone_op.tablegroup_id_ = 0;
alter_zone_op.table_id_ = 0;
alter_zone_op.op_type_ = OB_DDL_ALTER_ZONE;
alter_zone_op.schema_version_ = new_schema_version;
alter_zone_op.ddl_stmt_str_ = ddl_stmt_str != NULL ? *ddl_stmt_str : ObString();
if (OB_FAIL(log_operation(alter_zone_op, sql_client))) {
LOG_WARN("log alter zone ddl operation failed", K(alter_zone_op), K(ret));
}
}
return ret;
}
int ObZoneSqlService::add_zone(
const int64_t new_schema_version,
common::ObISQLClient &sql_client,
const ObString *ddl_stmt_str)
{
int ret = OB_SUCCESS;
if (OB_SUCC(ret)) {
ObSchemaOperation add_zone_op;
add_zone_op.tenant_id_ = OB_SYS_TENANT_ID;
add_zone_op.database_id_ = 0;
add_zone_op.tablegroup_id_ = 0;
add_zone_op.table_id_ = 0;
add_zone_op.op_type_ = OB_DDL_ADD_ZONE;
add_zone_op.schema_version_ = new_schema_version;
add_zone_op.ddl_stmt_str_ = ddl_stmt_str != NULL ? *ddl_stmt_str : ObString();
if (OB_FAIL(log_operation(add_zone_op, sql_client))) {
LOG_WARN("log add zone ddl operation failed", K(add_zone_op), K(ret));
}
}
return ret;
}
int ObZoneSqlService::delete_zone(
const int64_t new_schema_version,
common::ObISQLClient &sql_client,
const ObString *ddl_stmt_str)
{
int ret = OB_SUCCESS;
if (OB_SUCC(ret)) {
ObSchemaOperation delete_zone_op;
delete_zone_op.tenant_id_ = OB_SYS_TENANT_ID;
delete_zone_op.database_id_ = 0;
delete_zone_op.tablegroup_id_ = 0;
delete_zone_op.table_id_ = 0;
delete_zone_op.op_type_ = OB_DDL_DELETE_ZONE;
delete_zone_op.schema_version_ = new_schema_version;
delete_zone_op.ddl_stmt_str_ = ddl_stmt_str != NULL ? *ddl_stmt_str : ObString();
if (OB_FAIL(log_operation(delete_zone_op, sql_client))) {
LOG_WARN("log delete zone ddl operation failed", K(delete_zone_op), K(ret));
}
}
return ret;
}
int ObZoneSqlService::start_zone(
const int64_t new_schema_version,
common::ObISQLClient &sql_client,
const ObString *ddl_stmt_str)
{
int ret = OB_SUCCESS;
if (OB_SUCC(ret)) {
ObSchemaOperation start_zone_op;
start_zone_op.tenant_id_ = OB_SYS_TENANT_ID;
start_zone_op.database_id_ = 0;
start_zone_op.tablegroup_id_ = 0;
start_zone_op.table_id_ = 0;
start_zone_op.op_type_ = OB_DDL_START_ZONE;
start_zone_op.schema_version_ = new_schema_version;
start_zone_op.ddl_stmt_str_ = ddl_stmt_str != NULL ? *ddl_stmt_str : ObString();
if (OB_FAIL(log_operation(start_zone_op, sql_client))) {
LOG_WARN("log start zone ddl operation failed", K(start_zone_op), K(ret));
}
}
return ret;
}
int ObZoneSqlService::stop_zone(
const int64_t new_schema_version,
common::ObISQLClient &sql_client,
const ObString *ddl_stmt_str)
{
int ret = OB_SUCCESS;
if (OB_SUCC(ret)) {
ObSchemaOperation stop_zone_op;
stop_zone_op.tenant_id_ = OB_SYS_TENANT_ID;
stop_zone_op.database_id_ = 0;
stop_zone_op.tablegroup_id_ = 0;
stop_zone_op.table_id_ = 0;
stop_zone_op.op_type_ = OB_DDL_STOP_ZONE;
stop_zone_op.schema_version_ = new_schema_version;
stop_zone_op.ddl_stmt_str_ = ddl_stmt_str != NULL ? *ddl_stmt_str : ObString();
if (OB_FAIL(log_operation(stop_zone_op, sql_client))) {
LOG_WARN("log stop zone ddl operation failed", K(stop_zone_op), K(ret));
}
}
return ret;
}
} //end of schema
} //end of share
} //end of oceanbase

61
src/share/schema/ob_zone_sql_service.h
View File

@ -1,61 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_SHARE_SCHEMA_OB_ZONE_SQL_SERVICE_H_
#define OCEANBASE_SHARE_SCHEMA_OB_ZONE_SQL_SERVICE_H_
#include "share/schema/ob_ddl_sql_service.h"
namespace oceanbase
{
namespace common
{
class ObISQlClient;
}
namespace share
{
namespace schema
{
struct ObSchemaOperation;
class ObZoneSqlService : public ObDDLSqlService
{
public:
ObZoneSqlService(ObSchemaService &schema_service)
: ObDDLSqlService(schema_service) {}
virtual ~ObZoneSqlService() {}
virtual int alter_zone(const int64_t new_schema_version,
common::ObISQLClient &sql_client,
const common::ObString *ddl_stmt_str = NULL);
virtual int add_zone(const int64_t new_schema_version,
common::ObISQLClient &sql_client,
const common::ObString *ddl_stmt_str = NULL);
virtual int delete_zone(const int64_t new_schema_version,
common::ObISQLClient &sql_client,
const common::ObString *ddl_stmt_str = NULL);
virtual int start_zone(const int64_t new_schema_version,
common::ObISQLClient &sql_client,
const common::ObString *ddl_stmt_str = NULL);
virtual int stop_zone(const int64_t new_schema_version,
common::ObISQLClient &sql_client,
const common::ObString *ddl_stmt_str = NULL);
private:
DISALLOW_COPY_AND_ASSIGN(ObZoneSqlService);
};
} //end of namespace schema
} //end of namespace share
} //end of namespace oceanbase
#endif //OCEANBASE_SHARE_SCHEMA_OB_ZONE_SQL_SERVICE_H_

426
src/share/stat/ob_column_stat.cpp
View File

@ -1,426 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "share/stat/ob_column_stat.h"
#include <math.h>
#include "lib/utility/utility.h"
#include "lib/allocator/ob_malloc.h"
namespace oceanbase
{
namespace common
{
ObColumnStat::ObColumnStat()
: table_id_(0),
partition_id_(0),
column_id_(0),
version_(0),
last_rebuild_version_(0),
num_null_(0),
num_distinct_(0),
min_value_(),
max_value_(),
llc_bitmap_size_(0),
llc_bitmap_(NULL),
object_buf_(NULL),
is_modified_(false)
{
min_value_.set_min_value();
max_value_.set_max_value();
}
ObColumnStat::ObColumnStat(common::ObIAllocator &allocator)
: table_id_(0),
partition_id_(0),
column_id_(0),
version_(0),
last_rebuild_version_(0),
num_null_(0),
num_distinct_(0),
min_value_(),
max_value_(),
llc_bitmap_size_(0),
llc_bitmap_(NULL),
object_buf_(NULL),
is_modified_(false)
{
min_value_.set_min_value();
max_value_.set_max_value();
if (NULL == (llc_bitmap_ = static_cast<char*>(allocator.alloc(NUM_LLC_BUCKET)))) {
COMMON_LOG_RET(WARN, OB_ALLOCATE_MEMORY_FAILED, "allocate memory for llc_bitmap_ failed.");
} else if (NULL == (object_buf_ = static_cast<char*>(allocator.alloc(MAX_OBJECT_SERIALIZE_SIZE * 2)))) {
COMMON_LOG_RET(WARN, OB_ALLOCATE_MEMORY_FAILED, "allocate memory for object_buf_ failed.");
} else {
llc_bitmap_size_ = NUM_LLC_BUCKET;
MEMSET(llc_bitmap_, 0, llc_bitmap_size_);
MEMSET(object_buf_, 0, MAX_OBJECT_SERIALIZE_SIZE * 2);
}
}
ObColumnStat::~ObColumnStat()
{
// free llc_bitmap_;
// free object_buf_;
}
void ObColumnStat::reset()
{
table_id_ = 0;
partition_id_ = 0;
column_id_ = 0;
version_ = 0;
last_rebuild_version_ = 0;
num_null_ = 0;
num_distinct_ = 0;
min_value_.set_min_value();
max_value_.set_max_value();
MEMSET(llc_bitmap_, 0, llc_bitmap_size_);
MEMSET(object_buf_, 0, MAX_OBJECT_SERIALIZE_SIZE * 2);
is_modified_ = false;
}
int64_t ObColumnStat::size() const
{
return get_deep_copy_size();
}
int ObColumnStat::deep_copy(char *buf, const int64_t buf_len, ObIKVCacheValue *&value) const
{
int ret = OB_SUCCESS;
if (NULL == buf || buf_len < size()) {
ret = OB_INVALID_ARGUMENT;
COMMON_LOG(WARN, "invalid arguments.",
KP(buf), K(buf_len), K(size()), K(ret));
} else {
ObColumnStat *stat = new (buf) ObColumnStat();
int64_t pos = sizeof(*this);
if (OB_FAIL(stat->deep_copy(*this, buf, buf_len, pos))) {
COMMON_LOG(WARN, "deep copy column stat failed.", K(ret));
} else {
value = stat;
}
}
return ret;
}
int ObColumnStat::add_value(const common::ObObj &value)
{
int ret = OB_SUCCESS;
if (!is_writable()) {
ret = OB_NOT_INIT;
COMMON_LOG(WARN, "CAN NOT add value to column stat only for read.",
KP_(llc_bitmap), K_(llc_bitmap_size), KP_(object_buf), K(ret));
} else {
if (value.is_null()) {
is_modified_ = true;
++num_null_;
} else {
bool need_comp = true;
if (OB_SUCC(ret)) {
if (min_value_.is_min_value() || value.compare(min_value_) < 0) {
if (OB_FAIL(store_min_value(value))) {
COMMON_LOG(WARN, "store min value object failed.", K(ret));
} else {
is_modified_ = true;
need_comp = false;
}
}
}
if (OB_SUCC(ret)) {
if (max_value_.is_max_value() || (need_comp && value.compare(max_value_) > 0)) {
if (OB_FAIL(store_max_value(value))) {
COMMON_LOG(WARN, "store min value object failed.", K(ret));
} else {
is_modified_ = true;
}
}
}
if (OB_SUCC(ret)) {
uint64_t hval = 0;
if (OB_FAIL(calc_llc_value(value, hval))) {
COMMON_LOG(WARN, "calc llc values failed.", K(ret));
}
}
}
// do not accumulate num_row_ which will be set after merge phase complete.
// do not calculate average_row_size_, same as above.
}
return ret;
}
int ObColumnStat::add(const ObColumnStat &other)
{
int ret = OB_SUCCESS;
if (!is_writable()) {
ret = OB_NOT_INIT;
COMMON_LOG(WARN, "CAN NOT add value to column stat only for read.",
KP_(llc_bitmap), K_(llc_bitmap_size), KP_(object_buf), K(ret));
} else {
if (0 != other.num_null_) {
is_modified_ = true;
num_null_ += other.num_null_;
}
if (OB_SUCC(ret)) {
if (!other.max_value_.is_max_value() && !max_value_.is_max_value()) {
if (max_value_.compare(other.max_value_) < 0) {
if (OB_FAIL(store_max_value(other.max_value_))) {
COMMON_LOG(WARN, "store max value object failed.", K(ret));
} else {
is_modified_ = true;
}
}
} else if (other.max_value_.is_max_value() && max_value_.is_max_value()) {
//do nothing
} else {
if (max_value_.is_max_value()) {
if (OB_FAIL(store_max_value(other.max_value_))) {
COMMON_LOG(WARN, "store max value object failed.", K(ret));
} else {
is_modified_ = true;
}
}
}
}
if (OB_SUCC(ret)) {
if (!other.min_value_.is_min_value() && !min_value_.is_min_value()) {
if (min_value_.compare(other.min_value_) > 0) {
if (OB_FAIL(store_min_value(other.min_value_))) {
COMMON_LOG(WARN, "store min value object failed.", K(ret));
} else {
is_modified_ = true;
}
}
} else if (other.min_value_.is_min_value() && min_value_.is_min_value()) {
//do nothing
} else {
if (min_value_.is_min_value()) {
if (OB_FAIL(store_min_value(other.min_value_))) {
COMMON_LOG(WARN, "store min value object failed.", K(ret));
} else {
is_modified_ = true;
}
}
}
}
if (OB_SUCC(ret)) {
for (int64_t i = 0; i < NUM_LLC_BUCKET; i++) {
if (static_cast<uint8_t>(other.llc_bitmap_[i]) > static_cast<uint8_t>(llc_bitmap_[i])) {
llc_bitmap_[i] = other.llc_bitmap_[i];
is_modified_ = true;
}
}
}
}
return ret;
}
int ObColumnStat::finish()
{
int ret = OB_SUCCESS;
if (NULL == llc_bitmap_) {
ret = OB_NOT_INIT;
COMMON_LOG(WARN, "CAN NOT set column stat only for read.",
KP_(llc_bitmap), K(ret));
} else if (is_modified_) {
// use HyperLogLog Counting estimate number of distinct values.
double sum_of_pmax = 0;
double alpha = select_alpha_value(NUM_LLC_BUCKET);
int64_t empty_bucket_num = 0;
for (int64_t i = 0; i < NUM_LLC_BUCKET; ++i) {
sum_of_pmax += (1 / pow(2, (llc_bitmap_[i])));
if (llc_bitmap_[i] == 0) {
++empty_bucket_num;
}
}
double estimate_ndv = (alpha * NUM_LLC_BUCKET * NUM_LLC_BUCKET) / sum_of_pmax;
num_distinct_ = static_cast<int64_t>(estimate_ndv);
// check if estimate result too tiny or large.
if (estimate_ndv <= 5 * NUM_LLC_BUCKET / 2) {
if (0 != empty_bucket_num) {
// use linear count
num_distinct_ = static_cast<int64_t>(NUM_LLC_BUCKET
* log(NUM_LLC_BUCKET / double(empty_bucket_num)));
}
}
if (estimate_ndv > (static_cast<double>(LARGE_NDV_NUMBER) / 30)) {
num_distinct_ = static_cast<int64_t>((0-pow(2, 32)) * log(1 - estimate_ndv / LARGE_NDV_NUMBER));
}
}
return ret;
}
int ObColumnStat::deep_copy(const ObColumnStat &src, char *buf, const int64_t size, int64_t &pos)
{
int ret = OB_SUCCESS;
// assign base members no need to handle memory..
table_id_ = src.table_id_;
partition_id_ = src.partition_id_;
column_id_ = src.column_id_;
version_ = src.version_;
last_rebuild_version_ = src.last_rebuild_version_;
num_null_ = src.num_null_;
num_distinct_ = src.num_distinct_;
if (!src.is_valid() || NULL == buf || size <= 0) {
ret = OB_INVALID_ARGUMENT;
COMMON_LOG(WARN, "invalid arguments.", K(src), KP(buf), K(size), K(ret));
} else if (OB_FAIL(min_value_.deep_copy(src.min_value_, buf, size, pos))) {
COMMON_LOG(WARN, "deep copy min_value_ failed.", K_(src.min_value), K(ret));
} else if (OB_FAIL(max_value_.deep_copy(src.max_value_, buf, size, pos))) {
COMMON_LOG(WARN, "deep copy max_value_ failed.", K_(src.max_value), K(ret));
} else if (pos + src.llc_bitmap_size_ > size) {
ret = OB_BUF_NOT_ENOUGH;
COMMON_LOG(WARN, "llc_bitmap_size_ overflow.",
K_(src.llc_bitmap_size), K(ret));
} else {
// copy llc bitmap.
llc_bitmap_ = buf + pos;
llc_bitmap_size_ = src.llc_bitmap_size_;
MEMCPY(llc_bitmap_, src.llc_bitmap_, src.llc_bitmap_size_);
pos += llc_bitmap_size_;
}
return ret;
}
int64_t ObColumnStat::get_deep_copy_size() const
{
int64_t base_size = sizeof(ObColumnStat);
base_size += min_value_.get_deep_copy_size();
base_size += max_value_.get_deep_copy_size();
base_size += llc_bitmap_size_;
return base_size;
}
int ObColumnStat::store_llc_bitmap(const char *bitmap, const int64_t size)
{
int ret = OB_SUCCESS;
if (NULL == llc_bitmap_ || NULL == bitmap || size > llc_bitmap_size_) {
ret = common::OB_INVALID_ARGUMENT;
COMMON_LOG(WARN, "invalid arguments.",
KP_(llc_bitmap), K_(llc_bitmap_size),
KP(bitmap), K(size), K(ret));
} else {
MEMCPY(llc_bitmap_, bitmap, size);
llc_bitmap_size_ = size;
}
return ret;
}
int ObColumnStat::store_max_value(const common::ObObj &max)
{
int ret = OB_SUCCESS;
int64_t pos = 0;
if (NULL == object_buf_) {
ret = common::OB_INVALID_ARGUMENT;
COMMON_LOG(WARN, "invalid arguments.", KP_(object_buf), K(ret));
} else if (max.get_serialize_size() > MAX_OBJECT_SERIALIZE_SIZE) {
//ignore
} else if (OB_FAIL(max_value_.deep_copy(
max, object_buf_ + MAX_OBJECT_SERIALIZE_SIZE,
MAX_OBJECT_SERIALIZE_SIZE, pos))) {
COMMON_LOG(WARN, "deep copy max object failed.",
KP_(object_buf), K(max), K(pos), K(ret));
}
return ret;
}
int ObColumnStat::store_min_value(const common::ObObj &min)
{
int ret = OB_SUCCESS;
int64_t pos = 0;
if (NULL == object_buf_) {
ret = common::OB_INVALID_ARGUMENT;
COMMON_LOG(WARN, "invalid arguments.", KP_(object_buf), K(ret));
} else if (min.get_serialize_size() > MAX_OBJECT_SERIALIZE_SIZE) {
//ignore
} else if (OB_FAIL(min_value_.deep_copy(
min, object_buf_, MAX_OBJECT_SERIALIZE_SIZE, pos))) {
COMMON_LOG(WARN, "deep copy max object failed.",
KP_(object_buf), K(min), K(pos), K(ret));
}
return ret;
}
int ObColumnStat::calc_llc_value(const common::ObObj &value, uint64_t &h)
{
int ret = OB_SUCCESS;
if (OB_FAIL(value.hash(h, 0))) {
COMMON_LOG(WARN, "fail to hash.", K(ret), K(value));
} else {
// Mask out the k least significant bits as bucket NO
uint64_t hash_bucket = h;
uint64_t total_bucket_bits = TOTAL_BUCKET_BITS;
uint64_t bucket = 0;
while (total_bucket_bits > 0) {
bucket ^= (hash_bucket & (NUM_LLC_BUCKET -1));
hash_bucket >>= BUCKET_BITS;
total_bucket_bits -= BUCKET_BITS;
}
const uint64_t pmax = trailing_zeroes(h >> BUCKET_BITS); // pmax <= 64;
if (pmax > static_cast<uint8_t>(llc_bitmap_[bucket])) {
llc_bitmap_[bucket] = static_cast<uint8_t>(pmax);
is_modified_ = true;
}
}
return ret;
}
uint64_t ObColumnStat::trailing_zeroes(const uint64_t num)
{
uint64_t pos = 0;
if (0 == num) {
pos = HASH_VALUE_MAX_BITS;
} else {
while (((num >> pos) & 0x1) == 0) {
pos += 1;
}
}
return pos + 1;
}
double ObColumnStat::select_alpha_value(const int64_t num_bucket)
{
double ret = 0.0;
switch (num_bucket) {
case 16:
ret = 0.673;
break;
case 32:
ret = 0.697;
break;
case 64:
ret = 0.709;
break;
default:
ret = 0.7213 / (1 + 1.079 / double(num_bucket));
break;
}
return ret;
}
} // end of namespace common
} // end of namespace oceanbase

276
src/share/stat/ob_column_stat.h
View File

@ -1,276 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef _OB_COLUMN_STAT_H_
#define _OB_COLUMN_STAT_H_
#include <stdint.h>
#include "common/object/ob_object.h"
#include "lib/hash_func/murmur_hash.h"
#include "share/cache/ob_kvcache_struct.h"
namespace oceanbase
{
namespace common
{
class ObColumnStat : public common::ObIKVCacheValue
{
public:
static const int64_t BUCKET_BITS = 10; // ln2(1024) = 10;
static const int64_t TOTAL_BUCKET_BITS = 40; // 6 groups
static const int64_t NUM_LLC_BUCKET = (1 << BUCKET_BITS);
static const int64_t HASH_VALUE_MAX_BITS = 64; // 64 bits hash value.
static const int64_t LARGE_NDV_NUMBER = 2LL << 61; // 2 << 64 is too large for int64_t, and 2 << 61 is enough for ndv
static const int64_t MAX_OBJECT_SERIALIZE_SIZE = 512;
public:
struct Key : public common::ObIKVCacheKey
{
uint64_t table_id_;
uint64_t partition_id_;
uint64_t column_id_;
Key() : table_id_(0), partition_id_(0), column_id_(0)
{
}
Key(const uint64_t tid, const uint64_t pid, const uint64_t cid)
: table_id_(tid), partition_id_(pid), column_id_(cid)
{
}
uint64_t hash() const
{
return common::murmurhash(this, sizeof(Key), 0);
}
bool operator==(const ObIKVCacheKey &other) const
{
const Key &other_key = reinterpret_cast<const Key&>(other);
return table_id_ == other_key.table_id_
&& partition_id_ == other_key.partition_id_
&& column_id_ == other_key.column_id_;
}
uint64_t get_tenant_id() const
{
return common::OB_SYS_TENANT_ID;
}
int64_t size() const
{
return sizeof(*this);
}
int deep_copy(char *buf, const int64_t buf_len, ObIKVCacheKey *&key) const
{
int ret = OB_SUCCESS;
Key *tmp = NULL;
if (NULL == buf || buf_len < size()) {
ret = OB_INVALID_ARGUMENT;
COMMON_LOG(WARN, "invalid arguments.",
KP(buf), K(buf_len), K(size()), K(ret));
} else {
tmp = new (buf) Key();
*tmp = *this;
key = tmp;
}
return ret;
}
bool is_valid() const
{
return table_id_ > 0 && column_id_ > 0;
}
TO_STRING_KV(K(table_id_),
K(partition_id_),
K(column_id_));
};
public:
ObColumnStat();
//construct object to write, maybe failed when dont't have enough memory for llc_bitmap_ or object_buf_
explicit ObColumnStat(common::ObIAllocator &allocator);
~ObColumnStat();
void reset();
virtual int64_t size() const;
virtual int deep_copy(char *buf, const int64_t buf_len, ObIKVCacheValue *&value) const;
TO_STRING_KV(K(table_id_),
K(partition_id_),
K(column_id_),
K(version_),
K(last_rebuild_version_),
K(num_distinct_),
K(num_null_),
K(min_value_),
K(max_value_),
K(is_modified_));
int add_value(const common::ObObj &value);
int add(const ObColumnStat &other);
int finish();
int deep_copy(const ObColumnStat &src, char *buf, const int64_t size, int64_t &pos);
int64_t get_deep_copy_size() const;
int store_llc_bitmap(const char *bitmap, const int64_t size);
int store_min_value(const common::ObObj& min);
int store_max_value(const common::ObObj& max);
bool is_valid() const
{
return common::OB_INVALID_ID != table_id_
//&& partition_id_ >= 0
//&& column_id_ >= 0
&& version_ >= 0
&& last_rebuild_version_ >= 0
&& num_distinct_ >= 0
&& num_null_ >= 0;
}
// check ColumnStat Object if allocates object buffer for write.
bool is_writable() const
{
return (NULL != llc_bitmap_ && 0 != llc_bitmap_size_ && NULL != object_buf_ );
}
const Key get_key() const
{
return Key(table_id_, partition_id_, column_id_);
}
uint64_t get_column_id() const
{
return column_id_;
}
void set_column_id(uint64_t cid)
{
column_id_ = cid;
}
int64_t get_llc_bitmap_size() const
{
return llc_bitmap_size_;
}
const char *get_llc_bitmap() const
{
return llc_bitmap_;
}
void set_llc_bitmap(char *bitmap, const int64_t size)
{
llc_bitmap_ = bitmap;
llc_bitmap_size_ = size;
}
const common::ObObj &get_max_value() const
{
return max_value_;
}
void set_max_value(const common::ObObj &max)
{
max_value_ = max;
}
const common::ObObj &get_min_value() const
{
return min_value_;
}
void set_min_value(const common::ObObj &min)
{
min_value_ = min;
}
int64_t get_num_distinct() const
{
return num_distinct_;
}
void set_num_distinct(int64_t numDistinct)
{
num_distinct_ = numDistinct;
}
int64_t get_num_null() const
{
return num_null_;
}
void set_num_null(int64_t num_null)
{
num_null_ = num_null;
}
uint64_t get_partition_id() const
{
return partition_id_;
}
void set_partition_id(uint64_t pid)
{
partition_id_ = pid;
}
uint64_t get_table_id() const
{
return table_id_;
}
void set_table_id(uint64_t tid)
{
table_id_ = tid;
}
int64_t get_version() const
{
return version_;
}
void set_version(int64_t version)
{
version_ = version;
}
int64_t get_last_rebuild_version() const
{
return last_rebuild_version_;
}
void set_last_rebuild_version(int64_t last_rebuild_version)
{
last_rebuild_version_ = last_rebuild_version;
}
bool is_modified() const
{
return is_modified_;
}
private:
inline int calc_llc_value(const common::ObObj &value, uint64_t &h);
inline uint64_t trailing_zeroes(const uint64_t num);
inline double select_alpha_value(const int64_t num_bucket);
private:
DISALLOW_COPY_AND_ASSIGN(ObColumnStat);
uint64_t table_id_;
uint64_t partition_id_;
uint64_t column_id_;
int64_t version_;
int64_t last_rebuild_version_;
int64_t num_null_;
int64_t num_distinct_;
common::ObObj min_value_;
common::ObObj max_value_;
int64_t llc_bitmap_size_;
char *llc_bitmap_;
char *object_buf_;
bool is_modified_;
}; // end of class ObColumnStat
} // end of namespace common
} // end of namespace oceanbase
#endif /* _OB_COLUMN_STAT_H_ */

427
src/sql/das/ob_das_batch_scan_op.cpp
View File

@ -1,427 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_DAS
#include "sql/das/ob_das_batch_scan_op.h"
#include "sql/engine/ob_bit_vector.h"
#include "storage/access/ob_table_scan_iterator.h"
namespace oceanbase
{
using namespace common;
using namespace storage;
namespace sql
{
ObDASBatchScanOp::ObDASBatchScanOp(ObIAllocator &op_alloc)
: ObDASScanOp(op_alloc),
batch_result_(nullptr),
group_flags_(nullptr),
flag_memory_size_(0),
next_group_idx_(0),
group_cnt_(0),
batch_iter_creator_(op_alloc),
result_outputs_(nullptr)
{
}
int ObDASBatchScanOp::init_group_flags(int64_t max_group_cnt)
{
int ret = OB_SUCCESS;
int64_t memory_size = ObBitVector::memory_size(max_group_cnt);
if (memory_size > flag_memory_size_) {
void *buf = op_alloc_.alloc(memory_size);
if (OB_ISNULL(buf)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("allocate flag memory size failed", K(ret), K(memory_size));
} else {
group_flags_ = to_bit_vector(buf);
flag_memory_size_ = memory_size;
}
}
if (OB_SUCC(ret)) {
group_flags_->init(max_group_cnt);
}
return ret;
}
int ObDASBatchScanOp::open_op()
{
int ret = OB_SUCCESS;
ObITabletScan &tsc_service = get_tsc_service();
reset_access_datums_ptr();
if (OB_FAIL(init_scan_param())) {
LOG_WARN("init scan param failed", K(ret));
} else if (OB_FAIL(tsc_service.table_scan(scan_param_, batch_result_))) {
if (OB_SNAPSHOT_DISCARDED == ret && scan_param_.fb_snapshot_.is_valid()) {
ret = OB_INVALID_QUERY_TIMESTAMP;
} else if (OB_TRY_LOCK_ROW_CONFLICT != ret) {
LOG_WARN("fail to scan table", K(scan_param_), K(ret));
}
} else if (OB_FAIL(switch_batch_iter())) {
LOG_WARN("get result iterator form batch result iter failed", K(ret));
} else {
LOG_DEBUG("batch table scan begin", K_(scan_param), KPC_(result));
}
return ret;
}
int ObDASBatchScanOp::release_op()
{
int ret = OB_NOT_SUPPORTED;
scan_param_.partition_guard_ = nullptr;
scan_param_.destroy_schema_guard();
batch_result_ = nullptr;
result_ = nullptr;
batch_iter_creator_.destory();
return ret;
}
int ObDASBatchScanOp::add_group_flag(int64_t group_idx)
{
int ret = OB_SUCCESS;
ObPosArray &pos_array = scan_param_.range_array_pos_;
int64_t range_pos_cnt = pos_array.count();
int64_t last_range_cnt = range_pos_cnt > 0 ? pos_array.at(range_pos_cnt - 1) + 1 : 0;
int64_t range_cnt = scan_param_.key_ranges_.count();
bool empty_range = (range_cnt == last_range_cnt);
if (!empty_range) {
if (OB_FAIL(scan_param_.range_array_pos_.push_back(range_cnt - 1))) {
LOG_WARN("store key range array pos failed", K(ret), K(scan_param_.key_ranges_));
} else {
group_flags_->set(group_idx);
}
}
if (OB_SUCC(ret)) {
group_cnt_ = group_idx + 1;
}
return ret;
}
int ObDASBatchScanOp::rescan()
{
int &ret = errcode_;
//The first batched das rescan need to rescan ObTableScanIterIterator
//and then only need to switch iterator
ObITabletScan &tsc_service = get_tsc_service();
reset_access_datums_ptr();
if (OB_FAIL(tsc_service.table_rescan(scan_param_, batch_result_))) {
LOG_WARN("rescan the table iterator failed", K(ret));
} else if (OB_FAIL(switch_batch_iter())) {
LOG_WARN("get next iter from batch result failed", K(ret));
}
return ret;
}
int ObDASBatchScanOp::reuse_iter()
{
int &ret = errcode_;
ObITabletScan &tsc_service = get_tsc_service();
if (OB_FAIL(tsc_service.reuse_scan_iter(scan_param_.need_switch_param_, batch_result_))) {
LOG_WARN("reuse scan iterator failed", K(ret));
} else {
scan_param_.key_ranges_.reuse();
scan_param_.range_array_pos_.reuse();
result_ = nullptr;
next_group_idx_ = 0;
group_cnt_ = 0;
}
return ret;
}
int ObDASBatchScanOp::decode_task_result(ObIDASTaskResult *task_result)
{
int ret = OB_SUCCESS;
DASFoldIterator *fold_iter = nullptr;
int64_t group_cnt = scan_param_.range_array_pos_.count();
if (OB_FAIL(batch_iter_creator_.get_fold_iter(fold_iter))) {
LOG_WARN("allocate fold iterator failed", K(ret));
} else if (OB_FAIL(fold_iter->init_iter(*scan_ctdef_, *scan_rtdef_, group_cnt))) {
LOG_WARN("init iterator failed", K(ret));
} else {
result_outputs_ = &fold_iter->get_output_exprs();
if (OB_FAIL(ObDASScanOp::decode_task_result(task_result))) {
LOG_WARN("decode das scan task result failed", K(ret));
} else {
fold_iter->set_result_iter(result_);
batch_result_ = fold_iter;
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(switch_batch_iter())) {
LOG_WARN("switch batch iter failed", K(ret));
}
}
return ret;
}
int ObDASBatchScanOp::fill_task_result(ObIDASTaskResult &task_result, bool &has_more, int64_t &memory_limit)
{
int ret = OB_SUCCESS;
DASExpandIterator *expand_iter = nullptr;
if (OB_FAIL(batch_iter_creator_.get_expand_iter(expand_iter))) {
LOG_WARN("allocate expand iterator failed", K(ret));
} else if (OB_FAIL(expand_iter->init_iter(*scan_ctdef_, *scan_rtdef_, batch_result_, result_))) {
LOG_WARN("init iterator failed", K(ret));
} else {
result_ = expand_iter;
result_outputs_ = &(expand_iter->get_output_exprs());
if (OB_FAIL(ObDASScanOp::fill_task_result(task_result, has_more, memory_limit))) {
LOG_WARN("fill task result failed", K(ret));
}
}
return ret;
}
int ObDASBatchScanOp::switch_batch_iter()
{
int ret = OB_SUCCESS;
//When group_cnt<=0 means that das batch scan does not rely on group flags to determine
//whether the group range is empty.
//At this time, the parameter boundary of das batch scan is consistent with
//the boundary of range_array_pos,
//such as remote execution of das batch scan
if (OB_UNLIKELY(group_cnt_ <= 0)) {
ret = batch_result_->get_next_iter(result_);
} else if (OB_UNLIKELY(next_group_idx_ >= group_cnt_)) {
ret = OB_ITER_END;
LOG_DEBUG("group iterator reach end", K(ret), K(next_group_idx_), K(group_cnt_));
} else if (OB_LIKELY(group_flags_->at(next_group_idx_))) {
ret = batch_result_->get_next_iter(result_);
} else if (OB_FAIL(batch_iter_creator_.get_empty_iter(result_))) {
LOG_WARN("create empty iter failed", K(ret));
}
if (OB_SUCC(ret)) {
++next_group_idx_;
}
return ret;
}
ObArrayWrap<uint8_t> ObDASBatchScanOp::to_bit_array() const
{
int64_t count = flag_memory_size_ / sizeof(uint8_t);
uint8_t *data = reinterpret_cast<uint8_t*>(group_flags_);
return ObArrayWrap<uint8_t>(data, count);
}
OB_SERIALIZE_MEMBER((ObDASBatchScanOp, ObDASScanOp),
scan_param_.range_array_pos_);
int ObDASBatchScanOp::DASExpandIterator::init_iter(const ObDASScanCtDef &scan_ctdef,
ObDASScanRtDef &scan_rtdef,
ObNewIterIterator *batch_iter,
ObNewRowIterator *iter)
{
int ret = OB_SUCCESS;
//append group id pseudo column to output exprs
int64_t output_cnt = scan_ctdef.pd_expr_spec_.access_exprs_.count() + 1;
output_exprs_.set_allocator(&scan_rtdef.stmt_allocator_);
output_exprs_.set_capacity(output_cnt);
if (OB_FAIL(append(output_exprs_, scan_ctdef.pd_expr_spec_.access_exprs_))) {
LOG_WARN("append storage output exprs to expand iterator failed", K(ret));
} else if (OB_FAIL(output_exprs_.push_back(scan_ctdef.group_id_expr_))) {
LOG_WARN("store group id expr to output exprs failed", K(ret));
} else {
eval_ctx_ = &scan_rtdef.p_pd_expr_op_->get_eval_ctx();
batch_iter_ = batch_iter;
iter_ = iter;
}
return ret;
}
int ObDASBatchScanOp::DASExpandIterator::get_next_row()
{
int ret = OB_SUCCESS;
bool got_row = false;
while (OB_SUCC(ret) && !got_row) {
if (OB_FAIL(iter_->get_next_row())) {
if (OB_ITER_END != ret) {
LOG_WARN("get next row from result failed", K(ret));
} else if (OB_FAIL(batch_iter_->get_next_iter(iter_))) {
if (OB_ITER_END != ret) {
LOG_WARN("get next iterator from batch iter failed", K(ret));
} else {
LOG_DEBUG("reach the batch iterator end", K(ret));
}
} else {
++group_idx_;
}
} else {
got_row = true;
ObExpr *expr = output_exprs_.at(output_exprs_.count() - 1);
if (OB_UNLIKELY(expr->type_ != T_PSEUDO_GROUP_ID)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("expr type is invalid", K(ret), K(expr->type_));
} else {
// store group index at expand iterator's the last column
expr->locate_datum_for_write(*eval_ctx_).set_int(group_idx_);
expr->get_eval_info(*eval_ctx_).evaluated_ = true;
LOG_DEBUG("store group idx to output exprs", K(ret), K(group_idx_));
}
}
}
return ret;
}
int ObDASBatchScanOp::DASExpandIterator::get_next_rows(int64_t &count, int64_t capacity)
{
int ret = OB_SUCCESS;
bool got_row = false;
while (OB_SUCC(ret) && !got_row) {
if (OB_FAIL(iter_->get_next_rows(count, capacity))) {
if (OB_ITER_END != ret) {
LOG_WARN("get next rows from result failed", K(ret), K(count), K(capacity));
} else if (OB_FAIL(batch_iter_->get_next_iter(iter_))) {
if (OB_ITER_END != ret) {
LOG_WARN("get next iterator from batch iter failed", K(ret));
} else {
LOG_DEBUG("reach the batch iterator end", K(ret));
}
} else {
++group_idx_;
}
} else {
got_row = true;
ObExpr *expr = output_exprs_.at(output_exprs_.count() - 1);
if (OB_UNLIKELY(expr->type_ != T_PSEUDO_GROUP_ID)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("expr type is invalid", K(ret), K(expr->type_));
} else {
// store group index at expand iterator's the last column
ObDatumVector group_id_datums = expr->locate_expr_datumvector(*eval_ctx_);
for (int64_t i = 0; OB_SUCC(ret) && i < count; ++i) {
group_id_datums.at(i)->set_int(group_idx_);
}
expr->get_eval_info(*eval_ctx_).cnt_ = count;
expr->get_eval_info(*eval_ctx_).evaluated_ = true;
LOG_DEBUG("store group idx to output exprs", K(ret), K(group_idx_), K(count));
}
}
}
return ret;
}
int ObDASBatchScanOp::DASFoldIterator::init_iter(const ObDASScanCtDef &scan_ctdef,
ObDASScanRtDef &scan_rtdef,
int64_t group_cnt)
{
int ret = OB_SUCCESS;
//append group id pseudo column to output exprs
int64_t output_cnt = scan_ctdef.pd_expr_spec_.access_exprs_.count() + 1;
output_exprs_.set_allocator(&scan_rtdef.stmt_allocator_);
output_exprs_.set_capacity(output_cnt);
group_cnt_ = group_cnt;
if (OB_FAIL(append(output_exprs_, scan_ctdef.pd_expr_spec_.access_exprs_))) {
LOG_WARN("append storage output exprs to expand iterator failed", K(ret));
} else if (OB_FAIL(output_exprs_.push_back(scan_ctdef.group_id_expr_))) {
LOG_WARN("store group id expr to output exprs failed", K(ret));
} else {
eval_ctx_ = &scan_rtdef.p_pd_expr_op_->get_eval_ctx();
new(&last_row_) LastDASStoreRow(scan_rtdef.scan_allocator_);
last_row_.reuse_ = true;
}
return ret;
}
int ObDASBatchScanOp::DASFoldIterator::get_next_row()
{
int ret = OB_SUCCESS;
int64_t last_group_id = OB_NOT_NULL(last_row_.store_row_) ?
last_row_.store_row_->cells()[output_exprs_.count() - 1].get_int() : OB_INVALID_INDEX;
if (OB_UNLIKELY(OB_INVALID_INDEX == last_group_id)) {
const ObExpr *expr = output_exprs_.at(output_exprs_.count() - 1);
ObDatum &current_group_id = expr->locate_expr_datum(*eval_ctx_);
if (OB_FAIL(iter_->get_next_row())) {
if (OB_ITER_END != ret) {
LOG_WARN("get next row from iter failed", K(ret));
}
} else if (OB_LIKELY(current_group_id.get_int() > group_id_)) {
//no row belong to current group iterator, save current row and return OB_ITER_END
if (OB_FAIL(last_row_.save_store_row(output_exprs_, *eval_ctx_))) {
LOG_WARN("save store row failed", K(ret));
} else {
ret = OB_ITER_END;
LOG_DEBUG("current iterator reach end", K(ret), K_(group_id), K(current_group_id));
}
} else if (expr->locate_expr_datum(*eval_ctx_).get_int() < group_id_) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("last iterator has not reach iter end", K(ret),
K(group_id_), "current_group_id", expr->locate_expr_datum(*eval_ctx_));
} else {
LOG_DEBUG("current row match current group id", K_(group_id), K(current_group_id),
"current_row", ROWEXPR2STR(*eval_ctx_, output_exprs_));
}
} else if (OB_UNLIKELY(group_id_ < last_group_id)) {
ret = OB_ITER_END;
LOG_DEBUG("current iterator reach end", K(ret), K(last_group_id), K(group_id_));
} else if (OB_UNLIKELY(group_id_ > last_group_id)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("last iterator has not reach iter end", K(ret), K(group_id_), K(last_group_id));
} else if (OB_FAIL(last_row_.store_row_->to_expr(output_exprs_, *eval_ctx_))) {
LOG_WARN("to expr skip const failed", K(ret));
} else {
last_row_.store_row_->cells()[output_exprs_.count() - 1].set_int(OB_INVALID_INDEX);
LOG_DEBUG("last row match current group id", K_(group_id),
"current_row", ROWEXPR2STR(*eval_ctx_, output_exprs_));
}
return ret;
}
int ObDASBatchScanOp::DASFoldIterator::get_next_iter(ObNewRowIterator *&iter)
{
int ret = OB_SUCCESS;
if (group_id_ >= group_cnt_) {
ret = OB_ITER_END;
LOG_DEBUG("get next iter reach iter end", K(ret), K(group_id_), K(group_cnt_));
} else {
++group_id_;
iter = this;
}
return ret;
}
int ObDASBatchScanOp::DASBatchIterCreator::get_empty_iter(ObNewRowIterator *&empty_iter)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(empty_iter_)) {
ret = alloc_iter(empty_iter_);
}
if (OB_SUCC(ret)) {
empty_iter = empty_iter_;
}
return ret;
}
int ObDASBatchScanOp::DASBatchIterCreator::get_expand_iter(DASExpandIterator *&expand_iter)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(expand_iter_)) {
ret = alloc_iter(expand_iter_);
}
if (OB_SUCC(ret)) {
expand_iter = expand_iter_;
}
return ret;
}
int ObDASBatchScanOp::DASBatchIterCreator::get_fold_iter(DASFoldIterator *&fold_iter)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(fold_iter_)) {
ret = alloc_iter(fold_iter_);
}
if (OB_SUCC(ret)) {
fold_iter = fold_iter_;
}
return ret;
}
} // namespace sql
} // namespace oceanbase

4053
src/sql/engine/aggregate/ob_aggregate_function.cpp
View File

File diff suppressed because it is too large Load Diff

247
src/sql/engine/aggregate/ob_merge_groupby.cpp
View File

@ -1,247 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_ENG
#include "sql/engine/aggregate/ob_merge_groupby.h"
#include "lib/utility/utility.h"
#include "sql/session/ob_sql_session_info.h"
#include "sql/engine/ob_exec_context.h"
namespace oceanbase
{
using namespace common;
namespace sql
{
//REGISTER_PHY_OPERATOR(ObMergeGroupBy, PHY_MERGE_GROUP_BY);
class ObMergeGroupBy::ObMergeGroupByCtx : public ObGroupByCtx
{
public:
explicit ObMergeGroupByCtx(ObExecContext &exec_ctx)
: ObGroupByCtx(exec_ctx),
last_input_row_(NULL),
is_end_(false),
cur_output_group_id (-1),
first_output_group_id (0)
{
}
virtual void destroy() { ObGroupByCtx::destroy(); }
private:
DISALLOW_COPY_AND_ASSIGN(ObMergeGroupByCtx);
private:
const ObNewRow *last_input_row_;
bool is_end_;
// added to support groupby with rollup
int64_t cur_output_group_id;
int64_t first_output_group_id;
friend class ObMergeGroupBy;
};
ObMergeGroupBy::ObMergeGroupBy(ObIAllocator &alloc)
: ObGroupBy(alloc)
{
}
ObMergeGroupBy::~ObMergeGroupBy()
{
}
int ObMergeGroupBy::inner_create_operator_ctx(ObExecContext &ctx, ObPhyOperatorCtx *&op_ctx) const
{
return CREATE_PHY_OPERATOR_CTX(ObMergeGroupByCtx, ctx, get_id(), get_type(), op_ctx);
}
int ObMergeGroupBy::inner_open(ObExecContext &ctx) const
{
int ret = OB_SUCCESS;
if (OB_FAIL(ObGroupBy::init_group_by(ctx))) {
LOG_WARN("init group by failed", K(ret));
} else {
ObMergeGroupByCtx *groupby_ctx = GET_PHY_OPERATOR_CTX(ObMergeGroupByCtx, ctx, get_id());
if (OB_ISNULL(groupby_ctx)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("group by ctx is NULL", K(ret));
} else {
groupby_ctx->get_aggr_func().set_sort_based_gby();
}
}
return ret;
}
int ObMergeGroupBy::rescan(ObExecContext &ctx) const
{
int ret = OB_SUCCESS;
ObMergeGroupByCtx *groupby_ctx = NULL;
if (OB_FAIL(ObGroupBy::rescan(ctx))) {
LOG_WARN("rescan child operator failed", K(ret));
} else if (OB_ISNULL(groupby_ctx = GET_PHY_OPERATOR_CTX(ObMergeGroupByCtx, ctx, get_id()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("groupby_ctx is null");
} else {
groupby_ctx->aggr_func_.reuse();
groupby_ctx->last_input_row_ = NULL;
groupby_ctx->is_end_ = false;
groupby_ctx->cur_output_group_id = -1;
groupby_ctx->first_output_group_id = 0;
}
return ret;
}
int ObMergeGroupBy::inner_close(ObExecContext &ctx) const
{
UNUSED(ctx);
int ret = OB_SUCCESS;
return ret;
}
int ObMergeGroupBy::inner_get_next_row(ObExecContext &ctx, const ObNewRow *&row) const
{
int ret = OB_SUCCESS;
ObMergeGroupByCtx *groupby_ctx = NULL;
int64_t stop_output_group_id = group_col_idxs_.count();
int64_t col_count = group_col_idxs_.count()+rollup_col_idxs_.count();
if (OB_ISNULL(child_op_) || OB_ISNULL(groupby_ctx = GET_PHY_OPERATOR_CTX(ObMergeGroupByCtx, ctx, get_id()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get ObMergeGroupByCtx failed", K_(child_op));
} else if (has_rollup_ &&
groupby_ctx->cur_output_group_id >= groupby_ctx->first_output_group_id &&
groupby_ctx->cur_output_group_id >= stop_output_group_id) {
// output roll-up results here
if (OB_FAIL(rollup_and_calc_results(groupby_ctx->cur_output_group_id, groupby_ctx))) {
LOG_WARN("failed to rollup and calculate results", K(groupby_ctx->cur_output_group_id), K(ret));
} else {
--groupby_ctx->cur_output_group_id;
}
} else {
// output group by results here
if (groupby_ctx->is_end_) {
ret = OB_ITER_END;
} else if (NULL == groupby_ctx->last_input_row_) {
// get the first input row
if (OB_FAIL(child_op_->get_next_row(ctx, groupby_ctx->last_input_row_))) {
if (ret != OB_ITER_END) {
LOG_WARN("failed to get next row", K(ret));
}
} else if (has_rollup_ && OB_FAIL(groupby_ctx->aggr_func_.rollup_init(col_count))) {
LOG_WARN("failed to initialize roll up", K(ret));
}
}
int64_t group_id = 0;
if (OB_SUCC(ret) && has_rollup_) {
group_id = col_count;
}
if (OB_SUCC(ret) && groupby_ctx->last_input_row_ != NULL) {
if (OB_FAIL(groupby_ctx->aggr_func_.prepare(*groupby_ctx->last_input_row_, group_id))) {
LOG_WARN("failed to init aggr cells", K(ret));
} else {
groupby_ctx->last_input_row_ = NULL;
}
}
if (OB_SUCC(ret)) {
bool same_group = false;
const ObNewRow *input_row = NULL;
const ObRowStore::StoredRow *stored_row = NULL;
ObSQLSessionInfo *my_session = groupby_ctx->exec_ctx_.get_my_session();
const ObTimeZoneInfo *tz_info = (my_session != NULL) ? my_session->get_timezone_info() : NULL;
bool is_break = false;
int64_t first_diff_pos = 0;
while (OB_SUCC(ret) && !is_break && OB_SUCC(child_op_->get_next_row(ctx, input_row))) {
if (OB_FAIL(try_check_status(ctx))) {
LOG_WARN("check status failed", K(ret));
} else if (OB_FAIL(groupby_ctx->aggr_func_.get_cur_row(stored_row, group_id))) {
LOG_WARN("fail to get cur row from aggr_func", K(ret));
} else if (OB_ISNULL(stored_row) || OB_ISNULL(input_row)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("the stored row is NULL");
} else if (OB_FAIL(is_same_group(*stored_row, *input_row, same_group, first_diff_pos))) {
LOG_WARN("failed to check group", K(ret));
} else if (same_group) {
if (OB_FAIL(groupby_ctx->aggr_func_.process(*input_row, tz_info, group_id))) {
LOG_WARN("failed to calc aggr", K(ret));
} else if (mem_size_limit_ > 0 && mem_size_limit_ < groupby_ctx->aggr_func_.get_used_mem_size()) {
ret = OB_EXCEED_MEM_LIMIT;
LOG_WARN("merge group by has exceeded the mem limit", K_(mem_size_limit),
"aggr mem size", groupby_ctx->aggr_func_.get_used_mem_size());
}
} else if (OB_FAIL(rollup_and_calc_results(group_id, groupby_ctx))) {
LOG_WARN("failed to rollup and calculate results", K(group_id), K(ret));
} else {
groupby_ctx->last_input_row_ = input_row;
is_break = true;
if(has_rollup_) {
groupby_ctx->first_output_group_id = first_diff_pos + 1;
groupby_ctx->cur_output_group_id = group_id - 1;
}
}
} // end while
if (OB_ITER_END == ret) {
// the last group
groupby_ctx->is_end_ = true;
if (OB_FAIL(rollup_and_calc_results(group_id, groupby_ctx))) {
LOG_WARN("failed to rollup and calculate results", K(group_id), K(ret));
} else if(has_rollup_) {
groupby_ctx->first_output_group_id = 0;
groupby_ctx->cur_output_group_id = group_id - 1;
}
}
}
}
if (OB_SUCC(ret)) {
row = &groupby_ctx->get_cur_row();
}
return ret;
}
int ObMergeGroupBy::rollup_and_calc_results(const int64_t group_id, ObMergeGroupByCtx *groupby_ctx) const
{
int ret = OB_SUCCESS;
int64_t col_count = group_col_idxs_.count()+rollup_col_idxs_.count();
if (OB_UNLIKELY(OB_ISNULL(groupby_ctx) || group_id < 0 || group_id > col_count)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid argument", K(group_id), K(ret));
} else {
ObSQLSessionInfo *my_session = groupby_ctx->exec_ctx_.get_my_session();
const ObTimeZoneInfo *tz_info = (my_session != NULL) ? my_session->get_timezone_info() : NULL;
//for: SELECT GROUPING(z0_test0) FROM Z0CASE GROUP BY z0_test0, ROLLUP(z0_test0);
//issue:
bool set_grouping = true;
if (group_id > group_col_idxs_.count()) {
int64_t rollup_id = rollup_col_idxs_[group_id - group_col_idxs_.count() - 1].index_;
for (int64_t i = 0; set_grouping && i < group_col_idxs_.count(); ++i) {
if (rollup_id == group_col_idxs_[i].index_) {
set_grouping = false;
}
}
}
if (has_rollup_ && group_id > 0 &&
OB_FAIL(groupby_ctx->aggr_func_.rollup_process(tz_info, group_id - 1, group_id,
group_id <= group_col_idxs_.count() ?
group_col_idxs_[group_id - 1].index_ :
rollup_col_idxs_[group_id - group_col_idxs_.count() - 1].index_,
set_grouping))) {
//ret = OB_ERR_UNEXPECTED;
LOG_WARN("failed to rollup aggregation results", K(ret));
} else if (OB_FAIL(groupby_ctx->aggr_func_.get_result(groupby_ctx->get_cur_row(), tz_info, group_id))) {
LOG_WARN("failed to get aggr result", K(group_id), K(ret));
} else if (OB_FAIL(groupby_ctx->aggr_func_.reuse_group(group_id))) {
LOG_WARN("failed to reuse group", K(group_id), K(ret));
} else {
// do nothing
}
}
return ret;
}
} // namespace sql
} // namespace oceanbase

599
src/sql/engine/basic/ob_limit.cpp
View File

@ -1,599 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_ENG
#include "sql/engine/basic/ob_limit.h"
#include "lib/utility/utility.h"
#include "share/object/ob_obj_cast.h"
#include "sql/session/ob_sql_session_info.h"
#include "sql/engine/ob_physical_plan.h"
#include "sql/engine/expr/ob_sql_expression.h"
#include "sql/engine/ob_exec_context.h"
namespace oceanbase
{
using namespace common;
namespace sql
{
//REGISTER_PHY_OPERATOR(ObLimit, PHY_LIMIT);
class ObLimit::ObLimitCtx : public ObPhyOperatorCtx
{
public:
explicit ObLimitCtx(ObExecContext &ctx)
: ObPhyOperatorCtx(ctx),
limit_(-1),
offset_(0),
input_count_(0),
output_count_(0),
total_count_(0),
is_percent_first_(false),
limit_last_row_(NULL)
{
}
virtual void destroy() { ObPhyOperatorCtx::destroy_base(); }
private:
int64_t limit_;
int64_t offset_;
int64_t input_count_;
int64_t output_count_;
int64_t total_count_;
bool is_percent_first_;
ObNewRow *limit_last_row_;
friend class ObLimit;
};
ObLimit::ObLimit(ObIAllocator &alloc)
: ObSingleChildPhyOperator(alloc),
org_limit_(NULL),
org_offset_(NULL),
org_percent_(NULL),
is_calc_found_rows_(false),
is_top_limit_(false),
is_fetch_with_ties_(false),
sort_columns_(alloc)
{
}
ObLimit::~ObLimit()
{
reset();
}
void ObLimit::reset()
{
org_limit_ = NULL;
org_offset_ = NULL;
org_percent_ = NULL;
is_calc_found_rows_ = false;
is_top_limit_ = false;
is_fetch_with_ties_ = false;
sort_columns_.reset();
ObSingleChildPhyOperator::reset();
}
void ObLimit::reuse()
{
reset();
}
int ObLimit::set_limit(ObSqlExpression *limit, ObSqlExpression *offset, ObSqlExpression *percent)
{
int ret = OB_SUCCESS;
if (limit) {
org_limit_ = limit;
}
if (offset) {
org_offset_ = offset;
}
if (percent) {
org_percent_ = percent;
}
return ret;
}
int ObLimit::get_int_value(ObExecContext &ctx,
const ObSqlExpression *in_val,
int64_t &out_val,
bool &is_null_value) const
{
int ret = OB_SUCCESS;
ObNewRow input_row;
ObObj result;
ObExprCtx expr_ctx;
is_null_value = false;
if (in_val != NULL && !in_val->is_empty()) {
if (OB_FAIL(wrap_expr_ctx(ctx, expr_ctx))) {
LOG_WARN("wrap expr context failed", K(ret));
} else if (OB_FAIL(in_val->calc(expr_ctx, input_row, result))) {
LOG_WARN("Failed to calculate expression", K(ret));
} else if (result.is_int()) {
if (OB_FAIL(result.get_int(out_val))) {
LOG_WARN("get_int error", K(ret), K(result));
}
} else if (result.is_null()) {
out_val = 0;
is_null_value = true;
} else {
EXPR_DEFINE_CAST_CTX(expr_ctx, CM_NONE);
EXPR_GET_INT64_V2(result, out_val);
if (OB_FAIL(ret)) {
LOG_WARN("get_int error", K(ret), K(result));
}
}
}
return ret;
}
int ObLimit::get_double_value(ObExecContext &ctx,
const ObSqlExpression *double_val,
double &out_val) const
{
int ret = OB_SUCCESS;
ObNewRow input_row;
ObObj result;
ObExprCtx expr_ctx;
if (double_val != NULL && !double_val->is_empty()) {
if (OB_FAIL(wrap_expr_ctx(ctx, expr_ctx))) {
LOG_WARN("wrap expr context failed", K(ret));
} else if (OB_FAIL(double_val->calc(expr_ctx, input_row, result))) {
LOG_WARN("Failed to calculate expression", K(ret));
} else if (result.is_double()) {
if (OB_FAIL(result.get_double(out_val))) {
LOG_WARN("get_double error", K(ret), K(result));
}
} else if (result.is_null()) {
out_val = 0.0;
} else {
EXPR_DEFINE_CAST_CTX(expr_ctx, CM_NONE);
EXPR_GET_DOUBLE_V2(result, out_val);
if (OB_FAIL(ret)) {
LOG_WARN("get_double error", K(ret), K(result));
}
}
}
return ret;
}
int ObLimit::get_limit(ObExecContext &ctx, int64_t &limit, int64_t &offset, bool &is_percent_first) const
{
int ret = OB_SUCCESS;
double percent = 0.0;
bool is_null_value = false;
limit = -1;
offset = 0;
if (OB_FAIL(get_int_value(ctx, org_limit_, limit, is_null_value))) {
LOG_WARN("Get limit value failed", K(ret));
} else if (!is_null_value && OB_FAIL(get_int_value(ctx, org_offset_, offset, is_null_value))) {
LOG_WARN("Get offset value failed", K(ret));
} else if (is_null_value) {
offset = 0;
limit = 0;
} else {
//由于下层的block算子大多数是在inner_get_next_row才计算出总的行数，因此这里也需要这样设置
is_percent_first = (org_percent_ != NULL && !org_percent_->is_empty());
//revise limit, offset because rownum < -1 is rewritten as limit -1
//offset 2 rows fetch next -3 rows only --> is meaningless
offset = offset < 0 ? 0 : offset;
if (org_limit_ != NULL && !org_limit_->is_empty()) {//不能统一直接设置为0,因为需要支持仅仅只有offset情形
limit = limit < 0 ? 0 : limit;
}
}
return ret;
}
bool ObLimit::is_valid() const
{
return (get_column_count() > 0 && child_op_ != NULL && child_op_->get_column_count() > 0);
}
int ObLimit::inner_open(ObExecContext &ctx) const
{
int ret = OB_SUCCESS;
ObLimitCtx *limit_ctx = NULL;
if (OB_UNLIKELY(!is_valid())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("limit operator is invalid");
} else if (OB_FAIL(init_op_ctx(ctx))) {
LOG_WARN("initialize operator context failed", K(ret));
} else if (OB_ISNULL(limit_ctx = GET_PHY_OPERATOR_CTX(ObLimitCtx, ctx, get_id()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get physical operator context failed", K_(id));
} else if (OB_FAIL(get_limit(ctx, limit_ctx->limit_, limit_ctx->offset_, limit_ctx->is_percent_first_))) {
LOG_WARN("Failed to instantiate limit/offset", K(ret));
} else { }
return ret;
}
int ObLimit::rescan(ObExecContext &ctx) const
{
int ret = OB_SUCCESS;
ObLimitCtx *limit_ctx = NULL;
if (OB_FAIL(ObSingleChildPhyOperator::rescan(ctx))) {
LOG_WARN("rescan child physical operator failed", K(ret));
} else if (OB_ISNULL(limit_ctx = GET_PHY_OPERATOR_CTX(ObLimitCtx, ctx, get_id()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("limit_ctx is null");
} else {
limit_ctx->input_count_ = 0;
limit_ctx->output_count_ = 0;
}
return ret;
}
int ObLimit::inner_close(ObExecContext &ctx) const
{
UNUSED(ctx);
return OB_SUCCESS;
}
int ObLimit::init_op_ctx(ObExecContext &ctx) const
{
int ret = OB_SUCCESS;
ObPhyOperatorCtx *op_ctx = NULL;
if(OB_FAIL(CREATE_PHY_OPERATOR_CTX(ObLimitCtx, ctx, get_id(), get_type(), op_ctx))) {
LOG_WARN("failed to create LimitCtx", K(ret));
} else if (OB_ISNULL(op_ctx)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("op_ctx is null");
} else if (OB_FAIL(init_cur_row(*op_ctx, need_copy_row_for_compute()))) {
LOG_WARN("init current row failed", K(ret));
}
return ret;
}
int ObLimit::inner_get_next_row(ObExecContext &ctx, const ObNewRow *&row) const
{
int ret = OB_SUCCESS;
ObLimitCtx *limit_ctx = NULL;
ObSQLSessionInfo *my_session = NULL;
const ObNewRow *input_row = NULL;
if (!is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("limit operator is invalid", K(ret));
} else if (OB_ISNULL(limit_ctx = GET_PHY_OPERATOR_CTX(ObLimitCtx, ctx, get_id()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get physical operator ctx failed");
} else if (OB_ISNULL(my_session = GET_MY_SESSION(ctx))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("fail to get my session", K(ret));
} else { }
while (OB_SUCC(ret) && limit_ctx->input_count_ < limit_ctx->offset_) {
if (OB_FAIL(child_op_->get_next_row(ctx, input_row))) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("child_op failed to get next row",
K(limit_ctx->input_count_), K(limit_ctx->offset_), K(ret));
}
} else if (limit_ctx->is_percent_first_ && OB_FAIL(convert_limit_percent(ctx, limit_ctx))) {
LOG_WARN("failed to convert limit percent", K(ret));
} else {
++limit_ctx->input_count_;
}
} // end while
/*由于支持了oracle 12c的fetch功能，因此下面的执行流程比较复杂，这里简单解释一下：
* 1.is_percent_first_：代表的fetch是否指定的百分比取行数，比如：select * from t1 fetch next 50 percent rows only;
* 取总行数的50%出来，这个时候需要is_percent_first_来表明是否用的百分比，同时我们的下层block算子(sort、hash group by等)
* 都是在get_next_row时指定设置的，因此需要在第一次时去设置对应的limit数量，同时设置完后将is_percent_first_重新设置为false；
* 2.is_fetch_with_ties_：表示在拿到所需要的limit数量时，需要继续下探是否存在按照order by排序列值相等的情形，
* 比如表t1有3行数据 c1 c2 c3
* 1 2 3
* 1 2 4
* 2 2 3
* 这个时候假如按照表t1的c1列排序，同时设置了只输出一列，但是指定了with ties(sql为:select * from t1 order by c1 fetch next 1 rows with ties);
* 那么需要将每次从child op拿取rows同时保存拿到的最后一行数据，等拿到了指定的数量之后，继续按照保存好的最后一行数据
* 下探child op的rows,直到拿到按照order by排序列的值不相等或者拿完了child op的rows为止；比如上述例子中拿到行：1 2 3
* 会继续下探行：1 2 4，发现排序列c1值相等，会继续下探拿行：2 2 3，这个时候排序列c1值不等，整个get_next_row结束。
*
*/
int64_t left_count = 0;
if (OB_SUCC(ret)) {
if (limit_ctx->is_percent_first_ || limit_ctx->output_count_ < limit_ctx->limit_ || limit_ctx->limit_ < 0) {
if (OB_FAIL(child_op_->get_next_row(ctx, input_row))) {
if (OB_ITER_END != ret) {
LOG_WARN("child_op failed to get next row",
K(ret), K_(limit_ctx->limit), K_(limit_ctx->offset),
K_(limit_ctx->input_count), K_(limit_ctx->output_count));
}
} else if (limit_ctx->is_percent_first_ && OB_FAIL(convert_limit_percent(ctx, limit_ctx))) {
LOG_WARN("failed to convert limit percent", K(ret));
} else if (limit_ctx->limit_ == 0) {
ret = OB_ITER_END;
} else {
++limit_ctx->output_count_;
row = input_row;
//如果需要支持fetch with ties功能，需要拷贝limit拿出的最后一行保存下来供后续使用
if (is_fetch_with_ties_ && limit_ctx->output_count_ == limit_ctx->limit_ &&
OB_FAIL(deep_copy_limit_last_rows(limit_ctx, *row))) {
LOG_WARN("failed to deep copy limit last rows");
} else if (OB_FAIL(copy_cur_row(*limit_ctx, row))) {
LOG_WARN("copy current row failed", K(ret));
} else {/*do nothing*/}
}
//说明需要继续判断input rows能否按照order by items等值输出
} else if (limit_ctx->limit_ > 0 && is_fetch_with_ties_) {
bool is_equal = false;
if (OB_FAIL(child_op_->get_next_row(ctx, input_row))) {
if (OB_ITER_END != ret) {
LOG_WARN("child_op failed to get next row",
K(ret), K_(limit_ctx->limit), K_(limit_ctx->offset),
K_(limit_ctx->input_count), K_(limit_ctx->output_count));
}
} else if (OB_FAIL(is_row_order_by_item_value_equal(limit_ctx, input_row, is_equal))) {
LOG_WARN("failed to is row order by item value equal", K(ret));
} else if (is_equal) {
++limit_ctx->output_count_;
row = input_row;
if (OB_FAIL(copy_cur_row(*limit_ctx, row))) {
LOG_WARN("copy current row failed", K(ret));
} else {
LOG_DEBUG("copy cur row", K(*row));
}
} else {
//溢出的按照order by排序相等的row已经找完
ret = OB_ITER_END;
}
} else {
//结果条数已经满足
ret = OB_ITER_END;
if (is_calc_found_rows_) {
const ObNewRow *tmp_row = NULL;
while (OB_SUCC(child_op_->get_next_row(ctx, tmp_row))) {
++left_count;
}
if (OB_ITER_END != ret) {
LOG_WARN("fail to get next row from child", K(ret));
}
}
}
}
if (OB_ITER_END == ret) {
if (is_top_limit_) {
limit_ctx->total_count_ = left_count + limit_ctx->output_count_ + limit_ctx->input_count_;
ObPhysicalPlanCtx *plan_ctx = NULL;
if (OB_ISNULL(plan_ctx = ctx.get_physical_plan_ctx())) {
ret = OB_ERR_NULL_VALUE;
LOG_WARN("get physical plan context failed");
} else {
NG_TRACE_EXT(found_rows, OB_ID(total_count), limit_ctx->total_count_,
OB_ID(input_count), limit_ctx->input_count_);
plan_ctx->set_found_rows(limit_ctx->total_count_);
}
}
}
return ret;
}
int ObLimit::deep_copy_limit_last_rows(ObLimitCtx *limit_ctx, const ObNewRow row) const
{
int ret = OB_SUCCESS;
if (OB_ISNULL(limit_ctx) || OB_ISNULL(limit_ctx->calc_buf_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected null", K(ret), K(limit_ctx));
} else {
const int64_t buf_len = sizeof(ObNewRow) + row.get_deep_copy_size();
int64_t pos = sizeof(ObNewRow);
char *buf = NULL;
if (OB_ISNULL(buf = static_cast<char *>(limit_ctx->calc_buf_->alloc(buf_len)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("alloc new row failed", K(ret), K(buf_len));
} else if (OB_ISNULL(limit_ctx->limit_last_row_ = new (buf) ObNewRow())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("new_row is null", K(ret), K(buf_len));
} else if (OB_FAIL(limit_ctx->limit_last_row_ ->deep_copy(row, buf, buf_len, pos))) {
LOG_WARN("deep copy row failed", K(ret), K(buf_len), K(pos));
} else {/*do nothing*/}
}
return ret;
}
int ObLimit::is_row_order_by_item_value_equal(ObLimitCtx *limit_ctx,
const ObNewRow *input_row,
bool &is_equal) const
{
int ret = OB_SUCCESS;
ObNewRow *limit_last_row = NULL;
is_equal = false;
if (OB_ISNULL(limit_ctx) || OB_ISNULL(limit_last_row = limit_ctx->limit_last_row_) ||
OB_ISNULL(input_row)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected null", K(ret), K(limit_ctx), K(input_row), K(limit_last_row));
} else {
is_equal = true;
for (int64_t i = 0; OB_SUCC(ret) && is_equal && i < sort_columns_.count(); ++i) {
if (OB_UNLIKELY(sort_columns_.at(i).index_ >= input_row->count_ ||
sort_columns_.at(i).index_ >= limit_last_row->count_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("order by projector is invalid", K(ret), K(sort_columns_.at(i).index_),
K(input_row->count_), K(limit_last_row->count_));
} else {
is_equal = 0 == input_row->cells_[sort_columns_.at(i).index_].compare(
limit_last_row->cells_[sort_columns_.at(i).index_],
sort_columns_.at(i).cs_type_);
}
}
}
return ret;
}
int64_t ObLimit::to_string_kv(char *buf, const int64_t buf_len) const
{
int64_t pos = 0;
if (org_limit_ && org_offset_) {
J_KV(N_LIMIT, org_limit_, N_OFFSET, org_offset_);
} else if (org_limit_) {
J_KV(N_LIMIT, org_limit_);
} else if (org_offset_) {
J_KV(N_OFFSET, org_offset_);
}
return pos;
}
int ObLimit::add_filter(ObSqlExpression *expr)
{
UNUSED(expr);
LOG_ERROR_RET(OB_NOT_SUPPORTED, "limit operator should have no filter expr");
return OB_NOT_SUPPORTED;
}
int ObLimit::add_sort_columns(ObSortColumn sort_column) {
return sort_columns_.push_back(sort_column);
}
//针对percent需要这里根据总行数转换为对应的limit count
int ObLimit::convert_limit_percent(ObExecContext &ctx, ObLimitCtx *limit_ctx) const
{
int ret = OB_SUCCESS;
double percent = 0.0;
if (OB_ISNULL(limit_ctx)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected null", K(ret), K(limit_ctx));
} else if (OB_FAIL(get_double_value(ctx, org_percent_, percent))) {
LOG_WARN("failed to get double value", K(ret));
} else if (percent > 0) {
int64_t tot_count = 0;
if (OB_UNLIKELY(limit_ctx->limit_ != -1) || OB_ISNULL(child_op_) ||
OB_UNLIKELY(child_op_->get_type() != PHY_MATERIAL &&
child_op_->get_type() != PHY_SORT)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get unexpected error", K(ret), K(limit_ctx->limit_), K(child_op_));
} else if (child_op_->get_type() == PHY_MATERIAL &&
OB_FAIL(static_cast<ObMaterial *>(child_op_)->get_material_row_count(ctx, tot_count))) {
LOG_WARN("failed to get op row count", K(ret));
} else if (child_op_->get_type() == PHY_SORT &&
OB_FAIL(static_cast<ObSort *>(child_op_)->get_sort_row_count(ctx, tot_count))) {
LOG_WARN("failed to get op row count", K(ret));
} else if (OB_UNLIKELY(tot_count < 0)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get invalid child op row count", K(tot_count), K(ret));
} else if (percent < 100) {
//兼容oracle,向上取整
int64_t percent_int64 = static_cast<int64_t>(percent);
int64_t offset = (tot_count * percent / 100 - tot_count * percent_int64 / 100) > 0 ? 1 : 0;
limit_ctx->limit_ = tot_count * percent_int64 / 100 + offset;
limit_ctx->is_percent_first_ = false;
} else {
limit_ctx->limit_ = tot_count;
limit_ctx->is_percent_first_ = false;
}
} else {
limit_ctx->limit_ = 0;
}
return ret;
}
OB_DEF_SERIALIZE(ObLimit)
{
int ret = OB_SUCCESS;
bool has_limit_count = (org_limit_ != NULL && !org_limit_->is_empty());
bool has_limit_offset = (org_offset_ != NULL && !org_offset_->is_empty());
bool has_limit_percent = (org_percent_ != NULL && !org_percent_->is_empty());
OB_UNIS_ENCODE(is_calc_found_rows_);
OB_UNIS_ENCODE(has_limit_count);
OB_UNIS_ENCODE(has_limit_offset);
OB_UNIS_ENCODE(is_top_limit_);
if (has_limit_count) {
OB_UNIS_ENCODE(*org_limit_);
}
if (has_limit_offset) {
OB_UNIS_ENCODE(*org_offset_);
}
if (OB_SUCC(ret)) {
if (OB_FAIL(ObSingleChildPhyOperator::serialize(buf, buf_len, pos))) {
LOG_WARN("serialize child operator failed", K(ret));
}
}
if (OB_SUCC(ret)) {
OB_UNIS_ENCODE(has_limit_percent);
OB_UNIS_ENCODE(is_fetch_with_ties_);
OB_UNIS_ENCODE(sort_columns_);
if (has_limit_percent) {
OB_UNIS_ENCODE(*org_percent_);
}
}
return ret;
}
OB_DEF_SERIALIZE_SIZE(ObLimit)
{
int64_t len = 0;
bool has_limit_count = (org_limit_ != NULL && !org_limit_->is_empty());
bool has_limit_offset = (org_offset_ != NULL && !org_offset_->is_empty());
bool has_limit_percent = (org_percent_ != NULL && !org_percent_->is_empty());
OB_UNIS_ADD_LEN(is_calc_found_rows_);
OB_UNIS_ADD_LEN(has_limit_count);
OB_UNIS_ADD_LEN(has_limit_offset);
OB_UNIS_ADD_LEN(is_top_limit_);
if (has_limit_count) {
OB_UNIS_ADD_LEN(*org_limit_);
}
if (has_limit_offset) {
OB_UNIS_ADD_LEN(*org_offset_);
}
len += ObSingleChildPhyOperator::get_serialize_size();
OB_UNIS_ADD_LEN(has_limit_percent);
OB_UNIS_ADD_LEN(is_fetch_with_ties_);
OB_UNIS_ADD_LEN(sort_columns_);
if (has_limit_percent) {
OB_UNIS_ADD_LEN(*org_percent_);
}
return len;
}
OB_DEF_DESERIALIZE(ObLimit)
{
int ret = OB_SUCCESS;
bool has_limit_count = false;
bool has_limit_offset = false;
bool has_limit_percent = false;
OB_UNIS_DECODE(is_calc_found_rows_);
OB_UNIS_DECODE(has_limit_count);
OB_UNIS_DECODE(has_limit_offset);
OB_UNIS_DECODE(is_top_limit_);
if (OB_SUCC(ret) && has_limit_count) {
if (OB_FAIL(ObSqlExpressionUtil::make_sql_expr(my_phy_plan_, org_limit_))) {
LOG_WARN("make sql expression failed", K(ret));
} else if (OB_LIKELY(org_limit_ != NULL)) {
OB_UNIS_DECODE(*org_limit_);
}
}
if (OB_SUCC(ret) && has_limit_offset) {
if (OB_FAIL(ObSqlExpressionUtil::make_sql_expr(my_phy_plan_, org_offset_))) {
LOG_WARN("make sql expression failed", K(ret));
} else if (OB_LIKELY(org_offset_ != NULL)) {
OB_UNIS_DECODE(*org_offset_);
}
}
if (OB_SUCC(ret)) {
ret = ObSingleChildPhyOperator::deserialize(buf, data_len, pos);
}
if (OB_SUCC(ret)) {
OB_UNIS_DECODE(has_limit_percent);
OB_UNIS_DECODE(is_fetch_with_ties_);
OB_UNIS_DECODE(sort_columns_);
if (OB_SUCC(ret) && has_limit_percent) {
if (OB_FAIL(ObSqlExpressionUtil::make_sql_expr(my_phy_plan_, org_percent_))) {
LOG_WARN("make sql expression failed", K(ret));
} else if (OB_LIKELY(org_percent_ != NULL)) {
OB_UNIS_DECODE(*org_percent_);
}
}
}
return ret;
}
} // namespace sql
} // namespace oceanbase

391
src/sql/engine/basic/ob_topk.cpp
View File

@ -1,391 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_ENG
#include "sql/engine/basic/ob_topk.h"
#include "lib/utility/utility.h"
#include "share/object/ob_obj_cast.h"
#include "sql/session/ob_sql_session_info.h"
#include "sql/engine/ob_physical_plan.h"
#include "sql/engine/expr/ob_sql_expression.h"
#include "sql/engine/sort/ob_sort.h"
#include "sql/engine/basic/ob_material.h"
#include "sql/engine/aggregate/ob_hash_groupby.h"
#include "sql/engine/ob_exec_context.h"
namespace oceanbase
{
using namespace common;
namespace sql
{
class ObTopK::ObTopKCtx : public ObPhyOperatorCtx
{
public:
explicit ObTopKCtx(ObExecContext &ctx)
: ObPhyOperatorCtx(ctx),
topk_final_count_(-1),
output_count_(0)
{
}
virtual void destroy() { ObPhyOperatorCtx::destroy_base(); }
private:
int64_t topk_final_count_;//count of rows that need to be output upforward
int64_t output_count_;
friend class ObTopK;
};
ObTopK::ObTopK(ObIAllocator &alloc)
: ObSingleChildPhyOperator(alloc),
minimum_row_count_(-1),
topk_precision_(-1),
org_limit_(NULL),
org_offset_(NULL)
{
}
ObTopK::~ObTopK()
{
reset();
}
void ObTopK::reset()
{
org_limit_ = NULL;
org_offset_ = NULL;
minimum_row_count_ = -1;
topk_precision_ = -1;
ObSingleChildPhyOperator::reset();
}
void ObTopK::reuse()
{
reset();
}
int ObTopK::set_topk_params(ObSqlExpression *limit, ObSqlExpression *offset,
int64_t minimum_row_count, int64_t topk_precision)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(limit) || minimum_row_count < 0 || topk_precision < 0) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", KP(limit), K(minimum_row_count), K(topk_precision), K(ret));
} else {
org_limit_ = limit;
org_offset_ = offset;
minimum_row_count_ = minimum_row_count;
topk_precision_ = topk_precision;
}
return ret;
}
int ObTopK::get_int_value(ObExecContext &ctx,
const ObSqlExpression *in_val,
int64_t &out_val,
bool &is_null_value) const
{
int ret = OB_SUCCESS;
ObNewRow input_row;
ObObj result;
ObExprCtx expr_ctx;
is_null_value = false;
if (in_val != NULL && !in_val->is_empty()) {
if (OB_FAIL(wrap_expr_ctx(ctx, expr_ctx))) {
LOG_WARN("wrap expr context failed", K(ret));
} else if (OB_FAIL(in_val->calc(expr_ctx, input_row, result))) {
LOG_WARN("Failed to calculate expression", K(ret));
} else if (OB_LIKELY(result.is_int())) {
if (OB_FAIL(result.get_int(out_val))) {
LOG_WARN("get_int error", K(ret), K(result));
}
} else if (result.is_null()) {
out_val = 0;
is_null_value = true;
} else {
EXPR_DEFINE_CAST_CTX(expr_ctx, CM_NONE);
EXPR_GET_INT64_V2(result, out_val);
if (OB_FAIL(ret)) {
LOG_WARN("get_int error", K(ret), K(result));
}
}
}
return ret;
}
int ObTopK::get_topk_final_count(ObExecContext &ctx, int64_t &topk_final_count) const
{
int ret = OB_SUCCESS;
int64_t limit = -1;
int64_t offset = 0;
bool is_null_value = false;
ObPhysicalPlanCtx *plan_ctx = ctx.get_physical_plan_ctx();
if (OB_ISNULL(plan_ctx)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid plan ctx is NULL", K(ret));
} else if (OB_ISNULL(child_op_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Invalid child_op_ is NULL", K(ret));
} else if (OB_FAIL(get_int_value(ctx, org_limit_, limit, is_null_value))) {
LOG_WARN("Get limit value failed", K(ret));
} else if (!is_null_value && OB_FAIL(get_int_value(ctx, org_offset_, offset, is_null_value))) {
LOG_WARN("Get offset value failed", K(ret));
} else {
//revise limit, offset because rownum < -1 is rewritten as limit -1
limit = (is_null_value || limit < 0) ? 0 : limit;
offset = (is_null_value || offset < 0) ? 0 : offset;
topk_final_count = std::max(minimum_row_count_, limit + offset);
int64_t row_count = 0;
ObPhyOperatorType op_type = child_op_->get_type();
//TODO(yaoying.yyy): may be we should add one func to paremt class
if (PHY_SORT == op_type) {
ObSort *sort_op = static_cast<ObSort *>(child_op_);
if (OB_ISNULL(sort_op)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("casted sort_op is NULL", K(ret));
} else if (OB_FAIL(sort_op->get_sort_row_count(ctx, row_count))){
LOG_WARN("failed to get sort row count", K(ret));
} else {/*do nothing*/}
} else if (PHY_MATERIAL == op_type) {
ObMaterial *material_op = static_cast<ObMaterial *>(child_op_);
if (OB_ISNULL(material_op)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("casted material_op is NULL", K(ret));
} else if (OB_FAIL(material_op->get_material_row_count(ctx, row_count))){
LOG_WARN("failed to get material row count", K(ret));
} else {/*do nothing*/}
} else if (PHY_HASH_GROUP_BY == op_type) {
ObHashGroupBy *hash_groupby_op = static_cast<ObHashGroupBy *>(child_op_);
if (OB_ISNULL(hash_groupby_op)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("casted hash_groupby_op is NULL", K(ret));
} else if (OB_FAIL(hash_groupby_op->get_hash_groupby_row_count(ctx, row_count))){
LOG_WARN("failed to get material row count", K(ret));
} else {/*do nothing*/}
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Invalid child_op_", K(op_type), K(ret));
}
if (OB_SUCC(ret)) {
topk_final_count = std::max(topk_final_count, static_cast<int64_t>(row_count * topk_precision_ / 100));
if (topk_final_count >= row_count) {
plan_ctx->set_is_result_accurate(true);
} else {
plan_ctx->set_is_result_accurate(false);
}
}
}
return ret;
}
bool ObTopK::is_valid() const
{
return (get_column_count() > 0 && (NULL != org_limit_)
&& (NULL != child_op_) && child_op_->get_column_count() > 0);
}
int ObTopK::inner_open(ObExecContext &ctx) const
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_valid())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("limit operator is invalid");
} else if (OB_FAIL(init_op_ctx(ctx))) {
LOG_WARN("initialize operator context failed", K(ret));
} else {/*do nothing*/}
return ret;
}
int ObTopK::rescan(ObExecContext &ctx) const
{
int ret = OB_SUCCESS;
ObTopKCtx *topk_ctx = NULL;
if (OB_FAIL(ObSingleChildPhyOperator::rescan(ctx))) {
LOG_WARN("rescan child physical operator failed", K(ret));
} else if (OB_ISNULL(topk_ctx = GET_PHY_OPERATOR_CTX(ObTopKCtx, ctx, get_id()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("topk_ctx is null");
} else {
topk_ctx->output_count_ = 0;
}
return ret;
}
int ObTopK::inner_close(ObExecContext &ctx) const
{
UNUSED(ctx);
return OB_SUCCESS;
}
int ObTopK::init_op_ctx(ObExecContext &ctx) const
{
int ret = OB_SUCCESS;
ObPhyOperatorCtx *op_ctx = NULL;
if(OB_FAIL(CREATE_PHY_OPERATOR_CTX(ObTopKCtx, ctx, get_id(), get_type(), op_ctx))) {
LOG_WARN("failed to create TopKCtx", K(ret));
} else if (OB_ISNULL(op_ctx)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("op_ctx is null");
} else if (OB_FAIL(init_cur_row(*op_ctx, need_copy_row_for_compute()))) {
LOG_WARN("init current row failed", K(ret));
}
return ret;
}
int ObTopK::inner_get_next_row(ObExecContext &ctx, const ObNewRow *&row) const
{
int ret = OB_SUCCESS;
ObTopKCtx *topk_ctx = NULL;
ObSQLSessionInfo *my_session = NULL;
const ObNewRow *input_row = NULL;
if (!is_valid()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("limit operator is invalid");
} else if (OB_ISNULL(topk_ctx = GET_PHY_OPERATOR_CTX(ObTopKCtx, ctx, get_id()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("get physical operator ctx failed");
} else if (OB_ISNULL(my_session = GET_MY_SESSION(ctx))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("fail to get my session", K(ret));
} else {
if (0 == topk_ctx->output_count_ || topk_ctx->output_count_ < topk_ctx->topk_final_count_) {
if (OB_FAIL(child_op_->get_next_row(ctx, input_row))) {
if (OB_ITER_END != ret) {
LOG_WARN("child_op failed to get next row", K_(topk_ctx->topk_final_count),
K_(topk_ctx->output_count), K(ret));
}
} else {
if (0 == topk_ctx->output_count_) {
if (OB_FAIL(get_topk_final_count(ctx, topk_ctx->topk_final_count_))) {
LOG_WARN("failed to get_topk_final_count", K(ret));
} else if (OB_UNLIKELY(0 == topk_ctx->topk_final_count_)) {
//结果条数已经满足
ret = OB_ITER_END;
} else {/*do nothing*/}
}
if (OB_SUCC(ret)) {
++topk_ctx->output_count_;
row = input_row;
if (OB_FAIL(copy_cur_row(*topk_ctx, row))) {
LOG_WARN("copy current row failed", K(ret));
} else {
LOG_DEBUG("copy cur row", K(*row));
}
}
}
} else {
//结果条数已经满足
ret = OB_ITER_END;
}
}
return ret;
}
int64_t ObTopK::to_string_kv(char *buf, const int64_t buf_len) const
{
int64_t pos = 0;
//TODO(yaoying.yyy):macro define N_TOPK_PRECISION
if (org_limit_ && org_offset_) {
J_KV(N_LIMIT, org_limit_, N_OFFSET, org_offset_, "minimum_row_count", minimum_row_count_, "topk_precision", topk_precision_);
} else if (NULL != org_limit_) {
J_KV(N_LIMIT, org_limit_, "minimum_row_count", minimum_row_count_, "topk_precision", topk_precision_);
} else if (NULL != org_offset_) {
J_KV(N_OFFSET, org_offset_, "minimum_row_count", minimum_row_count_, "topk_precision", topk_precision_);
} else{/*do nothing*/}
return pos;
}
int ObTopK::add_filter(ObSqlExpression *expr)
{
UNUSED(expr);
LOG_ERROR_RET(OB_NOT_SUPPORTED, "limit operator should have no filter expr");
return OB_NOT_SUPPORTED;
}
OB_DEF_SERIALIZE(ObTopK)
{
int ret = OB_SUCCESS;
bool has_limit_count = (org_limit_ != NULL && !org_limit_->is_empty());
bool has_limit_offset = (org_offset_ != NULL && !org_offset_->is_empty());
OB_UNIS_ENCODE(minimum_row_count_);
OB_UNIS_ENCODE(topk_precision_);
OB_UNIS_ENCODE(has_limit_count);
OB_UNIS_ENCODE(has_limit_offset);
if (has_limit_count) {
OB_UNIS_ENCODE(*org_limit_);
}
if (has_limit_offset) {
OB_UNIS_ENCODE(*org_offset_);
}
if (OB_SUCC(ret)) {
if (OB_FAIL(ObSingleChildPhyOperator::serialize(buf, buf_len, pos))) {
LOG_WARN("serialize child operator failed", K(ret));
}
}
return ret;
}
OB_DEF_SERIALIZE_SIZE(ObTopK)
{
int64_t len = 0;
bool has_limit_count = (org_limit_ != NULL && !org_limit_->is_empty());
bool has_limit_offset = (org_offset_ != NULL && !org_offset_->is_empty());
OB_UNIS_ADD_LEN(minimum_row_count_);
OB_UNIS_ADD_LEN(topk_precision_);
OB_UNIS_ADD_LEN(has_limit_count);
OB_UNIS_ADD_LEN(has_limit_offset);
if (has_limit_count) {
OB_UNIS_ADD_LEN(*org_limit_);
}
if (has_limit_offset) {
OB_UNIS_ADD_LEN(*org_offset_);
}
len += ObSingleChildPhyOperator::get_serialize_size();
return len;
}
OB_DEF_DESERIALIZE(ObTopK)
{
int ret = OB_SUCCESS;
bool has_limit_count = false;
bool has_limit_offset = false;
OB_UNIS_DECODE(minimum_row_count_);
OB_UNIS_DECODE(topk_precision_);
OB_UNIS_DECODE(has_limit_count);
OB_UNIS_DECODE(has_limit_offset);
if (OB_SUCC(ret) && has_limit_count) {
if (OB_FAIL(ObSqlExpressionUtil::make_sql_expr(my_phy_plan_, org_limit_))) {
LOG_WARN("make sql expression failed", K(ret));
} else if (OB_LIKELY(org_limit_ != NULL)) {
OB_UNIS_DECODE(*org_limit_);
}
}
if (OB_SUCC(ret) && has_limit_offset) {
if (OB_FAIL(ObSqlExpressionUtil::make_sql_expr(my_phy_plan_, org_offset_))) {
LOG_WARN("make sql expression failed", K(ret));
} else if (OB_LIKELY(org_offset_ != NULL)) {
OB_UNIS_DECODE(*org_offset_);
}
}
if (OB_SUCC(ret)) {
ret = ObSingleChildPhyOperator::deserialize(buf, data_len, pos);
}
return ret;
}
} // namespace sql
} // namespace oceanbase

1
src/sql/engine/cmd/ob_database_executor.cpp
View File

@ -22,7 +22,6 @@
#include "sql/session/ob_sql_session_info.h"
#include "share/ob_common_rpc_proxy.h"
#include "lib/worker.h"
#include "share/ob_cluster_info_proxy.h"
#include "rootserver/ob_root_utils.h"
namespace oceanbase

669
src/sql/engine/dml/ob_table_update.cpp
View File

@ -1,669 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_ENG
#include "common/sql_mode/ob_sql_mode_utils.h"
#include "sql/engine/dml/ob_table_update.h"
#include "sql/engine/ob_physical_plan_ctx.h"
#include "sql/engine/ob_physical_plan.h"
#include "sql/session/ob_sql_session_info.h"
#include "sql/engine/ob_exec_context.h"
#include "lib/profile/ob_perf_event.h"
namespace oceanbase
{
using namespace common;
using namespace storage;
using namespace share;
using namespace share::schema;
namespace sql
{
ObTableUpdate::ObTableUpdate(ObIAllocator &alloc)
: ObTableModify(alloc),
updated_column_ids_(alloc),
updated_column_infos_(alloc),
is_global_index_(false),
new_spk_exprs_()
{
}
ObTableUpdate::~ObTableUpdate()
{
}
void ObTableUpdate::reset()
{
updated_column_infos_.reset();
updated_column_ids_.reset();
is_global_index_ = false;
ObTableModify::reset();
}
void ObTableUpdate::reuse()
{
updated_column_infos_.reuse();
updated_column_ids_.reuse();
is_global_index_ = false;
ObTableModify::reuse();
}
int ObTableUpdate::set_updated_column_info(int64_t array_index,
uint64_t column_id,
uint64_t project_index,
bool auto_filled_timestamp,
bool is_implicit,
const ObString *column_name)
{
int ret = OB_SUCCESS;
ColumnContent column;
column.projector_index_ = project_index;
column.auto_filled_timestamp_ = auto_filled_timestamp;
column.is_implicit_ = is_implicit;
if (OB_NOT_NULL(column_name) && column.column_name_.empty()) {
column.column_name_ = *column_name;
}
CK(array_index >= 0 && array_index < updated_column_ids_.count());
CK(array_index >= 0 && array_index < updated_column_infos_.count());
if (OB_SUCC(ret)) {
updated_column_ids_.at(array_index) = column_id;
updated_column_infos_.at(array_index) = column;
}
return ret;
}
int ObTableUpdate::inner_open(ObExecContext &ctx) const
{
int ret = OB_SUCCESS;
ObTableUpdateCtx *update_ctx = NULL;
ObPhysicalPlanCtx *plan_ctx = NULL;
ObSQLSessionInfo *my_session = NULL;
int64_t schema_version = 0;
int64_t binlog_row_image = ObBinlogRowImage::FULL;
NG_TRACE(update_open);
if (OB_FAIL(ObTableModify::inner_open(ctx))) {
LOG_WARN("open child operator failed", K(ret));
} else if (OB_ISNULL(update_ctx = GET_PHY_OPERATOR_CTX(ObTableUpdateCtx, ctx, get_id()))) {
ret = OB_ERR_NULL_VALUE;
LOG_WARN("get physical operator context failed", K_(id));
} else if (OB_ISNULL(plan_ctx = ctx.get_physical_plan_ctx())) {
ret = OB_ERR_NULL_VALUE;
LOG_WARN("get physical plan context failed");
} else if (OB_ISNULL(my_session = GET_MY_SESSION(ctx))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("fail to get my session", K(ret));
} else if (OB_ISNULL(my_phy_plan_)) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("invalid physical plan", K(ret), K(my_phy_plan_));
} else if (OB_FAIL(my_phy_plan_->get_base_table_version(index_tid_, schema_version))) {
LOG_WARN("failed to get base table version", K(ret));
} else if (OB_FAIL(my_session->get_binlog_row_image(binlog_row_image))) {
LOG_WARN("fail to get binlog row image", K(ret));
} else {
update_ctx->dml_param_.timeout_ = plan_ctx->get_ps_timeout_timestamp();
update_ctx->dml_param_.schema_version_ = schema_version;
update_ctx->dml_param_.is_total_quantity_log_ = (ObBinlogRowImage::FULL == binlog_row_image);
update_ctx->dml_param_.tz_info_ = TZ_INFO(my_session);
update_ctx->dml_param_.sql_mode_ = my_session->get_sql_mode();
update_ctx->dml_param_.prelock_ = my_session->get_prelock();
update_ctx->dml_param_.table_param_ = &table_param_;
update_ctx->dml_param_.tenant_schema_version_ = plan_ctx->get_tenant_schema_version();
update_ctx->dml_param_.is_ignore_ = is_ignore_;
OZ(my_phy_plan_->get_encrypt_meta(index_tid_, update_ctx->dml_param_.encrypt_meta_legacy_,
update_ctx->dml_param_.encrypt_meta_));
if (OB_FAIL(ret)){
} else if (gi_above_) {
if (OB_FAIL(get_gi_task(ctx))) {
LOG_WARN("get granule iterator task failed", K(ret));
}
}
if (OB_SUCC(ret) && !from_multi_table_dml()) {
OZ (TriggerHandle::do_handle_before_stmt(*this, *update_ctx,
ObTriggerEvents::get_update_event()));
}
if (!has_instead_of_trigger_) {
OZ (do_table_update(ctx), ret);
} else {
OZ (do_instead_of_trigger_update(ctx));
}
}
NG_TRACE(update_end);
return ret;
}
int ObTableUpdate::do_table_update(ObExecContext &ctx) const
{
int ret = OB_SUCCESS;
int64_t affected_rows = 0;
ObTableUpdateCtx *update_ctx = NULL;
ObPhysicalPlanCtx *plan_ctx = NULL;
ObSQLSessionInfo *my_session = NULL;
if (OB_ISNULL(update_ctx = GET_PHY_OPERATOR_CTX(ObTableUpdateCtx, ctx, get_id()))) {
ret = OB_ERR_NULL_VALUE;
LOG_WARN("get physical operator context failed", K(ret), K_(id));
} else if (update_ctx->iter_end_) {
LOG_DEBUG("can't get gi task, iter end", K(update_ctx->iter_end_), K(get_id()));
} else if (OB_ISNULL(my_session = GET_MY_SESSION(ctx))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("fail to get my session", K(ret));
} else if (OB_FAIL(get_part_location(ctx, update_ctx->part_infos_))) {
LOG_WARN("get part location failed", K(ret));
} else if (is_returning()) {
// do nothing, handle in ObTableUpdateReturning
} else if (OB_ISNULL(plan_ctx = ctx.get_physical_plan_ctx())) {
ret = OB_ERR_NULL_VALUE;
LOG_WARN("get physical plan context failed", K(ret));
} else if (OB_FAIL(update_rows(ctx, affected_rows))) {
if (OB_TRY_LOCK_ROW_CONFLICT != ret && OB_TRANSACTION_SET_VIOLATION != ret) {
LOG_WARN("update rows to partition storage failed",
K(ret), K(column_ids_), K(updated_column_ids_));
}
} else if (!from_multi_table_dml()) {
//dml meta info will be counted in multiple table dml operator, not here
plan_ctx->add_row_matched_count(update_ctx->get_found_rows());
plan_ctx->add_row_duplicated_count(update_ctx->get_changed_rows());
plan_ctx->add_affected_rows(my_session->get_capability().cap_flags_.OB_CLIENT_FOUND_ROWS ?
update_ctx->get_found_rows() : update_ctx->get_affected_rows());
}
SQL_ENG_LOG(DEBUG, "update rows end",
K(ret), K(affected_rows),
K(column_ids_), K(updated_column_ids_));
return ret;
}
int ObTableUpdate::do_instead_of_trigger_update(ObExecContext &ctx) const
{
int ret = OB_SUCCESS;
ObTableUpdateCtx *update_ctx = GET_PHY_OPERATOR_CTX(ObTableUpdateCtx, ctx, get_id());
ObPhysicalPlanCtx *plan_ctx = ctx.get_physical_plan_ctx();
OV (OB_NOT_NULL(update_ctx), OB_ERR_NULL_VALUE);
OV (OB_NOT_NULL(plan_ctx), OB_ERR_NULL_VALUE);
OX (update_ctx->expr_ctx_.calc_buf_->reuse());
OX (update_ctx->expr_ctx_.row_ctx_.reset());
if (OB_SUCC(ret)) {
const ObNewRow *full_row = NULL;
int64_t affected_rows = 0;
while (OB_SUCC(ret) && OB_SUCC(inner_get_next_row(ctx, full_row))) {
CK (OB_NOT_NULL(full_row));
OX (affected_rows += 1);
if (OB_SUCC(ret)) {
project_old_and_new_row(*update_ctx, *full_row);
ObNewRow &old_row = update_ctx->old_row_;
ObNewRow &new_row = update_ctx->new_row_;
OZ (TriggerHandle::init_param_rows(*this, *update_ctx, old_row, new_row), old_row, new_row);
OZ (TriggerHandle::do_handle_before_row(*this, *update_ctx, &new_row,
ObTriggerEvents::get_update_event()),
old_row, new_row);
}
}
if (OB_ITER_END == ret) {
plan_ctx->set_affected_rows(affected_rows);
LOG_TRACE("update for instead of trigger success", K(plan_ctx->get_affected_rows()), K(ret));
ret = OB_SUCCESS;
} else {
LOG_WARN("update for instead of trigger failed", K(ret));
}
}
return ret;
}
int ObTableUpdate::inner_close(ObExecContext &ctx) const
{
int ret = OB_SUCCESS;
ObTableUpdateCtx *update_ctx = GET_PHY_OPERATOR_CTX(ObTableUpdateCtx, ctx, get_id());
OV (OB_NOT_NULL(update_ctx), OB_ERR_NULL_VALUE);
if (!from_multi_table_dml()) {
OZ (TriggerHandle::do_handle_after_stmt(*this, *update_ctx,
ObTriggerEvents::get_update_event()));
}
int close_ret = ObTableModify::inner_close(ctx);
return (OB_SUCCESS == ret) ? close_ret : ret;
}
int ObTableUpdate::rescan(ObExecContext &ctx) const
{
int ret = OB_SUCCESS;
ObTableUpdateCtx *update_ctx = NULL;
if (!gi_above_ || from_multi_table_dml()) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("table update rescan not supported", K(ret));
LOG_USER_ERROR(OB_NOT_SUPPORTED, "table update rescan");
} else if (OB_ISNULL(update_ctx = GET_PHY_OPERATOR_CTX(ObTableUpdateCtx, ctx, get_id()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("table update context is null", K(ret), K(get_id()));
} else if (OB_FAIL(ObTableModify::rescan(ctx))) {
LOG_WARN("rescan child operator failed", K(ret));
} else {
update_ctx->found_rows_ = 0;
update_ctx->changed_rows_ = 0;
update_ctx->affected_rows_ = 0;
update_ctx->has_got_old_row_ = false;
update_ctx->part_infos_.reset();
update_ctx->part_key_.reset();
if (nullptr != update_ctx->rowkey_dist_ctx_) {
update_ctx->rowkey_dist_ctx_->clear();
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(get_gi_task(ctx))) {
LOG_WARN("get granule task failed", K(ret));
} else if (OB_FAIL(do_table_update(ctx))) {
LOG_WARN("do table update failed", K(ret));
}
}
return ret;
}
// TRICK:
// ObTableUpdate按照设计是不对外吐数据的，如果ObResultSet调用get_next_row，岂不是会得到数据？
// FACT：
// ObTableUpdate::open()中会把自己封装成一个Iterator传递给存储层，存储层会将这里的数据消耗
// 干净，直到返回ITER_END。那么，当轮到ObResultSet调用get_next_row的时候，它只能得到ITER_END了。
// 一切，只是巧合而已。
int ObTableUpdate::get_next_row(ObExecContext &ctx, const ObNewRow *&row) const
{
int ret = OB_SUCCESS;
ObTableUpdateCtx *update_ctx = NULL;
//update operator must has project operation
if (OB_FAIL(try_check_status(ctx))) {
LOG_WARN("check status failed", K(ret));
} else if (OB_ISNULL(update_ctx = GET_PHY_OPERATOR_CTX(ObTableUpdateCtx, ctx, get_id()))) {
ret = OB_ERR_NULL_VALUE;
LOG_WARN("get physical operator context failed", K_(id));
} else {
update_ctx->expr_ctx_.calc_buf_->reuse();
update_ctx->expr_ctx_.row_ctx_.reset();
}
if (OB_FAIL(ret)) {
//do nothing
} else if (!update_ctx->has_got_old_row_) {
NG_TRACE_TIMES(2, update_start_next_row);
const ObNewRow *full_row = NULL;
bool is_updated = false;
// 过滤未变化的行
while(OB_SUCC(ret) && !is_updated && OB_SUCC(inner_get_next_row(ctx, full_row))) {
CK(OB_NOT_NULL(full_row));
if (OB_SUCC(ret)) {
project_old_and_new_row(*update_ctx, *full_row);
// check_rowkey_is_null and check_rowkey_whether_distinct only works for
// mysql mode, mainly for the following cases
// update t2 left outer join t1 on t1.c1 = t2.c1 set t1.c2 = t2.c2;
if (OB_SUCC(ret) && !from_multi_table_dml()) {
bool is_null = false;
if (need_filter_null_row_) {
if (OB_FAIL(check_rowkey_is_null(update_ctx->old_row_, primary_key_ids_.count(), is_null))) {
LOG_WARN("check rowkey is null failed", K(ret), K(update_ctx->get_cur_row()), K(primary_key_ids_));
} else if (is_null) {
continue;
}
} else {
#if !defined(NDEBUG)
if (need_check_pk_is_null()) {
if (OB_FAIL(check_rowkey_is_null(update_ctx->old_row_,
primary_key_ids_.count(), is_null))) {
LOG_WARN("failed to check rowkey is null", K(ret));
} else if (OB_UNLIKELY(is_null)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("update row failed validity check", K(ret));
}
}
#endif
}
}
if (OB_SUCC(ret) && !from_multi_table_dml()) {
bool is_distinct = false;
if (OB_FAIL(check_rowkey_whether_distinct(ctx,
update_ctx->old_row_,
primary_key_ids_.count(),
distinct_algo_,
update_ctx->rowkey_dist_ctx_,
is_distinct))) {
LOG_WARN("check rowkey whether distinct failed", K(ret));
} else if (!is_distinct) {
continue;
}
}
if (OB_SUCC(ret) && !from_multi_table_dml()) {
ObNewRow &old_row = update_ctx->old_row_;
ObNewRow &new_row = update_ctx->new_row_;
OZ (TriggerHandle::init_param_rows(*this, *update_ctx, old_row, new_row), old_row, new_row);
OZ (TriggerHandle::do_handle_before_row(*this, *update_ctx, &new_row,
ObTriggerEvents::get_update_event()),
old_row, new_row);
OZ (check_row_null(ctx, new_row, column_infos_, updated_column_infos_), new_row);
}
}
if (OB_SUCC(ret)) {
//check update row whether changed
if (OB_LIKELY(!from_multi_table_dml() && !is_returning())) {
//if update operator from multi table dml,
//the row value will be check in multiple table dml operator
//dml meta info will also be counted in multiple table dml operator
OZ(check_updated_value(*update_ctx, *this, update_ctx->old_row_, update_ctx->new_row_, is_updated));
} else if (OB_LIKELY(check_row_whether_changed(update_ctx->new_row_))) {
is_updated = true;
}
}
if (OB_SUCC(ret)) {
ObNewRow &old_row = update_ctx->old_row_;
ObNewRow &new_row = update_ctx->new_row_;
if (is_updated) {
if (!from_multi_table_dml()) {
bool is_filtered = false;
OZ (ForeignKeyHandle::do_handle(*update_ctx, fk_args_, old_row, new_row),
old_row, new_row);
OZ(ObPhyOperator::filter_row_for_view_check(update_ctx->expr_ctx_, new_row,
view_check_exprs_, is_filtered));
OV(!is_filtered, OB_ERR_CHECK_OPTION_VIOLATED);
OZ (ObPhyOperator::filter_row_for_check_cst(update_ctx->expr_ctx_, new_row,
check_constraint_exprs_, is_filtered));
if (is_filtered && OB_SUCC(ret)) {
ret = OB_ERR_CHECK_CONSTRAINT_VIOLATED;
LOG_WARN("row is filtered by check filters, running is stopped", K(ret));
}
}
} else {
if (OB_FAIL(build_lock_row(*update_ctx, update_ctx->old_row_))) {
LOG_WARN("build lock row failed", K(ret), K(update_ctx->old_row_));
} else if (OB_FAIL(lock_row(ctx,
update_ctx->lock_row_,
update_ctx->dml_param_,
update_ctx->part_key_))) {
//没有发生更新，对当前行进行加锁
if (OB_TRY_LOCK_ROW_CONFLICT != ret) {
LOG_WARN("fail to lock row", K(ret), K(update_ctx->lock_row_), K(update_ctx->part_key_));
}
} else {
LOG_DEBUG("lock row", K(ret), K(update_ctx->lock_row_), K(update_ctx->part_key_));
}
}
if (!from_multi_table_dml()) {
OZ (TriggerHandle::do_handle_after_row(*this, *update_ctx,
ObTriggerEvents::get_update_event()),
old_row, new_row);
}
}
} // while
if (OB_SUCCESS != ret && OB_ITER_END != ret) {
LOG_WARN("get next row from child operator failed", K(ret));
}
if (OB_SUCC(ret)) {
// old row, keep old projector
row = &(update_ctx->old_row_);
LOG_DEBUG("get old row", K(update_ctx->old_row_));
update_ctx->has_got_old_row_ = true;
}
} else {
// new row
const ObNewRow &new_row = update_ctx->new_row_;
DLIST_FOREACH(cur_expr, new_spk_exprs_) {
const ObColumnExpression *new_spk_expr = static_cast<const ObColumnExpression*>(cur_expr);
int64_t result_idx = new_spk_expr->get_result_index();
if (OB_UNLIKELY(result_idx < 0)
|| OB_UNLIKELY(result_idx >= new_row.count_)) {
LOG_WARN("result index is invalid", K(ret), K(result_idx),
K(new_row.count_));
} else if (OB_FAIL(new_spk_expr->calc(update_ctx->expr_ctx_,
new_row,
new_row.cells_[result_idx]))) {
LOG_WARN("calc new spk expr failed", K(ret),
KPC(new_spk_expr), K(result_idx), K(new_row));
}
}
if (OB_SUCC(ret)) {
LOG_DEBUG("get new row", K(update_ctx->new_row_));
row = &(update_ctx->new_row_);
update_ctx->has_got_old_row_ = false;
}
}
NG_TRACE_TIMES(2, update_end_next_row);
return ret;
}
int ObTableUpdate::switch_iterator(ObExecContext &ctx) const
{
int ret = OB_SUCCESS;
ObTableUpdateCtx *update_ctx = NULL;
if (OB_FAIL(ObTableModify::switch_iterator(ctx))) {
if (OB_ITER_END != ret) {
LOG_WARN("switch single child operator iterator failed", K(ret));
}
} else if (OB_ISNULL(update_ctx = GET_PHY_OPERATOR_CTX(ObTableUpdateCtx, ctx, get_id()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("update ctx is null", K(ret));
} else {
update_ctx->has_got_old_row_ = false;
}
return ret;
}
bool ObTableUpdate::check_row_whether_changed(const ObNewRow &new_row) const
{
bool bret = false;
if (updated_column_infos_.count() > 0 && new_row.is_valid()) {
int64_t projector_index = updated_column_infos_.at(0).projector_index_;
if (projector_index >= 0 && projector_index < new_row.get_count()) {
const ObObj &updated_value = new_row.get_cell(projector_index);
bret = !(updated_value.is_ext() && ObActionFlag::OP_LOCK_ROW == updated_value.get_ext());
}
}
return bret;
}
int ObTableUpdate::inner_get_next_row(ObExecContext &ctx, const ObNewRow *&row) const
{
int ret = OB_SUCCESS;
ObTableUpdateCtx *update_ctx = NULL;
if (OB_ISNULL(child_op_)) {
ret = OB_NOT_INIT;
LOG_WARN("child_op_ is null");
} else if (OB_ISNULL(update_ctx = GET_PHY_OPERATOR_CTX(ObTableUpdateCtx, ctx, get_id()))) {
ret = OB_ERR_NULL_VALUE;
LOG_WARN("get physical operator context failed", K_(id));
} else if (update_ctx->iter_end_) {
LOG_DEBUG("can't get gi task, iter end", K(get_id()), K(update_ctx->iter_end_));
ret = OB_ITER_END;
} else {
if (from_multi_table_dml()) {
if (update_ctx->part_row_cnt_ <= 0) {
if (update_ctx->cur_part_idx_ < update_ctx->part_infos_.count()) {
update_ctx->part_row_cnt_ = update_ctx->part_infos_.at(update_ctx->cur_part_idx_).part_row_cnt_;
update_ctx->part_key_ = update_ctx->part_infos_.at(update_ctx->cur_part_idx_).partition_key_;
++update_ctx->cur_part_idx_;
}
}
if (OB_SUCC(ret)) {
--update_ctx->part_row_cnt_;
}
}
if (OB_SUCC(ret)) {
ret = child_op_->get_next_row(ctx, row);
}
}
if (OB_ITER_END == ret) {
NG_TRACE(update_iter_end);
}
return ret;
}
void ObTableUpdate::project_old_and_new_row(ObTableUpdateCtx &ctx,
const ObNewRow &full_row) const
{
ctx.full_row_ = full_row;
ObNewRow &new_row = ctx.new_row_;
ObNewRow &old_row = ctx.old_row_;
new_row.cells_ = full_row.cells_;
new_row.count_ = full_row.count_;
new_row.projector_ = ctx.new_row_projector_;
new_row.projector_size_ = ctx.new_row_projector_size_;
old_row = full_row;
//old row and new row have the same projector size
//but full row projector size is old row projector size + updated column count
//so old row copy from full row and must reset projector size
old_row.projector_size_ = ctx.new_row_projector_size_;
}
int ObTableUpdate::build_lock_row(ObTableUpdateCtx &update_ctx, const ObNewRow &old_row) const
{
int ret = OB_SUCCESS;
CK(OB_NOT_NULL(update_ctx.lock_row_.cells_));
CK(update_ctx.lock_row_.count_ <= old_row.get_count());
for (int64_t i = 0; OB_SUCC(ret) && i < update_ctx.lock_row_.count_; ++i) {
update_ctx.lock_row_.cells_[i] = old_row.get_cell(i);
}
return ret;
}
int64_t ObTableUpdate::to_string_kv(char *buf, const int64_t buf_len) const
{
int64_t pos = 0;
J_KV(N_TID, table_id_,
K_(index_tid),
N_CID, column_ids_,
N_UPDATED_CID, updated_column_ids_,
K_(updated_column_infos),
K_(is_global_index));
return pos;
}
int ObTableUpdate::init_op_ctx(ObExecContext &ctx) const
{
int ret = OB_SUCCESS;
ObTableUpdateCtx *op_ctx = NULL;
OZ(CREATE_PHY_OPERATOR_CTX(ObTableUpdateCtx, ctx, get_id(), get_type(), op_ctx));
CK(OB_NOT_NULL(op_ctx));
OZ(op_ctx->alloc_row_cells(projector_size_, op_ctx->lock_row_));
if (OB_SUCC(ret)) {
op_ctx->new_row_projector_ = projector_;
op_ctx->new_row_projector_size_ = projector_size_;
}
OX (ctx.set_dml_event(ObDmlEventType::DE_UPDATING));
OZ (op_ctx->init_update_columns(updated_column_infos_));
if (tg_args_.count() > 0) {
OZ(op_ctx->init_trigger_params(tg_event_, all_tm_points_,
tg_columns_.get_projector(),
tg_columns_.get_count(),
tg_columns_.get_rowtype_count()));
}
return ret;
}
inline int ObTableUpdate::update_rows(ObExecContext &ctx, int64_t &affected_rows) const
{
UNUSEDx(ctx, affected_rows);
int ret = OB_NOT_SUPPORTED;
LOG_USER_ERROR(OB_NOT_SUPPORTED, "table update rows");
// ObTableUpdateCtx *update_ctx = NULL;
// ObTaskExecutorCtx *executor_ctx = NULL;
// ObSQLSessionInfo *my_session = ctx.get_my_session();
// ObPartitionService *partition_service = NULL;
// if (OB_ISNULL(my_session)) {
// ret = OB_ERR_UNEXPECTED;
// LOG_WARN("my_session is null");
// } else if (OB_ISNULL(executor_ctx = GET_TASK_EXECUTOR_CTX(ctx))) {
// ret = OB_ERR_UNEXPECTED;
// LOG_WARN("fail to get task executor ctx", K(ret));
// } else if (OB_ISNULL(partition_service = executor_ctx->get_partition_service())) {
// ret = OB_ERR_UNEXPECTED;
// LOG_WARN("fail to get partition service", K(ret));
// } else if (OB_ISNULL(update_ctx = GET_PHY_OPERATOR_CTX(ObTableUpdateCtx, ctx, get_id()))) {
// ret = OB_ERR_NULL_VALUE;
// LOG_WARN("get physical operator context failed", K_(id));
// } else if (OB_UNLIKELY(update_ctx->part_infos_.empty())) {
// ret = OB_ERR_UNEXPECTED;
// LOG_WARN("part infos is empty", K(ret));
// } else if (OB_LIKELY(update_ctx->part_infos_.count() == 1)) {
// update_ctx->part_key_ = update_ctx->part_infos_.at(0).partition_key_;
// ObDMLRowIterator dml_row_iter(ctx, *this);
// if (OB_FAIL(dml_row_iter.init())) {
// LOG_WARN("init dml row iterator", K(ret));
// } else if (OB_FAIL(partition_service->update_rows(*my_session->get_tx_desc(),
// update_ctx->dml_param_,
// update_ctx->part_key_,
// column_ids_,
// updated_column_ids_,
// &dml_row_iter,
// affected_rows))) {
// if (OB_TRY_LOCK_ROW_CONFLICT != ret) {
// LOG_WARN("insert row to partition storage failed", K(ret));
// }
// }
// } else {
// //多分区插入
// //给UPDATE语句建两个row的buffer用来去除row上的projector
// const ObNewRow *old_row = NULL;
// const ObNewRow *new_row = NULL;
// if (OB_FAIL(update_ctx->create_cur_rows(2, update_ctx->new_row_projector_size_, NULL, 0))) {
// ret = OB_ERR_UNEXPECTED;
// LOG_WARN("create current rows failed", K(ret), K_(projector_size));
// }
// while (OB_SUCC(ret) && OB_SUCC(get_next_row(ctx, old_row)) && OB_SUCC(get_next_row(ctx, new_row))) {
// if (OB_FAIL(copy_cur_row_by_projector(update_ctx->cur_rows_[0], old_row))) {
// LOG_WARN("copy old row failed", K(ret));
// } else if (OB_FAIL(copy_cur_row_by_projector(update_ctx->cur_rows_[1], new_row))) {
// LOG_WARN("copy new row failed", K(ret));
// } else if (OB_FAIL(partition_service->update_row(*my_session->get_tx_desc(),
// update_ctx->dml_param_,
// update_ctx->part_key_,
// column_ids_,
// updated_column_ids_,
// *old_row,
// *new_row))) {
// if (OB_TRY_LOCK_ROW_CONFLICT != ret) {
// LOG_WARN("update row to partition storage failed", K(ret));
// }
// } else {
// LOG_DEBUG("update multi part", K_(update_ctx->part_key), K(*old_row), K(*new_row));
// }
// }
// if (OB_ITER_END == ret) {
// ret = OB_SUCCESS;
// } else if (OB_FAIL(ret)) {
// LOG_WARN("process update row failed", K(ret));
// }
// }
return ret;
}
OB_DEF_SERIALIZE(ObTableUpdate)
{
int ret = OB_SUCCESS;
BASE_SER((ObTableUpdate, ObTableModify));
LST_DO_CODE(OB_UNIS_ENCODE, updated_column_ids_, updated_column_infos_, is_global_index_);
if (OB_SUCC(ret)) {
if (OB_FAIL(serialize_dlist(new_spk_exprs_, buf, buf_len, pos))) {
LOG_WARN("failed to serialize calc_exprs_", K(ret));
}
}
return ret;
}
OB_DEF_DESERIALIZE(ObTableUpdate)
{
int ret = OB_SUCCESS;
BASE_DESER((ObTableUpdate, ObTableModify));
LST_DO_CODE(OB_UNIS_DECODE, updated_column_ids_, updated_column_infos_, is_global_index_);
OB_UNIS_DECODE_EXPR_DLIST(ObColumnExpression, new_spk_exprs_, my_phy_plan_);
return ret;
}
OB_DEF_SERIALIZE_SIZE(ObTableUpdate)
{
int64_t len = 0;
BASE_ADD_LEN((ObTableUpdate, ObTableModify));
LST_DO_CODE(OB_UNIS_ADD_LEN, updated_column_ids_, updated_column_infos_, is_global_index_);
len += get_dlist_serialize_size(new_spk_exprs_);
return len;
}
} // namespace sql
} // namespace oceanbase

88
src/sql/engine/ob_single_child_phy_operator.cpp
View File

@ -1,88 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_ENG
#include "sql/engine/ob_single_child_phy_operator.h"
using namespace oceanbase::sql;
using namespace oceanbase::common;
ObSingleChildPhyOperator::ObSingleChildPhyOperator(ObIAllocator &alloc)
: ObPhyOperator(alloc),
child_op_(NULL)
{
}
ObSingleChildPhyOperator::~ObSingleChildPhyOperator()
{
}
void ObSingleChildPhyOperator::reset()
{
child_op_ = NULL;
ObPhyOperator::reset();
}
void ObSingleChildPhyOperator::reuse()
{
child_op_ = NULL;
ObPhyOperator::reuse();
}
int ObSingleChildPhyOperator::set_child(int32_t child_idx, ObPhyOperator &child_operator)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(NULL != child_op_)) {
ret = OB_INIT_TWICE;
LOG_ERROR("child_op_ already init", "op_type", get_type());
} else if (OB_UNLIKELY(0 != child_idx)) {
ret = OB_INVALID_ARGUMENT;
LOG_ERROR("invalid child", K(child_idx));
} else {
child_op_ = &child_operator;
child_operator.set_parent(this);
}
return ret;
}
ObPhyOperator *ObSingleChildPhyOperator::get_child(int32_t child_idx) const
{
ObPhyOperator *ret = NULL;
if (OB_LIKELY(0 == child_idx)) {
ret = child_op_;
}
return ret;
}
int ObSingleChildPhyOperator::accept(ObPhyOperatorVisitor &visitor) const
{
int ret = OB_SUCCESS;
if (OB_FAIL(visitor.pre_visit(*this))) {
LOG_WARN("fail to pre-visit single child operator", K(*this));
}
if (OB_SUCC(ret)) {
if (OB_ISNULL(child_op_)) {
ret = OB_ERR_UNEXPECTED;
} else if (OB_FAIL(child_op_->accept(visitor))) {
LOG_WARN("fail to visit child", K(*child_op_));
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(visitor.post_visit(*this))) {
LOG_WARN("fail to post-visit single child operator", K(*this));
} else {
// done
}
}
return ret;
}

43
src/sql/engine/ob_single_child_phy_operator.h
View File

@ -1,43 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef _OB_SINGLE_CHILD_PHY_OPERATOR_H
#define _OB_SINGLE_CHILD_PHY_OPERATOR_H 1
#include "sql/engine/ob_phy_operator.h"
namespace oceanbase
{
namespace sql
{
class ObSingleChildPhyOperator : public ObPhyOperator
{
public:
explicit ObSingleChildPhyOperator(common::ObIAllocator &alloc);
virtual ~ObSingleChildPhyOperator();
/// set the only one child
virtual int set_child(int32_t child_idx, ObPhyOperator &child_operator);
/// get the only one child
virtual ObPhyOperator *get_child(int32_t child_idx) const;
virtual int32_t get_child_num() const { return 1; }
virtual void reset();
virtual void reuse();
virtual int accept(ObPhyOperatorVisitor &visitor) const;
private:
// disallow copy
DISALLOW_COPY_AND_ASSIGN(ObSingleChildPhyOperator);
protected:
// data members
ObPhyOperator *child_op_;
};
} // end namespace sql
} // end namespace oceanbase
#endif /* _OB_SINGLE_CHILD_PHY_OPERATOR_H */

616
src/sql/engine/px/exchange/ob_px_transmit.cpp
View File

@ -1,616 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_ENG
#include "sql/engine/px/exchange/ob_px_transmit.h"
#include "sql/engine/px/ob_px_util.h"
#include "sql/dtl/ob_dtl_channel_group.h"
#include "sql/dtl/ob_dtl_rpc_channel.h"
#include "sql/executor/ob_range_hash_key_getter.h"
#include "sql/executor/ob_slice_calc.h"
#include "sql/dtl/ob_dtl.h"
#include "share/config/ob_server_config.h"
using namespace oceanbase::common;
using namespace oceanbase::sql;
using namespace oceanbase::sql::dtl;
using namespace oceanbase::share::schema;
// ObPxExchangeInput do NOT need serialize, filled by TaskProcessor
OB_SERIALIZE_MEMBER(ObPxTransmitInput, ch_provider_ptr_);
int ObPxTransmitInput::get_part_ch_map(ObPxPartChInfo &map, int64_t timeout_ts)
{
int ret = OB_SUCCESS;
map.part_ch_array_.reset();
ObPxSQCProxy *ch_provider = reinterpret_cast<ObPxSQCProxy *>(ch_provider_ptr_);
if (OB_ISNULL(ch_provider)) {
ret = OB_NOT_INIT;
LOG_WARN("ch provider not init", K(ret));
} else if (OB_FAIL(ch_provider->get_part_ch_map(map, timeout_ts))) {
LOG_WARN("fail get affinity map from provider", K(ret));
}
return ret;
}
int ObPxTransmitInput::get_parent_dfo_key(ObDtlDfoKey &key)
{
int ret = OB_SUCCESS;
ObPxSQCProxy *ch_provider = reinterpret_cast<ObPxSQCProxy *>(ch_provider_ptr_);
if (OB_ISNULL(ch_provider)) {
ret = OB_NOT_INIT;
LOG_WARN("ch provider not init", K(ret));
} else {
ch_provider->get_parent_dfo_key(key);
if (!ObDfo::is_valid_dfo_id(key.get_dfo_id())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected status", K(key.get_dfo_id()));
}
}
return ret;
}
int ObPxTransmitInput::get_data_ch(ObPxTaskChSet &task_ch_set, int64_t timeout_ts)
{
int ret = OB_SUCCESS;
ObPxSQCProxy *ch_provider = reinterpret_cast<ObPxSQCProxy *>(ch_provider_ptr_);
if (OB_ISNULL(ch_provider)) {
ret = OB_NOT_INIT;
LOG_WARN("ch provider not init", K(ret));
} else if (OB_FAIL(ch_provider->get_transmit_data_ch(
get_sqc_id(), get_task_id(), timeout_ts, task_ch_set, nullptr))) {
LOG_WARN("fail get data ch sets from provider", K(ret));
}
return ret;
}
//////////////////////////////////////////
// 不能实现这个序列化继承，因为存在兼容性问题。
// 建议：ObPxTransmit 不要再添加任何需要序列化的成员
// OB_SERIALIZE_MEMBER((ObPxTransmit, ObTransmit), partition_id_idx_);
ObPxTransmit::ObPxTransmit(common::ObIAllocator &alloc)
: ObTransmit(alloc)
{
}
ObPxTransmit::~ObPxTransmit()
{
}
ObPxTransmit::ObPxTransmitCtx::ObPxTransmitCtx(ObExecContext &ctx)
: ObTransmit::ObTransmitCtx(ctx),
px_row_allocator_(common::ObModIds::OB_SQL_PX),
transmited_(false),
first_row_(nullptr),
iter_end_(false),
consume_first_row_(false),
dfc_unblock_msg_proc_(dfc_),
chs_agent_(),
use_bcast_opt_(false),
part_ch_info_()
{
}
ObPxTransmit::ObPxTransmitCtx::~ObPxTransmitCtx()
{
destroy();
}
void ObPxTransmit::ObPxTransmitCtx::destroy(){
ObTransmitCtx::destroy();
task_ch_set_.reset();
px_row_allocator_.reset();
task_channels_.reset();
dfc_.destroy();
loop_.reset();
chs_agent_.~ObDtlChanAgent();
part_ch_info_.~ObPxPartChInfo();
}
int ObPxTransmit::inner_open(ObExecContext &exec_ctx) const
{
int ret = OB_SUCCESS;
ObPhysicalPlanCtx *phy_plan_ctx = NULL;
ObPxTransmitInput *trans_input = NULL;
ObPxTransmitCtx *transmit_ctx = NULL;
const ObNewRow *row = nullptr;
if (OB_ISNULL(child_op_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("child op is NULL", K(ret));
} else if (OB_UNLIKELY(filter_exprs_.get_size() > 0)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("filter exprs should be empty", K(ret), K(filter_exprs_.get_size()));
} else if (OB_FAIL(ObTransmit::inner_open(exec_ctx))) {
LOG_WARN("initialize operator context failed", K(ret));
} else if (OB_ISNULL(phy_plan_ctx = GET_PHY_PLAN_CTX(exec_ctx)) ||
OB_ISNULL(transmit_ctx = GET_PHY_OPERATOR_CTX(ObPxTransmitCtx, exec_ctx, get_id())) ||
OB_ISNULL(trans_input = GET_PHY_OP_INPUT(ObPxTransmitInput, exec_ctx, get_id()))) {
ret = OB_ERR_UNEXPECTED;
} else {
//
// 这个拿第一行需要保留.
if (child_op_->is_dml_operator() && !child_op_->is_pdml_operator()) {
//如果下游是dml操作，dml操作不向上层算子迭代行，直接返回iter end
transmit_ctx->iter_end_ = true;
LOG_TRACE("transmit iter end", K(ret), K(transmit_ctx->iter_end_));
} else if ((ret = get_next_row(exec_ctx, row)) != OB_SUCCESS && (ret != OB_ITER_END)) {
LOG_WARN("fail to get next row", K(ret));
} else {
transmit_ctx->iter_end_ = ret == OB_SUCCESS ? false : true;
LOG_TRACE("transmit iter end", K(ret), K(transmit_ctx->iter_end_));
ret = OB_SUCCESS;
if (OB_NOT_NULL(row)) {
transmit_ctx->first_row_ = row;
}
}
}
if (OB_SUCC(ret)) {
LOG_TRACE("TIMERECORD ", "reserve:=1 name:=TASK dfoid:", trans_input->get_dfo_id(),"sqcid:", trans_input->get_sqc_id(), "taskid:", trans_input->get_task_id(), "start:", ObTimeUtility::current_time());
if (OB_FAIL(transmit_ctx->init_channel(exec_ctx, *trans_input))) {
LOG_WARN("failed to init channel", K(ret));
} else {
transmit_ctx->metric_.set_id(get_id());
}
LOG_TRACE("TIMERECORD ", "reserve:=1 name:=TASK dfoid:", trans_input->get_dfo_id(),"sqcid:", trans_input->get_sqc_id(), "taskid:", trans_input->get_task_id(), "end:", ObTimeUtility::current_time());
}
return ret;
}
int ObPxTransmit::ObPxTransmitCtx::init_dfc(ObExecContext &ctx, ObDtlDfoKey &key)
{
int ret = OB_SUCCESS;
ObPhysicalPlanCtx *phy_plan_ctx = NULL;
if (OB_ISNULL(phy_plan_ctx = GET_PHY_PLAN_CTX(ctx))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("The phy plan ctx is null", K(ret));
} else if (OB_FAIL(dfc_.init(ctx.get_my_session()->get_effective_tenant_id(),
task_ch_set_.count()))) {
LOG_WARN("Fail to init dfc", K(ret));
} else {
dfc_.set_timeout_ts(phy_plan_ctx->get_timeout_timestamp());
dfc_.set_transmit();
dfc_.set_dfo_key(key);
dfc_.set_op_metric(&metric_);
dfc_.set_dtl_channel_watcher(&loop_);
DTL.get_dfc_server().register_dfc(dfc_);
LOG_TRACE("Worker init dfc", K(key), K(dfc_.is_receive()));
}
return ret;
}
int ObPxTransmit::ObPxTransmitCtx::init_channel(
ObExecContext &ctx,
ObPxTransmitInput &trans_input)
{
int ret = OB_SUCCESS;
ObPhysicalPlanCtx *phy_plan_ctx = NULL;
ObDtlDfoKey key;
LOG_TRACE("Try to get channel information from SQC");
if (OB_ISNULL(phy_plan_ctx = GET_PHY_PLAN_CTX(ctx))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("The phy plan ctx is null", K(ret));
} else if (OB_FAIL(trans_input.get_data_ch(
task_ch_set_, phy_plan_ctx->get_timeout_timestamp()))) {
LOG_WARN("Fail to get data dtl channel", K(ret));
} else if (OB_FAIL(trans_input.get_parent_dfo_key(key))) {
LOG_WARN("Failed to get parent dfo key", K(ret));
} else if (OB_FAIL(init_dfc(ctx, key))) {
LOG_WARN("Failed to init dfc", K(ret));
} else if (OB_FAIL(ObPxTransmit::link_ch_sets(task_ch_set_, task_channels_, &dfc_))) {
LOG_WARN("Fail to link data channel", K(ret));
} else {
bool enable_audit = GCONF.enable_sql_audit && ctx.get_my_session()->get_local_ob_enable_sql_audit();
metric_.init(enable_audit);
common::ObIArray<dtl::ObDtlChannel*> &channels = task_channels_;
loop_.set_tenant_id(ctx.get_my_session()->get_effective_tenant_id());
bool use_interm_result = false;
int64_t px_batch_id = ctx.get_px_batch_id();
loop_.register_processor(dfc_unblock_msg_proc_)
.register_interrupt_processor(interrupt_proc_);
if (OB_SUCC(ret)) {
ObPxSQCProxy *sqc_proxy = NULL;
if (OB_ISNULL(sqc_proxy = reinterpret_cast<ObPxSQCProxy *>(
trans_input.get_ch_provider_ptr()))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("fail to get ch provider ptr", K(ret));
} else {
use_interm_result = sqc_proxy->get_transmit_use_interm_result();
}
}
loop_.set_interm_result(use_interm_result);
ARRAY_FOREACH_X(channels, idx, cnt, OB_SUCC(ret)) {
dtl::ObDtlChannel *ch = channels.at(idx);
if (OB_ISNULL(ch)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("NULL unexpected", K(ch), K(ret));
} else {
ch->set_audit(enable_audit);
ch->set_interm_result(use_interm_result);
ch->set_batch_id(px_batch_id);
}
LOG_TRACE("Transmit channel", K(ch), KP(ch->get_id()), K(ch->get_peer()));
}
LOG_TRACE("Get transmit channel ok",
"task_id", trans_input.get_task_id(),
"ch_cnt", channels.count(),
K(ret));
}
return ret;
}
int ObPxTransmit::get_next_row(ObExecContext &exec_ctx, const ObNewRow *&row) const
{
int ret = OB_SUCCESS;
// 调用 ObPhyOperator 方法，支持投影
ret = ObPhyOperator::get_next_row(exec_ctx, row);
return ret;
}
int ObPxTransmit::inner_get_next_row(ObExecContext &exec_ctx, const ObNewRow *&row) const
{
int ret = OB_SUCCESS;
if (OB_ISNULL(child_op_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("child op is NULL", K(ret));
} else if (OB_FAIL(child_op_->get_next_row(exec_ctx, row))) {
if (OB_ITER_END != ret) {
LOG_WARN("fail get next row from child", K(ret)); // 非 iter_end 错误再打印WARN日志
}
}
return ret;
}
int ObPxTransmit::rescan(ObExecContext &exec_ctx) const
{
int ret = OB_SUCCESS;
auto op_ctx = GET_PHY_OPERATOR_CTX(ObPxTransmitCtx, exec_ctx, get_id());
if (OB_ISNULL(op_ctx)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("NULL operator ctx", K(ret), K_(id));
} else if (exec_ctx.is_restart_plan() && exec_ctx.is_dfo_rescan_plan()) {
op_ctx->iter_end_ = false;
op_ctx->transmited_ = false;
dtl::ObDtlChannel *ch = NULL;
common::ObIArray<dtl::ObDtlChannel*> &channels = op_ctx->task_channels_;
for (int i = 0; i < channels.count(); ++i) {
ch = channels.at(i);
ch->reset_state();
ch->set_channel_is_eof(false);
ch->set_batch_id(exec_ctx.get_px_batch_id());
}
OZ(ObPhyOperator::rescan(exec_ctx));
} else {
ret = OB_NOT_SUPPORTED;
LOG_USER_ERROR(OB_NOT_SUPPORTED, "transmit rescan");
LOG_WARN("transmit rescan is not supported", K(ret));
}
return ret;
}
int ObPxTransmit::inner_close(ObExecContext &exec_ctx) const
{
int ret = OB_SUCCESS;
// 注意：不能再 inner_open 中调用 flush rows，因为它会阻塞 inner_open 执行完成
// 最好不要在inner_close中flush data，这会导致出错情况下，也send数据，应该send_rows中send_eof_row直接flush掉数据
ObPxTransmitCtx *transmit_ctx = NULL;
if (OB_ISNULL(transmit_ctx = GET_PHY_OPERATOR_CTX(ObPxTransmitCtx, exec_ctx, get_id()))) {
LOG_DEBUG("The operator has not been opened.", K(ret), K_(id), "op_type",
ob_phy_operator_type_str(get_type()));
} else if (OB_FAIL(ObTransmit::inner_close(exec_ctx))) {
LOG_WARN("fail close op", K(ret));
}
/* we must release channel even if there is some error happen before */
if (OB_NOT_NULL(transmit_ctx)) {
transmit_ctx->chs_agent_.destroy();
dtl::ObDtlChannelLoop &loop = transmit_ctx->loop_;
int release_channel_ret = loop.unregister_all_channel();
if (release_channel_ret != common::OB_SUCCESS) {
// the following unlink actions is not safe is any unregister failure happened
LOG_ERROR("fail unregister all channel from msg_loop", KR(release_channel_ret));
}
release_channel_ret = ObPxChannelUtil::unlink_ch_set(transmit_ctx->task_ch_set_, &transmit_ctx->dfc_, false);
if (release_channel_ret != common::OB_SUCCESS) {
LOG_WARN("release dtl channel failed", K(release_channel_ret));
}
}
return ret;
}
int ObPxTransmit::send_rows(ObExecContext &exec_ctx,
ObPxTransmitCtx &transmit_ctx,
ObSliceIdxCalc &slice_calc) const
{
int ret = OB_SUCCESS;
const ObNewRow *row = NULL;
int64_t send_row_time_recorder = 0;
int64_t top_level_get_next_time_used = 0;
int64_t slice_time = 0;
int64_t row_count = 0;
ObObj tablet_id;
ObSliceIdxCalc::SliceIdxArray slice_idx_array;
while (OB_SUCC(ret)) {
if (OB_FAIL(next_row(exec_ctx, row, transmit_ctx))) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("fail to get next row from child op",
K(ret), K(child_op_->get_type()));
} else {
// iter end
if (OB_FAIL(send_eof_row(transmit_ctx, exec_ctx))) { // overwrite err code
LOG_WARN("fail send eof rows to channels", K(ret));
}
break;
}
} else if (OB_ISNULL(row)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("row is NULL", K(ret));
}
row_count++;
transmit_ctx.metric_.count();
if (OB_FAIL(ret)) {
LOG_WARN("fail to get next row", K(ret));
} else if (OB_FAIL(slice_calc.get_slice_indexes(*row, slice_idx_array))) {
LOG_WARN("fail get slice idx", K(ret));
} else if (transmit_ctx.dfc_.all_ch_drained()) {
ret = OB_ITER_END;
LOG_DEBUG("all channel has been drained");
} else if (has_partition_id_column_idx()
&& OB_FAIL(slice_calc.get_previous_row_tablet_id(tablet_id))) {
LOG_WARN("failed to get previous row partition_id", K(ret));
}
FOREACH_CNT_X(slice_idx, slice_idx_array, OB_SUCC(ret)) {
if (OB_FAIL(send_row(exec_ctx,
transmit_ctx,
*slice_idx,
tablet_id,
*row,
send_row_time_recorder))) { // via dtl
if (OB_ITER_END != ret) {
LOG_WARN("fail emit row to interm result", K(ret), K(slice_idx_array));
}
}
}
}
if (OB_ITER_END == ret) {
ret = OB_SUCCESS;
LOG_TRACE("transmit meet a iter end");
}
LOG_TRACE("Transmit time record", K(top_level_get_next_time_used), K(slice_time), K(row_count), K(ret));
return ret;
}
int ObPxTransmit::broadcast_rows(ObExecContext &exec_ctx,
ObPxTransmitCtx &transmit_ctx,
ObSliceIdxCalc &slice_calc) const
{
int ret = OB_SUCCESS;
UNUSED(slice_calc);
const ObNewRow *row = NULL;
int64_t top_level_get_next_time_used = 0;
int64_t slice_time = 0;
int64_t row_count = 0;
ObSliceIdxCalc::SliceIdxArray slice_idx_array;
while (OB_SUCC(ret)) {
if (OB_FAIL(next_row(exec_ctx, row, transmit_ctx))) {
if (OB_UNLIKELY(OB_ITER_END != ret)) {
LOG_WARN("fail to get next row from child op",
K(ret), K(child_op_->get_type()));
} else {
// iter end
if (OB_FAIL(broadcast_eof_row(transmit_ctx))) { // overwrite err code
LOG_WARN("fail send eof rows to channels", K(ret));
}
break;
}
} else if (OB_ISNULL(row)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("row is NULL", K(ret));
}
row_count++;
transmit_ctx.metric_.count();
if (OB_FAIL(ret)) {
LOG_WARN("fail to get next row", K(ret));
} else if (transmit_ctx.dfc_.all_ch_drained()) {
ret = OB_ITER_END;
LOG_DEBUG("all channel has been drained");
} else {
ObPxNewRow px_row(*row);
ret = transmit_ctx.chs_agent_.broadcast_row(px_row);
}
}
if (OB_ITER_END == ret) {
ret = OB_SUCCESS;
LOG_TRACE("transmit meet a iter end");
}
LOG_TRACE("Transmit time record", K(top_level_get_next_time_used), K(slice_time), K(row_count), K(ret));
return ret;
}
int ObPxTransmit::send_row(ObExecContext &ctx,
ObPxTransmitCtx &transmit_ctx,
int64_t slice_idx,
ObObj &partition_id,
const ObNewRow &row,
int64_t &time_recorder) const
{
UNUSED(time_recorder);
int ret = OB_SUCCESS;
const ObNewRow *sending_row = &row;
dtl::ObDtlChannel *ch = NULL;
LOG_DEBUG("Send row begin", K(row), K(ret));
ObPhysicalPlanCtx *phy_plan_ctx = NULL;
common::ObIArray<dtl::ObDtlChannel*> &channels = transmit_ctx.task_channels_;
if (OB_ISNULL(phy_plan_ctx = GET_PHY_PLAN_CTX(ctx))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("phy plan ctx is null", K(ret));
} else if (ObSliceIdxCalc::DEFAULT_CHANNEL_IDX_TO_DROP_ROW == slice_idx) {
// do nothing, just drop this row
transmit_ctx.op_monitor_info_.otherstat_1_value_++;
transmit_ctx.op_monitor_info_.otherstat_1_id_ = ObSqlMonitorStatIds::EXCHANGE_DROP_ROW_COUNT;
} else if (OB_UNLIKELY(slice_idx < 0 || slice_idx >= channels.count())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid slice idx", K(ret), K(lbt()),
K(slice_idx), "channels", channels.count(), K(row));
} else if (OB_ISNULL(ch = channels.at(slice_idx))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected NULL ptr", K(ret));
} else if (has_partition_id_column_idx()
&& OB_FAIL(copy_cur_row(transmit_ctx.get_cur_row(), sending_row, true))) {
LOG_WARN("failed to copy cur row", K(ret));
} else if (has_partition_id_column_idx()
&& OB_FAIL(update_row(transmit_ctx.get_cur_row(),
get_partition_id_column_idx(),
partition_id))) {
LOG_WARN("failed to update row", K(ret), K(get_partition_id_column_idx()), K(partition_id));
} else {
LOG_DEBUG("send row", K(get_name()), K(has_partition_id_column_idx()),
K(get_partition_id_column_idx()), K(partition_id), K(*sending_row));
ObPxNewRow px_row(*sending_row);
if (OB_FAIL(ret)) {
} else if (OB_FAIL(ch->send(px_row, phy_plan_ctx->get_timeout_timestamp()))) {
if (OB_ITER_END != ret) {
LOG_WARN("fail send row to slice channel", K(px_row), K(slice_idx), K(ret));
}
} else {
LOG_DEBUG("send row to slice channel", K(px_row), K(slice_idx));
}
}
LOG_DEBUG("Send row", K(slice_idx), K(row), K(ret));
return ret;
}
int ObPxTransmit::send_eof_row(ObPxTransmitCtx &transmit_ctx, ObExecContext &ctx) const
{
int ret = OB_SUCCESS;
dtl::ObDtlChannel *ch = NULL;
ObPhysicalPlanCtx *phy_plan_ctx = NULL;
common::ObIArray<dtl::ObDtlChannel*> &channels = transmit_ctx.task_channels_;
LOG_TRACE("Send eof row",
"op_id", get_id(),
"ch_cnt", channels.count(),
K(ret));
if (OB_ISNULL(phy_plan_ctx = GET_PHY_PLAN_CTX(ctx))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("phy plan ctx is null", K(ret));
} else {
for (int64_t slice_idx = 0; (OB_SUCCESS == ret) && slice_idx < channels.count(); ++slice_idx) {
if (NULL == (ch = channels.at(slice_idx))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected NULL ptr", K(ret));
} else {
ObPxNewRow px_eof_row;
px_eof_row.set_eof_row();
px_eof_row.set_data_type(ObDtlMsgType::PX_CHUNK_ROW);
if (OB_FAIL(ch->send(px_eof_row, phy_plan_ctx->get_timeout_timestamp(),
nullptr, true))) {
LOG_WARN("fail send eof row to slice channel", K(px_eof_row), K(slice_idx), K(ret));
} else if (OB_FAIL(ch->flush())) {
LOG_WARN("failed to flush send msg", K(px_eof_row), K(slice_idx), K(ret));
}
}
}
}
return ret;
}
int ObPxTransmit::broadcast_eof_row(ObPxTransmitCtx &transmit_ctx) const
{
int ret = OB_SUCCESS;
common::ObIArray<dtl::ObDtlChannel*> &channels = transmit_ctx.task_channels_;
LOG_TRACE("broadcast eof row",
"op_id", get_id(),
"ch_cnt", channels.count(),
K(ret));
ObPxNewRow px_eof_row;
px_eof_row.set_eof_row();
px_eof_row.set_data_type(ObDtlMsgType::PX_CHUNK_ROW);
if (OB_FAIL(transmit_ctx.chs_agent_.broadcast_row(px_eof_row, nullptr, true))) {
LOG_WARN("unexpected NULL ptr", K(ret));
} else if (OB_FAIL(transmit_ctx.chs_agent_.flush())) {
LOG_WARN("fail flush row to slice channel", K(ret));
}
return ret;
}
int ObPxTransmit::update_row(ObNewRow &row, int partition_id_column_idx, ObObj &partition_id) const
{
int ret = OB_SUCCESS;
// 这里要做一个投影的转换：
// partition_id_column_idx的值表示的是partition id在output exprs中的索引位置，
// 但是当前row并非是output row，需要按照project进行投影后的cells才是最终的output row
// 因此在update partition id的时候需要使用`row.get_cell()`方法访问
if (projector_size_ > partition_id_column_idx && partition_id_column_idx >= 0) {
row.get_cell(partition_id_column_idx) = partition_id;
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid partition id column idx",
K(partition_id_column_idx), K(projector_size_), K(*projector_));
}
return ret;
}
int ObPxTransmit::next_row(ObExecContext &exec_ctx,
const ObNewRow *&row,
ObPxTransmitCtx &transmit_ctx) const
{
int ret = OB_SUCCESS;
if (OB_ISNULL(child_op_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("child op is null");
} else if ((child_op_->is_dml_operator() && !child_op_->is_pdml_operator())
|| transmit_ctx.iter_end_) {
//如果下游是dml操作，dml操作不向上层算子迭代行，直接返回iter end
ret = OB_ITER_END;
transmit_ctx.consume_first_row_ = true;
LOG_TRACE("transmit iter end", K(ret), K(transmit_ctx.iter_end_));
} else if (!transmit_ctx.consume_first_row_) {
row = transmit_ctx.first_row_;
transmit_ctx.consume_first_row_ = true;
} else {
ret = get_next_row(exec_ctx, row);
}
return ret;
}
int ObPxTransmit::link_ch_sets(ObPxTaskChSet &ch_set,
common::ObIArray<dtl::ObDtlChannel*> &channels,
ObDtlFlowControl *dfc)
{
int ret = OB_SUCCESS;
dtl::ObDtlChannelInfo ci;
for (int64_t idx = 0; idx < ch_set.count() && OB_SUCC(ret); ++idx) {
dtl::ObDtlChannel *ch = NULL;
if (OB_FAIL(ch_set.get_channel_info(idx, ci))) {
LOG_WARN("fail get channel info", K(idx), K(ret));
} else if (OB_FAIL(dtl::ObDtlChannelGroup::link_channel(ci, ch, dfc))) {
LOG_WARN("fail link channel", K(ci), K(ret));
} else if (OB_ISNULL(ch)) {
LOG_WARN("fail add qc channel", K(ret));
} else if (OB_FAIL(channels.push_back(ch))) {
LOG_WARN("fail push back channel ptr", K(ci), K(ret));
}
}
return ret;
}

36
src/sql/engine/px/ob_dlt_data_channel_group.h
View File

@ -1,36 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef __OB_SQL_DTL_DATA_CHANNEL_GROUP_H__
#define __OB_SQL_DTL_DATA_CHANNEL_GROUP_H__
namespace oceanbase
{
namespace sql
{
class ObDTLDataChannelGroup
{
public:
ObDTLDataChannelGroup();
virtual ~ObDTLDataChannelGroup();
int wait(int64_t timeout_ts);
int notify();
int add_channel_info(ObDtlChannelInfo &producer, common::ObIArray<ObDtlChannelInfo> &consumers);
private:
/* functions */
/* variables */
DISALLOW_COPY_AND_ASSIGN(ObDTLDataChannelGroup);
};
}
}
#endif /* __OB_SQL_DTL_DATA_CHANNEL_GROUP_H__ */
//// end of header file

33
src/sql/engine/sort/ob_merge_sort_interface.h
View File

@ -1,33 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_SQL_ENGINE_SORT_OB_INTERFACE_MERGE_SORT_H_
#define OCEANBASE_SQL_ENGINE_SORT_OB_INTERFACE_MERGE_SORT_H_
#include "common/row/ob_row_iterator.h"
namespace oceanbase
{
namespace sql
{
class ObIMergeSort
{
public:
virtual int dump_base_run(common::ObOuterRowIterator &row_iterator, bool build_fragment = true) = 0;
virtual int build_cur_fragment() = 0;
};
} // end namespace sql
} // end namespace oceanbase
#endif

400
src/sql/engine/sort/ob_specific_columns_sort.cpp
View File

@ -1,400 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_ENG
#include "sql/engine/sort/ob_specific_columns_sort.h"
#include "common/row/ob_row_util.h"
#include "common/object/ob_obj_compare.h"
using namespace oceanbase::sql;
using namespace oceanbase::common;
//init static member value
ObSpecificColumnsSort::ObSpecificColumnsSort() :
row_alloc_(ObModIds::OB_SQL_SORT_ROW, OB_MALLOC_NORMAL_BLOCK_SIZE,
OB_SERVER_TENANT_ID, ObCtxIds::WORK_AREA),
row_count_(0),
prefix_keys_pos_(0),
row_array_pos_(0),
next_block_row_(NULL),
sort_array_(),
sort_columns_(NULL),
obobj_buffer_(NULL),
err_(OB_SUCCESS)
{
}
ObSpecificColumnsSort::ObSpecificColumnsSort(const char *label,
uint64_t malloc_block_size,
uint64_t tenant_id,
ObCtxIds::ObCtxIdEnum ctx_id) :
row_alloc_(label, malloc_block_size,
tenant_id, ctx_id),
row_count_(0),
prefix_keys_pos_(0),
row_array_pos_(0),
next_block_row_(NULL),
sort_array_(),
sort_columns_(NULL),
obobj_buffer_(NULL),
err_(OB_SUCCESS)
{
}
void ObSpecificColumnsSort::reset() {
row_count_ = 0;
prefix_keys_pos_ = 0;
row_array_pos_ = 0;
next_block_row_ = NULL;
sort_columns_ = NULL;
row_alloc_.reset();
sort_array_.reset();
obobj_buffer_ = NULL;
}
void ObSpecificColumnsSort::reuse() {
reset();
}
// just for roll up
void ObSpecificColumnsSort::rescan() {
row_array_pos_ = 0;
next_block_row_ = NULL;
}
int ObSpecificColumnsSort::set_sort_columns(
const ObIArray<ObSortColumn> &sort_columns, const int64_t prefix_pos) {
int ret = OB_SUCCESS;
sort_columns_ = &sort_columns;
ret = ObBaseSort::set_sort_columns(sort_columns, prefix_pos);
if(OB_SUCC(ret)){
prefix_keys_pos_ = prefix_pos;
}
else{
//do nothing
}
return ret;
}
int ObSpecificColumnsSort::deep_copy_new_row(const ObNewRow &row,
ObNewRow *&new_row, ObArenaAllocator &alloc) {
int ret = OB_SUCCESS;
char *buf = NULL;
int64_t pos = sizeof(ObNewRow);
int64_t buf_len = row.get_deep_copy_size() + sizeof(ObNewRow);
if (OB_ISNULL(buf = static_cast<char *>(alloc.alloc(buf_len)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("alloc new row failed", K(ret), K(buf_len));
} else if (OB_ISNULL(new_row = new (buf) ObNewRow())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("new_row is null", K(ret), K(buf_len));
} else if (OB_FAIL(new_row->deep_copy(row, buf, buf_len, pos))) {
LOG_WARN("deep copy row failed", K(ret), K(buf_len), K(pos));
} else {
// do nothing
}
return ret;
}
int ObSpecificColumnsSort::get_sort_result_array(common::ObArray<const common::ObNewRow *> &sort_result) {
sort_result = sort_array_;
return OB_SUCCESS;
};
int ObSpecificColumnsSort::add_row(const common::ObNewRow &row,
bool &need_sort) {
int ret = OB_SUCCESS;
bool is_cur_block = false;
need_sort = false;
const ObNewRow *last_row = NULL;
if (sort_array_.count() > 0) {
last_row = sort_array_.at(sort_array_.count() - 1);
}
if (OB_FAIL(check_block_row(row, last_row, is_cur_block))) {
LOG_WARN("check if row is belong to cur block failed", K(ret));
} else {
ObNewRow *new_row = NULL;
//if (need_deep_copy) {
if (OB_FAIL(deep_copy_new_row(row, new_row, row_alloc_))) {
LOG_WARN("deep copy a new row failed", K(ret));
} else if (OB_ISNULL(new_row)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("new row is null", K(ret));
} else if (is_cur_block) {
if (OB_FAIL(sort_array_.push_back(new_row))) {
LOG_WARN("push back new row failed", K(ret));
}
} else {
need_sort = true;
next_block_row_ = new_row; // new row is next sort block row
}
}
return ret;
}
int ObSpecificColumnsSort::add_row_without_copy(common::ObNewRow* row)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(row)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("new row is null", K(ret));
} else if (OB_FAIL(sort_array_.push_back(row))) {
LOG_WARN("push back new row failed", K(ret));
}
return ret;
}
// check row and last_row are belong to same sort block
// such as prefix column keys is (a,b)
// (1,2,3) and (1,2,4) are same block
int ObSpecificColumnsSort::check_block_row(const ObNewRow &row,
const ObNewRow *last_row, bool &is_cur_block) {
int ret = OB_SUCCESS;
is_cur_block = true;
if (!has_prefix_pos() || NULL == last_row) {
// do nothing, need add row
} else {
int cmp = 0;
const ObIArray<ObSortColumn> *sort_column = get_sort_columns();
if (OB_ISNULL(sort_column)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sort column is null", K(ret));
}
for (int64_t i = 0; 0 == cmp && OB_SUCC(ret) && i < prefix_keys_pos_;
i++) {
int64_t idx = sort_column->at(i).index_;
ObCompareCtx cmp_ctx(ObMaxType, sort_column->at(i).cs_type_, true,
INVALID_TZ_OFF, sort_column->at(i).get_cmp_null_pos());
cmp = row.get_cell(idx).compare(last_row->get_cell(idx),
cmp_ctx);
if ((cmp < 0 && sort_column->at(i).is_ascending())
|| (cmp > 0 && !sort_column->at(i).is_ascending())) { // check prefix_sort_columns
ret = OB_ERR_UNEXPECTED;
LOG_WARN("prefix sort ordering is invalid", K(ret), K(row),
K(*last_row), K(sort_column->at(i)));
}
}
if (OB_SUCC(ret)) {
if (0 == cmp) { // row is equal last_row with prefix sort keys
++row_count_;
} else {
is_cur_block = false;
}
}
}
return ret;
}
int ObSpecificColumnsSort::specific_columns_cmp(const void * p1,
const void * p2) {
bool bret = false;
ObSpecificColumnsSort** scs1 = (ObSpecificColumnsSort**) p1;
ObSpecificColumnsSort** scs2 = (ObSpecificColumnsSort**) p2;
if (OB_ISNULL(scs1) || OB_ISNULL(scs2)) {
LOG_WARN_RET(OB_INVALID_ARGUMENT, "scs1 is null or scs2 is null", K(bret));
} else if (OB_ISNULL(*scs1) || OB_ISNULL(*scs2)) {
LOG_WARN_RET(OB_INVALID_ARGUMENT, "*scs1 is null or *scs2 is null", K(bret));
} else if (OB_UNLIKELY(OB_SUCCESS != (*scs1)->err_)
|| OB_UNLIKELY(OB_SUCCESS != (*scs2)->err_)) {
// do nothing if we already have an error,
// so we can finish the sort process ASAP.
} else if (((*scs1) != (*scs1))) {
(*scs1)->err_ = OB_ERR_UNEXPECTED;
(*scs2)->err_ = OB_ERR_UNEXPECTED;
} else {
int cmp = 0;
int64_t prefix_sort_columns_count =
(*scs1)->sort_columns_->count() - (*scs1)->prefix_keys_pos_;
int64_t prefix_keys_pos = (*scs1)->prefix_keys_pos_;
ObSpecificColumnsSort* scs = *scs1;
ObObj* r1 = (ObObj*) (scs1 + 1);
ObObj* r2 = (ObObj*) (scs2 + 1);
ObCompareCtx cmp_ctx;
cmp_ctx.cmp_type_ = ObMaxType;
cmp_ctx.cmp_cs_type_ = CS_TYPE_INVALID;
cmp_ctx.is_null_safe_ = true;
cmp_ctx.tz_off_ = INVALID_TZ_OFF;
for (int64_t i = 0;
OB_SUCCESS == (scs)->err_ && 0 == cmp
&& i < prefix_sort_columns_count; ++i) {
//int64_t idx = ObSpecificColumnsSort::sort_columns_->at(i).index_;
cmp_ctx.cmp_cs_type_ = (scs)->sort_columns_->at(prefix_keys_pos + i ).cs_type_;
cmp_ctx.null_pos_ = (scs)->sort_columns_->at(prefix_keys_pos + i).get_cmp_null_pos();
cmp = r1[i].compare(r2[i], cmp_ctx);
if (cmp < 0) {
bret = (scs)->sort_columns_->at(prefix_keys_pos + i).is_ascending();
} else if (cmp > 0) {
bret = !((scs)->sort_columns_->at(prefix_keys_pos + i).is_ascending());
} else {
//do nothing
}
} // end for
}
return bret ? -1 : 1;
}
int ObSpecificColumnsSort::init_specific_columns() {
int ret = OB_SUCCESS;
ObObj* rows_buf = NULL;
ObSpecificColumnsSort** scs = NULL;
if (sort_array_.count() <= 0) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("the sort_array_ is empty", K(ret));
} else if (OB_ISNULL(
obobj_buffer_ = static_cast<char *>(row_alloc_.alloc(
(sizeof(ObObj) * (sort_columns_->count() - prefix_keys_pos_) + sizeof(ObNewRow*)
+ sizeof(ObSpecificColumnsSort*)) * sort_array_.count())))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to alloc memory for obobj buffer", K(ret));
} else if (OB_ISNULL(scs =
reinterpret_cast<ObSpecificColumnsSort**>(obobj_buffer_))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("the first ptr in obobj buffer is null", K(ret));
} else {
//mem view [ this obj obj obj obnewrow*]
for (int64_t i = 0; OB_SUCC(ret) && i < sort_array_.count(); i++) {
//only contain the columns that should be sorted.
int64_t buf_offset = 0;
*scs = this;
scs = scs + 1;
rows_buf = reinterpret_cast<ObObj *>(scs);
ObNewRow* row = const_cast<ObNewRow*>(sort_array_.at(i));
if (OB_ISNULL(rows_buf)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("the pointer is null", K(ret));
} else if (OB_ISNULL(row)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("some of the ObNewRow* in sort_array_ is null", K(ret), K(i));
} else {
int64_t idx = 0;
for (int64_t j = 0; OB_SUCC(ret) && j < (sort_columns_->count() - prefix_keys_pos_); j++) {
idx = sort_columns_->at(prefix_keys_pos_ + j).index_;
MEMCPY(rows_buf + buf_offset, &(row->get_cell(idx)), sizeof(ObObj));
++buf_offset;
}
if (OB_SUCC(ret)) {
rows_buf = rows_buf + (sort_columns_->count() - prefix_keys_pos_);
//add raw obnewrow ptr to the space's tail
ObNewRow** temp_ob_new_row = reinterpret_cast<ObNewRow **>(rows_buf);
if (OB_ISNULL(temp_ob_new_row)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("the pointer is null", K(ret));
} else {
*temp_ob_new_row = row;
if (OB_ISNULL(
scs = reinterpret_cast<ObSpecificColumnsSort**>(temp_ob_new_row
+ 1))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("the pointer is null", K(ret));
}
}
}
else {
//do nothing
}
}
}
}
return ret;
}
int ObSpecificColumnsSort::sort_rows() {
//LOG_INFO("Specific Columns Working");
int ret = OB_SUCCESS;
void* first_row = NULL;
const ObIArray<ObSortColumn> *sort_column = NULL;
if (OB_ISNULL(sort_column = get_sort_columns())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("sort column is null", K(ret));
} else if (prefix_keys_pos_ > sort_column->count()
|| 0 == sort_column->count()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN(
"unexpected err, prefix_keys_pos should be smaller than column count",
K(prefix_keys_pos_), K(sort_column->count()));
} else if (0 == sort_array_.count()) {
// do nothing
} else if (OB_FAIL(init_specific_columns())) {
LOG_WARN("err happen in specific column sort init, failed to do sort",
K(ret));
} else if (OB_ISNULL(first_row = static_cast<void *>(obobj_buffer_))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected NULL ptr", K(ret));
} else {
int64_t fixed_offset = (sort_columns_->count() - prefix_keys_pos_) * sizeof(ObObj)
+ sizeof(ObNewRow**) + sizeof(ObSpecificColumnsSort**);
std::qsort(first_row, sort_array_.count(), fixed_offset,
specific_columns_cmp);
if (OB_SUCCESS != err_) {
ret = err_;
LOG_WARN("failed to do qsort", K(ret));
} else {
char* cur_buf = obobj_buffer_;
cur_buf = cur_buf + (sort_columns_->count() - prefix_keys_pos_) * sizeof(ObObj)
+ sizeof(ObSpecificColumnsSort**);
for (int64_t i = 0; i < sort_array_.count(); i++) {
ObNewRow** tmp = (reinterpret_cast<ObNewRow**>(cur_buf));
if (OB_ISNULL(tmp)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected NULL ptr", K(ret));
break;
} else {
sort_array_.at(i) = *tmp;
cur_buf = cur_buf + fixed_offset;
}
}
}
}
if (OB_ISNULL(obobj_buffer_)) {
} else {
row_alloc_.free(obobj_buffer_);
obobj_buffer_ = NULL;
}
return ret;
}
int ObSpecificColumnsSort::get_next_row(common::ObNewRow &row) {
int ret = OB_SUCCESS;
const ObNewRow *tmp_row = NULL;
if (row_array_pos_ < 0) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("row array pos is invalid", K(row_array_pos_),
K(sort_array_.count()));
} else if (row_array_pos_ >= sort_array_.count()) {
if (NULL != next_block_row_) {
row_array_pos_ = 0;
sort_array_.reset();
sort_array_.push_back(next_block_row_);
next_block_row_ = NULL;
}
ret = OB_ITER_END;
} else if (OB_ISNULL(tmp_row = sort_array_.at(row_array_pos_))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("tmp row is null", K(ret), K(row_array_pos_));
} else if (tmp_row->get_count() > row.count_) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("array row count is invalid", K(ret), K(tmp_row->count_),
K(row.count_));
} else {
int64_t idx = 0;
while (idx < tmp_row->get_count()) {
row.cells_[idx] = tmp_row->get_cell(idx);
++idx;
}
++row_array_pos_;
}
return ret;
}

165
src/sql/ob_sql_character_set.cpp
View File

@ -1,165 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "sql/ob_sql_character_set.h"
#include <strings.h>
#include <stdio.h>
#include "share/ob_define.h"
namespace oceanbase
{
namespace sql
{
typedef struct CharacterSet
{
const char *name_;
int32_t number_;
} ObCharacterSet;
typedef struct CharacterSetR
{
int32_t number_;
const char *name_;
} ObCharacterSetR;
static const ObCharacterSet charset[] =
{
{"armscii8", 32},
{"ascii", 11},
{"big5", 1},
{"binary", 63},
{"cp1250", 26},
{"cp1251", 14},
{"cp1256", 57},
{"cp1257", 29},
{"cp850", 4},
{"cp852", 40},
{"cp866", 36},
{"cp932", 95},
{"dec8", 3},
{"eucjpms", 97},
{"euckr", 19},
{"gb2312", 24},
{"gbk", 28},
{"geostd8", 92},
{"greek", 25},
{"hebrew", 16},
{"hp8", 6},
{"keybcs2", 37},
{"koi8r", 7},
{"koi8u", 22},
{"latin1", 5},
{"latin2", 2},
{"latin5", 30},
{"latin7", 20},
{"macce", 38},
{"macroman", 39},
{"sjis", 13},
{"swe7", 10},
{"tis620", 18},
{"ucs2", 35},
{"ujis", 12},
{"utf8", 33}
};
static const ObCharacterSetR charsetR[] =
{
{1, "big5"},
{2, "latin2"},
{3, "dec8"},
{4, "cp850"},
{5, "latin1"},
{6, "hp8"},
{7, "koi8r"},
{10, "swe7"},
{11, "ascii"},
{12, "ujis"},
{13, "sjis"},
{14, "cp1251"},
{16, "hebrew"},
{18, "tis620"},
{19, "euckr"},
{20, "latin7"},
{22, "koi8u"},
{24, "gb2312"},
{25, "greek"},
{26, "cp1250"},
{28, "gbk"},
{29, "cp1257"},
{30, "latin5"},
{32, "armscii8"},
{33, "utf8"},
{35, "ucs2"},
{36, "cp866"},
{37, "keybcs2"},
{38, "macce"},
{39, "macroman"},
{40, "cp852"},
{57, "cp1256"},
{63, "binary"},
{92, "geostd8"},
{95, "cp932"},
{97, "eucjpms"},
};
int32_t get_char_number_from_name(const common::ObString &name)
{
int32_t ret = 0;
int32_t low = 0;
int32_t high = ARRAYSIZEOF(charset) - 1;
int cret = 0;
while (low <= high) {
ret = low + (high - low) / 2;
cret = name.case_compare(charset[ret].name_);
if (0 == cret) {
break;
} else if (0 > cret) {
high = ret - 1;
} else {
low = ret + 1;
}
}
if (low > high) {
ret = -1;
} else {
ret = charset[ret].number_;
}
return ret;
}
const char *get_char_name_from_number(int32_t number)
{
const char *name = "gbk";
int32_t ret = 0;
int32_t low = 0;
int32_t high = ARRAYSIZEOF(charsetR) - 1;
while (low <= high) {
ret = low + (high - low) / 2;
if (charsetR[ret].number_ == number) {
break;
} else if (charsetR[ret].number_ > number) {
high = ret - 1;
} else {
low = ret + 1;
}
}
if (low > high) {
ret = -1;
} else {
name = charsetR[ret].name_;
}
return name;
}
}
}

25
src/sql/ob_sql_character_set.h
View File

@ -1,25 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OB_SQL_CHARACTER_SET_H_
#define OB_SQL_CHARACTER_SET_H_
#include <stdint.h>
#include "lib/string/ob_string.h"
namespace oceanbase
{
namespace sql
{
extern int32_t get_char_number_from_name(const common::ObString &name);
extern const char *get_char_name_from_number(const int32_t number);
}
}
#endif

2
src/sql/optimizer/ob_phy_table_location_info.cpp
View File

@ -27,7 +27,7 @@ namespace sql
ObOptTabletLoc::ObOptTabletLoc()
: partition_id_(OB_INVALID_INDEX),
first_level_part_id_(OB_INVALID_INDEX),
replica_locations_(ObModIds::OB_SQL_OPTIMIZER_LOCATION_CACHE, OB_MALLOC_NORMAL_BLOCK_SIZE),
replica_locations_("SqlOptimLocaCac", OB_MALLOC_NORMAL_BLOCK_SIZE),
renew_time_(0)
{
}

238
src/sql/plan_cache/ob_plan_cache2.cpp
View File

@ -1,238 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_PC
#include "ob_plan_cache.h"
#include "sql/ob_sql_context.h"
#include "sql/plan_cache/ob_ps_sql_utils.h"
#include "sql/plan_cache/ob_ps_plan_cache_callback.h"
#include "sql/plan_cache/ob_pcv_set.h"
namespace oceanbase
{
using namespace common;
namespace sql
{
//添加plan到plan cache
// 1.判断plan cache内存是否达到上限；
// 2.判断plan大小是否超出限制
// 3.通过plan cache key获取pcv：
// 如果获取pcv成功：则将plan 加入pcv
// 如果获取pcv失败：则新生成pcv; 将plan 加入该pcv; 最后将该pcv 加入到key->pcv map中，
int ObPlanCache::add_ps_plan(const ObPsStmtId stmt_id,
ObPhysicalPlan *plan,
ObPlanCacheCtx &pc_ctx)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(plan)) {
ret = OB_INVALID_ARGUMENT;
SQL_PC_LOG(WARN, "invalid physical plan", K(ret));
} else if (is_reach_memory_limit()) {
ret = OB_REACH_MEMORY_LIMIT;
SQL_PC_LOG(DEBUG, "plan cache memory used reach the high water mark", K(mem_used_), K(get_mem_limit()), K(ret));
} else if (plan->get_mem_size() > ObPlanCache::MAX_PLAN_SIZE) {
ret = OB_SQL_PC_PLAN_SIZE_LIMIT;
SQL_PC_LOG(DEBUG, "plan size too large, don't add to plan cache", "plan_size", plan->get_mem_size(), K(ret));
} else {
if (OB_FAIL(inner_add_ps_plan(stmt_id, plan, pc_ctx))) {
if (!is_not_supported_err(ret) && OB_SQL_PC_PLAN_DUPLICATE != ret) {
SQL_PC_LOG(WARN, "fail to add plan", K(ret));
}
}
}
// if (OB_SUCC(ret)) {
// if (OB_FAIL(add_plan_stat(pc_ctx, plan))) {
// LOG_WARN("Failed to add plan cache stat", K(ret));
// }
// }
return ret;
}
// 通过plan cache key, 从stmt_id -> pcv map中获取pcv
int ObPlanCache::get_ps_pcv_set(const ObPsStmtId stmt_id,
ObPCVSet *&pcv_set,
ObPsPCVSetAtomicOp &op)
{
int ret = OB_SUCCESS;
//get pcv and inc ref count
int hash_err = ps_pcvs_map_.read_atomic(stmt_id, op);
switch (hash_err) {
case OB_SUCCESS: {
//get pcv and lock
if (OB_FAIL(op.get_value(pcv_set))) {
SQL_PC_LOG(WARN, "failed to lock pcv set", K(ret), K(stmt_id));
}
break;
}
case OB_HASH_NOT_EXIST: { //返回时 pcv_set = NULL; ret = OB_SUCCESS;
SQL_PC_LOG(DEBUG, "entry does not exist.", K(stmt_id));
break;
}
default: {
SQL_PC_LOG(WARN, "failed to get pcv set", K(ret), K(stmt_id));
ret = hash_err;
break;
}
}
return ret;
}
//only create pcv_set for stmt_id, not add plan
int ObPlanCache::get_or_create_pcv_set(const ObPsStmtId stmt_id, ObPlanCacheCtx &pc_ctx)
{
int ret = OB_SUCCESS;
ObPsPCVSetWlockAndRef w_ref_lock;
ObPCVSet *pcv_set = NULL;
if (OB_FAIL(get_ps_pcv_set(stmt_id, pcv_set, w_ref_lock/* write locked */))) {
SQL_PC_LOG(WARN, "failed to get pcv_set form plan_cache by id", K(ret), K(stmt_id));
} else if (NULL == pcv_set) {
if (OB_FAIL(create_ps_pcv_set(pc_ctx, pcv_set))) {
if (!is_not_supported_err(ret)) {
SQL_PC_LOG(WARN, "fail to create pcv_set and add plan", K(ret));
}
}
//set key value
if (OB_SUCC(ret)) {
int hash_err = ps_pcvs_map_.set_refactored(stmt_id, pcv_set);
if (OB_HASH_EXIST == hash_err) { //may be this pcv_set has been set by other thread。
//TODO
//pcv_set->remove_plan(exec_context, *plan);
pcv_set->dec_ref_count();//will clean auto
} else if (OB_SUCCESS == hash_err) {
SQL_PC_LOG(INFO, "succeed to set pcv_set to sql_pcvs_map");
} else {
SQL_PC_LOG(WARN, "failed to add pcv_set to sql_pcvs_map", K(ret));
//TODO
//pcv_set->remove_plan(exec_context, *plan);
pcv_set->dec_ref_count();
}
}
} else {
// release wlock whatever
pcv_set->unlock();
pcv_set->dec_ref_count();
//如果add plan时check到table or view的version已过期，则删除该pcv_set
// if (OB_OLD_SCHEMA_VERSION == ret) {
// SQL_PC_LOG(INFO, "table or view in plan cache value is old", K(ret));
// if (OB_FAIL(remove_ps_pcv_set(stmt_id))) {
// SQL_PC_LOG(WARN, "fail to remove plan cache value", K(ret));
// } else if (OB_FAIL(add_ps_pcv_set(stmt_id, pc_ctx))) {
// SQL_PC_LOG(DEBUG, "fail to add plan", K(ret), K(plan));
// }
// }
}
return ret;
}
int ObPlanCache::create_ps_pcv_set(ObPlanCacheCtx &pc_ctx,
ObPCVSet *&pcv_set)
{
int ret = OB_SUCCESS;
pcv_set = NULL;
char *ptr = NULL;
if (NULL == (ptr = (char *)inner_allocator_.alloc(sizeof(ObPCVSet)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to allocate memory for pcv set", K(ret));
} else {
pcv_set = new (ptr)ObPCVSet();
pcv_set->inc_ref_count();
if (OB_FAIL(pcv_set->init(pc_ctx, this, 0))) {//hualong @@todo
LOG_WARN("failed to init pcv set", K(ret));
}
}
if (OB_FAIL(ret) && NULL!=pcv_set) {
pcv_set->~ObPCVSet();
inner_allocator_.free(pcv_set);
pcv_set = NULL;
}
return ret;
}
int ObPlanCache::create_pcv_set_and_add_ps_plan(const ObPsStmtId stmt_id,
ObPhysicalPlan *plan,
ObPlanCacheCtx &pc_ctx,
ObPCVSet *&pcv_set)
{
int ret = OB_SUCCESS;
pcv_set = NULL;
char *ptr = NULL;
UNUSED(stmt_id);
if (OB_ISNULL(plan)) {
ret = OB_INVALID_ARGUMENT;
SQL_PC_LOG(WARN, "invalid argument", K(ret));
} else if (NULL == (ptr = (char *)inner_allocator_.alloc(sizeof(ObPCVSet)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to allocate memory for pcv set", K(ret));
} else {
pcv_set = new(ptr)ObPCVSet();
pcv_set->inc_ref_count();//构造ObPCVSet后的引用计数
if (OB_FAIL(pcv_set->init(pc_ctx, this, plan->get_merged_version()))) {
LOG_WARN("fail to init pcv set", K(ret));
}
}
//add plan
if (OB_SUCC(ret)) {
if (OB_FAIL(pcv_set->add_plan(plan, pc_ctx))) {
if (!is_not_supported_err(ret)) {
SQL_PC_LOG(WARN, "failed to add plan to plan cache value", K(ret));
}
} else {
// pcv_set->update_stmt_stat();
//如果该value在set进plan cache时失败，则在释放value时会减去该plan在plan cache中的内存占用量。
inc_mem_used(plan->get_mem_size());
}
}
if (OB_FAIL(ret) && NULL != pcv_set) {
pcv_set->~ObPCVSet();
inner_allocator_.free(pcv_set);
pcv_set = NULL;
}
return ret;
}
//淘汰指定plan cache key对应的pcv set
int ObPlanCache::remove_ps_pcv_set(const ObPsStmtId stmt_id)
{
int ret = OB_SUCCESS;
int hash_err = OB_SUCCESS;
ObPCVSet *pcv_set = NULL;
hash_err = ps_pcvs_map_.erase_refactored(stmt_id, &pcv_set);
if (OB_SUCCESS == hash_err) {
if (NULL != pcv_set) {
// remove plan cache reference, even remove_plan_stat() failed
SQL_PC_LOG(WARN, "xxx pcv_set ref_count", K(pcv_set->get_ref_count()));
pcv_set->dec_ref_count();
} else {
ret = OB_ERR_UNEXPECTED;
SQL_PC_LOG(ERROR, "pcv_set should not be null", K(stmt_id));
}
} else if (OB_HASH_NOT_EXIST == hash_err) {
// do nothing
SQL_PC_LOG(INFO, "plan cache key is alreay be deleted", K(stmt_id));
} else {
ret = hash_err;
SQL_PC_LOG(WARN, "failed to erase pcv_set from plan cache by key", K(stmt_id), K(hash_err));
}
return ret;
}
} //end of namespace sql
} //end of namespace oceanbase

779
src/sql/plan_cache/ob_plan_cache_manager.cpp
View File

@ -1,779 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_PC
#include "ob_plan_cache_manager.h"
#include "lib/mysqlclient/ob_mysql_proxy.h"
#include "lib/thread_local/ob_tsi_factory.h"
#include "share/config/ob_server_config.h"
#include "observer/ob_req_time_service.h"
#include "observer/omt/ob_tenant_config_mgr.h"
#include "pl/ob_pl.h"
#include "sql/plan_cache/ob_cache_object_factory.h"
#include "pl/pl_cache/ob_pl_cache_mgr.h"
namespace oceanbase
{
namespace sql
{
int ObPlanCacheManager::init(common::ObAddr addr)
{
int ret = OB_SUCCESS;
if (!inited_) {
if (OB_FAIL(pcm_.create(hash::cal_next_prime(512),
ObModIds::OB_HASH_BUCKET_PLAN_CACHE,
ObModIds::OB_HASH_NODE_PLAN_CACHE))) {
SQL_PC_LOG(WARN, "Failed to init plan cache manager", K(ret));
} else if (OB_FAIL(ps_pcm_.create(hash::cal_next_prime(512),
ObModIds::OB_HASH_BUCKET_PLAN_CACHE,
ObModIds::OB_HASH_NODE_PS_CACHE))) {
SQL_PC_LOG(WARN, "Failed to init ps plan cache manager", K(ret));
} else if (OB_FAIL(TG_CREATE(lib::TGDefIDs::PlanCacheEvict, tg_id_))) {
SQL_PC_LOG(WARN, "tg create failed", K(ret));
} else if (OB_FAIL(TG_START(tg_id_))) {
//TODO shengle, maybe exist memory leak
SQL_PC_LOG(WARN, "Failed to init timer", K(ret));
} else {
self_addr_ = addr;
elimination_task_.plan_cache_manager_ = this;
if (OB_FAIL(TG_SCHEDULE(tg_id_, elimination_task_,
GCONF.plan_cache_evict_interval, false))) {
SQL_PC_LOG(WARN, "Failed to schedule cache elimination task", K(ret));
}
}
}
if (OB_SUCC(ret)) {
inited_ = true;
destroyed_ = false;
}
return ret;
}
void ObPlanCacheManager::destroy()
{
if (inited_ && !destroyed_) {
destroyed_ = true;
inited_ = false;
observer::ObReqTimeGuard req_timeinfo_guard;
TG_DESTROY(tg_id_);
for (PlanCacheMap::iterator it = pcm_.begin();
it != pcm_.end();
it++) {
if (OB_ISNULL(it->second)) {
BACKTRACE_RET(ERROR, OB_ERR_UNEXPECTED, true, "plan_cache is null");
} else {
it->second->destroy();
}
}
}
}
ObPlanCache *ObPlanCacheManager::get_plan_cache(uint64_t tenant_id)
{
ObPlanCache *pc = NULL;
if (!destroyed_) {
ObPlanCacheManagerAtomic op;
int ret = pcm_.atomic_refactored(tenant_id, op);
if (OB_SUCCESS == ret) {
pc = op.get_plan_cache();
SQL_PC_LOG(DEBUG, "Get Plan Cache Success", K(tenant_id), K(pc));
} else if (OB_HASH_NOT_EXIST != ret) {
SQL_PC_LOG(WARN, "fait to do atomic", K(ret));
}
}
return pc;
}
ObPsCache *ObPlanCacheManager::get_ps_cache(const uint64_t tenant_id)
{
ObPsCache *ps_cache = NULL;
if (!destroyed_) {
ObPsCacheManagerAtomic op;
int ret = ps_pcm_.atomic_refactored(tenant_id, op);
if (OB_SUCC(ret)) {
SQL_PC_LOG(DEBUG, "Get ps plan cache success", K(tenant_id), K(ps_cache));
ps_cache = op.get_ps_cache();
} else if (OB_HASH_NOT_EXIST != ret) {
SQL_PC_LOG(WARN, "failed to do atomic", K(ret));
}
}
return ps_cache;
}
// maybe get plan_cache = NULL;
// this thread other thread
//
// get but not exist
// set tenand_id->plan_cache to map
// alloc plan_cache
// set to map but exist now
// free plan_cache
// erase tenand_id->plan_cache
// get but not exist
// return NULL
ObPlanCache *ObPlanCacheManager::get_or_create_plan_cache(uint64_t tenant_id, const ObPCMemPctConf &pc_mem_conf)
{
ObPlanCache *pc = NULL;
if (!destroyed_) {
ObPlanCacheManagerAtomic op;
int ret = pcm_.read_atomic(tenant_id, op);
if (OB_SUCCESS == ret) {
pc = op.get_plan_cache();
SQL_PC_LOG(DEBUG, "Get Plan Cache Success", K(tenant_id), K(pc));
} else if (OB_HASH_NOT_EXIST == ret) {
// create plan cache
void *buff = ob_malloc(sizeof(ObPlanCache), ObNewModIds::OB_SQL_PLAN_CACHE);
if (OB_ISNULL(buff)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
SQL_PC_LOG(ERROR, "Alloc mem for plan cache failed");
} else if (NULL == (pc = new(buff)ObPlanCache())) {
ret = OB_NOT_INIT;
SQL_PC_LOG(WARN, "fail to constructor plan cache");
} else {
if (OB_FAIL(pc->init(common::OB_PLAN_CACHE_BUCKET_NUMBER,
self_addr_,
tenant_id,
&pcm_))) {
SQL_PC_LOG(WARN, "init plan cache fail", K(tenant_id));
} else if (OB_FAIL(pc->set_mem_conf(pc_mem_conf))) {
SQL_PC_LOG(WARN, "fail to set plan cache memory conf", K(ret));
}
if (OB_FAIL(ret)) {
SQL_PC_LOG(WARN, "Failed to init plan cache", K(tenant_id), K(ret));
pc->~ObPlanCache();
ob_free(buff);
buff = NULL;
pc = NULL;
} else if (OB_SUCCESS == ret) {// add new plan cache to sql engine
pc->inc_ref_count(); //hash表引用计数
pc->inc_ref_count(); //外部代码引用计数
ret = pcm_.set_refactored(tenant_id, pc);
if (OB_SUCC(ret)) {
SQL_PC_LOG(DEBUG, "Add Plan Cache Success", K(tenant_id), K(pc));
} else {
// someone else add plan cache already
// or error when add plan cache to hashmap
pc->~ObPlanCache();
pc = NULL;
ob_free(buff);
buff = NULL;
if (OB_SUCCESS == pcm_.atomic_refactored(tenant_id, op)) {
pc = op.get_plan_cache();
SQL_PC_LOG(DEBUG, "Get Plan Cache Success", K(tenant_id), K(pc));
} else {
//this case is describle on function name;
SQL_PC_LOG(DEBUG, "tenant erase by others",K(tenant_id), K(ret));
}
}
}
}
} else {
SQL_PC_LOG(WARN, "unexpectd error", K(ret));
}
}
return pc;
}
ObPsCache *ObPlanCacheManager::get_or_create_ps_cache(
const uint64_t tenant_id, const ObPCMemPctConf &pc_mem_conf)
{
ObPsCache *ps_cache = NULL;
if (!destroyed_) {
ObPsCacheManagerAtomic op;
int ret = ps_pcm_.atomic_refactored(tenant_id, op);
if (OB_SUCCESS == ret) {
ps_cache = op.get_ps_cache();
SQL_PC_LOG(DEBUG, "Get ps Plan Cache Success", K(tenant_id), K(ps_cache));
} else if (OB_HASH_NOT_EXIST == ret) {
// create ps plan cache
void *buff = ob_malloc(sizeof(ObPsCache), ObModIds::OB_SQL_PS_CACHE);
if (OB_ISNULL(buff)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
SQL_PC_LOG(ERROR, "Alloc mem for ps plan cache failed");
} else if (NULL == (ps_cache = new(buff)ObPsCache())) {
ret = OB_NOT_INIT;
SQL_PC_LOG(WARN, "fail to constructor ps plan cache");
} else {
if (OB_SUCC(ps_cache->init(common::OB_PLAN_CACHE_BUCKET_NUMBER,
self_addr_,
tenant_id))) {
SQL_PC_LOG(INFO, "Init Ps Plan Cache Success", K(tenant_id));
} else if (OB_FAIL(ps_cache->set_mem_conf(pc_mem_conf))) {
SQL_PC_LOG(WARN, "fail to set plan cache memory conf", K(ret));
} else {
SQL_PC_LOG(WARN, "Failed to init plan cache", K(tenant_id), K(ret));
ps_cache->~ObPsCache();
ob_free(buff);
buff = NULL;
ps_cache = NULL;
}
// add new plan cache to sql engine
if (OB_SUCCESS == ret && NULL != ps_cache) {
ps_cache->inc_ref_count(); //hash表引用计数
ps_cache->inc_ref_count(); //外部代码引用计数
ret = ps_pcm_.set_refactored(tenant_id, ps_cache);
if (OB_SUCC(ret)) {
SQL_PC_LOG(INFO, "Add ps Plan Cache Success", K(tenant_id), K(ps_cache),
"ref_count", ps_cache->get_ref_count());
} else {
// someone else add plan cache already
// or error when add plan cache to hashmap
ps_cache->~ObPsCache();
ps_cache = NULL;
ob_free(buff);
buff = NULL;
if (OB_SUCCESS == ps_pcm_.atomic_refactored(tenant_id, op)) {
ps_cache = op.get_ps_cache();
SQL_PC_LOG(DEBUG, "Get Ps Plan Cache Success", K(tenant_id), K(ps_cache));
} else {
//this case is describle on function name;
SQL_PC_LOG(INFO, "tenant erase by others",K(tenant_id), K(ret));
}
}
}
}
} else {
SQL_PC_LOG(WARN, "unexpectd error", K(ret));
}
}
return ps_cache;
}
int ObPlanCacheManager::revert_plan_cache(const uint64_t &tenant_id)
{
int ret = OB_SUCCESS;
ObPlanCache *ppc = NULL;
observer::ObReqTimeGuard req_timeinfo_guard;
int tmp_ret = pcm_.get_refactored(tenant_id, ppc);
if (OB_SUCCESS == tmp_ret && NULL != ppc) {
SQL_PC_LOG(INFO, "plan_cache_manager revert plan cache",
"pc ref_count", ppc->get_ref_count(),
K(tenant_id));
//cancel scheduled task
ppc->dec_ref_count();
} else if (OB_HASH_NOT_EXIST == tmp_ret) { // maybe erase by other thread
SQL_PC_LOG(INFO, "Plan Cache not exist", K(tenant_id));
} else {
ret = tmp_ret;
SQL_PC_LOG(ERROR, "unexpected error when erase plan cache",
K(tenant_id), K(ppc), K(ret));
}
return ret;
}
void ObPlanCacheManager::ObPlanCacheEliminationTask::runTimerTask()
{
#define NEED_AUTO_FLUSH_PC(v) (0 == v ? false : true)
int ret = OB_SUCCESS;
if (OB_ISNULL(plan_cache_manager_)) {
ret = OB_NOT_INIT;
SQL_PC_LOG(WARN, "plan_cache_manager not inited", K(ret));
} else {
++run_task_counter_;
const int64_t auto_flush_pc_interval = (int64_t)(GCONF._ob_plan_cache_auto_flush_interval) / (1000 * 1000L); // second
{
// 在调用plan cache接口前引用plan资源前必须定义guard
observer::ObReqTimeGuard req_timeinfo_guard;
run_plan_cache_task();
run_ps_cache_task();
if (NEED_AUTO_FLUSH_PC(auto_flush_pc_interval)
&& 0 == run_task_counter_ % auto_flush_pc_interval) {
run_auto_flush_plan_cache_task();
}
SQL_PC_LOG(INFO, "schedule next cache evict task",
"evict_interval", (int64_t)(GCONF.plan_cache_evict_interval));
}
// free cache obj in deleted map
if (get_plan_cache_gc_strategy() > 0) {
observer::ObReqTimeGuard req_timeinfo_guard;
run_free_cache_obj_task();
}
SQL_PC_LOG(INFO, "schedule next cache evict task",
"evict_interval", (int64_t)(GCONF.plan_cache_evict_interval));
if (OB_FAIL(TG_SCHEDULE(plan_cache_manager_->tg_id_, *this,
GCONF.plan_cache_evict_interval,
false))) {
SQL_PC_LOG(WARN, "Schedule new elimination failed", K(ret));
}
}
}
void ObPlanCacheManager::ObPlanCacheEliminationTask::run_auto_flush_plan_cache_task()
{
if (OB_ISNULL(plan_cache_manager_)) {
// do nothing
} else {
IGNORE_RETURN plan_cache_manager_->flush_all_plan_cache();
}
}
void ObPlanCacheManager::ObPlanCacheEliminationTask::run_plan_cache_task()
{
int ret = OB_SUCCESS;
ObArray<uint64_t> tenant_id_array;
ObGetAllCacheKeyOp op(&tenant_id_array);
if (OB_ISNULL(plan_cache_manager_)) {
ret = OB_NOT_INIT;
SQL_PC_LOG(WARN, "plan_cache_manager not inited", K(ret));
} else if (OB_FAIL(plan_cache_manager_->pcm_.foreach_refactored(op))) {
SQL_PC_LOG(ERROR, "fail to traverse pcm", K(ret));
} else {
for (int64_t i = 0; i < tenant_id_array.count(); i++) { //循环忽略错误码,保证所有plan cache启动淘汰
uint64_t tenant_id = tenant_id_array.at(i);
MAKE_TENANT_SWITCH_SCOPE_GUARD(guard);
// 由于 tenat_id_array的tenant移除 与 drop tenant存在一定时延。因此，在这里需要检查一下tenant是否存在。
// bug fix:
if (!GCTX.omt_->is_available_tenant(tenant_id)) {
// do nothing
} else if (OB_FAIL(guard.switch_to(tenant_id))) {
ret = OB_SUCCESS;
LOG_DEBUG("switch tenant fail", K(tenant_id));
} else {
ObPlanCache *plan_cache = plan_cache_manager_->get_plan_cache(tenant_id);
if (NULL != plan_cache) {
if (OB_FAIL(plan_cache->update_memory_conf())) { //如果失败, 则不更新设置, 也不影响其他流程
SQL_PC_LOG(WARN, "fail to update plan cache memory sys val", K(ret));
}
if (OB_FAIL(plan_cache->cache_evict())) {
SQL_PC_LOG(ERROR, "Plan cache evict failed, please check", K(ret));
}
plan_cache->dec_ref_count();
}
}
}
}
}
void ObPlanCacheManager::ObPlanCacheEliminationTask::run_ps_cache_task()
{
int ret = OB_SUCCESS;
ObArray<uint64_t> tenant_id_array;
ObGetAllCacheKeyOp op(&tenant_id_array);
if (OB_ISNULL(plan_cache_manager_)) {
ret = OB_NOT_INIT;
SQL_PC_LOG(WARN, "plan_cache_manager not inited", K(ret));
} else if (OB_FAIL(plan_cache_manager_->ps_pcm_.foreach_refactored(op))) {
SQL_PC_LOG(ERROR, "fail to traverse pcm", K(ret));
} else {
for (int64_t i = 0; i < tenant_id_array.count(); i++) { //循环忽略错误码,保证所有ps cache启动淘汰
uint64_t tenant_id = tenant_id_array.at(i);
ObPCMemPctConf conf;
ObPsCache *ps_cache = plan_cache_manager_->get_ps_cache(tenant_id); // inc ps_cache ref
ObPlanCache *plan_cache = plan_cache_manager_->get_plan_cache(tenant_id);
if (NULL != plan_cache) {
conf.limit_pct_ = plan_cache->get_mem_limit_pct();
conf.high_pct_ = plan_cache->get_mem_high_pct();
conf.low_pct_ = plan_cache->get_mem_low_pct();
}
if (NULL != ps_cache) {
ps_cache->set_mem_conf(conf);
ps_cache->cache_evict();
ps_cache->dec_ref_count();
}
}
}
}
void ObPlanCacheManager::ObPlanCacheEliminationTask::run_free_cache_obj_task()
{
int ret = OB_SUCCESS;
ObArray<uint64_t> tenant_id_array;
ObGetAllCacheKeyOp op(&tenant_id_array);
int64_t safe_timestamp = INT64_MAX;
if (OB_ISNULL(plan_cache_manager_)) {
ret = OB_NOT_INIT;
SQL_PC_LOG(WARN, "plan_cache_manager not inited", K(ret));
} else if (OB_FAIL(plan_cache_manager_->pcm_.foreach_refactored(op))) {
SQL_PC_LOG(ERROR, "failed to traverse pcm", K(ret));
} else if (observer::ObGlobalReqTimeService::get_instance()
.get_global_safe_timestamp(safe_timestamp)) {
SQL_PC_LOG(ERROR, "failed to get global safe timestamp", K(ret));
} else {
for (int i = 0; i < tenant_id_array.count(); i++) {
uint64_t tenant_id = tenant_id_array.at(i);
// inc plan cache ref
ObPlanCache *plan_cache = plan_cache_manager_->get_plan_cache(tenant_id);
if (NULL != plan_cache) {
ObArray<AllocCacheObjInfo> deleted_objs;
MAKE_TENANT_SWITCH_SCOPE_GUARD(guard);
if (!GCTX.omt_->is_available_tenant(tenant_id)) {
// do nothing
} else if (OB_FAIL(guard.switch_to(tenant_id))) {
ret = OB_SUCCESS;
LOG_DEBUG("switch tenant fail", K(tenant_id));
} else if (OB_FAIL(plan_cache->dump_deleted_objs<DUMP_ALL>(deleted_objs, safe_timestamp))) {
SQL_PC_LOG(WARN, "failed to traverse hashmap", K(ret));
} else {
int64_t tot_mem_used = 0;
for (int k = 0; k < deleted_objs.count(); k++) {
tot_mem_used += deleted_objs.at(k).mem_used_;
} // end for
if (tot_mem_used >= ((plan_cache->get_mem_limit() / 100) * 30)) {
LOG_ERROR("Cache Object Memory Leaked Much!!!", K(tot_mem_used),
K(plan_cache->get_mem_limit()), K(deleted_objs),
K(tenant_id), K(safe_timestamp));
} else if (deleted_objs.count() > 0) {
LOG_WARN("Cache Object Memory Leaked Much!!!", K(deleted_objs), K(tenant_id),
K(safe_timestamp), K(plan_cache->get_mem_limit()));
}
#if defined(NDEBUG)
LOG_DEBUG("AUTO Mode is not active for now");
#else
if (AUTO == get_plan_cache_gc_strategy()) {
for (int j = 0; j < deleted_objs.count(); j++) {
if (OB_FAIL(ObCacheObjectFactory::destroy_cache_obj(true,
deleted_objs.at(j).obj_id_,
plan_cache))) {
LOG_WARN("failde to destroy cache obj", K(ret));
}
} // end inner loop
}
#endif
}
#if !defined(NDEBUG)
if (OB_SUCC(ret) && OB_FAIL(plan_cache->dump_all_objs())) {
LOG_WARN("failed to dump deleted map", K(ret));
}
#endif
if(NULL != plan_cache) {
plan_cache->dec_ref_count();
plan_cache = NULL;
}
}
} // end outter loop
}
}
int ObPlanCacheManager::evict_plan_by_table_name(uint64_t tenant_id, uint64_t database_id, ObString tab_name)
{
int ret = OB_SUCCESS;
observer::ObReqTimeGuard req_timeinfo_guard;
ObPlanCache *plan_cache = get_plan_cache(tenant_id);
if (NULL != plan_cache) {
if (OB_FAIL(plan_cache->evict_plan_by_table_name(tenant_id, database_id, tab_name))) {
SQL_PC_LOG(WARN, "fail to evict plan by table name", K(ret));
}
plan_cache->dec_ref_count();
}
return ret;
}
int ObPlanCacheManager::flush_plan_cache_by_sql_id(uint64_t tenant_id,
uint64_t db_id,
common::ObString sql_id) {
int ret = OB_SUCCESS;
observer::ObReqTimeGuard req_timeinfo_guard;
ObPlanCache *plan_cache = get_plan_cache(tenant_id);
if (NULL != plan_cache) {
if (OB_FAIL(plan_cache->cache_evict_plan_by_sql_id(db_id, sql_id))) {
SQL_PC_LOG(ERROR, "Plan cache evict failed, please check", K(ret));
}
ObArray<AllocCacheObjInfo> deleted_objs;
int64_t safe_timestamp = INT64_MAX;
if (OB_FAIL(ret)) {
// do nothing
} else if (OB_FAIL(observer::ObGlobalReqTimeService::get_instance()
.get_global_safe_timestamp(safe_timestamp))) {
SQL_PC_LOG(ERROR, "failed to get global safe timestamp", K(ret));
} else if (OB_FAIL(plan_cache->dump_deleted_objs<DUMP_SQL>(deleted_objs, safe_timestamp))) {
SQL_PC_LOG(WARN, "failed to get deleted sql objs", K(ret));
} else {
LOG_INFO("Deleted Cache Objs", K(deleted_objs));
for (int64_t i = 0; i < deleted_objs.count(); i++) { // ignore error code and continue
if (OB_FAIL(ObCacheObjectFactory::destroy_cache_obj(true,
deleted_objs.at(i).obj_id_,
plan_cache))) {
LOG_WARN("failed to destroy cache obj", K(ret));
}
}
}
plan_cache->dec_ref_count();
}
return ret;
}
int ObPlanCacheManager::flush_all_plan_cache()
{
int ret = OB_SUCCESS;
observer::ObReqTimeGuard req_timeinfo_guard;
ObArray<uint64_t> tenant_id_array;
ObGetAllCacheKeyOp op(&tenant_id_array);
if (OB_FAIL(pcm_.foreach_refactored(op))) {
SQL_PC_LOG(ERROR, "fail to traverse pcm", K(ret));
}
for (int64_t i = 0; i < tenant_id_array.count(); i ++) { //ignore ret
uint64_t tenant_id = tenant_id_array.at(i);
if (OB_FAIL(flush_plan_cache(tenant_id))) {
SQL_PC_LOG(ERROR, "plan cache evict failed", K(tenant_id), K(ret));
}
}
return ret;
}
int ObPlanCacheManager::flush_all_lib_cache()
{
int ret = OB_SUCCESS;
observer::ObReqTimeGuard req_timeinfo_guard;
ObArray<uint64_t> tenant_id_array;
ObGetAllCacheKeyOp op(&tenant_id_array);
if (OB_FAIL(pcm_.foreach_refactored(op))) {
SQL_PC_LOG(ERROR, "fail to traverse pcm", K(ret));
}
for (int64_t i = 0; i < tenant_id_array.count(); i ++) { //ignore ret
uint64_t tenant_id = tenant_id_array.at(i);
if (OB_FAIL(flush_lib_cache(tenant_id))) {
SQL_PC_LOG(ERROR, "lib cache evict failed", K(tenant_id), K(ret));
}
}
return ret;
}
int ObPlanCacheManager::flush_all_lib_cache_by_ns(ObLibCacheNameSpace ns)
{
int ret = OB_SUCCESS;
observer::ObReqTimeGuard req_timeinfo_guard;
ObArray<uint64_t> tenant_id_array;
ObGetAllCacheKeyOp op(&tenant_id_array);
if (ns <= ObLibCacheNameSpace::NS_INVALID || ns >= ObLibCacheNameSpace::NS_MAX) {
ret = OB_ERR_UNEXPECTED;
SQL_PC_LOG(ERROR, "invalid namespace type", K(ns));
} else if (OB_FAIL(pcm_.foreach_refactored(op))) {
SQL_PC_LOG(ERROR, "fail to traverse pcm", K(ret));
}
for (int64_t i = 0; i < tenant_id_array.count(); i ++) { //ignore ret
uint64_t tenant_id = tenant_id_array.at(i);
if (OB_FAIL(flush_lib_cache_by_ns(tenant_id, ns))) {
SQL_PC_LOG(ERROR, "lib cache evict failed", K(tenant_id), K(ret));
}
}
return ret;
}
int ObPlanCacheManager::flush_all_pl_cache()
{
int ret = OB_SUCCESS;
observer::ObReqTimeGuard req_timeinfo_guard;
ObArray<uint64_t> tenant_id_array;
ObGetAllCacheKeyOp op(&tenant_id_array);
if (OB_FAIL(pcm_.foreach_refactored(op))) {
SQL_PC_LOG(ERROR, "fail to traverse pcm", K(ret));
}
for (int64_t i = 0; i < tenant_id_array.count(); i++) {
uint64_t tenant_id = tenant_id_array.at(i);
if (OB_FAIL(flush_pl_cache(tenant_id))) {
SQL_PC_LOG(ERROR, "pl cache evict failed", K(tenant_id), K(ret));
}
}
return ret;
}
int ObPlanCacheManager::flush_pl_cache(const uint64_t tenant_id)
{
int ret = OB_SUCCESS;
observer::ObReqTimeGuard req_timeinfo_guard;
ObPlanCache *plan_cache = get_plan_cache(tenant_id);
int64_t safe_timestamp = INT64_MAX;
ObArray<AllocCacheObjInfo> deleted_objs;
if (NULL != plan_cache) {
if (OB_FAIL(pl::ObPLCacheMgr::cache_evict_all_pl(plan_cache))) {
SQL_PC_LOG(ERROR, "PL cache evict failed, please check", K(ret));
} else if (OB_FAIL(observer::ObGlobalReqTimeService::get_instance()
.get_global_safe_timestamp(safe_timestamp))) {
SQL_PC_LOG(ERROR, "failed to get global safe timestamp", K(ret));
} else if (OB_FAIL(plan_cache->dump_deleted_objs<DUMP_PL>(deleted_objs, safe_timestamp))) {
SQL_PC_LOG(ERROR, "failed to dump deleted pl objs", K(ret));
} else {
LOG_INFO("Deleted Cache Objs", K(deleted_objs));
for (int i = 0; i < deleted_objs.count(); i++) { // ignore error code and continue
if (OB_FAIL(ObCacheObjectFactory::destroy_cache_obj(true,
deleted_objs.at(i).obj_id_,
plan_cache))) {
LOG_WARN("failed to destroy cache obj", K(ret));
}
}
}
plan_cache->dec_ref_count();
}
return ret;
}
int ObPlanCacheManager::flush_all_ps_cache()
{
int ret = OB_SUCCESS;
ObArray<uint64_t> tenant_id_array;
ObGetAllCacheKeyOp op(&tenant_id_array);
if (OB_FAIL(pcm_.foreach_refactored(op))) {
SQL_PC_LOG(ERROR, "fail to traverse pcm", K(ret));
}
for (int64_t i = 0; i < tenant_id_array.count(); i++) {
uint64_t tenant_id = tenant_id_array.at(i);
if (OB_FAIL(flush_ps_cache(tenant_id))) {
SQL_PC_LOG(ERROR, "ps cache evict failed", K(tenant_id), K(ret));
}
}
return ret;
}
int ObPlanCacheManager::flush_ps_cache(const uint64_t tenant_id)
{
int ret = OB_SUCCESS;
ObPsCache *ps_cache = get_ps_cache(tenant_id);
if (NULL != ps_cache) {
if (OB_FAIL(ps_cache->cache_evict_all_ps())) {
SQL_PC_LOG(ERROR, "ps cache evict failed, please check", K(ret));
}
ps_cache->dec_ref_count();
}
return ret;
}
int ObPlanCacheManager::flush_lib_cache_by_ns(const uint64_t tenant_id, const ObLibCacheNameSpace ns)
{
int ret = OB_SUCCESS;
observer::ObReqTimeGuard req_timeinfo_guard;
ObPlanCache *plan_cache = get_plan_cache(tenant_id);
int64_t safe_timestamp = INT64_MAX;
ObArray<AllocCacheObjInfo> deleted_objs;
if (NULL != plan_cache) {
if (OB_FAIL(plan_cache->cache_evict_by_ns(ns))) {
SQL_PC_LOG(ERROR, "cache evict by ns failed, please check", K(ret));
} else if (OB_FAIL(observer::ObGlobalReqTimeService::get_instance()
.get_global_safe_timestamp(safe_timestamp))) {
SQL_PC_LOG(ERROR, "failed to get global safe timestamp", K(ret));
} else if (OB_FAIL(plan_cache->dump_deleted_objs_by_ns(deleted_objs, safe_timestamp, ns))) {
SQL_PC_LOG(ERROR, "failed to dump deleted objs by ns", K(ret));
} else {
LOG_INFO("Deleted Cache Objs", K(deleted_objs));
for (int i = 0; i < deleted_objs.count(); i++) { // ignore error code and continue
if (OB_FAIL(ObCacheObjectFactory::destroy_cache_obj(true,
deleted_objs.at(i).obj_id_,
plan_cache))) {
LOG_WARN("failed to destroy cache obj", K(ret));
}
}
}
plan_cache->dec_ref_count();
}
return ret;
}
int ObPlanCacheManager::flush_lib_cache(const uint64_t tenant_id)
{
int ret = OB_SUCCESS;
observer::ObReqTimeGuard req_timeinfo_guard;
ObPlanCache *plan_cache = get_plan_cache(tenant_id);
if (NULL != plan_cache) {
if (OB_FAIL(plan_cache->cache_evict_all_obj())) {
SQL_PC_LOG(ERROR, "lib cache evict failed, please check", K(ret));
}
ObArray<AllocCacheObjInfo> deleted_objs;
int64_t safe_timestamp = INT64_MAX;
if (OB_FAIL(ret)) {
// do nothing
} else if (OB_FAIL(observer::ObGlobalReqTimeService::get_instance().get_global_safe_timestamp(safe_timestamp))) {
SQL_PC_LOG(ERROR, "failed to get global safe timestamp", K(ret));
} else if (OB_FAIL(plan_cache->dump_deleted_objs<DUMP_ALL>(deleted_objs, safe_timestamp))) {
SQL_PC_LOG(WARN, "failed to get deleted sql objs", K(ret));
} else {
LOG_INFO("Deleted Cache Objs", K(deleted_objs));
for (int64_t i = 0; i < deleted_objs.count(); i++) { // ignore error code and continue
if (OB_FAIL(ObCacheObjectFactory::destroy_cache_obj(true,
deleted_objs.at(i).obj_id_,
plan_cache))) {
LOG_WARN("failed to destroy cache obj", K(ret));
}
}
}
plan_cache->dec_ref_count();
}
return ret;
}
int ObPlanCacheManager::flush_plan_cache(const uint64_t tenant_id)
{
int ret = OB_SUCCESS;
observer::ObReqTimeGuard req_timeinfo_guard;
ObPlanCache *plan_cache = get_plan_cache(tenant_id);
if (NULL != plan_cache) {
if (OB_FAIL(plan_cache->cache_evict_all_plan())) {
SQL_PC_LOG(ERROR, "Plan cache evict failed, please check", K(ret));
}
ObArray<AllocCacheObjInfo> deleted_objs;
int64_t safe_timestamp = INT64_MAX;
if (OB_FAIL(ret)) {
// do nothing
} else if (OB_FAIL(observer::ObGlobalReqTimeService::get_instance().get_global_safe_timestamp(safe_timestamp))) {
SQL_PC_LOG(ERROR, "failed to get global safe timestamp", K(ret));
} else if (OB_FAIL(plan_cache->dump_deleted_objs<DUMP_SQL>(deleted_objs, safe_timestamp))) {
SQL_PC_LOG(WARN, "failed to get deleted sql objs", K(ret));
} else {
LOG_INFO("Deleted Cache Objs", K(deleted_objs));
for (int64_t i = 0; i < deleted_objs.count(); i++) { // ignore error code and continue
if (OB_FAIL(ObCacheObjectFactory::destroy_cache_obj(true,
deleted_objs.at(i).obj_id_,
plan_cache))) {
LOG_WARN("failed to destroy cache obj", K(ret));
}
}
}
plan_cache->dec_ref_count();
}
return ret;
}
int ObPlanCacheManager::revert_ps_cache(const uint64_t &tenant_id)
{
int ret = OB_SUCCESS;
ObPsCache *ppc = NULL;
observer::ObReqTimeGuard req_timeinfo_guard;
int tmp_ret = ps_pcm_.erase_refactored(tenant_id, &ppc);
if (OB_SUCCESS == tmp_ret && NULL != ppc) {
SQL_PC_LOG(INFO, "plan_cache_manager revert ps plan cache",
"pc ref_count", ppc->get_ref_count(),
K(tenant_id));
//cancel scheduled task
ppc->dec_ref_count();
} else if (OB_HASH_NOT_EXIST == tmp_ret) { // maybe erase by other thread
SQL_PC_LOG(INFO, "PS Plan Cache not exist", K(tenant_id));
} else {
ret = tmp_ret;
SQL_PC_LOG(ERROR, "unexpected error when erase plan cache",
K(tenant_id), K(ppc), K(ret));
}
return ret;
}
int ObPlanCacheManager::get_plan_cache_gc_strategy()
{
PlanCacheGCStrategy strategy = INVALID;
for (int i = 0; i < ARRAYSIZEOF(plan_cache_gc_confs) && strategy == INVALID; i++) {
if (0 == ObString::make_string(plan_cache_gc_confs[i])
.case_compare(GCONF._ob_plan_cache_gc_strategy)) {
strategy = static_cast<PlanCacheGCStrategy>(i);
}
}
return strategy;
}
} // end of namespace sql
} // end of namespace oceanbase

190
src/sql/plan_cache/ob_plan_cache_manager.h
View File

@ -1,190 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_SQL_PLAN_CACHE_OB_PLAN_CACHE_MANAGER_
#define OCEANBASE_SQL_PLAN_CACHE_OB_PLAN_CACHE_MANAGER_
#include "lib/net/ob_addr.h"
#include "lib/hash/ob_hashmap.h"
#include "ob_plan_cache.h"
#include "sql/plan_cache/ob_ps_cache.h"
namespace test
{
void test_plan_cache_manager();
}
namespace oceanbase
{
namespace sql
{
class ObPlanCache;
class ObPlanCacheManager
{
public:
typedef common::hash::ObHashMap<uint64_t, ObPlanCache *> PlanCacheMap;
typedef common::hash::ObHashMap<uint64_t, ObPsCache *> PsPlanCacheMap;
class ObPlanCacheEliminationTask : public common::ObTimerTask
{
public:
ObPlanCacheEliminationTask() : plan_cache_manager_(NULL),
run_task_counter_(0)
{
}
// main routine
void runTimerTask(void);
private:
void run_plan_cache_task();
void run_ps_cache_task();
void run_free_cache_obj_task();
void run_auto_flush_plan_cache_task();
public:
ObPlanCacheManager *plan_cache_manager_;
int64_t run_task_counter_;
};
struct ObGetAllCacheKeyOp
{
explicit ObGetAllCacheKeyOp(common::ObIArray<uint64_t> *key_array)
: key_array_(key_array)
{
}
template<class T>
int operator()(common::hash::HashMapPair<uint64_t, T *> &entry)
{
int ret = common::OB_SUCCESS;
if (OB_ISNULL(key_array_)) {
ret = common::OB_NOT_INIT;
SQL_PC_LOG(WARN, "key_array not inited", K(ret));
} else if (OB_FAIL(key_array_->push_back(entry.first))) {
SQL_PC_LOG(WARN, "fail to push back key", K(ret));
}
return ret;
}
common::ObIArray<uint64_t> *key_array_;
};
public:
ObPlanCacheManager():tg_id_(-1),
inited_(false),
destroyed_(false),
plan_cache_id_(0)
{};
virtual ~ObPlanCacheManager() { destroy(); }
int init(common::ObAddr addr);
void destroy();
// get tenant plan cache, if not exists, create a new plan cache
ObPlanCache *get_or_create_plan_cache(uint64_t tenant_id, const ObPCMemPctConf &pc_mem_conf);
ObPsCache *get_or_create_ps_cache(const uint64_t tenant_id, const ObPCMemPctConf &pc_mem_conf);
int revert_plan_cache(const uint64_t &tenant_id);
int revert_ps_cache(const uint64_t &tenant_id);
int flush_all_plan_cache();
int flush_all_lib_cache();
int flush_all_lib_cache_by_ns(ObLibCacheNameSpace ns);
int flush_plan_cache(const uint64_t tenant_id);
int flush_plan_cache_by_sql_id(uint64_t tenant_id, uint64_t db_id, common::ObString sql_id);
int flush_all_pl_cache();
int flush_pl_cache(const uint64_t tenant_id);
int flush_all_ps_cache();
int flush_ps_cache(const uint64_t tenant_id);
int flush_lib_cache(const uint64_t tenant_id);
int flush_lib_cache_by_ns(const uint64_t tenant_id, const ObLibCacheNameSpace ns);
int evict_plan_by_table_name(uint64_t tenant_id, uint64_t database_id, common::ObString tab_name);
PlanCacheMap &get_plan_cache_map() {return pcm_;}
PsPlanCacheMap &get_ps_cache_map() {return ps_pcm_; }
// get tenant plan cache, if not exists, return NULL
// only used when revert plan cache
ObPlanCache *get_plan_cache(uint64_t tenant_id);
ObPsCache *get_ps_cache(const uint64_t tenant_id);
private:
enum PlanCacheGCStrategy { INVALID = -1, OFF = 0, REPORT = 1, AUTO = 2};
static int get_plan_cache_gc_strategy();
friend void ::test::test_plan_cache_manager();
DISALLOW_COPY_AND_ASSIGN(ObPlanCacheManager);
private:
common::ObAddr self_addr_;
PlanCacheMap pcm_;
PsPlanCacheMap ps_pcm_;
int tg_id_;
ObPlanCacheEliminationTask elimination_task_;
bool inited_;
bool destroyed_;
volatile uint64_t plan_cache_id_;
}; // end of class ObPlanCaeManager
class ObPlanCacheManagerAtomic
{
public:
typedef common::hash::HashMapPair<uint64_t, ObPlanCache *> MapKV;
public:
ObPlanCacheManagerAtomic():plan_cache_(NULL) {};
virtual ~ObPlanCacheManagerAtomic() {};
int operator() (MapKV &entry) {
int ret = common::OB_SUCCESS;
if (OB_ISNULL(entry.second)) {
ret = common::OB_INVALID_ARGUMENT;
SQL_PC_LOG(WARN, "invalid argument", K(ret));
} else {
plan_cache_ = entry.second;
entry.second->inc_ref_count();
}
return ret;
}
ObPlanCache *get_plan_cache()
{
return plan_cache_;
}
private:
DISALLOW_COPY_AND_ASSIGN(ObPlanCacheManagerAtomic);
private:
ObPlanCache *plan_cache_;
}; // end of class ObPlanCacheManagerAtomic
class ObPsCacheManagerAtomic
{
public:
typedef common::hash::HashMapPair<uint64_t, ObPsCache *> MapKV;
ObPsCacheManagerAtomic() : ps_cache_(NULL) {};
virtual ~ObPsCacheManagerAtomic() {}
int operator() (MapKV &entry) {
int ret = common::OB_SUCCESS;
if (OB_ISNULL(entry.second)) {
ret = common::OB_INVALID_ARGUMENT;
SQL_PC_LOG(WARN, "invalid argument", K(ret));
} else {
ps_cache_ = entry.second;
entry.second->inc_ref_count();
}
return ret;
}
ObPsCache *get_ps_cache() const { return ps_cache_; }
private:
DISALLOW_COPY_AND_ASSIGN(ObPsCacheManagerAtomic);
ObPsCache *ps_cache_;
};
} // end of namespace sql
} // end of namespace oceanbase
#endif /* _OB_PLAN_CACHE_MANAGER_H_ */

40
src/sql/resolver/dcl/ob_dcl_stmt.h
View File

@ -1,40 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_SQL_RESOLVER_OB_DCL_STMT_H_
#define OCEANBASE_SQL_RESOLVER_OB_DCL_STMT_H_ 1
#include "share/ob_rpc_struct.h"
#include "sql/resolver/ob_stmt.h"
#include "sql/resolver/ob_cmd.h"
namespace oceanbase
{
namespace sql
{
class ObDCLStmt : public ObStmt, public ObICmd
{
public:
ObDCLStmt(common::ObIAllocator *name_pool, stmt::StmtType type)
: ObStmt(name_pool, type)
{
}
explicit ObDCLStmt(stmt::StmtType type): ObStmt(type)
{
}
virtual ~ObDCLStmt() {}
virtual int get_cmd_type() const { return get_stmt_type(); }
virtual bool cause_implicit_commit() const { return true; }
private:
DISALLOW_COPY_AND_ASSIGN(ObDCLStmt);
};
} // namespace sql
} // namespace oceanbase
#endif // OCEANBASE_SQL_RESOLVER_OB_DCL_STMT_H

99
src/sql/resolver/ddl/ob_create_view_stmt.cpp
View File

@ -1,99 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "sql/resolver/ddl/ob_create_view_stmt.h"
using namespace oceanbase::common;
using namespace oceanbase::sql;
ObCreateViewStmt::ObCreateViewStmt()
: ObStmt(stmt::T_CREATE_VIEW),
name_pool_(NULL), definition_stmt_(NULL)
{
}
ObCreateViewStmt::ObCreateViewStmt(ObIAllocator *name_pool)
: ObStmt(stmt::T_CREATE_VIEW),
name_pool_(name_pool), definition_stmt_(NULL)
{
}
ObCreateViewStmt::~ObCreateViewStmt()
{
}
void ObCreateViewStmt::set_name_pool(common::ObIAllocator *name_pool)
{
name_pool_ = name_pool;
}
int ObCreateViewStmt::set_view_name(const common::ObString &view_name)
{
OB_ASSERT(name_pool_);
OB_ASSERT(view_name.length() > 0 && view_name.ptr());
int ret = OB_SUCCESS;
if ((ret = ob_write_string(*name_pool_, view_name, view_name_)) != OB_SUCCESS) {
_OB_LOG(WARN, "write view name failed, ret=%d", ret);
}
return ret;
}
int ObCreateViewStmt::set_view_definition(const common::ObString &view_definition)
{
OB_ASSERT(name_pool_);
OB_ASSERT(view_definition.length() > 0 && view_definition.ptr());
int ret = OB_SUCCESS;
if (OB_SUCCESS != (ret = ob_write_string(*name_pool_, view_definition, view_definition_))) {
_OB_LOG(WARN, "write view definition failed, ret=%d", ret);
}
return ret;
}
void ObCreateViewStmt::set_definition_stmt(ObSelectStmt *definition_stmt)
{
OB_ASSERT(definition_stmt);
definition_stmt_ = definition_stmt;
}
const ObString &ObCreateViewStmt::get_view_name() const
{
return view_name_;
}
const ObString &ObCreateViewStmt::get_view_definition() const
{
return view_definition_;
}
const ObSelectStmt *ObCreateViewStmt::get_definition_stmt() const
{
return definition_stmt_;
}
void ObCreateViewStmt::print(FILE *fp, int32_t level, int32_t index)
{
OB_ASSERT(fp && definition_stmt_);
print_indentation(fp, level);
fprintf(fp, "ObCreateViewStmt %d Begin\n", index);
print_indentation(fp, level + 1);
fprintf(fp, "View Name ::= %.*s\n", view_name_.length(), view_name_.ptr());
print_indentation(fp, level + 1);
fprintf(fp, "View Definition ::='%.*s'", view_definition_.length(), view_definition_.ptr());
print_indentation(fp, level + 1);
fprintf(fp, "View Definition Stmt ::=\n");
definition_stmt_->print(fp, level + 2, index);
print_indentation(fp, level);
fprintf(fp, "ObCreateTableStmt %d End\n", index);
}

50
src/sql/resolver/ddl/ob_create_view_stmt.h
View File

@ -1,50 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_SQL_OB_CREATE_VIEW_STMT_H
#define OCEANBASE_SQL_OB_CREATE_VIEW_STMT_H
#include "sql/resolver/ob_stmt.h"
#include "sql/resolver/dml/ob_select_stmt.h"
#include "sql/parser/parse_node.h"
namespace oceanbase
{
namespace sql
{
class ObCreateViewStmt : public ObStmt
{
public:
explicit ObCreateViewStmt(common::ObIAllocator *name_pool);
ObCreateViewStmt();
virtual ~ObCreateViewStmt();
void set_name_pool(common::ObIAllocator *name_pool);
int set_view_name(const common::ObString &view_name);
int set_view_definition(const common::ObString &view_definition);
void set_definition_stmt(ObSelectStmt *definition_stmt);
const common::ObString &get_view_name() const;
const common::ObString &get_view_definition() const;
const ObSelectStmt *get_definition_stmt() const;
virtual void print(FILE *fp, int32_t level, int32_t index = 0);
virtual bool cause_implicit_commit() const { return true; }
protected:
common::ObIAllocator *name_pool_;
private:
ObSelectStmt *definition_stmt_;
common::ObString view_name_;
common::ObString view_definition_;
};
} //sql
} //oceanbase
#endif //OCEANBASE_SQL_OB_CREATE_VIEW_STMT_H

32
src/sql/session/ob_erase_session_callback.h
View File

@ -1,32 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_SQL_SESSION_ERASE_SESSION_CALLBACK_
#define OCEANBASE_SQL_SESSION_ERASE_SESSION_CALLBACK_
namespace oceanbase
{
namespace common
{
class ObEndTransReq;
}
namespace sql
{
class ObIEraseSessionCallback
{
public:
ObIEraseSessionCallback();
virtual ~ObIEraseSessionCallback();
virtual int callback(common::ObEndTransReq &req) = 0;
};
}
}
#endif /* OCEANBASE_SQL_SESSION_ERASE_SESSION_CALLBACK_ */

127
src/sql/session/ob_prepared_sql_store.cpp
View File

@ -1,127 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_PC
#include "ob_prepared_sql_store.h"
#include "lib/oblog/ob_log.h"
namespace oceanbase
{
using namespace common;
namespace sql
{
int ObPreparedSqlStore::init()
{
int ret = OB_SUCCESS;
//init allocator
if (OB_FAIL(allocator_.init(128))) {
LOG_WARN("Buddy allocator init failed", K(ret));
} else if (OB_FAIL(psmap_.create(hash::cal_next_prime(512),
ObModIds::OB_HASH_BUCKET_PLAN_CACHE,
ObModIds::OB_HASH_NODE_PLAN_CACHE))) {
LOG_WARN("Failed to init psmap", K(ret));
} else {
ObMemAttr attr(OB_SERVER_TENANT_ID, ObModIds::OB_SQL_PREPARED_SQL);
allocator_.set_attr(attr);
}
return ret;
}
int ObPreparedSqlStore::store_sql(const ObString &sql, ObString &osql)
{
int ret = OB_SUCCESS;
ObPreparedSqlValue *value = NULL;
ObPreparedSqlStoreAddRef op;
do {
ret = psmap_.atomic(sql, op);
if (OB_HASH_EXIST == ret) {
if (OB_SUCC(op.get_rc())) {
osql = op.get_sql();
}
} else if (OB_HASH_NOT_EXIST == ret) {
int64_t bsize = sizeof(ObPreparedSqlValue) + sql.length();
char *buff = reinterpret_cast<char *>(allocator_.alloc(bsize));
if (NULL != buff) {
MEMCPY(buff, reinterpret_cast<void *>(const_cast<char *>(sql.ptr())),
sql.length());
value = new(buff + sql.length())ObPreparedSqlValue();
ObString nsql(sql.length(), buff);
if (OB_SUCC(value->init())) {
ret = psmap_.set(nsql, value);
if (OB_HASH_EXIST == ret) {
value->~ObPreparedSqlValue();
allocator_.free(buff);
ret = OB_EAGAIN;
} else if (OB_HASH_INSERT_SUCC == ret) {
LOG_DEBUG("Insert new prepare sql success", K(sql));
osql = nsql;
ret = OB_SUCCESS;
} else {
LOG_ERROR("unexpected error", K(ret));
value->~ObPreparedSqlValue();
allocator_.free(buff);
}
} else {
LOG_WARN("Init ObPrepareSqlValue failed", K(ret));
value->~ObPreparedSqlValue();
allocator_.free(buff);
}
} else {
LOG_WARN("Alloc mem for new prepare sql failed");
ret = OB_ALLOCATE_MEMORY_FAILED;
}
} else {
LOG_ERROR("unexpected error", K(ret));
}
} while (OB_EAGAIN == ret);
return ret;
}
int ObPreparedSqlStore::free_sql(const ObString &sql)
{
int ret = OB_SUCCESS;
ObPreparedSqlValue *value = NULL;
ObPreparedSqlStoreDecRef op;
ret = psmap_.atomic(sql, op);
if (OB_HASH_EXIST == ret) {
if (OB_DEC_AND_LOCK == op.get_rc()) {
//delete sql
value = op.get_prepared_sql_value();
if (0 == value->ref_count_) {
ret = psmap_.erase(sql);
if (OB_HASH_EXIST == ret) {
LOG_DEBUG("Erase prepared sql success", K(sql));
value->~ObPreparedSqlValue();
value = NULL;
allocator_.free(reinterpret_cast<void *>(const_cast<char *>(sql.ptr())));
} else {
LOG_ERROR("Erase prepared sql");
}
}
} else if (OB_SUCCESS == op.get_rc()) {
ret = OB_SUCCESS;
} else {
LOG_WARN("unexpected error", K(ret));
ret = OB_ERR_UNEXPECTED;
}
} else if (OB_HASH_NOT_EXIST == ret) {
ret = OB_ENTRY_NOT_EXIST;
} else {
LOG_ERROR("unexpected error", K(ret));
ret = OB_ERR_UNEXPECTED;
}
return ret;
}
} // namespace sql
} // namespace oceanbase

182
src/sql/session/ob_prepared_sql_store.h
View File

@ -1,182 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_SQL_SESSION_OB_PREPARED_SQL_STORE_
#define OCEANBASE_SQL_SESSION_OB_PREPARED_SQL_STORE_
#include "lib/allocator/ob_pooled_allocator.h"
#include "lib/allocator/ob_buddy_allocator.h"
#include "share/ob_define.h"
#include "lib/hash/ob_hashmap.h"
namespace oceanbase
{
namespace sql
{
class ObPreparedSqlValue
{
public:
ObPreparedSqlValue(): ref_count_(0)
{
}
int init()
{
int ret = common::OB_SUCCESS;
if (0 != pthread_rwlock_init(&rwlock_, NULL)) {
ret = common::OB_ERROR;
}
return ret;
}
int wlock()
{
return pthread_rwlock_wrlock(&rwlock_);
}
int try_rlock()
{
return pthread_rwlock_tryrdlock(&rwlock_);
}
int64_t ref_count_;
pthread_rwlock_t rwlock_;
};
class ObPreparedSqlStoreAtomic
{
public:
ObPreparedSqlStoreAtomic(): rc_(common::OB_SUCCESS),
value_(NULL)
{
}
virtual ~ObPreparedSqlStoreAtomic()
{
}
virtual void operator()(common::hash::HashMapPair<common::ObString, ObPreparedSqlValue *> entry)
{
UNUSED(entry);
}
int get_rc() const
{
return rc_;
}
common::ObString &get_sql()
{
return sql_;
}
ObPreparedSqlValue *get_prepared_sql_value()
{
return value_;
}
protected:
int rc_;
ObPreparedSqlValue *value_;
common::ObString sql_;
};
class ObPreparedSqlStoreAddRef: public ObPreparedSqlStoreAtomic
{
public:
ObPreparedSqlStoreAddRef()
{
}
virtual ~ObPreparedSqlStoreAddRef()
{
}
virtual void operator()(common::hash::HashMapPair<common::ObString, ObPreparedSqlValue *> entry)
{
rc_ = entry.second->try_rlock();
if (0 == rc_) {
entry.second->ref_count_++;
sql_ = entry.first;
value_ = entry.second;
rc_ = common::OB_SUCCESS;
} else if (EBUSY == rc_) {
rc_ = common::OB_EAGAIN;
} else {
//LOG_ERROR("try rlock on ObPreparedSqlValue failed", K(rc_),
// K(entry.first));
rc_ = common::OB_ERROR;
}
}
};
class ObPreparedSqlStoreDecRef: public ObPreparedSqlStoreAtomic
{
public:
ObPreparedSqlStoreDecRef()
{
}
virtual ~ObPreparedSqlStoreDecRef()
{
}
virtual void operator()(common::hash::HashMapPair<common::ObString, ObPreparedSqlValue *> entry)
{
entry.second->ref_count_--;
if (0 == entry.second->ref_count_) {
if (0 == entry.second->wlock()) {
rc_ = common::OB_DEC_AND_LOCK;
sql_ = entry.first;
} else {
rc_ = common::OB_ERROR;
}
} else {
sql_ = entry.first;
value_ = entry.second;
rc_ = common::OB_SUCCESS;
}
}
};
class ObPreparedSqlStore
{
public:
ObPreparedSqlStore(): allocator_(common::get_global_tc_allocator())
{
}
~ObPreparedSqlStore()
{
}
int init();
public:
int store_sql(const common::ObString &sql, common::ObString &osql);
int free_sql(const common::ObString &sql);
//typedef common::ObPooledAllocator<common::hash::HashMapTypes<common::ObString,
// ObPreparedSqlValue*>::AllocType,
// common::ObWrapperAllocator>
//PreparedSqlMapAllocator;
typedef common::hash::ObHashMap<common::ObString,
ObPreparedSqlValue *> PreparedSqlMap;
private:
DISALLOW_COPY_AND_ASSIGN(ObPreparedSqlStore);
private:
PreparedSqlMap psmap_;
// common::ObBuddyAllocator allocator_;
};
} // namespace sql
} // namespace oceanbase
#endif // OCEANBASE_SQL_SESSION_OB_PREPARED_SQL_STORE_

271
src/sql/session/ob_ps_info_mgr.cpp
View File

@ -1,271 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL_SESSION
#include "sql/session/ob_ps_info_mgr.h"
#include "lib/oblog/ob_log.h"
#include "lib/oblog/ob_log_module.h"
using namespace oceanbase::sql;
using namespace oceanbase::common;
ObPsInfoMgr::ObPsInfoMgr()
:last_stmt_id_(0),
block_allocator_(SMALL_BLOCK_SIZE, common::OB_MALLOC_NORMAL_BLOCK_SIZE,
ObMalloc(ObModIds::OB_SQL_SESSION_SBLOCK)),
bucket_allocator_wrapper_(&block_allocator_),
id_psinfo_map_allocer_(SMALL_BLOCK_SIZE, ObWrapperAllocator(&block_allocator_)),
name_psinfo_map_allocer_(SMALL_BLOCK_SIZE, ObWrapperAllocator(&block_allocator_)),
id_psinfo_map_(),
name_psinfo_map_(),
ps_session_info_pool_(SMALL_BLOCK_SIZE, ObWrapperAllocator(&block_allocator_))
{
}
ObPsInfoMgr::~ObPsInfoMgr()
{
}
int ObPsInfoMgr::init()
{
int ret = OB_SUCCESS;
if (OB_FAIL(id_psinfo_map_.create(64,
&id_psinfo_map_allocer_,
&bucket_allocator_wrapper_))) {
LOG_WARN("init id_psinfo_map failed", K(ret));
} else if (OB_FAIL(name_psinfo_map_.create(64,
&name_psinfo_map_allocer_,
&bucket_allocator_wrapper_))) {
LOG_WARN("init name_psinfo_map failed", K(ret));
}
return ret;
}
void ObPsInfoMgr::reset()
{
last_stmt_id_ = 0;
ps_sql_store_ = NULL;
id_psinfo_map_.clear();
id_psinfo_map_allocer_.reset();
name_psinfo_map_.clear();
name_psinfo_map_allocer_.reset();
ps_session_info_pool_.reset();
}
int ObPsInfoMgr::add_ps_info(const uint64_t sql_id,
const ObString &sql,
const ObPhysicalPlanCtx *pctx,
const bool is_dml)
{
int ret = OB_SUCCESS;
ObPsSessionInfo *info = NULL;
uint64_t stmt_id = allocate_stmt_id();
ret = id_psinfo_map_.get(static_cast<uint32_t>(sql_id), info);
if (hash::HASH_NOT_EXIST == ret) {
info = ps_session_info_pool_.alloc();
if (OB_UNLIKELY(NULL == info)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR( "can not alloc mem for ObPsSessionInfo");
} else {
info->init(ObWrapperAllocator(&block_allocator_));
info->set_sql_id(sql_id);
info->set_params_count(pctx->get_param_store().count());
ObString osql;
if (OB_SUCC(ps_sql_store_->store_sql(sql, osql))) {
info->set_prepare_sql(osql);
//info->set_stmt_type(stmt_type);//TODO 这个有什么用
info->set_dml(is_dml);
ret = id_psinfo_map_.set(static_cast<uint32_t>(stmt_id), info);
if (hash::HASH_INSERT_SUCC == ret) {
LOG_DEBUG("insert item into id_psinfo_map success", K(stmt_id), K(info));
ret = OB_SUCCESS;
} else if (hash::HASH_EXIST == ret) {
LOG_WARN("exist in id_psinfo_map", K(stmt_id));
ps_session_info_pool_.free(info);
ret = OB_ERR_UNEXPECTED;
} else {
LOG_WARN("fail to insert item into id_psinfo_map", K(stmt_id), K(ret));
ps_session_info_pool_.free(info);
ret = OB_ERR_UNEXPECTED;
}
} else {
LOG_WARN("store prepare sql failed", K(ret));
}
}
} else if (hash::HASH_EXIST == ret) {
LOG_ERROR("stmt id exist", K(stmt_id), K(ret));
} else {
LOG_WARN("fail to get ps session info", K(stmt_id), K(ret));
}
return ret;
}
int ObPsInfoMgr::add_ps_info(const ObString &pname,
const uint64_t sql_id,
const ObString &sql,
const ObPhysicalPlanCtx *pctx,
const bool is_dml)
{
int ret = OB_SUCCESS;
ObPsSessionInfo *info = NULL;
ret = name_psinfo_map_.get(pname, info);
if (hash::HASH_NOT_EXIST == ret) {
info = ps_session_info_pool_.alloc();
if (OB_UNLIKELY(NULL == info)) {
LOG_WARN("can not alloc mem for ObPsSessionInfo");
ret = OB_ALLOCATE_MEMORY_FAILED;
} else {
info->init(ObWrapperAllocator(&block_allocator_));
info->set_sql_id(sql_id);
info->set_params_count(pctx->get_param_store().count());
ObString osql;
if (OB_SUCC(ps_sql_store_->store_sql(sql, osql))) {
info->set_prepare_sql(osql);
//info->set_stmt_type(stmt_type);//TODO 这个有什么用
info->set_dml(is_dml);
ret = name_psinfo_map_.set(pname, info);
if (hash::HASH_INSERT_SUCC == ret) {
LOG_DEBUG("insert item into name_psinfo_map success", K(pname), K(info));
ret = OB_SUCCESS;
} else if (hash::HASH_EXIST == ret) {
LOG_WARN("exist in name_psinfo_map", K(pname));
ps_session_info_pool_.free(info);
ret = OB_ERR_UNEXPECTED;
} else {
LOG_WARN("fail to insert item into name_psinfo_map", K(pname), K(ret));
ps_session_info_pool_.free(info);
ret = OB_ERR_UNEXPECTED;
}
} else {
LOG_WARN("store prepare sql failed", K(ret));
}
}
} else if (hash::HASH_EXIST == ret) {
LOG_ERROR("stmt id exist", K(pname), K(ret));
} else {
LOG_WARN("fail to get ps session info", K(pname), K(ret));
}
return ret;
}
int ObPsInfoMgr::close_all_stmt()
{
int ret = OB_SUCCESS;
//int err = OB_SUCCESS;
//IdPsInfoMap::iterator iter;
//ObPsSessionInfo *info = NULL;
//uint64_t sql_id = 0;
//for (iter = id_psinfo_map_.begin(); iter != id_psinfo_map_.end(); iter++) {
// if (hash::HASH_EXIST != (err = id_psinfo_map_.get(iter->first, info))) {
// LOG_WARN("not found ObPsSessionInfo", "whose key", iter->first);
// } else {
// sql_id = info->get_sql_id();
// if (OB_SUCCESS != (err = ps_store_->remove_plan(sql_id))) {
// LOG_WARN("close prepared statement failed", K_(session_key), K(sql_id));
// } else {
// LOG_INFO("close prepared statement when session quit", K_(session_key), K(sql_id),
// "stmt_id", iter->first);
// OB_STAT_INC(SQL, OB_SYS_TENANT_ID, SQL_PS_COUNT, -1);
// }
// ps_session_info_pool_.free(info);//free ps session info when session quit
// }
//}
//id_psinfo_map_.clear(); //clear id_psinfo_map
//
//NamePsInfoMap::Iterator niter;
//for (niter = name_psinfo_map_.begin(); niter != name_psinfo_map_.end(); niter++) {
// if (hash::HASH_EXIST != (err = name_psinfo_map_.get(niter->first, info))) {
// LOG_WARN("not found ObPsSessionInfo", "whose key", niter->first);
// } else {
// sql_id = info->get_sql_id();
// if (OB_SUCCESS != (err = ps_store_->remove_plan(sql_id))) {
// LOG_WARN("close prepared statement failed", K_(session_key), K(sql_id));
// } else {
// LOG_INFO("close prepared statement when session quit", K_(session_key), K(sql_id),
// "stmt_id", iter->first);
// OB_STAT_INC(SQL, OB_SYS_TENANT_ID, SQL_PS_COUNT, -1);
// }
// ps_session_info_pool_.free(info);//free ps session info when session quit
// }
//}
//name_psinfo_map_.clear(); //clear name_psinfo_map
return ret;
}
int ObPsInfoMgr::remove_ps_info(const uint64_t stmt_id)
{
int ret = OB_SUCCESS;
ObPsSessionInfo *info = NULL;
if (id_psinfo_map_.get(static_cast<uint32_t>(stmt_id), info) != hash::HASH_EXIST) {
ret = OB_ERR_PREPARE_STMT_NOT_FOUND;
LOG_WARN("prepare statement id not found in id_psinfo_map", K(stmt_id));
} else {
if (OB_SUCC(ps_sql_store_->free_sql(info->get_prepare_sql()))) {
ps_session_info_pool_.free(info);
if (id_psinfo_map_.erase(static_cast<uint32_t>(stmt_id)) != hash::HASH_EXIST) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("drop prepare statement error", K(stmt_id));
} else {
LOG_DEBUG("remove ps statement", K(stmt_id));
}
} else {
LOG_WARN("remove prepared sql failed", K(ret));
}
}
return ret;
}
int ObPsInfoMgr::remove_ps_info(const ObString &name)
{
int ret = OB_SUCCESS;
ObPsSessionInfo *info = NULL;
if (name_psinfo_map_.get(name, info) != hash::HASH_EXIST) {
ret = OB_ERR_PREPARE_STMT_NOT_FOUND;
LOG_WARN("prepare statement id not found in name_psinfo_map", K(ret), K(name));
} else {
ps_sql_store_->free_sql(info->get_prepare_sql());
ps_session_info_pool_.free(info);
info = NULL;
if (name_psinfo_map_.erase(name) != hash::HASH_EXIST) {
ret = OB_ERR_UNEXPECTED;
LOG_ERROR("drop prepare statement error", K(ret), K(name));
} else {
LOG_DEBUG("remove ps statement", K(name));
}
}
return ret;
}
ObPsSessionInfo* ObPsInfoMgr::get_psinfo(const ObString &name)
{
ObPsSessionInfo *sinfo = NULL;
int ret = name_psinfo_map_.get(name, sinfo);
if (hash::HASH_EXIST != ret) {
LOG_WARN("Get ps session info failed", K(name), K(ret));
sinfo = NULL;
} else {
LOG_DEBUG("Get ps session info success", K(name));
}
return sinfo;
}
ObPsSessionInfo* ObPsInfoMgr::get_psinfo(const uint64_t stmt_id)
{
ObPsSessionInfo *sinfo = NULL;
int ret = id_psinfo_map_.get(static_cast<uint32_t>(stmt_id), sinfo);
if (hash::HASH_EXIST != ret) {
LOG_WARN("Get ps session info failed", K(stmt_id), K(ret));
sinfo = NULL;
} else {
LOG_DEBUG("Get ps session info success", K(stmt_id));
}
return sinfo;
}

92
src/sql/session/ob_ps_info_mgr.h
View File

@ -1,92 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef _OB_PS_SESSION_MGR_H
#define _OB_PS_SESSION_MGR_H 1
#include "lib/allocator/ob_pooled_allocator.h"
#include "lib/hash/ob_hashmap.h"
#include "sql/session/ob_ps_session_info.h"
#include "sql/engine/ob_physical_plan_ctx.h"
#include "sql/session/ob_prepared_sql_store.h"
namespace oceanbase
{
namespace sql
{
// manage all prepared statements of one session
class ObPsInfoMgr
{
public:
typedef common::ObPooledAllocator<common::hash::HashMapTypes<uint32_t, ObPsSessionInfo *>::AllocType, common::ObWrapperAllocator>
IdPsInfoMapAllocer;
typedef common::ObPooledAllocator<common::hash::HashMapTypes<common::ObString, ObPsSessionInfo *>::AllocType, common::ObWrapperAllocator>
NamePsInfoMapAllocer;
typedef common::hash::ObHashMap<uint32_t,
ObPsSessionInfo *,
common::hash::NoPthreadDefendMode,
common::hash::hash_func<uint32_t>,
common::hash::equal_to<uint32_t>,
IdPsInfoMapAllocer,
common::hash::NormalPointer,
common::ObWrapperAllocator
> IdPsInfoMap;
typedef common::hash::ObHashMap<common::ObString,
ObPsSessionInfo *,
common::hash::NoPthreadDefendMode,
common::hash::hash_func<common::ObString>,
common::hash::equal_to<common::ObString>,
NamePsInfoMapAllocer,
common::hash::NormalPointer,
common::ObWrapperAllocator
> NamePsInfoMap;
public:
ObPsInfoMgr();
virtual ~ObPsInfoMgr();
int init();
void reset();
int add_ps_info(const uint64_t sql_id, const common::ObString &sql,
const ObPhysicalPlanCtx *pctx, const bool is_dml);
int add_ps_info(const common::ObString &pname, const uint64_t sql_id, const common::ObString &sql,
const ObPhysicalPlanCtx *pctx, const bool is_dml);
int remove_ps_info(const common::ObString &name);
int remove_ps_info(const uint64_t stmt_id);
ObPsSessionInfo *get_psinfo(const common::ObString &name);
ObPsSessionInfo *get_psinfo(const uint64_t stmt_id);
int close_all_stmt();
int64_t get_ps_mem_size() { return block_allocator_.get_total_mem_size(); }
private:
static const int64_t SMALL_BLOCK_SIZE = OB_SESSION_SMALL_BLOCK_SIZE - 8;
// disallow copy
DISALLOW_COPY_AND_ASSIGN(ObPsInfoMgr);
// function members
uint64_t allocate_stmt_id() {return __sync_add_and_fetch(&last_stmt_id_, 1);}
private:
// data members
uint64_t last_stmt_id_;
common::ObSmallBlockAllocator<> block_allocator_;
common::ObWrapperAllocator bucket_allocator_wrapper_;
IdPsInfoMapAllocer id_psinfo_map_allocer_;
NamePsInfoMapAllocer name_psinfo_map_allocer_;
IdPsInfoMap id_psinfo_map_; // stmt-id -> ObPsSessionInfo
NamePsInfoMap name_psinfo_map_; // name -> ObPsSessionInfo for text prepare
common::ObPooledAllocator<ObPsSessionInfo, common::ObWrapperAllocator> ps_session_info_pool_;
};
} // end namespace sql
} // end namespace oceanbase
#endif /* _OB_PS_SESSION_MGR_H */

166
src/storage/direct_load/ob_direct_load_external_fragment_builder.h
View File

@ -1,166 +0,0 @@
// Copyright (c) 2022-present Oceanbase Inc. All Rights Reserved.
// Author:
// suzhi.yt <>
#pragma once
#include "lib/container/ob_vector.h"
#include "storage/direct_load/ob_direct_load_external_fragment.h"
#include "storage/direct_load/ob_direct_load_external_interface.h"
namespace oceanbase
{
namespace storage
{
template <typename T>
class ObDirectLoadExternalFragmentBuilder
{
typedef ObDirectLoadExternalIterator<T> ExternalIterator;
typedef ObDirectLoadExternalWriter<T> ExternalWriter;
public:
ObDirectLoadExternalFragmentBuilder();
~ObDirectLoadExternalFragmentBuilder();
void reuse();
void reset();
int init(ObDirectLoadTmpFileManager *file_mgr, ExternalWriter *external_writer);
int build_fragment(const common::ObVector<T *> &item_list);
int build_fragment(ExternalIterator &iter);
const ObDirectLoadExternalFragmentArray &get_fragments() const { return fragments_; }
protected:
ObDirectLoadTmpFileManager *file_mgr_;
int64_t dir_id_;
ExternalWriter *external_writer_;
ObDirectLoadExternalFragmentArray fragments_;
bool is_inited_;
};
template <typename T>
ObDirectLoadExternalFragmentBuilder<T>::ObDirectLoadExternalFragmentBuilder()
: file_mgr_(nullptr), dir_id_(-1), external_writer_(nullptr), is_inited_(false)
{
}
template <typename T>
ObDirectLoadExternalFragmentBuilder<T>::~ObDirectLoadExternalFragmentBuilder()
{
reset();
}
template <typename T>
void ObDirectLoadExternalFragmentBuilder<T>::reuse()
{
fragments_.reset();
}
template <typename T>
void ObDirectLoadExternalFragmentBuilder<T>::reset()
{
file_mgr_ = nullptr;
dir_id_ = -1;
external_writer_ = nullptr;
fragments_.reset();
is_inited_ = false;
}
template <typename T>
int ObDirectLoadExternalFragmentBuilder<T>::init(ObDirectLoadTmpFileManager *file_mgr,
ExternalWriter *external_writer)
{
int ret = common::OB_SUCCESS;
if (IS_INIT) {
ret = common::OB_INIT_TWICE;
STORAGE_LOG(WARN, "ObDirectLoadExternalSortRound init twice", KR(ret), KP(this));
} else if (OB_UNLIKELY(nullptr == file_mgr || nullptr == external_writer)) {
ret = common::OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid args", KR(ret), KP(file_mgr), KP(external_writer));
} else {
if (OB_FAIL(file_mgr->alloc_dir(dir_id_))) {
STORAGE_LOG(WARN, "fail to alloc dir", KR(ret));
} else {
file_mgr_ = file_mgr;
external_writer_ = external_writer;
is_inited_ = true;
}
}
return ret;
}
template <typename T>
int ObDirectLoadExternalFragmentBuilder<T>::build_fragment(const common::ObVector<T *> &item_list)
{
int ret = common::OB_SUCCESS;
if (IS_NOT_INIT) {
ret = common::OB_NOT_INIT;
STORAGE_LOG(WARN, "ObDirectLoadExternalFragmentBuilder not init", KR(ret));
} else {
ObDirectLoadExternalFragment fragment;
if (OB_FAIL(file_mgr_->alloc_file(dir_id_, fragment.file_handle_))) {
STORAGE_LOG(WARN, "fail to alloc file", KR(ret));
} else if (OB_FAIL(external_writer_->open(fragment.file_handle_))) {
STORAGE_LOG(WARN, "fail to open tmp file", KR(ret));
}
for (int64_t i = 0; OB_SUCC(ret) && i < item_list.size(); ++i) {
if (OB_FAIL(external_writer_->write_item(*item_list[i]))) {
STORAGE_LOG(WARN, "fail to write item", KR(ret));
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(external_writer_->close())) {
STORAGE_LOG(WARN, "fail to close external writer", KR(ret));
}
}
if (OB_SUCC(ret)) {
fragment.file_size_ = external_writer_->get_file_size();
fragment.row_count_ = item_list.size();
fragment.max_data_block_size_ = external_writer_->get_max_block_size();
if (OB_FAIL(fragments_.push_back(fragment))) {
STORAGE_LOG(WARN, "fail to push back fragment", KR(ret));
}
}
}
return ret;
}
template <typename T>
int ObDirectLoadExternalFragmentBuilder<T>::build_fragment(ExternalIterator &iter)
{
int ret = common::OB_SUCCESS;
if (IS_NOT_INIT) {
ret = common::OB_NOT_INIT;
STORAGE_LOG(WARN, "ObDirectLoadExternalFragmentBuilder not init", KR(ret));
} else {
ObDirectLoadExternalFragment fragment;
const T *item = nullptr;
if (OB_FAIL(file_mgr_->alloc_file(dir_id_, fragment.file_handle_))) {
STORAGE_LOG(WARN, "fail to alloc file", KR(ret));
} else if (OB_FAIL(external_writer_->open(fragment.file_handle_))) {
STORAGE_LOG(WARN, "fail to open tmp file", KR(ret));
}
while (OB_SUCC(ret)) {
if (OB_FAIL(iter.get_next_item(item))) {
if (OB_UNLIKELY(common::OB_ITER_END != ret)) {
STORAGE_LOG(WARN, "fail to get next item", KR(ret));
} else {
ret = common::OB_SUCCESS;
break;
}
} else if (OB_FAIL(external_writer_->write_item(*item))) {
STORAGE_LOG(WARN, "fail to write item", KR(ret));
} else {
++fragment.row_count_;
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(external_writer_->close())) {
STORAGE_LOG(WARN, "fail to close external writer", KR(ret));
} else if (OB_FAIL(fragments_.push_back(fragment))) {
STORAGE_LOG(WARN, "fail to push back fragment", KR(ret));
}
}
}
return ret;
}
} // namespace storage
} // namespace oceanbase

178
src/storage/direct_load/ob_direct_load_memory_writer.h
View File

@ -1,178 +0,0 @@
// Copyright (c) 2022-present Oceanbase Inc. All Rights Reserved.
// Author:
// suzhi.yt <>
#pragma once
#include "lib/allocator/page_arena.h"
#include "lib/container/ob_vector.h"
#include "observer/table_load/ob_table_load_stat.h"
#include "share/ob_errno.h"
namespace oceanbase
{
namespace storage
{
template <typename T, typename Compare, typename FramgentBuilder>
class ObDirectLoadMemoryWriter
{
static const int64_t MIN_MEMORY_LIMIT = 8 * 1024LL * 1024LL; // min memory limit is 8M
public:
ObDirectLoadMemoryWriter();
~ObDirectLoadMemoryWriter();
void reuse();
void reset();
int init(uint64_t tenant_id, int64_t mem_limit, Compare *compare,
FramgentBuilder *fragment_builder);
int add_item(const T &item);
int close();
TO_STRING_KV(K(is_inited_), K(buf_mem_limit_), KP(compare_), KP(fragment_builder_));
private:
int build_fragment();
private:
int64_t buf_mem_limit_;
Compare *compare_;
FramgentBuilder *fragment_builder_;
common::ObArenaAllocator allocator_;
common::ObVector<T *> item_list_;
bool is_inited_;
DISALLOW_COPY_AND_ASSIGN(ObDirectLoadMemoryWriter);
};
template <typename T, typename Compare, typename FramgentBuilder>
ObDirectLoadMemoryWriter<T, Compare, FramgentBuilder>::ObDirectLoadMemoryWriter()
: buf_mem_limit_(0),
compare_(nullptr),
fragment_builder_(nullptr),
allocator_("TLD_MemWriter"),
is_inited_(false)
{
}
template <typename T, typename Compare, typename FramgentBuilder>
ObDirectLoadMemoryWriter<T, Compare, FramgentBuilder>::~ObDirectLoadMemoryWriter()
{
reset();
}
template <typename T, typename Compare, typename FramgentBuilder>
void ObDirectLoadMemoryWriter<T, Compare, FramgentBuilder>::reuse()
{
for (int64_t i = 0; i < item_list_.size(); ++i) {
item_list_[i]->~T();
}
item_list_.reset();
allocator_.reset();
}
template <typename T, typename Compare, typename FramgentBuilder>
void ObDirectLoadMemoryWriter<T, Compare, FramgentBuilder>::reset()
{
buf_mem_limit_ = 0;
compare_ = nullptr;
fragment_builder_ = nullptr;
for (int64_t i = 0; i < item_list_.size(); ++i) {
item_list_[i]->~T();
}
item_list_.reset();
allocator_.reset();
is_inited_ = false;
}
template <typename T, typename Compare, typename FramgentBuilder>
int ObDirectLoadMemoryWriter<T, Compare, FramgentBuilder>::init(uint64_t tenant_id,
int64_t mem_limit, Compare *compare,
FramgentBuilder *fragment_builder)
{
int ret = common::OB_SUCCESS;
if (IS_INIT) {
ret = common::OB_INIT_TWICE;
STORAGE_LOG(WARN, "ObDirectLoadMemoryWriter init twice", KR(ret), KP(this));
} else if (OB_UNLIKELY(common::OB_INVALID_ID == tenant_id || mem_limit < MIN_MEMORY_LIMIT ||
nullptr == compare || nullptr == fragment_builder)) {
ret = common::OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid args", KR(ret), K(mem_limit), KP(compare), KP(fragment_builder));
} else {
buf_mem_limit_ = mem_limit;
compare_ = compare;
fragment_builder_ = fragment_builder;
allocator_.set_tenant_id(tenant_id);
is_inited_ = true;
}
return ret;
}
template <typename T, typename Compare, typename FramgentBuilder>
int ObDirectLoadMemoryWriter<T, Compare, FramgentBuilder>::add_item(const T &item)
{
int ret = common::OB_SUCCESS;
if (IS_NOT_INIT) {
ret = common::OB_NOT_INIT;
STORAGE_LOG(WARN, "ObDirectLoadMemoryWriter not init", KR(ret), KP(this));
} else {
const int64_t item_size = sizeof(T) + item.get_deep_copy_size();
if (item_size > buf_mem_limit_) {
ret = common::OB_SIZE_OVERFLOW;
STORAGE_LOG(WARN, "invalid item size, must not larger than buf memory limit", KR(ret),
K(item_size), K(buf_mem_limit_));
} else if (allocator_.used() + item_size > buf_mem_limit_ && OB_FAIL(build_fragment())) {
STORAGE_LOG(WARN, "fail to build fragment", KR(ret));
} else {
OB_TABLE_LOAD_STATISTICS_TIME_COST(memory_add_item_time_us);
char *buf = nullptr;
T *new_item = nullptr;
if (OB_ISNULL(buf = static_cast<char *>(allocator_.alloc(item_size)))) {
ret = common::OB_ALLOCATE_MEMORY_FAILED;
STORAGE_LOG(WARN, "fail to allocate memory", KR(ret), K(item_size));
} else {
new_item = new (buf) T();
int64_t buf_pos = sizeof(T);
if (OB_FAIL(new_item->deep_copy(item, buf, item_size, buf_pos))) {
STORAGE_LOG(WARN, "fail to deep copy item", KR(ret));
} else if (OB_FAIL(item_list_.push_back(new_item))) {
STORAGE_LOG(WARN, "fail to push back new item", KR(ret));
}
}
}
}
return ret;
}
template <typename T, typename Compare, typename FramgentBuilder>
int ObDirectLoadMemoryWriter<T, Compare, FramgentBuilder>::build_fragment()
{
int ret = common::OB_SUCCESS;
if (item_list_.size() > 1) {
OB_TABLE_LOAD_STATISTICS_TIME_COST(memory_sort_item_time_us);
std::sort(item_list_.begin(), item_list_.end(), *compare_);
if (OB_FAIL(compare_->get_error_code())) {
ret = compare_->get_error_code();
STORAGE_LOG(WARN, "fail to sort memory item list", KR(ret));
}
}
if (OB_SUCC(ret)) {
if (OB_FAIL(fragment_builder_->build_fragment(item_list_))) {
STORAGE_LOG(WARN, "fail to build fragment", KR(ret));
} else {
reuse();
}
}
return ret;
}
template <typename T, typename Compare, typename FramgentBuilder>
int ObDirectLoadMemoryWriter<T, Compare, FramgentBuilder>::close()
{
int ret = common::OB_SUCCESS;
if (IS_NOT_INIT) {
ret = common::OB_NOT_INIT;
STORAGE_LOG(WARN, "ObDirectLoadMemoryWriter not init", KR(ret), KP(this));
} else if (item_list_.size() > 0 && OB_FAIL(build_fragment())) {
STORAGE_LOG(WARN, "fail to build fragment", KR(ret));
}
return ret;
}
} // namespace storage
} // namespace oceanbase

32
src/storage/gts/ob_ha_gts_define.h
View File

@ -1,32 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_GTS_OB_HA_GTS_DEFINE_H_
#define OCEANBASE_GTS_OB_HA_GTS_DEFINE_H_
#include "lib/utility/utility.h"
namespace oceanbase
{
namespace gts
{
typedef common::ObIntWarp ObGtsID;
typedef common::ObIntWarp ObTenantID;
typedef common::ObIntWarp ObGtsReqID;
const int64_t HA_GTS_SOURCE_LEASE = 60 * 1000 * 1000 - 2 * 1000 * 1000; // 58s
const int64_t AUTO_CHANGE_MEMBER_INTERVAL = 1 * 1000 * 1000; // 1s
const int64_t GTS_OFFLINE_THRESHOLD = 1 * 1000 * 1000;
const int64_t INTERVAL_WITH_CREATED_TS = 30 * 1000 * 1000; // 30s
} // namespace gts
} // namespace oceanbase
#endif // OCEANBASE_GTS_OB_HA_GTS_DEFINE_H_

107
src/storage/ls/ob_ls_data_access_service.h
View File

@ -1,107 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_STORAGE_OB_LS_DATA_ACCESS_SERVICE_
#define OCEANBASE_STORAGE_OB_LS_DATA_ACCESS_SERVICE_
namespace oceanbase
{
namespace storage
{
// TODO: SQL need change pkey in param to tablet_id
class ObLSAccessService
{
public:
int table_scan(
ObTableScanParam &param,
common::ObNewRowIterator *&result);
int table_scan(
ObTableScanParam &param,
common::ObNewIterIterator *&result);
int join_mv_scan(
ObTableScanParam &left_param,
ObTableScanParam &right_param,
ObIPartitionGroup &right_partition,
common::ObNewRowIterator *&result);
int delete_rows(
ObStoreCtx &ctx,
const ObDMLBaseParam &dml_param,
const common::ObIArray<uint64_t> &column_ids,
common::ObNewRowIterator *row_iter,
int64_t &affected_rows);
int delete_row(ObStoreCtx &ctx,
const ObDMLBaseParam &dml_param,
const common::ObIArray<uint64_t> &column_ids,
const common::ObNewRow &row);
int put_rows(
ObStoreCtx &ctx,
const ObDMLBaseParam &dml_param,
const common::ObIArray<uint64_t> &column_ids,
common::ObNewRowIterator *row_iter,
int64_t &affected_rows);
int insert_rows(
ObStoreCtx &ctx,
const ObDMLBaseParam &dml_param,
const common::ObIArray<uint64_t> &column_ids,
common::ObNewRowIterator *row_iter,
int64_t &affected_rows);
int insert_row(
ObStoreCtx &ctx,
const ObDMLBaseParam &dml_param,
const common::ObIArray<uint64_t> &column_ids,
const common::ObNewRow &row);
int insert_row(
ObStoreCtx &ctx,
const ObDMLBaseParam &dml_param,
const common::ObIArray<uint64_t> &column_ids,
const common::ObIArray<uint64_t> &duplicated_column_ids,
const common::ObNewRow &row,
const ObInsertFlag flag,
int64_t &affected_rows,
common::ObNewRowIterator *&duplicated_rows);
int revert_insert_iter(const share::ObLSID &id, common::ObNewRowIterator *iter);
int update_rows(
ObStoreCtx &ctx,
const ObDMLBaseParam &dml_param,
const common::ObIArray<uint64_t> &column_ids,
const common::ObIArray< uint64_t> &updated_column_ids,
common::ObNewRowIterator *row_iter,
int64_t &affected_rows);
int update_row(
ObStoreCtx &ctx,
const ObDMLBaseParam &dml_param,
const common::ObIArray<uint64_t> &column_ids,
const common::ObIArray<uint64_t> &updated_column_ids,
const common::ObNewRow &old_row,
const common::ObNewRow &new_row);
int lock_rows(
ObStoreCtx &ctx,
const ObDMLBaseParam &dml_param,
const int64_t abs_lock_timeout,
common::ObNewRowIterator *row_iter,
const ObLockFlag lock_flag,
const bool is_sfu,
int64_t &affected_rows);
int lock_rows(
ObStoreCtx &ctx,
const ObDMLBaseParam &dml_param,
const int64_t abs_lock_timeout,
const common::ObNewRow &row,
ObLockFlag lock_flag,
const bool is_sfu);
private:
ObLSTabletService *ls_tablet_svr_;
};
}
}
#endif

0
src/storage/memtable/mvcc/ob_tx_callback.h
View File

129
src/storage/ob_data_storage_info.h
View File

@ -1,129 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_STORAGE_OB_DATA_STORAGE_INFO_
#define OCEANBASE_STORAGE_OB_DATA_STORAGE_INFO_
#include <cstdint>
#include "lib/utility/ob_print_utils.h"
#include "lib/utility/ob_unify_serialize.h"
#include "share/ob_define.h"
namespace oceanbase
{
namespace storage
{
class ObDataStorageInfo
{
public:
ObDataStorageInfo()
: last_replay_log_id_(0),
publish_version_(0),
schema_version_(0),
for_filter_log_compat_(0),
created_by_new_minor_freeze_(false),
last_replay_log_ts_(0) {}
~ObDataStorageInfo() {}
bool is_valid() const;
void reset();
void reset_for_no_memtable_replica();
void set_last_replay_log_id(const uint64_t last_replay_log_id);
uint64_t get_last_replay_log_id() const;
void set_publish_version(const int64_t publish_version);
int64_t get_publish_version() const;
void set_schema_version(const int64_t schema_version);
int64_t get_schema_version() const;
void inc_update_schema_version(const int64_t schema_version);
bool is_created_by_new_minor_freeze() const { return created_by_new_minor_freeze_; }
void set_created_by_new_minor_freeze() { created_by_new_minor_freeze_ = true; }
int64_t get_last_replay_log_ts() const;
void set_last_replay_log_ts(const int64_t last_replay_log_ts);
TO_STRING_KV("last_replay_log_id", last_replay_log_id_,
"last_replay_log_ts", last_replay_log_ts_,
"publish_version", publish_version_,
"schema_version", schema_version_,
"created_by_new_minor_freeze", created_by_new_minor_freeze_);
OB_UNIS_VERSION(1);
private:
// Log point to start replay.
uint64_t last_replay_log_id_;
int64_t publish_version_;
int64_t schema_version_;
// In order to be compatible with the filter log id of 225.
uint64_t for_filter_log_compat_;
bool created_by_new_minor_freeze_;
int64_t last_replay_log_ts_;
};
inline bool ObDataStorageInfo::is_valid() const
{
return (common::is_valid_trans_version(publish_version_)
&& schema_version_ >= 0);
}
inline void ObDataStorageInfo::set_last_replay_log_id(const uint64_t last_replay_log_id)
{
last_replay_log_id_ = last_replay_log_id;
}
inline uint64_t ObDataStorageInfo::get_last_replay_log_id() const
{
return last_replay_log_id_;
}
inline void ObDataStorageInfo::set_publish_version(const int64_t publish_version)
{
publish_version_ = publish_version;
}
inline int64_t ObDataStorageInfo::get_publish_version() const
{
return publish_version_;
}
inline void ObDataStorageInfo::set_schema_version(const int64_t schema_version)
{
schema_version_ = schema_version;
}
inline int64_t ObDataStorageInfo::get_schema_version() const
{
return schema_version_;
}
inline void ObDataStorageInfo::inc_update_schema_version(const int64_t schema_version)
{
if (schema_version > schema_version_) {
schema_version_ = schema_version;
}
}
inline int64_t ObDataStorageInfo::get_last_replay_log_ts() const
{
return last_replay_log_ts_;
}
inline void ObDataStorageInfo::set_last_replay_log_ts(const int64_t last_replay_log_ts)
{
last_replay_log_ts_ = last_replay_log_ts;
}
}
}
#endif /* OCEANBASE_STORAGE_OB_DATA_STORAGE_INFO_ */

662
src/storage/ob_long_ops_monitor.cpp
View File

@ -1,662 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX STORAGE
#include "ob_long_ops_monitor.h"
#include "observer/ob_server_struct.h"
#include "lib/thread/thread_mgr.h"
#include "share/schema/ob_schema_struct.h"
#include "share/schema/ob_schema_getter_guard.h"
using namespace oceanbase::common;
using namespace oceanbase::common::hash;
using namespace oceanbase::share::schema;
using namespace oceanbase::storage;
ObILongOpsKey::ObILongOpsKey()
: tenant_id_(OB_INVALID_ID), sid_(OB_INVALID_ID)
{
MEMSET(name_, 0, sizeof(name_));
MEMSET(target_, 0 ,sizeof(target_));
}
int64_t ObILongOpsKey::hash() const
{
uint64_t hash_val = 0;
hash_val = murmurhash(&tenant_id_, sizeof(tenant_id_), hash_val);
hash_val = murmurhash(&sid_, sizeof(sid_), hash_val);
hash_val = murmurhash(name_, sizeof(name_), hash_val);
hash_val = murmurhash(target_, sizeof(target_), hash_val);
return hash_val;
}
bool ObILongOpsKey::operator==(const ObILongOpsKey &other) const
{
return tenant_id_ == other.tenant_id_ && sid_ == other.sid_
&& (0 == MEMCMP(name_, other.name_, sizeof(name_)))
&& (0 == MEMCMP(target_, other.target_, sizeof(target_)));
}
bool ObILongOpsKey::is_valid() const
{
return OB_INVALID_ID != tenant_id_ && '\0' != name_[0]
&& '\0' != target_[0];
}
int ObCreateIndexKey::to_key_string()
{
int ret = OB_SUCCESS;
int64_t name_pos = 0;
int64_t target_pos = 0;
if (OB_FAIL(databuff_printf(name_, MAX_LONG_OPS_NAME_LENGTH, name_pos, "CREATE INDEX"))) {
LOG_WARN("fail to set name string", K(ret));
} else if (OB_FAIL(databuff_printf(target_, MAX_LONG_OPS_NAME_LENGTH, target_pos, "index_table_id=%ld, ", index_table_id_))) {
LOG_WARN("fail to convert index_table_id to string", K(ret));
} else if (OB_FAIL(databuff_printf(target_, MAX_LONG_OPS_TARGET_LENGTH, target_pos, "partition_id=%ld", partition_id_))) {
LOG_WARN("fail to convert partition_id to string", K(ret));
}
return ret;
}
int ObILongOpsTaskStat::assign(const ObILongOpsTaskStat &other)
{
int ret = OB_SUCCESS;
if (this != &other) {
task_id_ = other.task_id_;
cpu_cost_ = other.cpu_cost_;
io_cost_ = other.io_cost_;
state_ = other.state_;
type_ = other.type_;
}
return ret;
}
int ObCreateIndexScanTaskStat::assign(const ObILongOpsTaskStat &other)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(ObILongOpsTaskStat::TaskType::SCAN != other.type_)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret), K(other));
} else {
const ObCreateIndexScanTaskStat &other_scan_stat = static_cast<const ObCreateIndexScanTaskStat &>(other);
if (this != &other) {
if (OB_FAIL(ObILongOpsTaskStat::assign(other))) {
LOG_WARN("fail to assign task", K(ret));
} else {
macro_count_ = other_scan_stat.macro_count_;
}
}
}
return ret;
}
int ObCreateIndexSortTaskStat::assign(const ObILongOpsTaskStat &other)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(ObILongOpsTaskStat::TaskType::SORT != other.type_)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret), K(other));
} else {
const ObCreateIndexSortTaskStat &other_sort_stat = static_cast<const ObCreateIndexSortTaskStat &>(other);
if (this != &other) {
if (OB_FAIL(ObILongOpsTaskStat::assign(other))) {
LOG_WARN("fail to assign task", K(ret));
} else {
macro_count_ = other_sort_stat.macro_count_;
run_count_ = other_sort_stat.run_count_;
}
}
}
return ret;
}
ObCommonOpsStatValue::ObCommonOpsStatValue()
: start_time_(0), finish_time_(0), elapsed_time_(0), remaining_time_(0),
percentage_(0), last_update_time_(0), is_updated_(false), message_(), lock_()
{
memset(message_, 0, sizeof(message_));
}
ObCommonOpsStatValue &ObCommonOpsStatValue::operator=(const ObCommonOpsStatValue &other)
{
if (this != &other) {
start_time_ = other.start_time_;
finish_time_ = other.finish_time_;
elapsed_time_ = other.elapsed_time_;
remaining_time_ = other.remaining_time_;
percentage_ = other.percentage_;
last_update_time_ = other.last_update_time_;
is_updated_ = other.is_updated_;
MEMCPY(message_, other.message_, sizeof(message_));
}
return *this;
}
void ObCommonOpsStatValue::reset()
{
start_time_ = 0;
finish_time_= 0;
elapsed_time_ = 0;
remaining_time_ = 0;
percentage_ = 0;
last_update_time_ = 0;
is_updated_ = 0;
memset(message_, 0, sizeof(message_));
}
ObCreateIndexPartitionStat::ObCreateIndexPartitionStat()
: ObILongOpsStat(), key_(), task_stats_(),
allocator_(ObModIds::OB_SSTABLE_LONG_OPS_MONITOR)
{
}
ObCreateIndexPartitionStat::~ObCreateIndexPartitionStat()
{
reset();
}
int ObCreateIndexPartitionStat::update_task_stat(const ObILongOpsTaskStat &task_stat)
{
int ret = OB_SUCCESS;
SpinWLockGuard guard(common_value_.lock_);
if (OB_UNLIKELY(!task_stat.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret), K(task_stat));
} else {
int64_t i = 0;
for (i = 0; OB_SUCC(ret) && i < task_stats_.count(); ++i) {
if (task_stat.task_id_ == task_stats_.at(i)->task_id_) {
break;
}
}
if (i == task_stats_.count()) {
ObILongOpsTaskStat *new_task_stat = NULL;
if (OB_FAIL(copy_task(&task_stat, new_task_stat))) {
LOG_WARN("fail to copy task", K(ret));
} else if (OB_FAIL(task_stats_.push_back(new_task_stat))) {
LOG_WARN("fail to push back task stat", K(ret));
}
} else {
if (OB_FAIL(task_stats_.at(i)->assign(task_stat))) {
LOG_WARN("fail to assign task", K(ret));
}
}
if (OB_SUCC(ret)) {
common_value_.is_updated_ = true;
common_value_.last_update_time_ = ObTimeUtility::current_time();
}
}
return ret;
}
int ObCreateIndexPartitionStat::copy_task(const ObILongOpsTaskStat *src,
ObILongOpsTaskStat *&dest)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(src)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret), KP(src));
} else {
void *buf = NULL;
ObCreateIndexScanTaskStat *scan_stat = NULL;
ObCreateIndexSortTaskStat *sort_stat = NULL;
switch (src->type_) {
case ObILongOpsTaskStat::TaskType::SCAN:
if (OB_ISNULL(buf = allocator_.alloc(sizeof(ObCreateIndexScanTaskStat)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to allocate memory", K(ret));
} else {
scan_stat = new (buf) ObCreateIndexScanTaskStat();
*scan_stat = *static_cast<const ObCreateIndexScanTaskStat *>(src);
dest = scan_stat;
}
break;
case ObILongOpsTaskStat::TaskType::SORT:
if (OB_ISNULL(buf = allocator_.alloc(sizeof(ObCreateIndexSortTaskStat)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to allocate memory", K(ret));
} else {
sort_stat = new (buf) ObCreateIndexSortTaskStat();
*sort_stat = *static_cast<const ObCreateIndexSortTaskStat *>(src);
dest = sort_stat;
}
break;
default:
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid task type", K(ret), K(src->type_));
}
}
return ret;
}
void ObCreateIndexPartitionStat::reset()
{
for (int64_t i = 0; i < task_stats_.count(); ++i) {
ObILongOpsTaskStat *stat = task_stats_.at(i);
if (NULL != stat) {
stat->~ObILongOpsTaskStat();
stat = NULL;
}
}
task_stats_.reset();
allocator_.reset();
}
int ObCreateIndexPartitionStat::estimate_cost()
{
int ret = OB_SUCCESS;
SpinRLockGuard guard(common_value_.lock_);
if (common_value_.is_updated_) {
for (int64_t i = 0; OB_SUCC(ret) && i < task_stats_.count(); ++i) {
ObILongOpsTaskStat *task_stat = task_stats_.at(i);
if (OB_ISNULL(task_stat)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("error unexpected, task stat must not be NULL", K(ret));
} else if (OB_FAIL(task_stat->estimate_cost())) {
LOG_WARN("fail to estimate task cost", K(ret));
}
}
if (OB_SUCC(ret)) {
double completed_tasks_cost = 0;
double uncompleted_tasks_cost = 0;
bool is_finish = true;
for (int64_t i = 0; OB_SUCC(ret) && i < task_stats_.count(); ++i) {
ObILongOpsTaskStat *task_stat = task_stats_.at(i);
if (ObILongOpsTaskStat::TaskState::FAIL == task_stat->state_) {
completed_tasks_cost = 1;
uncompleted_tasks_cost = 1;
is_finish = true;
break;
} else if (ObILongOpsTaskStat::TaskState::SUCCESS == task_stat->state_) {
completed_tasks_cost += task_stat->get_cost();
} else {
uncompleted_tasks_cost += task_stat->get_cost();
is_finish = false;
}
}
if (OB_SUCC(ret)) {
const int64_t now = ObTimeUtility::current_time();
double total_task_cost = completed_tasks_cost + uncompleted_tasks_cost;
int64_t estimate_total_time = 0;
if (is_finish) {
common_value_.percentage_ = 100;
}
if (total_task_cost > 0) {
double tmp_percentage = static_cast<double>(completed_tasks_cost) / total_task_cost;
common_value_.percentage_ = static_cast<int64_t>(tmp_percentage * 100);
}
if (100 == common_value_.percentage_) {
common_value_.finish_time_ = common_value_.last_update_time_;
}
common_value_.elapsed_time_ = common_value_.finish_time_ > 0 ? common_value_.finish_time_ - common_value_.start_time_ : now - common_value_.start_time_;
if (common_value_.percentage_ > 0 && common_value_.elapsed_time_ > 0) {
estimate_total_time = 100 * common_value_.elapsed_time_ / common_value_.percentage_;
common_value_.remaining_time_ = std::max(0L, estimate_total_time - common_value_.elapsed_time_);
}
}
}
common_value_.is_updated_ = false;
}
return ret;
}
int ObCreateIndexPartitionStat::deep_copy(char *buf, const int64_t buf_len, ObILongOpsStat *&value) const
{
int ret = OB_SUCCESS;
const int64_t deep_copy_size = get_deep_copy_size();
if (OB_ISNULL(buf) || buf_len < deep_copy_size) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(ret), KP(buf), K(buf_len));
} else {
ObCreateIndexPartitionStat *pvalue = new (buf) ObCreateIndexPartitionStat();
for (int64_t i = 0; OB_SUCC(ret) && i < task_stats_.count(); ++i) {
ObILongOpsTaskStat *src = task_stats_.at(i);
ObILongOpsTaskStat *dest = NULL;
if (OB_FAIL(pvalue->copy_task(src, dest))) {
STORAGE_LOG(WARN, "fail to copy task", K(ret));
} else if (OB_FAIL(pvalue->task_stats_.push_back(dest))) {
STORAGE_LOG(WARN, "fail to push back task", K(ret));
}
if (OB_FAIL(ret) && NULL != dest) {
dest->~ObILongOpsTaskStat();
dest = NULL;
}
}
if (OB_SUCC(ret)) {
pvalue->key_ = key_;
pvalue->common_value_ = common_value_;
value = pvalue;
}
}
return ret;
}
int ObCreateIndexPartitionStat::check_can_purge(bool &can_purge)
{
int ret = OB_SUCCESS;
can_purge = false;
const int64_t index_table_id = key_.index_table_id_;
const uint64_t tenant_id = key_.tenant_id_;
const ObTableSchema *index_schema = NULL;
share::schema::ObSchemaGetterGuard schema_guard;
if (OB_FAIL(GCTX.schema_service_->get_tenant_full_schema_guard(tenant_id, schema_guard))) {
LOG_WARN("fail to get schema guard", K(ret), K(index_table_id));
} else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, index_table_id, index_schema))) {
LOG_WARN("fail to get table schema", K(ret));
} else if (OB_ISNULL(index_schema)) {
// index has been dropped, need delete
can_purge = true;
} else {
const int64_t now = ObTimeUtility::current_time();
const share::schema::ObIndexStatus index_status= index_schema->get_index_status();
can_purge = is_final_index_status(index_status)
&& (common_value_.last_update_time_ / 1000 + MAX_LIFE_TIME < now / 1000);
}
return ret;
}
ObILongOpsStatHandle::ObILongOpsStatHandle()
: ObResourceHandle<ObILongOpsStat>()
{
}
ObILongOpsStatHandle::~ObILongOpsStatHandle()
{
reset();
}
void ObILongOpsStatHandle::reset()
{
if (NULL != ptr_) {
int tmp_ret = OB_SUCCESS;
if (OB_SUCCESS != (tmp_ret = ObLongOpsMonitor::get_instance().dec_handle_ref(*this))) {
LOG_WARN_RET(tmp_ret, "fail to dec handle ref", K(tmp_ret));
}
}
}
ObLongOpsMonitor::ObLongOpsMonitor()
: is_inited_(false), map_()
{
}
ObLongOpsMonitor::~ObLongOpsMonitor()
{
}
int ObLongOpsMonitor::init()
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(is_inited_)) {
ret = OB_INIT_TWICE;
LOG_WARN("ObLongOpsMonitor has been inited twice", K(ret));
} else if (OB_FAIL(map_.init(DEFAULT_BUCKET_NUM, OB_SERVER_TENANT_ID,
ObModIds::OB_SSTABLE_LONG_OPS_MONITOR,
TOTAL_LIMIT, HOLD_LIMIT, PAGE_SIZE))) {
LOG_WARN("fail to init map", K(ret));
} else {
is_inited_ = true;
}
return ret;
}
int ObLongOpsMonitor::add_long_ops_stat(const ObILongOpsKey &key,
ObILongOpsStat &stat)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ObLongOpsMonitor has not been inited", K(ret));
} else if (OB_UNLIKELY(!key.is_valid() || !stat.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret), K(key), K(stat));
} else if (OB_FAIL(map_.set(key, stat))) {
LOG_WARN("fail to set to map", K(ret), K(key));
}
return ret;
}
int ObLongOpsMonitor::get_long_ops_stat(const ObILongOpsKey &key,
ObILongOpsStatHandle &handle)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ObLongOpsMonitor has not been inited", K(ret));
} else if (OB_UNLIKELY(!key.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret), K(key));
} else if (OB_FAIL(map_.get(key, handle))) {
LOG_WARN("fail to get partition stat", K(ret));
}
return ret;
}
int ObLongOpsMonitor::remove_long_ops_stat(const ObILongOpsKey &key)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ObLongOpsMonitor has not been inited", K(ret));
} else if (OB_UNLIKELY(!key.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret));
} else if (OB_FAIL(map_.erase(key))) {
LOG_WARN("fail to get from map", K(ret), K(key));
}
return ret;
}
int ObLongOpsMonitor::update_task_stat(const ObILongOpsKey &key,
const ObILongOpsTaskStat &task_stat)
{
int ret = OB_SUCCESS;
ObILongOpsStatHandle handle;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ObLongOpsMonitor has not been inited", K(ret));
} else if (OB_UNLIKELY(!key.is_valid() || !task_stat.is_valid())) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret), K(key), K(task_stat));
} else if (OB_FAIL(map_.get(key, handle))) {
LOG_WARN("fail to get from map", K(ret), K(key));
} else if (OB_ISNULL(handle.get_resource_ptr())) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("error unexpected, partition stat must not be NULL", K(ret));
} else if (OB_FAIL(handle.get_resource_ptr()->update_task_stat(task_stat))) {
LOG_WARN("fail to update task stat", K(ret), K(task_stat));
}
return ret;
}
template <typename Callback>
int ObLongOpsMonitor::foreach(Callback &callback)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ObLongOpsMonitor has not been inited", K(ret));
} else if (OB_FAIL(map_.foreach(callback))) {
LOG_WARN("fail to foreach map", K(ret));
}
return ret;
}
int ObLongOpsMonitor::dec_handle_ref(ObILongOpsStatHandle &handle)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ObLongOpsMonitor has not been inited", K(ret));
} else if (OB_FAIL(map_.dec_handle_ref(handle.ptr_))) {
LOG_WARN("fail to dec handle ref", K(ret));
}
return ret;
}
ObLongOpsMonitorIterator::ObKeySnapshotCallback::ObKeySnapshotCallback(
ObIArray<ObILongOpsKey> &key_snapshot)
: key_snapshot_(key_snapshot)
{
}
int ObLongOpsMonitorIterator::ObKeySnapshotCallback::operator()(PAIR &pair)
{
int ret = OB_SUCCESS;
if (OB_FAIL(key_snapshot_.push_back(pair.first))) {
LOG_WARN("fail to push back key", K(ret));
}
return ret;
}
ObLongOpsMonitor &ObLongOpsMonitor::get_instance()
{
static ObLongOpsMonitor instance;
return instance;
}
ObLongOpsMonitorIterator::ObLongOpsMonitorIterator()
: is_inited_(false), key_snapshot_(), key_cursor_(0)
{
}
ObLongOpsMonitorIterator::~ObLongOpsMonitorIterator()
{
}
int ObLongOpsMonitorIterator::init()
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(is_inited_)) {
ret = OB_INIT_TWICE;
LOG_WARN("ObLongOpsMonitorIterator has been inited twice", K(ret));
} else {
is_inited_ = true;
key_cursor_ = 0;
if (OB_FAIL(make_key_snapshot())) {
LOG_WARN("fail to make key snapshot", K(ret));
}
}
return ret;
}
int ObLongOpsMonitorIterator::make_key_snapshot()
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ObLongOpsMonitorIterator has not been inited", K(ret));
} else {
ObKeySnapshotCallback callback(key_snapshot_);
if (OB_FAIL(ObLongOpsMonitor::get_instance().foreach(callback))) {
LOG_WARN("fail to do foreach map", K(ret));
}
}
return ret;
}
int ObLongOpsMonitorIterator::get_next_stat(ObILongOpsStatHandle &handle)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ObLongOpsMonitorIterator has not been inited", K(ret));
} else if (OB_UNLIKELY(key_cursor_ >= key_snapshot_.count())) {
ret = OB_ITER_END;
} else {
bool need_retry = true;
while (OB_SUCC(ret) && need_retry && key_cursor_ < key_snapshot_.count()) {
const ObILongOpsKey &key = key_snapshot_.at(key_cursor_);
if (OB_FAIL(ObLongOpsMonitor::get_instance().get_long_ops_stat(key, handle))) {
if (OB_UNLIKELY(OB_ENTRY_NOT_EXIST != ret)) {
LOG_WARN("fail to get parition stat", K(ret), K(key));
} else {
need_retry = true;
ret = OB_SUCCESS;
}
} else {
need_retry = false;
}
++key_cursor_;
}
}
return ret;
}
ObPurgeCompletedMonitorInfoTask::ObPurgeCompletedMonitorInfoTask()
: is_inited_(false), schema_service_(nullptr)
{
}
int ObPurgeCompletedMonitorInfoTask::init(share::schema::ObMultiVersionSchemaService *schema_service,
int tg_id)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(is_inited_)) {
ret = OB_INIT_TWICE;
LOG_WARN("ObPurgeCompletedMonitorInfoTask has already been inited", K(ret));
} else if (OB_ISNULL(schema_service)) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid arguments", K(ret), KP(schema_service));
} else {
schema_service_ = schema_service;
is_inited_ = true;
if (OB_FAIL(TG_SCHEDULE(tg_id, *this, SCHEDULE_INTERVAL, true /*schedule repeatly*/))) {
LOG_WARN("fail to schedule task", K(ret));
}
}
return ret;
}
void ObPurgeCompletedMonitorInfoTask::destroy()
{
is_inited_ = false;
schema_service_ = NULL;
}
void ObPurgeCompletedMonitorInfoTask::runTimerTask()
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ObPurgeCompletedMonitorInfoTask has not been inited", K(ret));
} else {
int64_t iterated_count = 0;
ObLongOpsMonitorIterator iter;
if (OB_FAIL(iter.init())) {
LOG_WARN("fail to init iterator", K(ret));
}
while (OB_SUCC(ret) && iterated_count < MAX_BATCH_COUNT) {
bool can_purge = false;
ObILongOpsStatHandle handle;
if (OB_FAIL(iter.get_next_stat(handle))) {
if (OB_ITER_END != ret) {
LOG_WARN("fail to get next stat", K(ret));
}
} else if (OB_FAIL(handle.get_resource_ptr()->check_can_purge(can_purge))) {
LOG_WARN("fail to check can purge", K(ret));
} else if (can_purge) {
if (OB_FAIL(ObLongOpsMonitor::get_instance().remove_long_ops_stat(handle.get_resource_ptr()->get_key()))) {
LOG_WARN("fail to remove long ops stat", K(ret));
}
}
++iterated_count;
}
}
}

30
src/storage/ob_non_trans_log.h
View File

@ -1,30 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_STORAGE_OB_NON_TRANS_LOG_
#define OCEANBASE_STORAGE_OB_NON_TRANS_LOG_
namespace oceanbase
{
namespace storage
{
class ObNonTransLog
{
public:
ObNonTransLog() {}
virtual ~ObNonTransLog() {}
virtual int replace_tenant_id(const uint64_t new_tenant_id) = 0;
};
} // storage
} // oceanbase
#endif //OCEANBASE_STORAGE_OB_NON_TRANS_LOG_

128
src/storage/ob_safe_ref.h
View File

@ -1,128 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_MEMTABLE_OB_SAFE_REF_H_
#define OCEANBASE_MEMTABLE_OB_SAFE_REF_H_
#include "lib/objectpool/ob_concurrency_objpool.h"
#include "lib/queue/ob_link_queue.h"
namespace oceanbase
{
namespace storage
{
struct SafeRef: public common::ObLink
{
SafeRef(): ref_(0), seq_(0), ptr_(NULL) {}
~SafeRef() {}
void xref(int64_t x) { ATOMIC_AAF(&ref_, x); }
int64_t ref() { return ATOMIC_LOAD(&ref_); }
int64_t seq() { return ATOMIC_LOAD(&seq_); }
void inc_seq() { ATOMIC_AAF(&seq_, 1); }
int64_t ref_;
int64_t seq_;
void* ptr_;
};
struct SafeRef2
{
SafeRef2(): sref_(NULL), seq_(0) {}
~SafeRef2() {}
SafeRef* sref_;
int64_t seq_;
};
class SafeRefAlloc
{
public:
typedef common::ObLinkQueue FreeList;
SafeRef* alloc() {
int ret = common::OB_SUCCESS;
SafeRef* p = NULL;
while(NULL == p) {
if (OB_FAIL(free_list_.pop((common::ObLink*&)p))) {
p = OB_NEW(SafeRef, "SafeRef");
}
if (NULL == p) {
ob_usleep(10 * 1000);
}
}
return p;
}
void free(SafeRef* p) {
(void)free_list_.push(p);
}
private:
FreeList free_list_;
};
class ObSafeRefKeeper
{
public:
typedef SafeRefAlloc Alloc;
ObSafeRefKeeper() {}
~ObSafeRefKeeper() {}
void reg(SafeRef2& ref, void* ptr) {
SafeRef* p = alloc_.alloc();
p->ptr_ = ptr;
ref.sref_ = p;
ref.seq_ = p->seq_;
}
void unreg(SafeRef2& ref) {
SafeRef* p = (SafeRef*)ref.sref_;
if (NULL != p) {
p->inc_seq();
ATOMIC_STORE(&ref.sref_, NULL);
while(p->ref() > 0) {
PAUSE();
}
ATOMIC_STORE(&p->ptr_, NULL);
alloc_.free(p);
}
}
void* lock(SafeRef2& ref) {
void* ret = NULL;
SafeRef* p = (SafeRef*)ref.sref_;
if (NULL != p) {
p->xref(1);
ret = p->ptr_;
if (NULL == ret) {
// unexpected
p->xref(-1);
} else if (p->seq() != ref.seq_) {
p->xref(-1);
ret = NULL;
}
}
return ret;
}
void unlock(SafeRef2& ref) {
SafeRef* p = (SafeRef*)ref.sref_;
if (NULL != p) {
p->xref(-1);
}
}
private:
Alloc alloc_;
};
inline ObSafeRefKeeper& get_safe_ref_keeper()
{
static ObSafeRefKeeper ref_keeper;
return ref_keeper;
}
#define REF_KEEPER get_safe_ref_keeper()
}; // end namespace memtable
}; // end namespace oceanbase
#endif /* OCEANBASE_MEMTABLE_OB_SAFE_REF_H_ */

48
src/storage/ob_store_row_filter.cpp
View File

@ -1,48 +0,0 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "storage/ob_store_row_filter.h"
#include "storage/ob_i_store.h"
#include "share/ob_errno.h"
#include "share/schema/ob_multi_version_schema_service.h"
#include "observer/ob_server_struct.h"
#include "sql/optimizer/ob_table_location.h"
namespace oceanbase
{
using namespace common;
using namespace sql;
namespace storage
{
int ObStoreRowFilter::init(const ObTableLocation *part_filter, ObExecContext *exec_ctx)
{
int ret = OB_SUCCESS;
if (NULL == part_filter || NULL == exec_ctx) {
ret = OB_INVALID_ARGUMENT;
STORAGE_LOG(WARN, "invalid argument", K(ret), KP(part_filter), KP(exec_ctx));
} else {
part_filter_ = part_filter;
exec_ctx_ = exec_ctx;
}
return ret;
}
int ObStoreRowFilter::check(const ObStoreRow &store_row, bool &is_filtered) const
{
// support for partition split, not used now
return OB_NOT_SUPPORTED;
}
} // namespace storage
} // namespace oceanbase

Some files were not shown because too many files have changed in this diff Show More