hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge branch 'master' into develop

Browse Source
This commit is contained in:
wangyunlai 2024-03-19 11:03:52 +08:00 committed by GitHub
commit a8d15b7669
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
418 changed files with 10931 additions and 4057 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/compile.yml vendored
View File

@ -32,7 +32,7 @@ jobs:
large-packages: true
swap-storage: true
docker-images: true
- uses: actions/checkout@v3
- name: Install ubuntu environment

2
.github/workflows/mkbook.yml vendored
View File

@ -36,7 +36,7 @@ jobs:
run: |
mkdir mdbook
mv docs mdbook/src
- name: Install mdBook
run: |
curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSf -y | sh

4
CMakeLists.txt
View File

@ -5,14 +5,14 @@ include(cmake/Env.cmake)
if(OB_BUILD_OPENSOURCE)
project("OceanBase_CE"
VERSION 4.3.0.1
VERSION 4.3.1.0
DESCRIPTION "OceanBase distributed database system"
HOMEPAGE_URL "https://open.oceanbase.com/"
LANGUAGES CXX C ASM)
message(STATUS "open source build enabled")
else()
project(OceanBase
VERSION 4.3.0.1
VERSION 4.3.1.0
DESCRIPTION "OceanBase distributed database system"
HOMEPAGE_URL "https://www.oceanbase.com/"
LANGUAGES CXX C ASM)

2
cmake/DEB.cmake
View File

@ -29,7 +29,6 @@ if(DEBIAN_NAME AND DEBIAN_VERSION)
endif()
set(CPACK_DEBIAN_PACKAGE_NAME ${CPACK_PACKAGE_NAME})
set(CPACK_DEBIAN_SERVER_PACKAGE_DEPENDS "oceanbase-ce-libs = ${CPACK_PACKAGE_VERSION}-${CPACK_DEBIAN_PACKAGE_RELEASE}")
set(CPACK_PACKAGE_DESCRIPTION ${CPACK_PACKAGE_DESCRIPTION})
set(CPACK_PACKAGE_CONTACT "${OceanBase_CE_HOMEPAGE_URL}")
set(CPACK_DEBIAN_PACKAGE_MAINTAINER "OceanBase")
@ -38,6 +37,7 @@ set(CPACK_DEBIAN_PACKAGE_PRIORITY "Optional")
# systemd define on deb
if (OB_BUILD_OPENSOURCE)
set(CPACK_DEBIAN_SERVER_PACKAGE_DEPENDS "oceanbase-ce-libs = ${CPACK_PACKAGE_VERSION}-${CPACK_DEBIAN_PACKAGE_RELEASE}")
set(CPACK_DEBIAN_SERVER_PACKAGE_DEPENDS "${CPACK_DEBIAN_SERVER_PACKAGE_DEPENDS}, jq, systemd")
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/tools/rpm/systemd/profile/post_install.sh.template

7
cmake/RPM.cmake
View File

@ -31,11 +31,6 @@ else()
endif()
set(CPACK_RPM_PACKAGE_GROUP "Applications/Databases")
if (CPACK_RPM_PACKAGE_RELEASE)
set(CPACK_RPM_SERVER_PACKAGE_REQUIRES "oceanbase-ce-libs = ${CPACK_PACKAGE_VERSION}-${CPACK_RPM_PACKAGE_RELEASE}")
else()
set(CPACK_RPM_SERVER_PACKAGE_REQUIRES "oceanbase-ce-libs = ${CPACK_PACKAGE_VERSION}")
endif()
set(CPACK_RPM_PACKAGE_DESCRIPTION ${CPACK_PACKAGE_DESCRIPTION})
set(CPACK_RPM_PACKAGE_LICENSE "Mulan PubL v2.")
set(CPACK_RPM_DEFAULT_USER "admin")
@ -59,7 +54,7 @@ set(CPACK_RPM_SPEC_MORE_DEFINE
# systemd define on rpm
if (OB_BUILD_OPENSOURCE)
set(CPACK_RPM_SERVER_PACKAGE_REQUIRES "${CPACK_RPM_SERVER_PACKAGE_REQUIRES}, jq, systemd")
set(CPACK_RPM_SERVER_PACKAGE_REQUIRES "oceanbase-ce-libs = ${CPACK_PACKAGE_VERSION}, jq, systemd")
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/tools/rpm/systemd/profile/post_install.sh.template
${CMAKE_CURRENT_SOURCE_DIR}/tools/rpm/systemd/profile/post_install.sh

2
deps/init/oceanbase.el7.aarch64.deps vendored
View File

@ -43,7 +43,7 @@ obdevtools-llvm-11.0.1-312022092921.el7.aarch64.rpm
[tools-deps]
devdeps-oblogmsg-1.0-182024030421.el7.aarch64.rpm
devdeps-rocksdb-6.22.1.1-52022100420.el7.aarch64.rpm
obshell-4.2.2.0-122024022614.el7.aarch64.rpm target=community
obshell-4.2.3.0-102024031414.el7.aarch64.rpm target=community
[test-utils]
ob-deploy-2.6.2-2.el7.aarch64.rpm target=community

2
deps/init/oceanbase.el7.x86_64.deps vendored
View File

@ -46,7 +46,7 @@ obdevtools-llvm-11.0.1-312022092921.el7.x86_64.rpm
[tools-deps]
devdeps-oblogmsg-1.0-182024030421.el7.x86_64.rpm
devdeps-rocksdb-6.22.1.1-52022100420.el7.x86_64.rpm
obshell-4.2.2.0-122024022614.el7.x86_64.rpm target=community
obshell-4.2.3.0-102024031414.el7.x86_64.rpm target=community
[test-utils]
ob-deploy-2.6.2-2.el7.x86_64.rpm target=community

2
deps/init/oceanbase.el8.aarch64.deps vendored
View File

@ -43,7 +43,7 @@ obdevtools-llvm-11.0.1-312022092921.el8.aarch64.rpm
[tools-deps]
devdeps-oblogmsg-1.0-182024030421.el8.aarch64.rpm
devdeps-rocksdb-6.22.1.1-52022100420.el8.aarch64.rpm
obshell-4.2.2.0-122024022614.el8.aarch64.rpm target=community
obshell-4.2.3.0-102024031414.el8.aarch64.rpm target=community
[test-utils]
ob-deploy-2.6.2-2.el8.aarch64.rpm target=community

2
deps/init/oceanbase.el8.x86_64.deps vendored
View File

@ -45,7 +45,7 @@ obdevtools-llvm-11.0.1-312022092921.el8.x86_64.rpm
[tools-deps]
devdeps-oblogmsg-1.0-182024030421.el8.x86_64.rpm
devdeps-rocksdb-6.22.1.1-52022100420.el8.x86_64.rpm
obshell-4.2.2.0-122024022614.el8.x86_64.rpm target=community
obshell-4.2.3.0-102024031414.el8.x86_64.rpm target=community
[test-utils]
ob-deploy-2.6.2-2.el8.x86_64.rpm target=community

2
deps/init/oceanbase.el9.aarch64.deps vendored
View File

@ -50,7 +50,7 @@ obdevtools-llvm-11.0.1-312022092921.el8.aarch64.rpm
[tools-deps]
devdeps-oblogmsg-1.0-182024030421.el8.aarch64.rpm
devdeps-rocksdb-6.22.1.1-52022100420.el8.aarch64.rpm
obshell-4.2.2.0-122024022614.el8.aarch64.rpm target=community
obshell-4.2.3.0-102024031414.el8.aarch64.rpm target=community
[test-utils]
ob-deploy-1.6.0-41.el8.aarch64.rpm target=community

2
deps/init/oceanbase.el9.x86_64.deps vendored
View File

@ -53,7 +53,7 @@ obdevtools-llvm-11.0.1-312022092921.el8.x86_64.rpm
[tools-deps]
devdeps-oblogmsg-1.0-182024030421.el8.x86_64.rpm
devdeps-rocksdb-6.22.1.1-52022100420.el8.x86_64.rpm
obshell-4.2.2.0-122024022614.el8.x86_64.rpm target=community
obshell-4.2.3.0-102024031414.el8.x86_64.rpm target=community
[test-utils]
ob-deploy-1.6.0-41.el8.x86_64.rpm target=community

1
deps/oblib/src/lib/CMakeLists.txt vendored
View File

@ -319,6 +319,7 @@ ob_set_subtarget(ob_malloc_object_list common_alloc
allocator/ob_mod_define.cpp
allocator/ob_page_manager.cpp
allocator/ob_slice_alloc.cpp
allocator/ob_sql_mem_leak_checker.cpp
allocator/ob_tc_malloc.cpp
)

5
deps/oblib/src/lib/alloc/alloc_struct.cpp vendored
View File

@ -24,13 +24,16 @@ using namespace common;
namespace lib
{
thread_local ObMemAttr ObMallocHookAttrGuard::tl_mem_attr(OB_SERVER_TENANT_ID,
"glibc_malloc",
ObCtxIds::GLIBC);
static int64_t g_divisive_mem_size[OB_MAX_CPU_NUM];
static thread_local bool g_is_ob_mem_mgr_path = false;
uint32_t ObMemVersionNode::global_version = 0;
__thread bool ObMemVersionNode::tl_ignore_node = true;
__thread ObMemVersionNode* ObMemVersionNode::tl_node = NULL;
ObMallocHookAttrGuard::ObMallocHookAttrGuard(const ObMemAttr& attr)
: old_attr_(tl_mem_attr)
{

90
deps/oblib/src/lib/alloc/alloc_struct.h vendored
View File

@ -23,6 +23,7 @@
#include "lib/alloc/alloc_assist.h"
#include "lib/alloc/abit_set.h"
#include "lib/allocator/ob_mod_define.h"
#include "lib/list/ob_dlink_node.h"
#ifndef NDEBUG
#define MEMCHK_LEVEL 1
@ -119,15 +120,24 @@ struct ObLabel
const char *str_;
};
struct ObMemVersionNode : public ObDLinkBase<ObMemVersionNode>
{
static uint32_t global_version;
static __thread bool tl_ignore_node;
static __thread ObMemVersionNode* tl_node;
uint32_t version_ = UINT32_MAX;
};
struct ObMemAttr
{
friend ObMemAttr DoNotUseMe(ObMemAttr &attr, bool expect_500);
friend ObMemAttr DoNotUseMe(ObMemAttr &attr);
friend ObMemAttr UseUnexpected500(ObMemAttr &attr);
friend ObMemAttr IgnoreVersion(ObMemAttr &attr);
uint64_t tenant_id_;
ObLabel label_;
uint64_t ctx_id_;
uint64_t sub_ctx_id_;
ObAllocPrio prio_;
explicit ObMemAttr(
uint64_t tenant_id = common::OB_SERVER_TENANT_ID,
ObLabel label = ObLabel(),
@ -137,42 +147,73 @@ struct ObMemAttr
label_(label),
ctx_id_(ctx_id),
sub_ctx_id_(ObSubCtxIds::MAX_SUB_CTX_ID),
prio_(prio) {}
prio_(prio),
use_500_(false),
expect_500_(true),
ignore_version_(ObMemVersionNode::tl_ignore_node)
{}
int64_t to_string(char* buf, const int64_t buf_len) const;
bool use_500() const { return use_500_; }
bool expect_500() const { return expect_500_; }
bool ignore_version() const { return ignore_version_; }
private:
bool use_500_ = false;
bool expect_500_ = true;
union {
char padding__[4];
struct {
struct {
uint8_t use_500_ : 1;
uint8_t expect_500_ : 1;
uint8_t ignore_version_ : 1;
};
};
};
};
inline ObMemAttr DoNotUseMe(ObMemAttr &attr, bool expect_500)
inline ObMemAttr DoNotUseMe(ObMemAttr &attr)
{
attr.use_500_ = true;
attr.expect_500_ = expect_500;
attr.ignore_version_ = true;
return attr;
}
inline ObMemAttr DoNotUseMe(const ObMemAttr &&attr, const bool expect_500)
inline ObMemAttr UseUnexpected500(ObMemAttr &attr)
{
ObMemAttr attr_cpy = attr;
return DoNotUseMe(attr_cpy, expect_500);
attr.use_500_ = true;
attr.expect_500_ = false;
attr.ignore_version_ = true;
return attr;
}
inline ObMemAttr DoNotUseMe(const ObLabel &label, const bool expect_500)
inline ObMemAttr IgnoreVersion(ObMemAttr &attr)
{
ObMemAttr attr(OB_SERVER_TENANT_ID, label);
return DoNotUseMe(attr, expect_500);
attr.ignore_version_ = true;
return attr;
}
inline ObMemAttr DoNotUseMe(const ObLabel &label, const uint64_t ctx_id, const bool expect_500)
{
ObMemAttr attr(OB_SERVER_TENANT_ID, label, ctx_id);
return DoNotUseMe(attr, expect_500);
}
#define ObMemAttrFriendFunc(func_name) \
inline ObMemAttr func_name(const ObMemAttr &&attr) \
{ \
ObMemAttr attr_cpy = attr; \
return func_name(attr_cpy); \
} \
inline ObMemAttr func_name(const ObLabel &label) \
{ \
ObMemAttr attr(OB_SERVER_TENANT_ID, label); \
return func_name(attr); \
} \
inline ObMemAttr func_name(const ObLabel &label, const uint64_t ctx_id) \
{ \
ObMemAttr attr(OB_SERVER_TENANT_ID, label, ctx_id); \
return func_name(attr); \
}
#define SET_USE_500(args...) ::oceanbase::lib::DoNotUseMe(args, true)
#define SET_USE_UNEXPECTED_500(args...) ::oceanbase::lib::DoNotUseMe(args, false)
ObMemAttrFriendFunc(DoNotUseMe);
ObMemAttrFriendFunc(UseUnexpected500);
ObMemAttrFriendFunc(IgnoreVersion);
#define SET_USE_500(args...) ::oceanbase::lib::DoNotUseMe(args)
#define SET_USE_UNEXPECTED_500(args...) ::oceanbase::lib::UseUnexpected500(args)
#define SET_IGNORE_MEM_VERSION(args...) ::oceanbase::lib::IgnoreVersion(args)
struct AllocHelper
{
@ -274,16 +315,18 @@ struct AObject {
uint16_t obj_offset_;
uint32_t alloc_bytes_;
uint64_t tenant_id_;
uint32_t version_;
char label_[AOBJECT_LABEL_SIZE + 1];
// padding to ensure data_ is 16x offset
union {
char padding__[4];
char padding__[16];
struct {
struct {
uint8_t on_leak_check_ : 1;
uint8_t on_malloc_sample_ : 1;
uint8_t ignore_version_ : 1;
};
};
};
@ -522,8 +565,7 @@ char *ABlock::data() const
AObject::AObject()
: MAGIC_CODE_(FREE_AOBJECT_MAGIC_CODE),
nobjs_(0), nobjs_prev_(0), obj_offset_(0),
alloc_bytes_(0), tenant_id_(0),
on_leak_check_(false), on_malloc_sample_(false)
alloc_bytes_(0), on_leak_check_(false), on_malloc_sample_(false)
{
}

107
deps/oblib/src/lib/alloc/memory_dump.cpp vendored
View File

@ -14,7 +14,7 @@
#include "lib/alloc/memory_dump.h"
#include <setjmp.h>
#include "lib/alloc/ob_free_log_printer.h"
#include "lib/allocator/ob_sql_mem_leak_checker.h"
#include "lib/signal/ob_signal_struct.h"
#include "lib/rc/context.h"
#include "lib/utility/utility.h"
@ -205,6 +205,45 @@ int ObMemoryDump::push(void *task)
return ret;
}
int ObMemoryDump::generate_mod_stat_task(ObMemoryCheckContext *memory_check_ctx)
{
int ret = OB_SUCCESS;
ObMemoryDumpTask *task = alloc_task();
if (OB_ISNULL(task)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("alloc task failed");
} else {
task->type_ = STAT_LABEL;
task->memory_check_ctx_ = memory_check_ctx;
COMMON_LOG(INFO, "task info", K(*task));
if (OB_FAIL(push(task))) {
LOG_WARN("push task failed", K(ret));
free_task(task);
}
}
return ret;
}
int ObMemoryDump::check_sql_memory_leak()
{
int ret = OB_SUCCESS;
ObMemoryCheckContext memory_check_ctx;
ObThreadCond &cond = memory_check_ctx.cond_;
if (OB_FAIL(cond.init(ObWaitEventIds::DEFAULT_COND_WAIT))) {
LOG_WARN("thread cond init failed", K(ret));
} else if (OB_FAIL(generate_mod_stat_task(&memory_check_ctx))) {
LOG_WARN("generate mod stat task", K(ret));
} else {
ObThreadCondGuard guard(cond);
if (OB_FAIL(cond.wait())) {
LOG_WARN("thread condition wait failed", K(ret));
} else {
ret = memory_check_ctx.ret_;
}
}
return ret;
}
int ObMemoryDump::load_malloc_sample_map(ObMallocSampleMap &malloc_sample_map)
{
int ret = OB_SUCCESS;
@ -231,19 +270,10 @@ void ObMemoryDump::run1()
} else if (OB_ENTRY_NOT_EXIST == ret) {
int64_t current_ts = common::ObClockGenerator::getClock();
if (current_ts - last_dump_ts > STAT_LABEL_INTERVAL) {
auto *task = alloc_task();
if (OB_ISNULL(task)) {
LOG_WARN("alloc task failed");
} else {
task->type_ = STAT_LABEL;
if (OB_FAIL(push(task))){
LOG_WARN("push task failed", K(ret));
free_task(task);
}
}
generate_mod_stat_task();
last_dump_ts = current_ts;
} else {
ob_usleep(current_ts - last_dump_ts);
ob_usleep(1000);
}
}
}
@ -485,6 +515,13 @@ void ObMemoryDump::handle(void *task)
"tenant_id", "ctx_id", "chunk_cnt", "label_cnt",
"segv_cnt");
const int64_t start_ts = ObTimeUtility::current_time();
static bool has_memory_leak = false;
// sql memory leak can diagnose the memleak of object whose mem_version belongs to [min_check_version, max_check_version).
uint32_t min_check_version;
uint32_t max_check_version;
ObMemoryCheckContext *memory_check_ctx = m_task->memory_check_ctx_;
ObSqlMemoryLeakChecker::get_instance().update_check_range(NULL == memory_check_ctx || !memory_check_ctx->is_sql_memory_leak(),
min_check_version, max_check_version);
for (int tenant_idx = 0; tenant_idx < tenant_cnt; tenant_idx++) {
uint64_t tenant_id = tenant_ids_[tenant_idx];
for (int ctx_id = 0; ctx_id < ObCtxIds::MAX_CTX_ID; ctx_id++) {
@ -517,28 +554,28 @@ void ObMemoryDump::handle(void *task)
UNUSEDx(chunk, block);
return OB_SUCCESS;
},
[tenant_id, ctx_id, &lmap, w_stat, &item_used]
[tenant_id, ctx_id, &lmap, w_stat, &item_used, min_check_version, max_check_version]
(AChunk *chunk, ABlock *block, AObject *object) {
int ret = OB_SUCCESS;
if (object->in_use_) {
bool expect = AOBJECT_TAIL_MAGIC_CODE ==
reinterpret_cast<uint64_t&>(object->data_[object->alloc_bytes_]);
if (!expect && object->is_valid() && object->in_use_
&& REACH_TIME_INTERVAL(1 * 1000 * 1000)) {
ObLabel label = object->label();
char *ptr = object->data_;
int32_t length = object->alloc_bytes_;
LOG_INFO("tail magic maybe broken!!!", K(tenant_id), KP(ptr),
K(length), K(label));
if (OB_FAIL(label_stat(chunk, block, object, lmap, w_stat->up2date_items_,
ARRAYSIZEOF(w_stat->up2date_items_), item_used))) {
// do-nothing
} else if (OB_FAIL(malloc_sample_stat(tenant_id, ctx_id, object,
w_stat->malloc_sample_map_))) {
// do-nothing
} else if (!object->ignore_version_ &&
object->version_ >= min_check_version &&
object->version_ < max_check_version) {
has_memory_leak = true;
char bt[MAX_BACKTRACE_LENGTH] = {'\0'};
if (object->on_malloc_sample_) {
parray(bt, sizeof(bt), (int64_t*)&object->data_[object->alloc_bytes_ - AOBJECT_BACKTRACE_SIZE], AOBJECT_BACKTRACE_COUNT);
}
allow_next_syslog();
LOG_WARN("SQL_MEMORY_LEAK", KP(object), K(tenant_id), K(ctx_id), K(object->version_), K(object->label_), K(bt));
}
}
ret = label_stat(chunk, block, object, lmap,
w_stat->up2date_items_, ARRAYSIZEOF(w_stat->up2date_items_),
item_used);
if (OB_SUCC(ret)) {
ret = malloc_sample_stat(tenant_id, ctx_id,
object, w_stat->malloc_sample_map_);
}
return ret;
});
if (OB_FAIL(ret)) {
@ -593,7 +630,17 @@ void ObMemoryDump::handle(void *task)
ObLatchWGuard guard(iter_lock_, common::ObLatchIds::MEM_DUMP_ITER_LOCK);
std::swap(r_stat_, w_stat_);
}
ObFreeLogPrinter::get_instance().disable_free_log();
if (NULL != memory_check_ctx) {
if (memory_check_ctx->is_sql_memory_leak() && has_memory_leak) {
memory_check_ctx->ret_ = OB_ERR_UNEXPECTED;
has_memory_leak = false;
LOG_WARN("there has sql memory leak");
}
if (OB_FAIL(memory_check_ctx->cond_.signal())) {
LOG_WARN("failed to signal condition", K(ret));
}
memory_check_ctx = NULL;
}
} else {
int fd = -1;
if (-1 == (fd = ::open(LOG_FILE,

20
deps/oblib/src/lib/alloc/memory_dump.h vendored
View File

@ -45,6 +45,21 @@ enum DumpType
STAT_LABEL
};
struct ObMemoryCheckContext
{
enum CheckType
{
SQL_MEMORY_LEAK,
};
ObMemoryCheckContext(CheckType type = SQL_MEMORY_LEAK)
: ret_(OB_SUCCESS), type_(type), cond_()
{}
bool is_sql_memory_leak() { return SQL_MEMORY_LEAK == type_; }
int ret_;
CheckType type_;
ObThreadCond cond_;
};
class ObMemoryDumpTask
{
public:
@ -68,6 +83,9 @@ public:
void *p_chunk_;
};
};
struct {
ObMemoryCheckContext *memory_check_ctx_;
};
};
};
@ -189,6 +207,8 @@ public:
lib::ObMutexGuard guard(task_mutex_);
avaliable_task_set_ |= (1 << pos);
}
int generate_mod_stat_task(ObMemoryCheckContext *memory_check_ctx = NULL);
int check_sql_memory_leak();
int load_malloc_sample_map(lib::ObMallocSampleMap& malloc_sample_map);
private:
void run1() override;

7
deps/oblib/src/lib/alloc/ob_tenant_ctx_allocator.cpp vendored
View File

@ -460,6 +460,10 @@ void* ObTenantCtxAllocator::common_realloc(const void *ptr, const int64_t size,
if (obj != NULL) {
obj->on_malloc_sample_ = sample_allowed;
ob_malloc_sample_backtrace(obj, size);
obj->ignore_version_ = attr.ignore_version() || ObMemVersionNode::tl_ignore_node;
if (!obj->ignore_version_) {
obj->version_ = ObMemVersionNode::tl_node->version_;
}
nptr = obj->data_;
get_mem_leak_checker().on_alloc(*obj, attr);
SANITY_POISON(obj, AOBJECT_HEADER_SIZE);
@ -467,9 +471,6 @@ void* ObTenantCtxAllocator::common_realloc(const void *ptr, const int64_t size,
SANITY_POISON((void*)upper_align((int64_t)obj->data_ + size, 8),
alloc_size - size + sizeof(AOBJECT_TAIL_MAGIC_CODE));
} else if (TC_REACH_TIME_INTERVAL(1 * 1000 * 1000)) {
int level = ObFreeLogPrinter::get_level();
ObFreeLogPrinter::get_instance().enable_free_log(attr.tenant_id_,
attr.ctx_id_, level);
const char *msg = is_errsim ? "[ERRSIM] errsim inject memory error" : alloc_failed_msg();
LOG_DBA_WARN(OB_ALLOCATE_MEMORY_FAILED, "[OOPS]", "alloc failed reason", KCSTRING(msg));
_OB_LOG_RET(WARN, OB_ALLOCATE_MEMORY_FAILED, "oops, alloc failed, tenant_id=%ld, ctx_id=%ld, ctx_name=%s, ctx_hold=%ld, "

1
deps/oblib/src/lib/alloc/object_mgr.cpp vendored
View File

@ -247,6 +247,7 @@ SubObjectMgr *ObjectMgr::create_sub_mgr()
auto *obj = root_mgr.alloc_object(sizeof(SubObjectMgr), attr);
root_mgr.unlock();
if (OB_NOT_NULL(obj)) {
obj->ignore_version_ = true;
SANITY_UNPOISON(obj->data_, obj->alloc_bytes_);
sub_mgr = new (obj->data_) SubObjectMgr(ta_,
enable_no_log_,

2
deps/oblib/src/lib/alloc/object_set.cpp vendored
View File

@ -84,10 +84,8 @@ AObject *ObjectSet::alloc_object(
if (NULL != obj) {
abort_unless(obj->in_use_);
abort_unless(obj->is_valid());
reinterpret_cast<uint64_t&>(obj->data_[size]) = AOBJECT_TAIL_MAGIC_CODE;
obj->alloc_bytes_ = static_cast<uint32_t>(size);
if (attr.label_.str_ != nullptr) {
STRNCPY(&obj->label_[0], attr.label_.str_, sizeof(obj->label_));
obj->label_[sizeof(obj->label_) - 1] = '\0';

66
deps/oblib/src/lib/allocator/ob_sql_mem_leak_checker.cpp vendored Normal file
View File

@ -0,0 +1,66 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "ob_sql_mem_leak_checker.h"
namespace oceanbase
{
using namespace lib;
namespace common
{
ObSqlMemoryLeakChecker &ObSqlMemoryLeakChecker::get_instance()
{
static ObSqlMemoryLeakChecker instance;
return instance;
}
void ObSqlMemoryLeakChecker::add(ObMemVersionNode &node)
{
lib::ObMutexGuard guard(lock_);
dlist_.add_first(&node);
}
void ObSqlMemoryLeakChecker::remove(ObMemVersionNode &node)
{
lib::ObMutexGuard guard(lock_);
dlist_.remove(&node);
}
uint32_t ObSqlMemoryLeakChecker::get_min_version()
{
uint32_t version = UINT32_MAX;
ObMutexGuard guard(lock_);
DLIST_FOREACH_NORET(node, dlist_) {
if (node->version_ < version) {
version = node->version_;
}
}
return version;
}
void ObSqlMemoryLeakChecker::update_check_range(const bool need_delay,
uint32_t &min_check_version,
uint32_t &max_check_version)
{
uint32_t global_version = ATOMIC_AAF(&ObMemVersionNode::global_version, 1);
min_check_version = min_check_version_;
max_check_version = max_check_version_;
max_check_version_ = std::min(get_min_version(), global_version);
if (!need_delay) {
max_check_version = max_check_version_;
}
min_check_version_ = max_check_version;
COMMON_LOG(INFO, "update_check_range", K(min_check_version), K(max_check_version), K(global_version));
}
} // end namespace common
} // end namespace oceanbase

89
deps/oblib/src/lib/allocator/ob_sql_mem_leak_checker.h vendored Normal file
View File

@ -0,0 +1,89 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_SQL_MEM_LEAK_CHECKER_H_
#define OCEANBASE_SQL_MEM_LEAK_CHECKER_H_
#include "lib/lock/ob_mutex.h"
#include "lib/alloc/alloc_struct.h"
namespace oceanbase
{
namespace common
{
class ObSqlMemoryLeakChecker
{
public:
ObSqlMemoryLeakChecker()
: lock_(), dlist_()
{}
static ObSqlMemoryLeakChecker &get_instance();
void add(lib::ObMemVersionNode &node);
void remove(lib::ObMemVersionNode &node);
uint32_t get_min_version();
void update_check_range(const bool need_delay,
uint32_t &min_check_version,
uint32_t &max_check_version);
private:
uint32_t min_check_version_;
uint32_t max_check_version_;
lib::ObMutex lock_;
ObDList<lib::ObMemVersionNode> dlist_;
};
struct ObMemVersionNodeGuard
{
public:
ObMemVersionNodeGuard()
{
lib::ObMemVersionNode::tl_node = &node_;
ObSqlMemoryLeakChecker::get_instance().add(node_);
}
~ObMemVersionNodeGuard()
{
lib::ObMemVersionNode::tl_node = NULL;
ObSqlMemoryLeakChecker::get_instance().remove(node_);
}
private:
lib::ObMemVersionNode node_;
};
struct ObSqlMemoryLeakGuard
{
public:
ObSqlMemoryLeakGuard(const bool enable_memory_leak)
: enable_memory_leak_(enable_memory_leak && NULL != lib::ObMemVersionNode::tl_node),
ignore_node_(lib::ObMemVersionNode::tl_ignore_node)
{
lib::ObMemVersionNode::tl_ignore_node = !enable_memory_leak_;
if (enable_memory_leak) {
version_ = lib::ObMemVersionNode::tl_node->version_;
lib::ObMemVersionNode::tl_node->version_ = ATOMIC_LOAD(&lib::ObMemVersionNode::global_version);
}
}
~ObSqlMemoryLeakGuard()
{
lib::ObMemVersionNode::tl_ignore_node = ignore_node_;
if (enable_memory_leak_) {
lib::ObMemVersionNode::tl_node->version_ = version_;
}
}
private:
bool enable_memory_leak_;
bool ignore_node_;
uint32_t version_;
};
#define DISABLE_SQL_MEMLEAK_GUARD ::oceanbase::common::ObSqlMemoryLeakGuard sql_memory_leak_guard(false);
#define ENABLE_SQL_MEMLEAK_GUARD ::oceanbase::common::ObSqlMemoryLeakGuard sql_memory_leak_guard(true);
} // end namespace common
} // end namespace oceanbase
#endif

1
deps/oblib/src/lib/hash/ob_linear_hash_map.h vendored
View File

@ -642,6 +642,7 @@ int ObLinearHashMap<Key, Value, MemMgrTag>::HashMapMemMgrCore::init(const int64_
int ret = OB_SUCCESS;
dir_alloc_.set_attr(ObMemAttr(tenant_id, "LinearHashMapDi"));
cnter_alloc_.set_attr(ObMemAttr(tenant_id, "LinearHashMapCn"));
map_array_.set_attr(SET_USE_500("HashMapArray"));
// Init node alloc.
ret = node_alloc_.init(static_cast<int64_t>(sizeof(Node)),
SET_USE_500(ObMemAttr(tenant_id, "LinearHashMapNo")));

2
deps/oblib/src/lib/mysqlclient/ob_connection_allocator.h vendored
View File

@ -350,7 +350,7 @@ int ObLruConnectionAllocator<T>::init()
int ret = OB_SUCCESS;
if (is_inited_) {
// do nothing
} else if (OB_FAIL(sessionid_to_conns_map_.create(HASH_BUCKET_NUM, "sessid_conn_map"))) {
} else if (OB_FAIL(sessionid_to_conns_map_.create(HASH_BUCKET_NUM, SET_IGNORE_MEM_VERSION("sessid_conn_map")))) {
_OB_LOG(WARN, "failed to init hashmap sessionid_to_conns_map_, ret=%d", ret);
} else {
is_inited_ = true;

236
deps/oblib/src/lib/mysqlclient/ob_dblink_error_trans.cpp vendored
View File

@ -44,10 +44,17 @@ bool __attribute__((weak)) get_enable_dblink_cfg()
return true;
}
uint64_t __attribute__((weak)) get_current_tenant_id_for_dblink()
{
return oceanbase::OB_INVALID_ID;
}
namespace oceanbase
{
namespace common
{
namespace sqlclient
{
int sqlclient::ObDblinkErrorTrans::external_errno_to_ob_errno(bool is_oracle_err,
int external_errno,
@ -88,5 +95,234 @@ int sqlclient::ObDblinkErrorTrans::external_errno_to_ob_errno(bool is_oracle_err
return ret;
}
#ifdef OB_BUILD_DBLINK
int ObTenantDblinkKeeper::clean_dblink_conn(uint32_t sessid, bool force_disconnect)
{
int ret = OB_SUCCESS;
int64_t value = 0;
obsys::ObRLockGuard wg(lock_);
if (!dblink_conn_map_.created()) {
ret = OB_NOT_INIT;
LOG_WARN("dblink_conn_map_ is not inited", K(ret), K(tenant_id_), K(sessid));
} else if (OB_FAIL(dblink_conn_map_.get_refactored(sessid, value))) {
if (OB_HASH_NOT_EXIST == ret) {
ret = OB_SUCCESS;
} else {
LOG_WARN("failed to get connection from map", K(ret), K(sessid));
}
} else if (0 == value) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("value is 0", K(ret), K(value), K(sessid));
} else {
common::sqlclient::ObISQLConnection *connection = reinterpret_cast<common::sqlclient::ObISQLConnection *>(value);
while (OB_NOT_NULL(connection) && OB_SUCC(ret)) {
common::sqlclient::ObISQLConnection *next = connection->get_next_conn();
connection->dblink_wlock(); //prevent connection still in use
connection->set_reverse_link_creadentials(false);
common::sqlclient::ObCommonServerConnectionPool * server_conn_pool = connection->get_common_server_pool();
const bool need_disconnect = force_disconnect || !connection->usable();
if (NULL == server_conn_pool) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("server_conn_pool of dblink connection is NULL", K(tenant_id_), K(sessid), K(connection), K(ret));
} else if (OB_FAIL(server_conn_pool->release(connection, !need_disconnect))) {
LOG_WARN("session failed to release dblink connection", K(tenant_id_), K(sessid), K(connection), K(ret));
} else {
LOG_TRACE("session succ to release dblink connection", K(tenant_id_), K(sessid), K(connection), K(ret));
}
connection->dblink_unwlock();
connection = next;
}
if (OB_SUCC(ret) && OB_FAIL(dblink_conn_map_.erase_refactored(sessid))) {
LOG_WARN("failed to erase_refactored", K(tenant_id_), K(sessid), K(ret));;
}
}
return ret;
}
int ObTenantDblinkKeeper::set_dblink_conn(uint32_t sessid, common::sqlclient::ObISQLConnection *dblink_conn)
{
int ret = OB_SUCCESS;
obsys::ObRLockGuard wg(lock_);
if (!dblink_conn_map_.created()) {
ret = OB_NOT_INIT;
LOG_WARN("dblink_conn_map_ is not inited", K(ret), K(tenant_id_), K(sessid));
} else if (OB_ISNULL(dblink_conn)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null ptr", K(ret), K(tenant_id_), K(sessid));
} else {
int64_t value = 0;
if (OB_FAIL(dblink_conn_map_.get_refactored(sessid, value))) {
if (OB_HASH_NOT_EXIST == ret) {
ret = OB_SUCCESS;
if (OB_FAIL(dblink_conn_map_.set_refactored(sessid, reinterpret_cast<int64_t>(dblink_conn)))) {
LOG_WARN("failed to set refactored", K(ret), K(tenant_id_), K(sessid));
} else {
dblink_conn->set_next_conn(NULL);
}
} else {
LOG_WARN("failed to get connection from map", K(ret), K(sessid));
}
} else if (0 == value) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("value is 0", K(ret), K(value), K(sessid));
} else {
common::sqlclient::ObISQLConnection *connection = reinterpret_cast<common::sqlclient::ObISQLConnection *>(value);
while (OB_NOT_NULL(connection)) {
if (dblink_conn == connection) {
break;
}
connection = connection->get_next_conn();
}
if (OB_NOT_NULL(connection)) {
//do nothing
} else {
common::sqlclient::ObISQLConnection *header = reinterpret_cast<common::sqlclient::ObISQLConnection *>(value);
common::sqlclient::ObISQLConnection *temp = header->get_next_conn();
header->set_next_conn(dblink_conn);
dblink_conn->set_next_conn(temp);
LOG_TRACE("session succ to hold a dblink connection", KP(dblink_conn), K(sessid), K(tenant_id_), K(ret));
}
}
}
return ret;
}
int ObTenantDblinkKeeper::get_dblink_conn(uint32_t sessid, uint64_t dblink_id,
common::sqlclient::ObISQLConnection *&dblink_conn)
{
int ret = OB_SUCCESS;
dblink_conn = NULL;
ObArray<int64_t> *dblink_conn_array = NULL;
int64_t value = 0;
obsys::ObRLockGuard wg(lock_);
if (!dblink_conn_map_.created()) {
ret = OB_NOT_INIT;
LOG_WARN("dblink_conn_map_ is not inited", K(ret), K(tenant_id_), K(sessid));
} else if (OB_FAIL(dblink_conn_map_.get_refactored(sessid, value))) {
if (OB_HASH_NOT_EXIST == ret) {
ret = OB_SUCCESS;
} else {
LOG_WARN("failed to get connection", K(ret), K(sessid));
}
} else if (0 == value) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("value is NULL", K(ret), K(value), K(sessid));
} else {
common::sqlclient::ObISQLConnection *connection = reinterpret_cast<common::sqlclient::ObISQLConnection *>(value);
common::sqlclient::ObISQLConnection *temp_conn = NULL;
while (OB_NOT_NULL(connection)) {
if (dblink_id == connection->get_dblink_id()) {
temp_conn = connection;
break;
}
connection = connection->get_next_conn();
}
if (OB_SUCC(ret) && OB_NOT_NULL(temp_conn)) {
if (OB_SUCCESS != temp_conn->ping()) {
ret = OB_ERR_DBLINK_SESSION_KILLED;
LOG_WARN("connection is invalid", K(ret), K(temp_conn->usable()), KP(temp_conn), K(sessid), K(tenant_id_));
} else {
dblink_conn = temp_conn;
dblink_conn->dblink_rlock();
}
}
LOG_TRACE("session get a dblink connection", K(ret), K(dblink_id), K(tenant_id_), KP(dblink_conn), K(sessid));
}
return ret;
}
int ObTenantDblinkKeeper::init(uint64_t tenant_id)
{
static int SESSION_COUNT = 1024;
int ret = OB_SUCCESS;
tenant_id_ = tenant_id;
ObMemAttr attr(tenant_id, "DblinkKeeperBkt");
if (OB_FAIL(dblink_conn_map_.create(SESSION_COUNT, attr, attr))) {
LOG_WARN("fail init pool map", K(ret), K(tenant_id));
}
return ret;
}
int ObTenantDblinkKeeper::mtl_new(ObTenantDblinkKeeper *&dblink_keeper)
{
int ret = OB_SUCCESS;
uint64_t tenant_id = tenant_id = get_current_tenant_id_for_dblink();
if (OB_INVALID_ID == tenant_id) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid tenant id", K(ret), KP(get_current_tenant_id_for_dblink));
} else if (FALSE_IT(dblink_keeper = OB_NEW(ObTenantDblinkKeeper, ObMemAttr(tenant_id, "DblinkKeeper")))) {
} else if (OB_ISNULL(dblink_keeper)) {
ret = common::OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("failed to alloc meme", K(ret));
}
return ret;
}
int ObTenantDblinkKeeper::mtl_init(ObTenantDblinkKeeper *&dblink_keeper)
{
int ret = OB_SUCCESS;
uint64_t tenant_id = tenant_id = get_current_tenant_id_for_dblink();
if (OB_INVALID_ID == tenant_id) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("invalid tenant id", K(ret), KP(get_current_tenant_id_for_dblink));
} else if (OB_ISNULL(dblink_keeper)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null ptr", K(ret));
} else {
dblink_keeper->init(tenant_id);
LOG_TRACE("init dblink_keeper", K(tenant_id), KP(dblink_keeper), K(ret));
}
return ret;
}
void ObTenantDblinkKeeper::mtl_destroy(ObTenantDblinkKeeper *&dblink_keeper)
{
common::ob_delete(dblink_keeper);
dblink_keeper = nullptr;
}
int ObTenantDblinkKeeper::destroy()
{
int ret = OB_SUCCESS;
CleanDblinkArrayFunc clean_dblink_func;
if (OB_FAIL(dblink_conn_map_.foreach_refactored(clean_dblink_func))) {
LOG_WARN("failed to do foreach", K(ret));
} else {
dblink_conn_map_.destroy();
tenant_id_ = OB_INVALID_ID;
}
return ret;
}
int ObTenantDblinkKeeper::CleanDblinkArrayFunc::operator() (common::hash::HashMapPair<uint32_t, int64_t> &kv)
{
int ret = OB_SUCCESS;
common::sqlclient::ObISQLConnection *connection = reinterpret_cast<common::sqlclient::ObISQLConnection *>(kv.second);
while (OB_SUCC(ret) && OB_NOT_NULL(connection)) {
common::sqlclient::ObISQLConnection *next = connection->get_next_conn();
connection->set_reverse_link_creadentials(false);
common::sqlclient::ObCommonServerConnectionPool * server_conn_pool = NULL;
server_conn_pool = connection->get_common_server_pool();
uint32_t sessid = connection->get_sessid();
if (NULL == server_conn_pool) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("server_conn_pool of dblink connection is NULL", K(sessid), KP(connection), K(ret));
} else {
if (OB_FAIL(server_conn_pool->release(connection, false))) {
LOG_WARN("session failed to release dblink connection", K(sessid), KP(connection), K(ret));
} else {
LOG_TRACE("session succ to release dblink connection", K(sessid), KP(connection), K(ret));
}
}
connection = next;
}
if (OB_SUCC(ret)) {
kv.second = 0;
}
return ret;
}
#endif
} // end namespace sqlclient
} // end namespace common
} // end namespace oceanbase

40
deps/oblib/src/lib/mysqlclient/ob_dblink_error_trans.h vendored
View File

@ -14,9 +14,11 @@
#define OBDBLINKERROR_H
#include "lib/utility/ob_edit_distance.h"
#include "lib/ob_errno.h"
#include "lib/mysqlclient/ob_isql_connection.h"
extern bool get_dblink_reuse_connection_cfg();
extern bool get_enable_dblink_cfg();
extern uint64_t get_current_tenant_id_for_dblink();
namespace oceanbase
{
@ -56,6 +58,44 @@ public:
int &ob_errno);
};
#ifdef OB_BUILD_DBLINK
class ObTenantDblinkKeeper
{
public:
class CleanDblinkArrayFunc
{
public:
CleanDblinkArrayFunc() {}
virtual ~CleanDblinkArrayFunc() = default;
int operator()(common::hash::HashMapPair<uint32_t, int64_t> &kv);
};
public:
static int mtl_new(ObTenantDblinkKeeper *&dblink_keeper);
static int mtl_init(ObTenantDblinkKeeper *&dblink_keeper);
static void mtl_destroy(ObTenantDblinkKeeper *&dblink_keeper);
public:
ObTenantDblinkKeeper()
{
tenant_id_ = common::OB_INVALID_ID;
}
~ObTenantDblinkKeeper()
{
destroy();
}
int init(uint64_t tenant_id);
int set_dblink_conn(uint32_t sessid, common::sqlclient::ObISQLConnection *dblink_conn);
int get_dblink_conn(uint32_t sessid, uint64_t dblink_id,
common::sqlclient::ObISQLConnection *&dblink_conn);
int clean_dblink_conn(uint32_t sessid, bool force_disconnect);
private:
int destroy();
private:
uint64_t tenant_id_;
obsys::ObRWLock lock_;
hash::ObHashMap<uint32_t, int64_t> dblink_conn_map_;
};
#endif
} // namespace sqlclient
} // namespace common
} // namespace oceanbase

17
deps/oblib/src/lib/mysqlclient/ob_isql_connection.h vendored
View File

@ -100,7 +100,8 @@ public:
last_set_sql_mode_cstr_buf_size_(0),
last_set_client_charset_cstr_(NULL),
last_set_connection_charset_cstr_(NULL),
last_set_results_charset_cstr_(NULL)
last_set_results_charset_cstr_(NULL),
next_conn_(NULL)
{}
virtual ~ObISQLConnection() {
allocator_.reset();
@ -109,6 +110,7 @@ public:
last_set_client_charset_cstr_ = NULL;
last_set_connection_charset_cstr_ = NULL;
last_set_results_charset_cstr_ = NULL;
next_conn_ = NULL;
}
// sql execute interface
@ -239,11 +241,13 @@ public:
}
if (param_ctx.set_client_charset_cstr_ != last_set_client_charset_cstr_ ||
param_ctx.set_connection_charset_cstr_ != last_set_connection_charset_cstr_ ||
param_ctx.set_results_charset_cstr_ != last_set_results_charset_cstr_) {
param_ctx.set_results_charset_cstr_ != last_set_results_charset_cstr_ ||
param_ctx.set_transaction_isolation_cstr_ != last_set_transaction_isolation_cstr_) {
is_inited = false;
last_set_client_charset_cstr_ = param_ctx.set_client_charset_cstr_;
last_set_connection_charset_cstr_ = param_ctx.set_connection_charset_cstr_;
last_set_results_charset_cstr_ = param_ctx.set_results_charset_cstr_;
last_set_transaction_isolation_cstr_ = param_ctx.set_transaction_isolation_cstr_;
}
return ret;
}
@ -254,6 +258,12 @@ public:
void set_usable(bool flag) { usable_ = flag; }
bool usable() { return usable_; }
virtual int ping() { return OB_SUCCESS; }
void dblink_rlock() { dblink_lock_.rlock()->lock(); }
void dblink_unrlock() { dblink_lock_.rlock()->unlock(); }
void dblink_wlock() { dblink_lock_.wlock()->lock(); }
void dblink_unwlock() { dblink_lock_.wlock()->unlock(); }
ObISQLConnection *get_next_conn() { return next_conn_; }
void set_next_conn(ObISQLConnection *conn) { next_conn_ = conn; }
protected:
bool oracle_mode_;
bool is_inited_; // for oracle dblink, we have to init remote env with some sql
@ -268,7 +278,10 @@ protected:
const char *last_set_client_charset_cstr_;
const char *last_set_connection_charset_cstr_;
const char *last_set_results_charset_cstr_;
const char *last_set_transaction_isolation_cstr_;
common::ObArenaAllocator allocator_;
obsys::ObRWLock dblink_lock_;
ObISQLConnection *next_conn_; // used in dblink_conn_map_
};
} // end namespace sqlclient

7
deps/oblib/src/lib/mysqlclient/ob_isql_connection_pool.h vendored
View File

@ -75,6 +75,7 @@ struct dblink_param_ctx{
const char *set_client_charset_cstr_;
const char *set_connection_charset_cstr_;
const char *set_results_charset_cstr_;
const char *set_transaction_isolation_cstr_;
dblink_param_ctx() :
charset_id_(static_cast<uint16_t>(common::ObNlsCharsetId::CHARSET_AL32UTF8_ID)), //utf8, deault value, don't modify it cause dblink pull meta need it
ncharset_id_(static_cast<uint16_t>(common::ObNlsCharsetId::CHARSET_AL32UTF8_ID)), //utf8, deault value, don't modify it cause dblink pull meta need it
@ -87,7 +88,8 @@ struct dblink_param_ctx{
set_sql_mode_cstr_(NULL),
set_client_charset_cstr_(NULL),
set_connection_charset_cstr_(NULL),
set_results_charset_cstr_(NULL)
set_results_charset_cstr_(NULL),
set_transaction_isolation_cstr_(NULL)
{ }
TO_STRING_KV(K_(charset_id),
K_(ncharset_id),
@ -100,7 +102,8 @@ struct dblink_param_ctx{
K_(set_sql_mode_cstr),
K_(set_client_charset_cstr),
K_(set_connection_charset_cstr),
K_(set_results_charset_cstr));
K_(set_results_charset_cstr),
K_(set_transaction_isolation_cstr));
};
class ObISQLConnectionPool

27
deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp vendored
View File

@ -461,19 +461,23 @@ int ObDbLinkProxy::create_dblink_pool(const dblink_param_ctx &param_ctx, const O
int ObDbLinkProxy::acquire_dblink(const dblink_param_ctx &param_ctx, ObISQLConnection *&dblink_conn)
{
int ret = OB_SUCCESS;
DISABLE_SQL_MEMLEAK_GUARD;
ObISQLConnectionPool *dblink_pool = NULL;
ObISQLConnection * conn = NULL;
if (!is_inited()) {
ret = OB_NOT_INIT;
LOG_WARN("dblink proxy not inited");
} else if (OB_FAIL(switch_dblink_conn_pool(param_ctx.link_type_, dblink_pool))) {
LOG_WARN("failed to get dblink interface", K(ret), K(param_ctx));
} else if (OB_FAIL(dblink_pool->acquire_dblink(param_ctx, dblink_conn))) {
} else if (OB_FAIL(dblink_pool->acquire_dblink(param_ctx, conn))) {
LOG_WARN("acquire dblink failed", K(ret), K(param_ctx));
} else if (OB_FAIL(prepare_enviroment(param_ctx, dblink_conn))) {
} else if (OB_FAIL(prepare_enviroment(param_ctx, conn))) {
LOG_WARN("failed to prepare dblink env", K(ret));
} else {
dblink_conn->set_dblink_id(param_ctx.dblink_id_);
dblink_conn->set_dblink_driver_proto(param_ctx.link_type_);
conn->set_dblink_id(param_ctx.dblink_id_);
conn->set_dblink_driver_proto(param_ctx.link_type_);
conn->set_next_conn(NULL);
dblink_conn = conn;
}
return ret;
}
@ -509,7 +513,8 @@ int ObDbLinkProxy::execute_init_sql(const sqlclient::dblink_param_ctx &param_ctx
sql_ptr_type sql_ptr[] = {param_ctx.set_sql_mode_cstr_,
param_ctx.set_client_charset_cstr_,
param_ctx.set_connection_charset_cstr_,
param_ctx.set_results_charset_cstr_};
param_ctx.set_results_charset_cstr_,
param_ctx.set_transaction_isolation_cstr_};
ObMySQLStatement stmt;
ObMySQLConnection *mysql_conn = static_cast<ObMySQLConnection *>(dblink_conn);
for (int i = 0; OB_SUCC(ret) && i < sizeof(sql_ptr) / sizeof(sql_ptr_type); ++i) {
@ -528,6 +533,7 @@ int ObDbLinkProxy::execute_init_sql(const sqlclient::dblink_param_ctx &param_ctx
param_ctx.set_client_charset_cstr_,
param_ctx.set_connection_charset_cstr_,
param_ctx.set_results_charset_cstr_,
param_ctx.set_transaction_isolation_cstr_,
"set nls_date_format='YYYY-MM-DD HH24:MI:SS'",
"set nls_timestamp_format = 'YYYY-MM-DD HH24:MI:SS.FF'",
"set nls_timestamp_tz_format = 'YYYY-MM-DD HH24:MI:SS.FF TZR TZD'"
@ -542,8 +548,6 @@ int ObDbLinkProxy::execute_init_sql(const sqlclient::dblink_param_ctx &param_ctx
LOG_WARN("create statement failed", K(ret), K(param_ctx));
} else if (OB_FAIL(stmt.execute_update())) {
LOG_WARN("execute sql failed", K(ret), K(param_ctx));
} else {
// do nothing
}
}
}
@ -579,10 +583,15 @@ int ObDbLinkProxy::execute_init_sql(const sqlclient::dblink_param_ctx &param_ctx
int ObDbLinkProxy::release_dblink(/*uint64_t dblink_id,*/ DblinkDriverProto dblink_type, ObISQLConnection *dblink_conn)
{
int ret = OB_SUCCESS;
DISABLE_SQL_MEMLEAK_GUARD;
ObISQLConnectionPool *dblink_pool = NULL;
if (!is_inited()) {
if (OB_ISNULL(dblink_conn)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexcepted null ptr", K(ret));
} else if (FALSE_IT(dblink_conn->set_next_conn(NULL))) {
} else if (!is_inited()) {
ret = OB_NOT_INIT;
LOG_WARN("dblink proxy not inited");
LOG_WARN("dblink proxy not inited", K(ret));
} else if (OB_FAIL(switch_dblink_conn_pool(dblink_type, dblink_pool))) {
LOG_WARN("failed to get dblink interface", K(ret));
} else if (OB_FAIL(dblink_pool->release_dblink(dblink_conn))) {

1
deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h vendored
View File

@ -13,6 +13,7 @@
#ifndef OCEANBASE_MYSQL_PROXY_H_
#define OCEANBASE_MYSQL_PROXY_H_
#include "lib/allocator/ob_sql_mem_leak_checker.h"
#include "lib/mysqlclient/ob_isql_client.h"
#include "lib/mysqlclient/ob_mysql_result.h"
#include "lib/mysqlclient/ob_mysql_statement.h"

2
deps/oblib/src/lib/mysqlclient/ob_mysql_statement.cpp vendored
View File

@ -162,7 +162,7 @@ ObMySQLResult *ObMySQLStatement::execute_query(bool enable_use_result)
"err_msg", errmsg, K(ret), K(sql_str_));
} else {
LOG_WARN("fail to query server", "host", stmt_->host, "port", stmt_->port, K(conn_->get_sessid()),
"err_msg", errmsg, K(ret), K(STRLEN(sql_str_)), K(sql_str_));
"err_msg", errmsg, K(ret), K(STRLEN(sql_str_)), K(sql_str_), K(lbt()));
}
if (OB_SUCCESS == ret) {
ret = OB_ERR_SQL_CLIENT;

9
deps/oblib/src/lib/mysqlclient/ob_tenant_oci_envs.h vendored
View File

@ -26,9 +26,16 @@ public:
static int mtl_init(ObTenantOciEnvs *&tenant_oci_envs) { return OB_SUCCESS; }
static void mtl_destroy(ObTenantOciEnvs *&tenant_oci_envs) { }
};
class ObTenantDblinkKeeper
{
public:
static int mtl_new(ObTenantDblinkKeeper *&dblink_keeper) { return OB_SUCCESS; }
static int mtl_init(ObTenantDblinkKeeper *&dblink_keeper) { return OB_SUCCESS; }
static void mtl_destroy(ObTenantDblinkKeeper *&dblink_keeper) { }
};
#endif
} //sqlclient
} // namespace common
} // namespace oceanbase
#endif //OBTENANOCIENVS_H
#endif //OBTENANOCIENVS_H

1
deps/oblib/src/lib/number/ob_number_v2.cpp vendored
View File

@ -3058,6 +3058,7 @@ const char *ObNumber::format() const
if (OB_ISNULL(buffers)) {
buffer = nullptr;
} else if(OB_UNLIKELY(OB_SUCCESS != format(buffer, BUFFER_SIZE, length, -1))) {
buffer = nullptr;
LOG_ERROR_RET(OB_ERROR, "fail to format buffer");
} else {
buffer[length] = '\0';

1
deps/oblib/src/lib/ob_name_def.h vendored
View File

@ -1076,4 +1076,5 @@
#define N_INNER_DECODE_LIKE "inner_decode_like"
#define N_EXTRACT_CERT_EXPIRED_TIME "extract_cert_expired_time"
#define N_INNER_ROW_CMP_VALUE "inner_row_cmp_value"
#define N_SYS_LAST_REFRESH_SCN "last_refresh_scn"
#endif //OCEANBASE_LIB_OB_NAME_DEF_H_

246
deps/oblib/src/lib/resource/achunk_mgr.cpp vendored
View File

@ -59,13 +59,15 @@ AChunkMgr &AChunkMgr::instance()
}
AChunkMgr::AChunkMgr()
: free_list_(), large_free_list_(),
chunk_bitmap_(nullptr), limit_(DEFAULT_LIMIT), urgent_(0), hold_(0),
total_hold_(0), maps_(0), unmaps_(0), large_maps_(0), large_unmaps_(0),
huge_maps_(0), huge_unmaps_(0),
shadow_hold_(0)
: limit_(DEFAULT_LIMIT), urgent_(0), hold_(0),
total_hold_(0), used_(0), shadow_hold_(0),
max_chunk_cache_size_(limit_)
{
large_free_list_.set_max_chunk_cache_size(0);
// only cache normal_chunk or large_chunk
for (int i = 0; i < ARRAYSIZEOF(slots_); ++i) {
new (slots_ + i) Slot();
}
slots_[HUGE_ACHUNK_INDEX]->set_max_chunk_cache_size(0);
}
void *AChunkMgr::direct_alloc(const uint64_t size, const bool can_use_huge_page, bool &huge_page_used, const bool alloc_shadow)
@ -96,14 +98,8 @@ void *AChunkMgr::direct_alloc(const uint64_t size, const bool can_use_huge_page,
// aligned address returned
}
}
if (ptr != nullptr) {
ATOMIC_FAA(&maps_, 1);
if (size > LARGE_ACHUNK_SIZE) {
ATOMIC_FAA(&huge_maps_, 1);
} else if (size > NORMAL_ACHUNK_SIZE) {
ATOMIC_FAA(&large_maps_, 1);
}
ATOMIC_FAA(&get_slot(size).maps_, 1);
IGNORE_RETURN ATOMIC_FAA(&total_hold_, size);
} else {
LOG_ERROR_RET(OB_ALLOCATE_MEMORY_FAILED, "low alloc fail", K(size), K(orig_errno), K(errno));
@ -120,12 +116,7 @@ void AChunkMgr::direct_free(const void *ptr, const uint64_t size)
{
common::ObTimeGuard time_guard(__func__, 1000 * 1000);
ATOMIC_FAA(&unmaps_, 1);
if (size > LARGE_ACHUNK_SIZE) {
ATOMIC_FAA(&huge_unmaps_, 1);
} else if (size > NORMAL_ACHUNK_SIZE) {
ATOMIC_FAA(&large_unmaps_, 1);
}
ATOMIC_FAA(&get_slot(size).unmaps_, 1);
IGNORE_RETURN ATOMIC_FAA(&total_hold_, -size);
low_free(ptr, size);
}
@ -215,94 +206,39 @@ AChunk *AChunkMgr::alloc_chunk(const uint64_t size, bool high_prio)
const int64_t all_size = aligned(size);
AChunk *chunk = nullptr;
if (NORMAL_ACHUNK_SIZE == all_size) {
// TODO by fengshuo.fs: chunk cached by freelist may not use all memory in it,
// so update_hold can use hold_size too.
if (free_list_.count() > 0) {
chunk = free_list_.pop();
}
if (OB_ISNULL(chunk)) {
bool updated = false;
while (!(updated = update_hold(hold_size, high_prio)) && large_free_list_.count() > 0) {
if (OB_NOT_NULL(chunk = large_free_list_.pop())) {
int64_t all_size = chunk->aligned();
int64_t hold_size = chunk->hold();
direct_free(chunk, all_size);
IGNORE_RETURN update_hold(-hold_size, false);
chunk = nullptr;
}
}
if (updated) {
bool hugetlb_used = false;
void *ptr = direct_alloc(all_size, true, hugetlb_used, SANITY_BOOL_EXPR(true));
if (ptr != nullptr) {
chunk = new (ptr) AChunk();
chunk->is_hugetlb_ = hugetlb_used;
} else {
IGNORE_RETURN update_hold(-hold_size, false);
}
}
}
} else if (LARGE_ACHUNK_SIZE == all_size) {
if (large_free_list_.count() > 0) {
chunk = large_free_list_.pop();
}
if (chunk != NULL) {
int64_t orig_hold_size = chunk->hold();
if (hold_size == orig_hold_size) {
// do-nothing
} else if (hold_size > orig_hold_size) {
if (!update_hold(hold_size - orig_hold_size, high_prio)) {
direct_free(chunk, all_size);
IGNORE_RETURN update_hold(-orig_hold_size, false);
chunk = nullptr;
}
} else {
int result = 0;
do {
result = this->madvise((char*)chunk + hold_size, orig_hold_size - hold_size, MADV_DONTNEED);
} while (result == -1 && errno == EAGAIN);
if (-1 == result) {
LOG_WARN_RET(OB_ERR_SYS, "madvise failed", K(errno));
direct_free(chunk, all_size);
IGNORE_RETURN update_hold(-orig_hold_size, false);
chunk = nullptr;
} else {
IGNORE_RETURN update_hold(hold_size - orig_hold_size, false);
}
}
}
if (OB_ISNULL(chunk)) {
bool updated = false;
while (!(updated = update_hold(hold_size, high_prio)) && free_list_.count() > 0) {
if (OB_NOT_NULL(chunk = free_list_.pop())) {
int64_t all_size = chunk->aligned();
int64_t hold_size = chunk->hold();
direct_free(chunk, all_size);
IGNORE_RETURN update_hold(-hold_size, false);
chunk = nullptr;
}
}
if (updated) {
bool hugetlb_used = false;
void *ptr = direct_alloc(all_size, true, hugetlb_used, SANITY_BOOL_EXPR(true));
if (ptr != nullptr) {
chunk = new (ptr) AChunk();
chunk->is_hugetlb_ = hugetlb_used;
} else {
IGNORE_RETURN update_hold(-hold_size, false);
}
}
}
} else {
bool updated = false;
while (!(updated = update_hold(hold_size, high_prio)) &&
(free_list_.count() > 0 || large_free_list_.count() > 0)) {
if (OB_NOT_NULL(chunk = free_list_.pop()) || OB_NOT_NULL(chunk = large_free_list_.pop())) {
int64_t all_size = chunk->aligned();
int64_t hold_size = chunk->hold();
// Reuse chunk from self-cache
if (OB_NOT_NULL(chunk = get_slot(all_size)->pop())) {
int64_t orig_hold_size = chunk->hold();
if (hold_size == orig_hold_size) {
// do-nothing
} else if (hold_size > orig_hold_size) {
if (!update_hold(hold_size - orig_hold_size, high_prio)) {
direct_free(chunk, all_size);
IGNORE_RETURN update_hold(-hold_size, false);
IGNORE_RETURN update_hold(-orig_hold_size, false);
chunk = nullptr;
}
} else {
int result = this->madvise((char*)chunk + hold_size, orig_hold_size - hold_size, MADV_DONTNEED);
if (-1 == result) {
LOG_WARN_RET(OB_ERR_SYS, "madvise failed", K(errno));
direct_free(chunk, all_size);
IGNORE_RETURN update_hold(-orig_hold_size, false);
chunk = nullptr;
} else {
IGNORE_RETURN update_hold(hold_size - orig_hold_size, false);
}
}
}
if (OB_ISNULL(chunk)) {
bool updated = false;
for (int i = MAX_LARGE_ACHUNK_INDEX; !updated && i >= 0; --i) {
while (!(updated = update_hold(hold_size, high_prio)) &&
OB_NOT_NULL(chunk = slots_[i]->pop())) {
// Wash chunk from all-cache when observer's hold reaches limit
int64_t orig_all_size = chunk->aligned();
int64_t orig_hold_size = chunk->hold();
direct_free(chunk, orig_all_size);
IGNORE_RETURN update_hold(-orig_hold_size, false);
chunk = nullptr;
}
}
@ -317,8 +253,8 @@ AChunk *AChunkMgr::alloc_chunk(const uint64_t size, bool high_prio)
}
}
}
if (OB_NOT_NULL(chunk)) {
IGNORE_RETURN ATOMIC_AAF(&used_, hold_size);
chunk->alloc_bytes_ = size;
SANITY_UNPOISON(chunk, all_size); // maybe no need?
} else if (REACH_TIME_INTERVAL(1 * 1000 * 1000)) {
@ -334,24 +270,18 @@ void AChunkMgr::free_chunk(AChunk *chunk)
if (OB_NOT_NULL(chunk)) {
const int64_t hold_size = chunk->hold();
const uint64_t all_size = chunk->aligned();
IGNORE_RETURN ATOMIC_AAF(&used_, -hold_size);
const double max_large_cache_ratio = 0.5;
int64_t max_large_cache_size = min(limit_ - used_, max_chunk_cache_size_) * max_large_cache_ratio;
const int64_t cache_hold = hold_ - used_;
const int64_t large_cache_hold = cache_hold - slots_[NORMAL_ACHUNK_INDEX]->hold();
bool freed = true;
if (NORMAL_ACHUNK_SIZE == hold_size) {
if (hold_ + hold_size <= limit_) {
freed = !free_list_.push(chunk);
}
if (freed) {
direct_free(chunk, all_size);
IGNORE_RETURN update_hold(-hold_size, false);
}
} else if (LARGE_ACHUNK_SIZE == all_size) {
if (hold_ + hold_size <= limit_) {
freed = !large_free_list_.push(chunk);
}
if (freed) {
direct_free(chunk, all_size);
IGNORE_RETURN update_hold(-hold_size, false);
}
} else {
if (cache_hold + hold_size <= max_chunk_cache_size_
&& (NORMAL_ACHUNK_SIZE == all_size || large_cache_hold <= max_large_cache_size)
&& 0 == chunk->washed_size_) {
freed = !get_slot(all_size)->push(chunk);
}
if (freed) {
direct_free(chunk, all_size);
IGNORE_RETURN update_hold(-hold_size, false);
}
@ -365,9 +295,9 @@ AChunk *AChunkMgr::alloc_co_chunk(const uint64_t size)
AChunk *chunk = nullptr;
bool updated = false;
while (!(updated = update_hold(hold_size, true)) &&
(free_list_.count() > 0 || large_free_list_.count() > 0)) {
if (OB_NOT_NULL(chunk = free_list_.pop()) || OB_NOT_NULL(chunk = large_free_list_.pop())) {
for (int i = MAX_LARGE_ACHUNK_INDEX; !updated && i >= 0; --i) {
while (!(updated = update_hold(hold_size, true)) &&
OB_NOT_NULL(chunk = slots_[i]->pop())) {
int64_t all_size = chunk->aligned();
int64_t hold_size = chunk->hold();
direct_free(chunk, all_size);
@ -388,6 +318,7 @@ AChunk *AChunkMgr::alloc_co_chunk(const uint64_t size)
}
if (OB_NOT_NULL(chunk)) {
IGNORE_RETURN ATOMIC_AAF(&used_, hold_size);
chunk->alloc_bytes_ = size;
//SANITY_UNPOISON(chunk, all_size); // maybe no need?
} else if (REACH_TIME_INTERVAL(1 * 1000 * 1000)) {
@ -403,6 +334,7 @@ void AChunkMgr::free_co_chunk(AChunk *chunk)
if (OB_NOT_NULL(chunk)) {
const int64_t hold_size = chunk->hold();
const uint64_t all_size = chunk->aligned();
IGNORE_RETURN ATOMIC_AAF(&used_, -hold_size);
direct_free(chunk, all_size);
IGNORE_RETURN update_hold(-hold_size, false);
}
@ -435,17 +367,65 @@ bool AChunkMgr::update_hold(int64_t bytes, bool high_prio)
int AChunkMgr::madvise(void *addr, size_t length, int advice)
{
return ::madvise(addr, length, advice);
int result = 0;
if (length > 0) {
do {
result = ::madvise(addr, length, advice);
} while (result == -1 && errno == EAGAIN);
}
return result;
}
int64_t AChunkMgr::to_string(char *buf, const int64_t buf_len) const
{
int ret = OB_SUCCESS;
int64_t pos = 0;
int64_t resident_size = 0;
int64_t memory_used = get_virtual_memory_used(&resident_size);
int64_t large_maps = 0;
int64_t large_unmaps = 0;
for (int i = MIN_LARGE_ACHUNK_INDEX; i <= MAX_LARGE_ACHUNK_INDEX; ++i) {
large_maps += slots_[i].maps_;
large_unmaps += slots_[i].unmaps_;
}
int64_t total_maps = slots_[NORMAL_ACHUNK_INDEX].maps_ + large_maps + slots_[HUGE_ACHUNK_INDEX].maps_;
int64_t total_unmaps = slots_[NORMAL_ACHUNK_INDEX].unmaps_ + large_unmaps + slots_[HUGE_ACHUNK_INDEX].unmaps_;
ret = databuff_printf(buf, buf_len, pos,
"[CHUNK_MGR] limit=%'15ld hold=%'15ld total_hold=%'15ld used=%'15ld freelists_hold=%'15ld"
" total_maps=%'15ld total_unmaps=%'15ld large_maps=%'15ld large_unmaps=%'15ld huge_maps=%'15ld huge_unmaps=%'15ld"
" memalign=%d resident_size=%'15ld"
#ifndef ENABLE_SANITY
" virtual_memory_used=%'15ld\n",
#else
" virtual_memory_used=%'15ld actual_virtual_memory_used=%'15ld\n",
#endif
limit_, hold_, total_hold_, used_, hold_ - used_,
total_maps, total_unmaps, large_maps, large_unmaps, slots_[HUGE_ACHUNK_INDEX].maps_, slots_[HUGE_ACHUNK_INDEX].unmaps_,
0, resident_size,
#ifndef ENABLE_SANITY
memory_used
#else
memory_used - shadow_hold_, memory_used
#endif
);
for (int i = 0; OB_SUCC(ret) && i <= MAX_LARGE_ACHUNK_INDEX; ++i) {
const AChunkList &free_list = slots_[i].free_list_;
const int64_t maps = slots_[i].maps_;
const int64_t unmaps = slots_[i].unmaps_;
ret = databuff_printf(buf, buf_len, pos,
"[CHUNK_MGR] %'2d MB_CACHE: hold=%'15ld free=%'15ld pushes=%'15ld pops=%'15ld maps=%'15ld unmaps=%'15ld\n",
(i + 1) * 2, free_list.hold(), free_list.count(),
free_list.get_pushes(), free_list.get_pops(),
maps, unmaps);
}
return pos;
}
int64_t AChunkMgr::sync_wash()
{
int64_t washed_size = 0;
AChunk *free_lists[2] = {};
free_lists[0] = free_list_.popall();
free_lists[1] = large_free_list_.popall();
for (int i = 0; i < ARRAYSIZEOF(free_lists); ++i) {
AChunk *head = free_lists[i];
for (int i = 0; i <= MAX_LARGE_ACHUNK_INDEX; ++i) {
AChunk *head = slots_[i]->popall();
if (OB_NOT_NULL(head)) {
AChunk *chunk = head;
do {

90
deps/oblib/src/lib/resource/achunk_mgr.h vendored
View File

@ -192,12 +192,26 @@ class AChunkMgr
friend class ProtectedStackAllocator;
friend class ObMemoryCutter;
private:
struct Slot
{
Slot(int64_t max_cache_size = INT64_MAX) : maps_(0), unmaps_(0), free_list_()
{
free_list_.set_max_chunk_cache_size(max_cache_size);
}
AChunkList* operator->() { return &free_list_; }
int64_t maps_;
int64_t unmaps_;
AChunkList free_list_;
};
static constexpr int64_t DEFAULT_LIMIT = 4L << 30; // 4GB
static constexpr int64_t ACHUNK_ALIGN_SIZE = INTACT_ACHUNK_SIZE;
static constexpr int64_t NORMAL_ACHUNK_SIZE = INTACT_ACHUNK_SIZE;
static constexpr int64_t LARGE_ACHUNK_SIZE = INTACT_ACHUNK_SIZE << 1;
static constexpr int32_t N = 10;
static constexpr int32_t NORMAL_ACHUNK_INDEX = 0;
static constexpr int32_t MIN_LARGE_ACHUNK_INDEX = NORMAL_ACHUNK_INDEX + 1;
static constexpr int32_t MAX_LARGE_ACHUNK_INDEX = N - 1;
static constexpr int32_t HUGE_ACHUNK_INDEX = MAX_LARGE_ACHUNK_INDEX + 1;
public:
static constexpr int64_t DEFAULT_LARGE_CHUNK_CACHE_SIZE = 128L << 20;
static AChunkMgr &instance();
public:
@ -211,14 +225,18 @@ public:
void free_co_chunk(AChunk *chunk);
static OB_INLINE uint64_t aligned(const uint64_t size);
static OB_INLINE uint64_t hold(const uint64_t size);
void set_max_chunk_cache_size(const int64_t max_cache_size)
{ free_list_.set_max_chunk_cache_size(max_cache_size); }
void set_max_large_chunk_cache_size(const int64_t max_cache_size)
{ large_free_list_.set_max_chunk_cache_size(max_cache_size); }
void set_max_chunk_cache_size(const int64_t max_cache_size, const bool use_large_chunk_cache = false)
{
max_chunk_cache_size_ = max_cache_size;
int64_t large_chunk_cache_size = use_large_chunk_cache ? INT64_MAX : 0;
for (int i = MIN_LARGE_ACHUNK_INDEX; i <= MAX_LARGE_ACHUNK_INDEX; ++i) {
slots_[i]->set_max_chunk_cache_size(large_chunk_cache_size);
}
}
inline static AChunk *ptr2chunk(const void *ptr);
bool update_hold(int64_t bytes, bool high_prio);
virtual int madvise(void *addr, size_t length, int advice);
int64_t to_string(char *buf, const int64_t buf_len) const;
inline void set_limit(int64_t limit);
inline int64_t get_limit() const;
@ -227,17 +245,7 @@ public:
inline int64_t get_hold() const;
inline int64_t get_total_hold() const { return ATOMIC_LOAD(&total_hold_); }
inline int64_t get_used() const;
inline int64_t get_free_chunk_count() const;
inline int64_t get_free_chunk_pushes() const;
inline int64_t get_free_chunk_pops() const;
inline int64_t get_freelist_hold() const;
inline int64_t get_large_freelist_hold() const;
inline int64_t get_maps() { return maps_; }
inline int64_t get_unmaps() { return unmaps_; }
inline int64_t get_large_maps() { return large_maps_; }
inline int64_t get_large_unmaps() { return large_unmaps_; }
inline int64_t get_huge_maps() { return huge_maps_; }
inline int64_t get_huge_unmaps() { return huge_unmaps_; }
inline int64_t get_shadow_hold() const { return ATOMIC_LOAD(&shadow_hold_); }
int64_t sync_wash();
@ -251,24 +259,23 @@ private:
// wrap for mmap
void *low_alloc(const uint64_t size, const bool can_use_huge_page, bool &huge_page_used, const bool alloc_shadow);
void low_free(const void *ptr, const uint64_t size);
Slot &get_slot(const uint64_t size)
{
int64_t index = (size - 1) / INTACT_ACHUNK_SIZE;
if (index > HUGE_ACHUNK_INDEX) {
index = HUGE_ACHUNK_INDEX;
}
return slots_[index];
}
protected:
AChunkList free_list_;
AChunkList large_free_list_;
ChunkBitMap *chunk_bitmap_;
int64_t limit_;
int64_t urgent_;
int64_t hold_; // Including the memory occupied by free_list, limited by memory_limit
int64_t total_hold_; // Including virtual memory, just for statifics.
int64_t maps_;
int64_t unmaps_;
int64_t large_maps_;
int64_t large_unmaps_;
int64_t huge_maps_;
int64_t huge_unmaps_;
int64_t used_;
int64_t shadow_hold_;
int64_t max_chunk_cache_size_;
Slot slots_[N + 1];
}; // end of class AChunkMgr
OB_INLINE AChunk *AChunkMgr::ptr2chunk(const void *ptr)
@ -313,34 +320,13 @@ inline int64_t AChunkMgr::get_hold() const
inline int64_t AChunkMgr::get_used() const
{
return hold_ - get_freelist_hold() - get_large_freelist_hold();
}
inline int64_t AChunkMgr::get_free_chunk_count() const
{
return free_list_.count();
}
inline int64_t AChunkMgr::get_free_chunk_pushes() const
{
return free_list_.get_pushes();
}
inline int64_t AChunkMgr::get_free_chunk_pops() const
{
return free_list_.get_pops();
return used_;
}
inline int64_t AChunkMgr::get_freelist_hold() const
{
return free_list_.hold();
return hold_ - used_;
}
inline int64_t AChunkMgr::get_large_freelist_hold() const
{
return large_free_list_.hold();
}
} // end of namespace lib
} // end of namespace oceanbase

21
deps/oblib/src/lib/restore/cos/ob_cos_wrapper.cpp vendored
View File

@ -38,6 +38,7 @@ constexpr int OB_SUCCESS = 0;
constexpr int OB_INVALID_ARGUMENT = -4002;
constexpr int OB_INIT_TWICE = -4005;
constexpr int OB_ALLOCATE_MEMORY_FAILED = -4013;
constexpr int OB_ERR_UNEXPECTED = -4016;
constexpr int OB_SIZE_OVERFLOW = -4019;
constexpr int OB_CHECKSUM_ERROR = -4103;
constexpr int OB_BACKUP_FILE_NOT_EXIST = -9011;
@ -1564,6 +1565,7 @@ int ObCosWrapper::complete_multipart_upload(
ret = OB_INVALID_ARGUMENT;
cos_warn_log("[COS]upload_id is null, ret=%d\n", ret);
} else {
int64_t total_parts = 0;
cos_string_t bucket;
cos_string_t object;
cos_string_t upload_id;
@ -1604,6 +1606,12 @@ int ObCosWrapper::complete_multipart_upload(
cos_str_set(&complete_part->etag, part_content->etag.data);
cos_list_add_tail(&complete_part->node, &complete_part_list);
}
if (OB_SUCCESS != ret) {
break;
} else {
total_parts++;
}
}
if (OB_SUCCESS == ret && COS_TRUE == params->truncated) {
@ -1628,7 +1636,14 @@ int ObCosWrapper::complete_multipart_upload(
} while (OB_SUCCESS == ret && COS_TRUE == params->truncated);
if (OB_SUCCESS == ret) {
if (NULL == (cos_ret = cos_complete_multipart_upload(ctx->options, &bucket, &object, &upload_id, &complete_part_list, NULL, &resp_headers))
if (total_parts == 0) {
// If 'complete' without uploading any data, COS will return the error
// 'MalformedXML, The XML you provided was not well-formed or did not validate against our published schema'
ret = OB_ERR_UNEXPECTED;
cos_warn_log("[COS]no parts have been uploaded, ret=%d, upload_id=%s\n", ret, upload_id.data);
} else if (NULL == (cos_ret = cos_complete_multipart_upload(ctx->options, &bucket, &object,
&upload_id, &complete_part_list,
NULL, &resp_headers))
|| !cos_status_is_ok(cos_ret)) {
convert_io_error(cos_ret, ret);
cos_warn_log("[COS]fail to complete multipart upload, ret=%d\n", ret);
@ -1756,6 +1771,10 @@ int ObCosWrapper::del_unmerged_parts(
cos_info_log("[COS]succeed to abort multipart upload, bucket=%s, object=%s, upload_id=%s\n",
bucket_name.data_, content->key.data, content->upload_id.data);
}
if (OB_SUCCESS != ret) {
break;
}
}
}

8
deps/oblib/src/lib/restore/ob_storage_file.cpp vendored
View File

@ -1179,6 +1179,7 @@ int ObStorageFileMultiPartWriter::complete()
{
int ret = OB_SUCCESS;
char errno_buf[OB_MAX_ERROR_MSG_LEN] = "";
struct stat64 file_info;
if (!is_opened_) {
ret = OB_NOT_INIT;
@ -1195,6 +1196,13 @@ int ObStorageFileMultiPartWriter::complete()
if (has_error_) {
STORAGE_LOG(WARN, "multipart writer has error, skip complete",
KCSTRING(path_), KCSTRING(real_path_));
} else if (0 != ::stat64(path_, &file_info)) {
convert_io_error(errno, ret);
STORAGE_LOG(WARN, "fail to get file length",
K(ret), K_(path), K(errno), "errno", strerror_r(errno, errno_buf, sizeof(errno_buf)));
} else if (OB_UNLIKELY(file_info.st_size == 0)) {
ret = OB_ERR_UNEXPECTED;
OB_LOG(WARN, "no parts have been uploaded!", K(ret), K(file_info.st_size), K_(path));
} else if (0 != ::rename(path_, real_path_)) {
convert_io_error(errno, ret);
STORAGE_LOG(WARN, "failed to complete", K(ret), KCSTRING(real_path_),

32
deps/oblib/src/lib/restore/ob_storage_oss_base.cpp vendored
View File

@ -896,6 +896,7 @@ int ObStorageOssMultiPartWriter::complete()
}
if (OB_SUCC(ret)) {
int64_t total_parts = 0;
aos_string_t bucket;
aos_string_t object;
aos_str_set(&bucket, bucket_.ptr());
@ -926,18 +927,22 @@ int ObStorageOssMultiPartWriter::complete()
if (OB_ISNULL(complete_part_content = oss_create_complete_part_content(aos_pool_))) {
ret = OB_OSS_ERROR;
OB_LOG(WARN, "fail to create complete part content", K_(bucket), K_(object), K(ret));
break;
} else if (OB_ISNULL(part_content->part_number.data)
|| OB_ISNULL(part_content->etag.data)) {
ret = OB_OSS_ERROR;
OB_LOG(WARN, "invalid part_number or etag",
K(ret), KP(part_content->part_number.data), KP(part_content->etag.data));
break;
} else {
aos_str_set(&complete_part_content->part_number, part_content->part_number.data);
aos_str_set(&complete_part_content->etag, part_content->etag.data);
aos_list_add_tail(&complete_part_content->node, &complete_part_list);
}
if (OB_FAIL(ret)) {
break;
} else {
total_parts++;
}
}
if (OB_SUCC(ret) && AOS_TRUE == params->truncated) {
@ -964,7 +969,12 @@ int ObStorageOssMultiPartWriter::complete()
}
//complete multipart upload
if (OB_SUCC(ret)) {
if (OB_FAIL(ret)) {
} else if (OB_UNLIKELY(total_parts == 0)) {
// If 'complete' without uploading any data, OSS will create an object with a size of 0
ret = OB_ERR_UNEXPECTED;
OB_LOG(WARN, "no parts have been uploaded!", K(ret), K(total_parts), K(upload_id_.data));
} else {
if (OB_ISNULL(aos_ret = oss_complete_multipart_upload(oss_option_, &bucket, &object, &upload_id_,
&complete_part_list, complete_headers, &resp_headers)) || !aos_status_is_ok(aos_ret)) {
convert_io_error(aos_ret, ret);
@ -988,6 +998,14 @@ int ObStorageOssMultiPartWriter::close()
{
int ret = OB_SUCCESS;
ObExternalIOCounterGuard io_guard;
mod_.reset();
base_buf_pos_ = 0;
partnum_ = 0;
base_buf_ = nullptr;
is_opened_ = false;
file_length_ = -1;
upload_id_.len = -1;
upload_id_.data = NULL;
reset();
return ret;
}
@ -1194,6 +1212,10 @@ int ObStorageOssReader::pread(
break;
}
}
if (OB_FAIL(ret)) {
break;
}
} // end aos_list_for_each_entry
}
}
@ -1908,6 +1930,10 @@ int ObStorageOssUtil::del_unmerged_parts(const ObString &uri)
OB_LOG(INFO, "succeed abort oss multipart upload",
K(bucket_str), K(content->key.data), K(content->upload_id.data));
}
if (OB_FAIL(ret)) {
break;
}
}
}

10
deps/oblib/src/lib/restore/ob_storage_s3_base.cpp vendored
View File

@ -2100,8 +2100,14 @@ int ObStorageS3MultiPartWriter::complete_()
complete_multipart_upload_request.WithMultipartUpload(completed_multipart_upload);
Aws::S3::Model::CompleteMultipartUploadOutcome complete_multipart_upload_outcome;
if (FAILEDx(s3_client_->complete_multipart_upload(complete_multipart_upload_request,
complete_multipart_upload_outcome))) {
if (OB_FAIL(ret)) {
} else if (OB_UNLIKELY(part_num == 0)) {
// If 'complete' without uploading any data, S3 will return the error
// 'InvalidRequest，You must specify at least one part'
ret = OB_ERR_UNEXPECTED;
OB_LOG(WARN, "no parts have been uploaded!", K(ret), K(part_num), K_(upload_id));
} else if (OB_FAIL(s3_client_->complete_multipart_upload(complete_multipart_upload_request,
complete_multipart_upload_outcome))) {
OB_LOG(WARN, "failed to complete s3 multipart upload", K(ret));
} else if (!complete_multipart_upload_outcome.IsSuccess()) {
handle_s3_outcome(complete_multipart_upload_outcome, ret);

23
deps/oblib/src/lib/signal/ob_signal_handlers.cpp vendored
View File

@ -224,19 +224,24 @@ void coredump_cb(volatile int sig, volatile int sig_code, void* volatile sig_add
crash_info, ip, bp, sig, sig_code, sig_addr, rlimit_core,
ts, GETTID(), tname, TRACE_ID_FORMAT_PARAM(uval),
(NULL == extra_info) ? NULL : to_cstring(*extra_info), bt);
const auto &si_guard = ObSqlInfoGuard::get_cur_guard();
char sql[] = "SQL=";
ObSqlInfo sql_info = ObSqlInfoGuard::get_tl_sql_info();
char sql_id[] = "SQL_ID=";
char sql_string[] = ", SQL_STRING=";
char end[] = "\n";
struct iovec iov[4];
struct iovec iov[6];
memset(iov, 0, sizeof(iov));
iov[0].iov_base = print_buf;
iov[0].iov_len = print_len;
iov[1].iov_base = sql;
iov[1].iov_len = strlen(sql);
iov[2].iov_base = NULL != si_guard ? si_guard->sql_.ptr() : NULL;
iov[2].iov_len = NULL != si_guard ? si_guard->sql_.length() : 0;
iov[3].iov_base = end;
iov[3].iov_len = strlen(end);
iov[1].iov_base = sql_id;
iov[1].iov_len = strlen(sql_id);
iov[2].iov_base = sql_info.sql_id_.ptr();
iov[2].iov_len = sql_info.sql_id_.length();
iov[3].iov_base = sql_string;
iov[3].iov_len = strlen(sql_string);
iov[4].iov_base = sql_info.sql_string_.ptr();
iov[4].iov_len = sql_info.sql_string_.length();
iov[5].iov_base = end;
iov[5].iov_len = strlen(end);
writev(STDERR_FILENO, iov, sizeof(iov) / sizeof(iov[0]));
if (OB_SUCC(ret)) {
int status = 0;

2
deps/oblib/src/lib/signal/ob_signal_struct.cpp vendored
View File

@ -27,6 +27,8 @@ const int MP_SIG = SIGURG;
const int SIG_STACK_SIZE = 16L<<10;
uint64_t g_rlimit_core = 0;
thread_local ObSqlInfo ObSqlInfoGuard::tl_sql_info;
DTraceId DTraceId::gen_trace_id()
{
static int64_t seq = 0;

30
deps/oblib/src/lib/signal/ob_signal_struct.h vendored
View File

@ -112,32 +112,38 @@ struct ObSigHandlerCtx
extern ObSigHandlerCtx g_sig_handler_ctx_;
struct ObSqlInfo
{
ObString sql_string_;
ObString sql_id_;
};
class ObSqlInfoGuard
{
public:
ObSqlInfoGuard(const ObString &sql)
: sql_(sql)
ObSqlInfoGuard(const ObString &sql_string, const ObString &sql_id)
: last_sql_info_(tl_sql_info)
{
last_ = get_cur_guard();
get_cur_guard() = this;
tl_sql_info.sql_string_ = sql_string;
tl_sql_info.sql_id_ = sql_id;
}
~ObSqlInfoGuard()
{
get_cur_guard() = last_;
tl_sql_info = last_sql_info_;
}
static ObSqlInfoGuard *&get_cur_guard()
static ObSqlInfo get_tl_sql_info()
{
static thread_local ObSqlInfoGuard *cur_guard = NULL;
return cur_guard;
return tl_sql_info;
}
public:
ObString sql_;
private:
ObSqlInfoGuard *last_;
static thread_local ObSqlInfo tl_sql_info;
ObSqlInfo last_sql_info_;
};
} // namespace common
} // namespace oceanbase
#define SQL_INFO_GUARD(sql_string, sql_id) \
oceanbase::common::ObSqlInfoGuard sql_info_guard(sql_string, sql_id);
#endif // OCEANBASE_SIGNAL_STRUCT_H_

7
deps/oblib/src/lib/trace/ob_trace.cpp vendored
View File

@ -66,15 +66,16 @@ void flush_trace()
int ret = OB_SUCCESS;
ObTagCtxBase* tag = span->tags_;
bool first = true;
char tagstr[] = "\"tags\":[";
INIT_SPAN(span);
while (OB_SUCC(ret) && OB_NOT_NULL(tag)) {
if (pos + 10 >= MAX_TRACE_LOG_SIZE) {
if (pos + sizeof(tagstr) + 1 >= MAX_TRACE_LOG_SIZE) {
ret = OB_BUF_NOT_ENOUGH;
} else {
buf[pos++] = ',';
if (first) {
strcpy(buf + pos, "\"tags\":[");
pos += 8;
strncpy(buf + pos, tagstr, MAX_TRACE_LOG_SIZE - pos);
pos += sizeof(tagstr) - 1;
first = false;
}
ret = tag->tostring(buf, MAX_TRACE_LOG_SIZE, pos);

2
deps/oblib/src/lib/trace/ob_trace.h vendored
View File

@ -151,7 +151,7 @@ struct ObTagCtxBase
} else {
buf[pos++] = '{';
buf[pos++] = '\"';
IGNORE_RETURN strcpy(buf + pos, __tag_name_mapper[tag_type_]);
IGNORE_RETURN strncpy(buf + pos, __tag_name_mapper[tag_type_], buf_len - pos);
pos += l;
buf[pos++] = '\"';
buf[pos++] = ':';

1
deps/oblib/src/rpc/ob_lock_wait_node.cpp vendored
View File

@ -45,7 +45,6 @@ void ObLockWaitNode::set(void* addr,
total_update_cnt_ = total_trans_node_cnt;
run_ts_ = 0;
snprintf(key_, sizeof(key_), "%s", key);
reset_need_wait();
}
void ObLockWaitNode::change_hash(const int64_t hash, const int64_t lock_seq)

1
deps/oblib/src/rpc/obrpc/ob_rpc_mem_pool.cpp vendored
View File

@ -42,6 +42,7 @@ static void* rpc_mem_pool_direct_alloc(int64_t tenant_id, const char* label, int
tenant_id = OB_SERVER_TENANT_ID;
}
ObMemAttr attr(tenant_id, label, common::ObCtxIds::RPC_CTX_ID);
SET_IGNORE_MEM_VERSION(attr);
lib::ObTenantCtxAllocatorGuard allocator = lib::ObMallocAllocator::get_instance()->get_tenant_ctx_allocator(tenant_id, common::ObCtxIds::RPC_CTX_ID);
if (OB_ISNULL(allocator)) {
attr.tenant_id_ = OB_SERVER_TENANT_ID;

3
deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h vendored
View File

@ -901,6 +901,7 @@ PCODE_DEF(OB_DROP_EVENT_JOB, 0x1204)
PCODE_DEF(OB_RUN_DBMS_JOB, 0x1205)
// DBMS Scheduler API
PCODE_DEF(OB_RUN_DBMS_SCHED_JOB, 0x1206)
//PCODE_DEF(OB_STOP_DBMS_SCHED_JOB, 0x1207)
// label security
PCODE_DEF(OB_HANDLE_LABEL_SE_POLICY_DDL, 0x1301)
@ -1136,6 +1137,8 @@ PCODE_DEF(OB_TRIM_KEY_LIST, 0x1618)
// PCODE_DEF(OB_CAL_UNIT_PHY_RESOURCE, 0x1622)
// PCODE_DEF(OB_CAL_STANDBY_TENANT_PHY_RESOURCE, 0x1623)
//PCODE_DEF(OB_UPDATE_MVIEW_REFERENCE_TABLE_STATUS, 0x1624)
//**** 注意：在此行之前增加新的RPC ID ******
//
//占位须知：

262
deps/oblib/unittest/lib/alloc/test_chunk_mgr.cpp vendored
View File

@ -31,9 +31,7 @@ class TestChunkMgr
{
public:
TestChunkMgr()
{
large_free_list_.set_max_chunk_cache_size(AChunkList::DEFAULT_MAX_CHUNK_CACHE_SIZE);
}
{}
virtual void SetUp()
{
}
@ -55,43 +53,45 @@ TEST_F(TestChunkMgr, NormalChunk)
{
int NORMAL_SIZE = OB_MALLOC_BIG_BLOCK_SIZE;
int LARGE_SIZE = INTACT_ACHUNK_SIZE + 100;
int normal_hold = 0;
int normal_hold = AChunkMgr::hold(NORMAL_SIZE);
int large_hold = AChunkMgr::hold(LARGE_SIZE);
{
int64_t hold = 0;
AChunk *chunks[1024] = {};
for (int i = 0; i < 1024; i++) {
AChunk *chunks[8] = {};
// direct alloc 2M
for (int i = 0; i < 8; ++i) {
chunks[i] = alloc_chunk(NORMAL_SIZE);
normal_hold = chunks[i]->hold();
hold += normal_hold;
}
set_max_chunk_cache_size(hold - normal_hold);
for (int i = 0; i < 1024; i++) {
// create 2M-cache
for (int i = 0; i < 8; ++i) {
free_chunk(chunks[i]);
chunks[i] = NULL;
}
EXPECT_EQ(1023, free_list_.get_pushes());
EXPECT_EQ(0, free_list_.get_pops());
EXPECT_EQ(0, large_free_list_.get_pushes());
EXPECT_EQ(0, large_free_list_.get_pops());
hold -= normal_hold;
// alloc chunk from 2M-cache
for (int i = 0; i < 8; ++i) {
chunks[i] = alloc_chunk(NORMAL_SIZE);
}
EXPECT_EQ(8, slots_[0]->get_pushes());
EXPECT_EQ(8, slots_[0]->get_pops());
EXPECT_EQ(0, slots_[1]->get_pushes());
EXPECT_EQ(0, slots_[1]->get_pops());
EXPECT_EQ(hold, hold_);
// direct alloc 4M
// alloc chunk by wash 4M-cache
{
auto *chunk = alloc_chunk(LARGE_SIZE);
AChunk *chunk = alloc_chunk(LARGE_SIZE);
EXPECT_TRUE(NULL != chunk);
EXPECT_EQ(0, free_list_.get_pops());
EXPECT_EQ(0, large_free_list_.get_pops());
hold += chunk->hold();
hold += large_hold;
free_chunk(chunk);
EXPECT_EQ(hold, hold_);
}
// wash alloc
{
set_limit(hold);
auto *chunk = alloc_chunk(LARGE_SIZE);
chunk = alloc_chunk(NORMAL_SIZE);
EXPECT_TRUE(NULL != chunk);
EXPECT_EQ(2, free_list_.get_pops());
hold = hold - normal_hold * 2 + chunk->hold();
hold -= (large_hold - normal_hold);
EXPECT_EQ(8, slots_[0]->get_pushes());
EXPECT_EQ(8, slots_[0]->get_pops());
EXPECT_EQ(1, slots_[1]->get_pushes());
EXPECT_EQ(1, slots_[1]->get_pops());
EXPECT_EQ(hold, hold_);
}
}
@ -101,43 +101,56 @@ TEST_F(TestChunkMgr, LargeChunk)
{
int NORMAL_SIZE = OB_MALLOC_BIG_BLOCK_SIZE;
int LARGE_SIZE = INTACT_ACHUNK_SIZE + 100;
int large_hold = 0;
int BIG_SIZE = INTACT_ACHUNK_SIZE * 2 + 100;
int normal_hold = AChunkMgr::hold(NORMAL_SIZE);
int large_hold = AChunkMgr::hold(LARGE_SIZE);
int big_hold = AChunkMgr::hold(BIG_SIZE);
{
int64_t hold = 0;
AChunk *chunks[1024] = {};
for (int i = 0; i < 1024; i++) {
AChunk *chunks[8] = {};
// direct alloc 4M
for (int i = 0; i < 8; ++i) {
chunks[i] = alloc_chunk(LARGE_SIZE);
large_hold = chunks[i]->hold();
hold += large_hold;
}
set_max_large_chunk_cache_size(hold - large_hold);
for (int i = 0; i < 1024; i++) {
// create 4M-cache
for (int i = 0; i < 8; ++i) {
free_chunk(chunks[i]);
}
EXPECT_EQ(1023, large_free_list_.get_pushes());
EXPECT_EQ(0, large_free_list_.get_pops());
EXPECT_EQ(0, free_list_.get_pushes());
EXPECT_EQ(0, free_list_.get_pops());
hold -= large_hold;
// alloc chunk from self-cache(4M-cache)
for (int i = 0; i < 8; ++i) {
chunks[i] = alloc_chunk(LARGE_SIZE);
}
EXPECT_EQ(8, slots_[1]->get_pushes());
EXPECT_EQ(8, slots_[1]->get_pops());
EXPECT_EQ(hold, hold_);
// direct alloc 2M
// alloc chunk by wash 6M-cache and 2M-cache
{
auto *chunk = alloc_chunk(NORMAL_SIZE);
AChunk *chunk = alloc_chunk(BIG_SIZE);
EXPECT_TRUE(NULL != chunk);
EXPECT_EQ(0, free_list_.get_pops());
EXPECT_EQ(0, large_free_list_.get_pops());
hold += chunk->hold();
hold += big_hold;
free_chunk(chunk);
chunk = alloc_chunk(NORMAL_SIZE);
EXPECT_TRUE(NULL != chunk);
hold += normal_hold;
free_chunk(chunk);
EXPECT_EQ(1, slots_[0]->get_pushes());
EXPECT_EQ(0, slots_[0]->get_pops());
EXPECT_EQ(1, slots_[2]->get_pushes());
EXPECT_EQ(0, slots_[2]->get_pops());
EXPECT_EQ(hold, hold_);
}
// wash alloc
{
set_limit(hold);
auto *chunk = alloc_chunk(NORMAL_SIZE);
chunk = alloc_chunk(LARGE_SIZE);
hold += (large_hold - big_hold);
EXPECT_TRUE(NULL != chunk);
EXPECT_EQ(1, large_free_list_.get_pops());
hold = hold - large_hold + chunk->hold();
chunk = alloc_chunk(LARGE_SIZE);
hold += (large_hold - normal_hold);
EXPECT_TRUE(NULL != chunk);
EXPECT_EQ(1, slots_[0]->get_pushes());
EXPECT_EQ(1, slots_[0]->get_pops());
EXPECT_EQ(1, slots_[2]->get_pushes());
EXPECT_EQ(1, slots_[2]->get_pops());
EXPECT_EQ(hold, hold_);
}
}
@ -147,55 +160,34 @@ TEST_F(TestChunkMgr, HugeChunk)
{
int NORMAL_SIZE = OB_MALLOC_BIG_BLOCK_SIZE;
int LARGE_SIZE = INTACT_ACHUNK_SIZE + 100;
int HUGE_SIZE = INTACT_ACHUNK_SIZE * 3;
set_limit(20L<<30);
int huge_hold = 0;
int HUGE_SIZE = INTACT_ACHUNK_SIZE * 10 + 100;
{
int64_t hold = 0;
{
int64_t temp_hold = 0;
AChunk *chunks[1024] = {};
for (int i = 0; i < 1024; i++) {
chunks[i] = alloc_chunk(LARGE_SIZE);
hold += chunks[i]->hold();
temp_hold += chunks[i]->hold();
AChunk *chunks[8][2] = {};
for (int i = 0; i < 8; ++i) {
chunks[i][0] = alloc_chunk(NORMAL_SIZE);
chunks[i][1] = alloc_chunk(LARGE_SIZE);
hold += chunks[i][0]->hold();
hold += chunks[i][1]->hold();
}
set_max_large_chunk_cache_size(temp_hold);
for (int i = 0; i < 1024; i++) {
free_chunk(chunks[i]);
for (int i = 0; i < 8; ++i) {
free_chunk(chunks[i][0]);
free_chunk(chunks[i][1]);
}
EXPECT_EQ(1024, large_free_list_.get_pushes());
EXPECT_EQ(0, large_free_list_.get_pops());
EXPECT_EQ(0, free_list_.get_pushes());
EXPECT_EQ(0, free_list_.get_pops());
EXPECT_EQ(hold, hold_);
}
{
int64_t temp_hold = 0;
AChunk *chunks[1024] = {};
for (int i = 0; i < 1024; i++) {
chunks[i] = alloc_chunk(NORMAL_SIZE);
hold += chunks[i]->hold();
temp_hold += chunks[i]->hold();
}
set_max_chunk_cache_size(temp_hold);
for (int i = 0; i < 1024; i++) {
free_chunk(chunks[i]);
}
EXPECT_EQ(1024, free_list_.get_pushes());
EXPECT_EQ(0, free_list_.get_pops());
EXPECT_EQ(1024, large_free_list_.get_pushes());
EXPECT_EQ(0, large_free_list_.get_pops());
EXPECT_EQ(8, slots_[0]->get_pushes());
EXPECT_EQ(0, slots_[0]->get_pops());
EXPECT_EQ(8, slots_[1]->get_pushes());
EXPECT_EQ(0, slots_[1]->get_pops());
EXPECT_EQ(hold, hold_);
}
// direct alloc huge
{
auto *chunk = alloc_chunk(HUGE_SIZE);
AChunk *chunk = alloc_chunk(HUGE_SIZE);
EXPECT_TRUE(NULL != chunk);
EXPECT_EQ(0, free_list_.get_pops());
EXPECT_EQ(0, large_free_list_.get_pops());
EXPECT_EQ(0, slots_[0]->get_pops());
EXPECT_EQ(0, slots_[1]->get_pops());
hold += chunk->hold();
EXPECT_EQ(hold, hold_);
}
@ -203,14 +195,14 @@ TEST_F(TestChunkMgr, HugeChunk)
// wash alloc
{
set_limit(hold);
auto *chunk = alloc_chunk(free_list_.hold() - 100);
AChunk *chunk = alloc_chunk(HUGE_SIZE);
EXPECT_TRUE(NULL != chunk);
EXPECT_EQ(0, free_list_.hold());
EXPECT_NE(0, large_free_list_.hold());
EXPECT_NE(0, slots_[0]->hold());
EXPECT_EQ(0, slots_[1]->hold());
chunk = alloc_chunk(large_free_list_.hold() - 100);
chunk = alloc_chunk(slots_[0]->hold());
EXPECT_TRUE(NULL != chunk);
EXPECT_EQ(0, large_free_list_.hold());
EXPECT_EQ(0, slots_[0]->hold());
}
}
}
@ -222,14 +214,14 @@ TEST_F(TestChunkMgr, BorderCase_advise_shrink)
auto *chunk = alloc_chunk(LARGE_SIZE);
// pollute chunk
memset(chunk->data_, 0xaa, chunk->hold());
EXPECT_EQ(0, large_free_list_.get_pushes());
EXPECT_EQ(0, slots_[1]->get_pushes());
int64_t orig_chunk_hold = chunk->hold();
int64_t orig_hold = hold_;
free_chunk(chunk);
EXPECT_EQ(1, large_free_list_.get_pushes());
EXPECT_EQ(0, large_free_list_.get_pops());
EXPECT_EQ(1, slots_[1]->get_pushes());
EXPECT_EQ(0, slots_[1]->get_pops());
chunk = alloc_chunk(LARGE_SIZE - ps * 3);
EXPECT_EQ(1, large_free_list_.get_pops());
EXPECT_EQ(1, slots_[1]->get_pops());
EXPECT_EQ(madvise_len_, ps * 3);
EXPECT_FALSE(0 == chunk->data_[0] && 0 == memcmp(chunk->data_, chunk->data_ + 1, chunk->hold() - 1));
EXPECT_EQ(orig_chunk_hold - chunk->hold(), orig_hold - hold_);
@ -242,14 +234,14 @@ TEST_F(TestChunkMgr, BorderCase_advise_expand)
auto *chunk = alloc_chunk(LARGE_SIZE);
// pollute chunk
memset(chunk->data_, 0xaa, chunk->hold());
EXPECT_EQ(0, large_free_list_.get_pushes());
EXPECT_EQ(0, slots_[1]->get_pushes());
int64_t orig_chunk_hold = chunk->hold();
int64_t orig_hold = hold_;
free_chunk(chunk);
EXPECT_EQ(1, large_free_list_.get_pushes());
EXPECT_EQ(0, large_free_list_.get_pops());
EXPECT_EQ(1, slots_[1]->get_pushes());
EXPECT_EQ(0, slots_[1]->get_pops());
chunk = alloc_chunk(LARGE_SIZE + ps * 3);
EXPECT_EQ(1, large_free_list_.get_pops());
EXPECT_EQ(1, slots_[1]->get_pops());
EXPECT_FALSE(0 == chunk->data_[0] && 0 == memcmp(chunk->data_, chunk->data_ + 1, chunk->hold() - 1));
EXPECT_EQ(orig_chunk_hold - (int64_t)chunk->hold(), orig_hold - hold_);
}
@ -261,16 +253,16 @@ TEST_F(TestChunkMgr, BorderCase_advise_fail)
auto *chunk = alloc_chunk(LARGE_SIZE);
// pollute chunk
memset(chunk->data_, 0xaa, chunk->hold());
EXPECT_EQ(0, large_free_list_.get_pushes());
EXPECT_EQ(0, slots_[1]->get_pushes());
int64_t orig_chunk_hold = chunk->hold();
int64_t orig_hold = hold_;
free_chunk(chunk);
EXPECT_EQ(1, large_free_list_.get_pushes());
EXPECT_EQ(0, large_free_list_.get_pops());
EXPECT_EQ(1, slots_[1]->get_pushes());
EXPECT_EQ(0, slots_[1]->get_pops());
need_fail_ = true;
chunk = alloc_chunk(LARGE_SIZE - ps * 3);
EXPECT_EQ(1, large_free_list_.get_pushes());
EXPECT_EQ(1, large_free_list_.get_pops());
EXPECT_EQ(1, slots_[1]->get_pushes());
EXPECT_EQ(1, slots_[1]->get_pops());
// check remap happened
EXPECT_TRUE(0 == chunk->data_[0] && 0 == memcmp(chunk->data_, chunk->data_ + 1, chunk->hold() - 1));
EXPECT_EQ(orig_chunk_hold - (int64_t)chunk->hold(), orig_hold - hold_);
@ -286,17 +278,17 @@ TEST_F(TestChunkMgr, alloc_co_chunk)
chunk = alloc_chunk(LARGE_SIZE);
free_chunk(chunk);
}
EXPECT_EQ(1, large_free_list_.get_pushes());
EXPECT_EQ(0, large_free_list_.get_pops());
EXPECT_EQ(1, free_list_.get_pushes());
EXPECT_EQ(0, free_list_.get_pops());
EXPECT_EQ(1, slots_[1]->get_pushes());
EXPECT_EQ(0, slots_[1]->get_pops());
EXPECT_EQ(1, slots_[0]->get_pushes());
EXPECT_EQ(0, slots_[0]->get_pops());
set_limit(hold_);
auto *chunk = alloc_co_chunk(NORMAL_SIZE);
EXPECT_TRUE(chunk != NULL);
EXPECT_EQ(1, free_list_.get_pops());
EXPECT_EQ(1, slots_[1]->get_pops());
chunk = alloc_co_chunk(NORMAL_SIZE);
EXPECT_TRUE(chunk != NULL);
EXPECT_EQ(1, large_free_list_.get_pops());
EXPECT_EQ(1, slots_[0]->get_pops());
}
TEST_F(TestChunkMgr, FreeListBasic)
@ -304,51 +296,29 @@ TEST_F(TestChunkMgr, FreeListBasic)
{
AChunk *chunk = alloc_chunk(0);
free_chunk(chunk);
EXPECT_EQ(1, free_list_.get_pushes());
EXPECT_EQ(1, slots_[0]->get_pushes());
}
{
AChunk *chunk = alloc_chunk(0);
free_chunk(chunk);
EXPECT_EQ(2, free_list_.get_pushes());
EXPECT_EQ(1, free_list_.get_pops());
EXPECT_EQ(2, slots_[0]->get_pushes());
EXPECT_EQ(1, slots_[0]->get_pops());
}
{
AChunk *chunk = alloc_chunk(OB_MALLOC_BIG_BLOCK_SIZE);
free_chunk(chunk);
EXPECT_EQ(3, free_list_.get_pushes());
EXPECT_EQ(2, free_list_.get_pops());
EXPECT_EQ(3, slots_[0]->get_pushes());
EXPECT_EQ(2, slots_[0]->get_pops());
}
}
TEST_F(TestChunkMgr, FreeListManyChunk)
{
AChunk *chunks[1024] = {};
for (int i = 0; i < 1024; i++) {
chunks[i] = alloc_chunk(OB_MALLOC_BIG_BLOCK_SIZE);
}
for (int i = 0; i < 1024; i++) {
free_chunk(chunks[i]);
}
EXPECT_EQ(AChunkList::DEFAULT_MAX_CHUNK_CACHE_SIZE/INTACT_ACHUNK_SIZE, free_list_.get_pushes());
EXPECT_EQ(0, free_list_.get_pops());
for (int i = 0; i < 1024; i++) {
chunks[i] = alloc_chunk(OB_MALLOC_BIG_BLOCK_SIZE);
}
for (int i = 0; i < 1024; i++) {
free_chunk(chunks[i]);
}
EXPECT_EQ(2* AChunkList::DEFAULT_MAX_CHUNK_CACHE_SIZE/INTACT_ACHUNK_SIZE, free_list_.get_pushes());
EXPECT_EQ(AChunkList::DEFAULT_MAX_CHUNK_CACHE_SIZE/INTACT_ACHUNK_SIZE, free_list_.get_pops());
}
TEST_F(TestChunkMgr, sync_wash)
{
set_limit(1LL<<30);
int NORMAL_SIZE = OB_MALLOC_BIG_BLOCK_SIZE;
int LARGE_SIZE = INTACT_ACHUNK_SIZE + 100;
free_list_.set_max_chunk_cache_size(1LL<<30);
large_free_list_.set_max_chunk_cache_size(1LL<<30);
slots_[0]->set_max_chunk_cache_size(1LL<<30);
slots_[1]->set_max_chunk_cache_size(1LL<<30);
AChunk *chunks[16][2] = {};
for (int i = 0; i < 16; ++i) {
chunks[i][0] = alloc_chunk(NORMAL_SIZE);
@ -360,13 +330,13 @@ TEST_F(TestChunkMgr, sync_wash)
chunks[i][j] = NULL;
}
}
int64_t hold = free_list_.hold() + large_free_list_.hold();
int64_t hold = get_freelist_hold();
EXPECT_EQ(hold, hold_);
EXPECT_EQ(16, free_list_.count());
EXPECT_EQ(16, large_free_list_.count());
EXPECT_EQ(16, slots_[0]->count());
EXPECT_EQ(16, slots_[1]->count());
int64_t washed_size = sync_wash();
EXPECT_EQ(hold, washed_size);
EXPECT_EQ(0, hold_);
EXPECT_EQ(0, free_list_.count());
EXPECT_EQ(0, large_free_list_.count());
EXPECT_EQ(0, slots_[0]->count());
EXPECT_EQ(0, slots_[1]->count());
}

1
deps/oblib/unittest/lib/alloc/test_tenant_ctx_allocator.cpp vendored
View File

@ -28,6 +28,7 @@ using namespace oceanbase::common;
TEST(TestTenantAllocator, CtxAlloc)
{
CHUNK_MGR.set_max_chunk_cache_size(1<<20);
ObTenantCtxAllocator ta(123, 1);
ta.set_tenant_memory_mgr();
ta.set_limit(INT64_MAX);

14
deps/oblib/unittest/lib/task/test_cancel_task.cpp vendored
View File

@ -112,10 +112,10 @@ TEST(TestCancelTask, cancel_running)
ASSERT_EQ(OB_SUCCESS, timer.init());
ASSERT_EQ(OB_SUCCESS, timer.start());
TaskCommon task;
task.exec_time_ = 100000; // 100ms
task.exec_time_ = 300000; // 300ms
ASSERT_EQ(OB_SUCCESS, timer.schedule(task, 50000, true)); // repeate = true , delay = 50ms
ASSERT_EQ(1, timer.get_tasks_num());
::usleep(70000);
::usleep(150000);
ASSERT_EQ(1, task.task_run_count_); // task is running
// the running task has been removed from the task array
ASSERT_EQ(0, timer.get_tasks_num());
@ -132,15 +132,15 @@ TEST(TestCancelTask, cancel_non_running)
ASSERT_EQ(OB_SUCCESS, timer.init());
ASSERT_EQ(OB_SUCCESS, timer.start());
TaskCommon task1;
task1.exec_time_ = 100000; // 100ms
task1.exec_time_ = 300000; // 300ms
TaskCommon task2;
task2.exec_time_ = 50000; //50ms
ASSERT_EQ(OB_SUCCESS, timer.schedule(task1, 50000, false)); // t1
ASSERT_EQ(OB_SUCCESS, timer.schedule(task2, 100000, true)); // t2
ASSERT_EQ(OB_SUCCESS, timer.schedule(task2, 300000, true)); // t2
ASSERT_EQ(OB_SUCCESS, timer.schedule(task2, 500000, true)); // t4
ASSERT_EQ(OB_SUCCESS, timer.schedule(task2, 400000, true)); // t2
ASSERT_EQ(OB_SUCCESS, timer.schedule(task2, 500000, true)); // t3
ASSERT_EQ(OB_SUCCESS, timer.schedule(task2, 600000, true)); // t4
ASSERT_EQ(4, timer.get_tasks_num()); // 4 tasks were scheduled
::usleep(70000);
::usleep(150000);
ASSERT_EQ(1, task1.task_run_count_); // t1 is running
// t1 (i.e., the running task) has been removed from the task array
ASSERT_EQ(3, timer.get_tasks_num());

2
docs/README.md
View File

@ -18,7 +18,7 @@ At present, the guide is composed of the following parts:
6. [Running MySQL test](mysqltest.md)
7. [Debug](debug.md)
8. Commit code and submit a pull request
More information before you start a big feature developing, you should read content below and it can help you understand oceanbase better.
1. [Logging System](logging.md)
2. [Memory Management](memory.md)

80
docs/container.md
View File

@ -36,7 +36,7 @@ Refer to the current memory maintenance mode of ObString and the commonly used i
*
* Construct the buffer data and effective data length of the string
*
* There are also some derived constructors, such as omitting the buffer length
* There are also some derived constructors, such as omitting the buffer length
* (the buffer length is consistent with the data length)
*/
ObString(const obstr_size_t size, const obstr_size_t length, char *ptr);
@ -69,8 +69,8 @@ const char *ptr() const;
/**
* Case insensitively comparison
*
* @NOTE: Although ObString does not specify that it ends with '\0',
* strncasecmp is used in the implementation here, so please pay attention
* @NOTE: Although ObString does not specify that it ends with '\0',
* strncasecmp is used in the implementation here, so please pay attention
* when using this function.
*/
int case_compare(const ObString &obstr) const;
@ -79,7 +79,7 @@ int case_compare(const char *str) const;
/**
* Case-sensitive comparison
*
* @NOTE: Compared with case_compare, strncmp is not used here,
* @NOTE: Compared with case_compare, strncmp is not used here,
* but memcmp is used to compare the buffer length.
*/
int compare(const ObString &obstr) const;
@ -107,7 +107,7 @@ ObIArray();
/**
* Accept the specified array
*
* The interface class will not take over data-related memory.
* The interface class will not take over data-related memory.
* Memory processing depends on the specific implementation class.
*/
ObIArray(T *data, const int64_t count);
@ -120,7 +120,7 @@ int push_back(const T &obj);
/**
* Remove the last element
* @NOTE It is very likely that the destructor will not be called.
* @NOTE It is very likely that the destructor will not be called.
* You need to look at the specific implementation class.
*/
void pop_back();
@ -138,7 +138,7 @@ int remove(int64_t idx);
/**
* Get the element at the specified position
* @return OB_SUCCESS is returned successfully.
* @return OB_SUCCESS is returned successfully.
* If the specified location does not exist, a failure will be returned.
*/
int at(int64_t idx, T &obj);
@ -164,7 +164,7 @@ void destroy();
int reserve(int64_t capacity);
/**
* Reserve a specified size of memory space, usually the implementation
* Reserve a specified size of memory space, usually the implementation
* class will execute the object's constructor
*/
int prepare_allocate(int64_t capacity);
@ -238,7 +238,7 @@ int push_front(const value_type &value);
int pop_back();
/**
* Both pop_front functions delete the first element.
* Both pop_front functions delete the first element.
* The difference is that one will copy the object and the other will not.
*/
int pop_front(value_type &value);
@ -350,7 +350,7 @@ void clear();
/// Insert another linked list at the beginning of the linked list
void push_range(ObDList<DLinkNode> &range);
/// Delete the specified number of elements from the beginning
/// Delete the specified number of elements from the beginning
/// and place the deleted elements in the range
void pop_range(int32_t num, ObDList<DLinkNode> &range);
@ -457,32 +457,32 @@ Among them, `_key_type`, `_value_type`, `_hashfunc`, `_equal` have the same mean
### ObHashMap Interface Introduction
```cpp
/**
* The constructor of ObHashMap does nothing.
* The constructor of ObHashMap does nothing.
* You must call create for actual initialization.
* The parameters of the create function are mainly the number of buckets
* The parameters of the create function are mainly the number of buckets
* (bucket_num) and the parameters of the memory allocator.
* Providing a reasonable number of buckets can make hashmap run more efficiently
* without wasting too much memory.
*
* As you can see from the following interfaces, two memory allocators can be
* As you can see from the following interfaces, two memory allocators can be
* provided, one is the allocator of the bucket array,
* and the other is the allocator of element nodes.
*/
int create(int64_t bucket_num,
int create(int64_t bucket_num,
const ObMemAttr &bucket_attr,
const ObMemAttr &node_attr);
int create(int64_t bucket_num, const ObMemAttr &bucket_attr);
int create(int64_t bucket_num,
int create(int64_t bucket_num,
const lib::ObLabel &bucket_label,
const lib::ObLabel &node_label = ObModIds::OB_HASH_NODE,
const lib::ObLabel &node_label = ObModIds::OB_HASH_NODE,
uint64_t tenant_id = OB_SERVER_TENANT_ID,
uint64_t ctx_id = ObCtxIds::DEFAULT_CTX_ID);
int create(int64_t bucket_num,
_allocer *allocer,
int create(int64_t bucket_num,
_allocer *allocer,
const lib::ObLabel &bucket_label,
const lib::ObLabel &node_label = ObModIds::OB_HASH_NODE);
int create(int64_t bucket_num,
_allocer *allocer,
int create(int64_t bucket_num,
_allocer *allocer,
_bucket_allocer *bucket_allocer);
/// Destroy the current object directly
@ -504,37 +504,37 @@ int get_refactored(const _key_type &key, _value_type &value, const int64_t timeo
/**
* Set the value of a certain key value
* @param flag: 0 means it already exists and will not be overwritten,
* @param flag: 0 means it already exists and will not be overwritten,
* otherwise the original value will be overwritten.
* @param broadcast: whether to wake up the thread waiting to obtain the
* @param broadcast: whether to wake up the thread waiting to obtain the
* current key
* @param overwrite_key: not used. Please refer to flag
* @param callback: After the insertion or update is successful, you can
* @param callback: After the insertion or update is successful, you can
* use callback to perform some additional operations on the value.
*/
template <typename _callback = void>
int set_refactored(const _key_type &key,
int set_refactored(const _key_type &key,
const _value_type &value,
int flag = 0,
int broadcast = 0,
int overwrite_key = 0,
int broadcast = 0,
int overwrite_key = 0,
_callback *callback = nullptr);
/**
* Traverse all elements
* @note
* 1. You cannot delete elements, insert, etc. during the traversal process.
* Because some locks will be added during the traversal process, and locks
* will also be added for insertion, deletion and other actions, lock
* Because some locks will be added during the traversal process, and locks
* will also be added for insertion, deletion and other actions, lock
* conflicts may occur;
* 2. The callback action should be as small as possible because it works
* 2. The callback action should be as small as possible because it works
* within the lock scope.
*/
template<class _callback>
int foreach_refactored(_callback &callback) const;
/**
* Delete the specified key value.
* Delete the specified key value.
* If the value pointer is not null, the corresponding element will be returned
* @return If the element does not exist, OB_HASH_NOT_EXIST will be returned
*/
@ -554,7 +554,7 @@ template<class _pred>
int erase_if(const _key_type &key, _pred &pred, bool &is_erased, _value_type *value = NULL);
/**
* There is no need to copy elements, directly access the elements with
* There is no need to copy elements, directly access the elements with
* specified key values through callback.
* @note callback executed under write lock protection
*/
@ -562,7 +562,7 @@ template <class _callback>
int atomic_refactored(const _key_type &key, _callback &callback);
/**
* There is no need to copy the element value, just get the element directly
* There is no need to copy the element value, just get the element directly
* and access it through callback.
* @note callback executed under write lock protection
*/
@ -625,13 +625,13 @@ Below are some commonly used interfaces and precautions when using them.
*Declaration of ObLinkHashMap
* Template parameters:
* @param Key Key type
* @param Value The type of value, which needs to be inherited from
* @param Value The type of value, which needs to be inherited from
* LinkHashValue (refer to ob_link_hashmap_deps.h)
* @param AllocHandle Class to allocate release values and nodes
* @param AllocHandle Class to allocate release values and nodes
* (refer to ob_link_hashmap_deps.h)
* @param RefHandle Reference counting function. Don't modify it if you
* @param RefHandle Reference counting function. Don't modify it if you
* don't deeply understand its principles.
* @param SHRINK_THRESHOLD When the number of current nodes is too many or too
* @param SHRINK_THRESHOLD When the number of current nodes is too many or too
* few, it will expand or shrink. Try to keep the current nodes at
* Between the ratio [1/SHRINK_THRESHOLD, 1] (non-precise control)
*/
@ -661,7 +661,7 @@ int del(const Key &key);
*/
int get(const Key &key, Value*& value);
/// Releases the introduction count of the specified element.
/// Releases the introduction count of the specified element.
/// Can be released across threads
void revert(Value* value);
@ -673,14 +673,14 @@ int contains_key(const Key &key);
/**
* Traverse all elements
* @param fn: bool fn(Key &key, Value *value); The bool return value
* @param fn: bool fn(Key &key, Value *value); The bool return value
* indicates whether to continue traversing
*/
template <typename Function> int for_each(Function &fn);
/**
* Delete elements that meet the conditions
* @param fn bool fn(Key &key, Value *value); The bool return value
* @param fn bool fn(Key &key, Value *value); The bool return value
* indicates whether it needs to be deleted
*/
template <typename Function> int remove_if(Function &fn);

4
mittest/logservice/test_ob_simple_log_throttling_member_change.cpp
View File

@ -90,6 +90,8 @@ TEST_F(TestObSimpleLogIOWorkerThrottlingV2, test_throttling_majority)
LSN follower_max_lsn = palf_list[1]->palf_handle_impl_->sw_.get_max_lsn();
PALF_LOG(INFO, "prepare max_lsn", K(max_lsn), K(follower_max_lsn));
wait_lsn_until_flushed(max_lsn, leader);
wait_lsn_until_flushed(max_lsn, *palf_list[1]);
wait_lsn_until_flushed(max_lsn, *palf_list[2]);
PALF_LOG(INFO, "[CASE 1]test throttling interval");
int64_t throttling_percentage = 60;
@ -218,6 +220,8 @@ TEST_F(TestObSimpleLogIOWorkerThrottlingV2, test_throttling_minor_leader)
LSN follower_max_lsn = palf_list[1]->palf_handle_impl_->sw_.get_max_lsn();
PALF_LOG(INFO, "prepare max_lsn", K(max_lsn), K(follower_max_lsn));
wait_lsn_until_flushed(max_lsn, leader);
wait_lsn_until_flushed(max_lsn, *palf_list[1]);
wait_lsn_until_flushed(max_lsn, *palf_list[2]);
PALF_LOG(INFO, "[CASE 2]test throttling interval");
int64_t throttling_percentage = 60;

2
mittest/mtlenv/storage/CMakeLists.txt
View File

@ -30,6 +30,8 @@ storage_dml_unittest(test_tablet_aggregated_info test_tablet_aggregated_info.cpp
# storage_unittest(test_speed_limit test_speed_limit.cpp)
storage_dml_unittest(test_tablet_block_id_list test_tablet_block_id_list.cpp)
storage_dml_unittest(test_ls_tablet_info_writer_and_reader test_ls_tablet_info_writer_and_reader.cpp)
storage_dml_unittest(test_transfer_barrier test_transfer_barrier.cpp)
add_subdirectory(checkpoint)
add_subdirectory(blocksstable)

2
mittest/mtlenv/storage/checkpoint/test_lock_memtable_checkpoint.cpp
View File

@ -151,7 +151,7 @@ TEST_F(TestLockMemtableCheckpoint, replay_disorder)
LOG_INFO("TestLockMemtableCheckpoint::replay_disorder 5");
ret = memtable_->recover_obj_lock(DEFAULT_OUT_TRANS_UNLOCK_OP);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, checkpoint->flush(share::SCN::max_scn()));
ASSERT_EQ(OB_SUCCESS, checkpoint->flush(share::SCN::max_scn(), 0));
commit_version.val_ = 1;
commit_scn.val_ = 1;
ret = memtable_->update_lock_status(DEFAULT_OUT_TRANS_UNLOCK_OP,

187
mittest/mtlenv/storage/test_transfer_barrier.cpp Normal file
View File

@ -0,0 +1,187 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX STORAGE
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#define protected public
#define private public
#include "storage/schema_utils.h"
#include "storage/ob_storage_schema.h"
#include "storage/blocksstable/ob_sstable_meta.h"
#include "storage/ls/ob_ls.h"
#include "storage/meta_mem/ob_meta_obj_struct.h"
#include "storage/meta_mem/ob_tenant_meta_mem_mgr.h"
#include "storage/tablet/ob_tablet_meta.h"
#include "storage/tablet/ob_tablet_table_store.h"
#include "storage/tx_storage/ob_ls_service.h"
#include "mtlenv/mock_tenant_module_env.h"
#include "storage/test_dml_common.h"
#include "storage/column_store/ob_column_oriented_sstable.h"
#include "storage/ob_storage_schema_util.h"
using namespace oceanbase::common;
using namespace oceanbase::share;
using namespace oceanbase::share::schema;
namespace oceanbase
{
namespace storage
{
class TestTransferBarrier : public ::testing::Test
{
public:
TestTransferBarrier();
virtual ~TestTransferBarrier() = default;
static void SetUpTestCase();
static void TearDownTestCase();
virtual void SetUp() override;
virtual void TearDown() override;
};
TestTransferBarrier::TestTransferBarrier()
{
}
void TestTransferBarrier::SetUpTestCase()
{
int ret = OB_SUCCESS;
ret = MockTenantModuleEnv::get_instance().init();
ASSERT_EQ(OB_SUCCESS, ret);
}
void TestTransferBarrier::TearDownTestCase()
{
MockTenantModuleEnv::get_instance().destroy();
}
void TestTransferBarrier::SetUp()
{
}
void TestTransferBarrier::TearDown()
{
}
TEST_F(TestTransferBarrier, test_transfer_barrier_redo)
{
int ret = OB_SUCCESS;
logservice::ObReplayBarrierType barrier_flag = logservice::ObReplayBarrierType::NO_NEED_BARRIER;
const ObTxLogType log_type = ObTxLogType::TX_MULTI_DATA_SOURCE_LOG;
//PRE_BARRIER
//START_TRANSFER_OUT
barrier_flag = ObTxLogTypeChecker::need_replay_barrier(log_type, ObTxDataSourceType::START_TRANSFER_OUT);
ASSERT_EQ(barrier_flag, logservice::ObReplayBarrierType::PRE_BARRIER);
//START_TRANSFER_OUT_PREPARE
barrier_flag = ObTxLogTypeChecker::need_replay_barrier(log_type, ObTxDataSourceType::START_TRANSFER_OUT_PREPARE);
ASSERT_EQ(barrier_flag, logservice::ObReplayBarrierType::PRE_BARRIER);
//FINISH_TRANSFER_OUT
barrier_flag = ObTxLogTypeChecker::need_replay_barrier(log_type, ObTxDataSourceType::FINISH_TRANSFER_OUT);
ASSERT_EQ(barrier_flag, logservice::ObReplayBarrierType::PRE_BARRIER);
//START_TRANSFER_IN
barrier_flag = ObTxLogTypeChecker::need_replay_barrier(log_type, ObTxDataSourceType::START_TRANSFER_IN);
ASSERT_EQ(barrier_flag, logservice::ObReplayBarrierType::PRE_BARRIER);
//STRICT_BARRIER
//FINISH_TRANSFER_IN
barrier_flag = ObTxLogTypeChecker::need_replay_barrier(log_type, ObTxDataSourceType::FINISH_TRANSFER_IN);
ASSERT_EQ(barrier_flag, logservice::ObReplayBarrierType::STRICT_BARRIER);
//START_TRANSFER_OUT_V2
barrier_flag = ObTxLogTypeChecker::need_replay_barrier(log_type, ObTxDataSourceType::START_TRANSFER_OUT_V2);
ASSERT_EQ(barrier_flag, logservice::ObReplayBarrierType::STRICT_BARRIER);
//TRANSFER_MOVE_TX_CTX
barrier_flag = ObTxLogTypeChecker::need_replay_barrier(log_type, ObTxDataSourceType::TRANSFER_MOVE_TX_CTX);
ASSERT_EQ(barrier_flag, logservice::ObReplayBarrierType::STRICT_BARRIER);
}
TEST_F(TestTransferBarrier, test_transfer_barrier_before_prepare)
{
int ret = OB_SUCCESS;
logservice::ObReplayBarrierType barrier_flag = logservice::ObReplayBarrierType::NO_NEED_BARRIER;
const ObTxLogType log_type = ObTxLogType::TX_COMMIT_INFO_LOG;
//STRICT_BARRIER
//START_TRANSFER_IN
barrier_flag = ObTxLogTypeChecker::need_replay_barrier(log_type, ObTxDataSourceType::START_TRANSFER_IN);
ASSERT_EQ(barrier_flag, logservice::ObReplayBarrierType::STRICT_BARRIER);
}
TEST_F(TestTransferBarrier, test_transfer_barrier_before_commit)
{
int ret = OB_SUCCESS;
logservice::ObReplayBarrierType barrier_flag = logservice::ObReplayBarrierType::NO_NEED_BARRIER;
const ObTxLogType log_type = ObTxLogType::TX_COMMIT_LOG;
//STRICT_BARRIER
//START_TRANSFER_IN
barrier_flag = ObTxLogTypeChecker::need_replay_barrier(log_type, ObTxDataSourceType::START_TRANSFER_IN);
ASSERT_EQ(barrier_flag, logservice::ObReplayBarrierType::STRICT_BARRIER);
//START_TRANSFER_OUT_V2
barrier_flag = ObTxLogTypeChecker::need_replay_barrier(log_type, ObTxDataSourceType::START_TRANSFER_OUT_V2);
ASSERT_EQ(barrier_flag, logservice::ObReplayBarrierType::STRICT_BARRIER);
//TRANSFER_MOVE_TX_CTX
barrier_flag = ObTxLogTypeChecker::need_replay_barrier(log_type, ObTxDataSourceType::TRANSFER_MOVE_TX_CTX);
ASSERT_EQ(barrier_flag, logservice::ObReplayBarrierType::STRICT_BARRIER);
//TRANSFER_MOVE_TX_CTX
barrier_flag = ObTxLogTypeChecker::need_replay_barrier(log_type, ObTxDataSourceType::TRANSFER_MOVE_TX_CTX);
ASSERT_EQ(barrier_flag, logservice::ObReplayBarrierType::STRICT_BARRIER);
}
TEST_F(TestTransferBarrier, test_transfer_barrier_before_abort)
{
int ret = OB_SUCCESS;
logservice::ObReplayBarrierType barrier_flag = logservice::ObReplayBarrierType::NO_NEED_BARRIER;
const ObTxLogType log_type = ObTxLogType::TX_ABORT_LOG;
//STRICT_BARRIER
//START_TRANSFER_IN
barrier_flag = ObTxLogTypeChecker::need_replay_barrier(log_type, ObTxDataSourceType::START_TRANSFER_IN);
ASSERT_EQ(barrier_flag, logservice::ObReplayBarrierType::STRICT_BARRIER);
//START_TRANSFER_OUT_V2
barrier_flag = ObTxLogTypeChecker::need_replay_barrier(log_type, ObTxDataSourceType::START_TRANSFER_OUT_V2);
ASSERT_EQ(barrier_flag, logservice::ObReplayBarrierType::STRICT_BARRIER);
//TRANSFER_MOVE_TX_CTX
barrier_flag = ObTxLogTypeChecker::need_replay_barrier(log_type, ObTxDataSourceType::TRANSFER_MOVE_TX_CTX);
ASSERT_EQ(barrier_flag, logservice::ObReplayBarrierType::STRICT_BARRIER);
//TRANSFER_MOVE_TX_CTX
barrier_flag = ObTxLogTypeChecker::need_replay_barrier(log_type, ObTxDataSourceType::TRANSFER_MOVE_TX_CTX);
ASSERT_EQ(barrier_flag, logservice::ObReplayBarrierType::STRICT_BARRIER);
}
} // namespace storage
} // namespace oceanbase
int main(int argc, char **argv)
{
system("rm -f test_transfer_barrier.log*");
OB_LOGGER.set_file_name("test_transfer_barrier.log", true);
OB_LOGGER.set_log_level("INFO");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

50
mittest/mtlenv/tablelock/test_lock_memtable.cpp
View File

@ -135,7 +135,7 @@ TEST_F(TestLockMemtable, lock)
ObOBJLock *obj_lock = NULL;
share::SCN min_commited_scn;
share::SCN flushed_scn;
unsigned char lock_mode_in_same_trans = 0x0;
uint64_t lock_mode_cnt_in_same_trans[TABLE_LOCK_MODE_COUNT] = {0, 0, 0, 0, 0};
MyTxCtx default_ctx;
ObStoreCtx store_ctx;
@ -163,7 +163,7 @@ TEST_F(TestLockMemtable, lock)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(lock_exist, true);
// 1.4 check lock at lock map
LOG_INFO("TestLockMemtable::lock 1.4");
@ -181,7 +181,7 @@ TEST_F(TestLockMemtable, lock)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(lock_exist, true);
// 1.7 remove lock op at memctx.
LOG_INFO("TestLockMemtable::lock 1.7");
@ -191,7 +191,7 @@ TEST_F(TestLockMemtable, lock)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(lock_exist, false);
// 2.1 lock
@ -210,7 +210,7 @@ TEST_F(TestLockMemtable, lock)
DEFAULT_OUT_TRANS_LOCK_OP.lock_mode_,
DEFAULT_OUT_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(lock_exist, true);
// 2.3 unlock not complete lock
LOG_INFO("TestLockMemtable::lock 2.3");
@ -283,7 +283,7 @@ TEST_F(TestLockMemtable, lock)
TEST_F(TestLockMemtable, replay)
{
LOG_INFO("TestLockMemtable::replay");
unsigned char lock_mode_in_same_trans = 0x0;
uint64_t lock_mode_cnt_in_same_trans[TABLE_LOCK_MODE_COUNT] = {0, 0, 0, 0, 0};
int ret = OB_SUCCESS;
bool is_try_lock = true;
int64_t expired_time = ObClockGenerator::getClock() + 1 * 1000 * 1000;
@ -333,7 +333,7 @@ TEST_F(TestLockMemtable, replay)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(lock_exist, true);
// 1.3 check lock at lock map
LOG_INFO("TestLockMemtable::replay 1.3");
@ -351,7 +351,7 @@ TEST_F(TestLockMemtable, replay)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(lock_exist, true);
// 1.6 remove lock op at memctx.
LOG_INFO("TestLockMemtable::replay 1.6");
@ -361,7 +361,7 @@ TEST_F(TestLockMemtable, replay)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(lock_exist, false);
// 2. REPLAY LOCK
// 2.1 replay lock
@ -378,7 +378,7 @@ TEST_F(TestLockMemtable, replay)
DEFAULT_OUT_TRANS_LOCK_OP.lock_mode_,
DEFAULT_OUT_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(lock_exist, true);
// 2.3 unlock not complete lock
LOG_INFO("TestLockMemtable::replay 2.3");
@ -540,7 +540,7 @@ TEST_F(TestLockMemtable, recover)
TEST_F(TestLockMemtable, pre_check_lock)
{
LOG_INFO("TestLockMemtable::pre_check_lock");
unsigned char lock_mode_in_same_trans = 0x0;
uint64_t lock_mode_cnt_in_same_trans[TABLE_LOCK_MODE_COUNT] = {0, 0, 0, 0, 0};
int ret = OB_SUCCESS;
bool is_try_lock = true;
bool lock_exist = false;
@ -595,7 +595,7 @@ TEST_F(TestLockMemtable, pre_check_lock)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(lock_exist, true);
// 1.8 remove lock op at memctx.
LOG_INFO("TestLockMemtable::pre_check_lock 1.8");
@ -605,7 +605,7 @@ TEST_F(TestLockMemtable, pre_check_lock)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(lock_exist, false);
}
@ -724,7 +724,7 @@ TEST_F(TestLockMemtable, out_trans_multi_source)
share::SCN commit_version = share::SCN::min_scn();
share::SCN commit_scn = share::SCN::min_scn();
ObMemtableCtx *mem_ctx = NULL;
unsigned char lock_mode_in_same_trans = 0x0;
uint64_t lock_mode_cnt_in_same_trans[TABLE_LOCK_MODE_COUNT] = {0, 0, 0, 0, 0};
ObTableLockOp lock_op = DEFAULT_OUT_TRANS_LOCK_OP;
ObTableLockOp unlock_op = DEFAULT_OUT_TRANS_UNLOCK_OP;
unlock_op.create_trans_id_ = TRANS_ID2;
@ -752,7 +752,7 @@ TEST_F(TestLockMemtable, out_trans_multi_source)
lock_op.lock_mode_,
lock_op.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(lock_exist, false);
// 1.3 check exist at multi source
@ -772,7 +772,7 @@ TEST_F(TestLockMemtable, out_trans_multi_source)
lock_op.lock_mode_,
lock_op.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(lock_exist, false);
// 2.3 check exist at multi source
@ -795,7 +795,7 @@ TEST_F(TestLockMemtable, out_trans_multi_source)
lock_op.lock_mode_,
lock_op.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(lock_exist, false);
// 3.2 notify ON_COMMIT/ON_ABORT
@ -807,7 +807,7 @@ TEST_F(TestLockMemtable, out_trans_multi_source)
lock_op.lock_mode_,
lock_op.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(lock_exist, false);
ret = default_ctx.tx_ctx_.notify_data_source_(NotifyType::ON_ABORT, scn, is_replay, mds_array);
@ -817,7 +817,7 @@ TEST_F(TestLockMemtable, out_trans_multi_source)
lock_op.lock_mode_,
lock_op.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(lock_exist, false);
@ -832,7 +832,7 @@ TEST_F(TestLockMemtable, out_trans_multi_source)
lock_op.lock_mode_,
lock_op.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(lock_exist, true);
@ -997,7 +997,7 @@ TEST_F(TestLockMemtable, test_lock_retry)
bool lock_exist = false;
share::SCN min_commited_scn;
share::SCN flushed_scn;
unsigned char lock_mode_in_same_trans = 0x0;
uint64_t lock_mode_cnt_in_same_trans[TABLE_LOCK_MODE_COUNT] = {0, 0, 0, 0, 0};
ObTableLockOp lock_op = DEFAULT_OUT_TRANS_LOCK_OP;
min_commited_scn.set_min();
@ -1037,7 +1037,7 @@ TEST_F(TestLockMemtable, test_lock_retry)
DEFAULT_OUT_TRANS_LOCK_OP.lock_mode_,
DEFAULT_OUT_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(lock_exist, false);
// 3.3 check lock at part ctx
@ -1073,7 +1073,7 @@ TEST_F(TestLockMemtable, test_lock_retry)
DEFAULT_OUT_TRANS_LOCK_OP.lock_mode_,
DEFAULT_OUT_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(lock_exist, true);
// 6.3 check lock at part ctx
@ -1157,7 +1157,7 @@ TEST_F(TestLockMemtable, test_lock_retry_lock_conflict)
bool lock_exist = false;
share::SCN min_commited_scn;
share::SCN flushed_scn;
unsigned char lock_mode_in_same_trans = 0x0;
uint64_t lock_mode_cnt_in_same_trans[TABLE_LOCK_MODE_COUNT] = {0, 0, 0, 0, 0};
ObTableLockOp lock_first = DEFAULT_OUT_TRANS_LOCK_OP; // RX, owner 0
ObTableLockOp lock_second = DEFAULT_OUT_TRANS_LOCK_OP;
@ -1257,7 +1257,7 @@ TEST_F(TestLockMemtable, test_lock_retry_lock_conflict)
lock_second.lock_mode_,
lock_second.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(lock_exist, true);
// 6. clean

14
mittest/mtlenv/tablelock/test_lock_table_callback.cpp
View File

@ -133,7 +133,7 @@ TEST_F(TestLockTableCallback, callback)
{
int ret = OB_SUCCESS;
bool lock_exist = false;
unsigned char lock_mode_in_same_trans = 0x0;
uint64_t lock_mode_cnt_in_same_trans[TABLE_LOCK_MODE_COUNT] = {0, 0, 0, 0, 0};
const bool for_replay = false;
ObIMemtable *memtable = nullptr;
ObOBJLockCallback *cb = nullptr;
@ -155,7 +155,7 @@ TEST_F(TestLockTableCallback, callback)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(lock_exist, true);
ret = cb->trans_commit();
ASSERT_EQ(OB_SUCCESS, ret);
@ -164,7 +164,7 @@ TEST_F(TestLockTableCallback, callback)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(lock_exist, false);
// 3.2 intrans abort
LOG_INFO("TestLockTableCallback::callback 3.2");
@ -174,7 +174,7 @@ TEST_F(TestLockTableCallback, callback)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(lock_exist, true);
ret = cb->trans_abort();
ASSERT_EQ(OB_SUCCESS, ret);
@ -183,7 +183,7 @@ TEST_F(TestLockTableCallback, callback)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(lock_exist, false);
// 3.3 intrans stmt abort
LOG_INFO("TestLockTableCallback::callback 3.3");
@ -193,7 +193,7 @@ TEST_F(TestLockTableCallback, callback)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(lock_exist, true);
ret = cb->rollback_callback();
ASSERT_EQ(OB_SUCCESS, ret);
@ -202,7 +202,7 @@ TEST_F(TestLockTableCallback, callback)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(lock_exist, false);
}

40
mittest/mtlenv/tablelock/test_mem_ctx_table_lock.cpp
View File

@ -111,7 +111,7 @@ TEST_F(TestMemCtxTableLock, add_lock_record)
LOG_INFO("TestMemCtxTableLock::add_lock_record");
int ret = OB_SUCCESS;
bool lock_exist = false;
unsigned char lock_mode_in_same_trans = 0x0;
uint64_t lock_mode_cnt_in_same_trans[TABLE_LOCK_MODE_COUNT] = {0, 0, 0, 0, 0};
ObMemCtxLockOpLinkNode *lock_op_node = nullptr;
// 1. IN TRANS LOCK
// 1.1 add lock record
@ -126,7 +126,7 @@ TEST_F(TestMemCtxTableLock, add_lock_record)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(lock_exist, true);
// 2. OUT TRANS LOCK
@ -142,7 +142,7 @@ TEST_F(TestMemCtxTableLock, add_lock_record)
DEFAULT_OUT_TRANS_LOCK_OP.lock_mode_,
DEFAULT_OUT_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(lock_exist, true);
// 3. REMOVE CHECK
@ -154,7 +154,7 @@ TEST_F(TestMemCtxTableLock, add_lock_record)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(lock_exist, false);
// 3.2 out trans lock remove test
@ -165,7 +165,7 @@ TEST_F(TestMemCtxTableLock, add_lock_record)
DEFAULT_OUT_TRANS_LOCK_OP.lock_mode_,
DEFAULT_OUT_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(lock_exist, false);
@ -180,7 +180,7 @@ TEST_F(TestMemCtxTableLock, add_lock_record)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(lock_exist, true);
// 4.2 remove lock record
@ -191,7 +191,7 @@ TEST_F(TestMemCtxTableLock, add_lock_record)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(lock_exist, false);
}
@ -201,7 +201,7 @@ TEST_F(TestMemCtxTableLock, clear_table_lock)
LOG_INFO("TestMemCtxTableLock::clear_table_lock");
int ret = OB_SUCCESS;
bool lock_exist = false;
unsigned char lock_mode_in_same_trans = 0x0;
uint64_t lock_mode_cnt_in_same_trans[TABLE_LOCK_MODE_COUNT] = {0, 0, 0, 0, 0};
ObMemCtxLockOpLinkNode *lock_op_node = nullptr;
bool is_committed = false;
share::SCN commit_version;
@ -227,7 +227,7 @@ TEST_F(TestMemCtxTableLock, clear_table_lock)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(lock_exist, false);
// 2.2 out trans lock record
@ -242,7 +242,7 @@ TEST_F(TestMemCtxTableLock, clear_table_lock)
DEFAULT_OUT_TRANS_LOCK_OP.lock_mode_,
DEFAULT_OUT_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(lock_exist, false);
// 2.3 out trans unlock record
@ -257,7 +257,7 @@ TEST_F(TestMemCtxTableLock, clear_table_lock)
DEFAULT_OUT_TRANS_UNLOCK_OP.lock_mode_,
DEFAULT_OUT_TRANS_UNLOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(lock_exist, false);
@ -273,7 +273,7 @@ TEST_F(TestMemCtxTableLock, clear_table_lock)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(lock_exist, false);
}
@ -284,7 +284,7 @@ TEST_F(TestMemCtxTableLock, rollback_table_lock)
int ret = OB_SUCCESS;
bool lock_exist = false;
ObMemCtxLockOpLinkNode *lock_op_node = nullptr;
unsigned char lock_mode_in_same_trans = 0x0;
uint64_t lock_mode_cnt_in_same_trans[TABLE_LOCK_MODE_COUNT] = {0, 0, 0, 0, 0};
// 1.1 add lock record
LOG_INFO("TestMemCtxTableLock::rollback_table_lock 1.1");
ObTableLockOp lock_op1 = DEFAULT_IN_TRANS_LOCK_OP;
@ -304,7 +304,7 @@ TEST_F(TestMemCtxTableLock, rollback_table_lock)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(lock_exist, true);
ret = mem_ctx_.check_lock_exist(DEFAULT_OUT_TRANS_LOCK_OP.lock_id_,
@ -312,7 +312,7 @@ TEST_F(TestMemCtxTableLock, rollback_table_lock)
DEFAULT_OUT_TRANS_LOCK_OP.lock_mode_,
DEFAULT_OUT_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(lock_exist, true);
// 1.3 rollback
@ -326,7 +326,7 @@ TEST_F(TestMemCtxTableLock, rollback_table_lock)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(lock_exist, true);
ret = mem_ctx_.check_lock_exist(DEFAULT_OUT_TRANS_LOCK_OP.lock_id_,
@ -334,7 +334,7 @@ TEST_F(TestMemCtxTableLock, rollback_table_lock)
DEFAULT_OUT_TRANS_LOCK_OP.lock_mode_,
DEFAULT_OUT_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(lock_exist, false);
// 1.4 rollback again
@ -347,7 +347,7 @@ TEST_F(TestMemCtxTableLock, rollback_table_lock)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(lock_exist, false);
}
@ -355,7 +355,7 @@ TEST_F(TestMemCtxTableLock, rollback_table_lock)
TEST_F(TestMemCtxTableLock, check_lock_need_replay)
{
LOG_INFO("TestMemCtxTableLock::check_lock_need_replay");
unsigned char lock_mode_in_same_trans = 0x0;
uint64_t lock_mode_cnt_in_same_trans[TABLE_LOCK_MODE_COUNT] = {0, 0, 0, 0, 0};
int ret = OB_SUCCESS;
bool lock_exist = false;
share::SCN scn;
@ -406,7 +406,7 @@ TEST_F(TestMemCtxTableLock, check_lock_need_replay)
DEFAULT_IN_TRANS_LOCK_OP.lock_mode_,
DEFAULT_IN_TRANS_LOCK_OP.op_type_,
lock_exist,
lock_mode_in_same_trans);
lock_mode_cnt_in_same_trans);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(lock_exist, false);
}

80
mittest/mtlenv/tablelock/test_obj_lock.cpp
View File

@ -110,7 +110,7 @@ TEST_F(TestObjLock, in_trans_lock)
bool is_try_lock = true;
int64_t expired_time = 0;
ObTxIDSet conflict_tx_set;
unsigned char lock_mode_in_same_trans = 0x0;
uint64_t lock_mode_cnt_in_same_trans[TABLE_LOCK_MODE_COUNT] = {0, 0, 0, 0, 0};
// int64_t expired_time = ObClockGenerator::getClock() + 1000;
@ -126,7 +126,7 @@ TEST_F(TestObjLock, in_trans_lock)
ret = obj_lock_.lock(param,
store_ctx,
DEFAULT_IN_TRANS_LOCK_OP,
lock_mode_in_same_trans,
lock_mode_cnt_in_same_trans,
allocator_,
conflict_tx_set);
ASSERT_EQ(OB_SUCCESS, ret);
@ -140,7 +140,7 @@ TEST_F(TestObjLock, in_trans_lock)
ret = obj_lock_.lock(param,
store_ctx,
DEFAULT_IN_TRANS_LOCK_OP,
lock_mode_in_same_trans,
lock_mode_cnt_in_same_trans,
allocator_,
conflict_tx_set);
ASSERT_EQ(OB_SUCCESS, ret);
@ -153,7 +153,7 @@ TEST_F(TestObjLock, in_trans_lock)
ret = obj_lock_.lock(param,
store_ctx,
DEFAULT_IN_TRANS_LOCK_OP,
lock_mode_in_same_trans,
lock_mode_cnt_in_same_trans,
allocator_,
conflict_tx_set);
ASSERT_EQ(OB_SUCCESS, ret);
@ -182,7 +182,7 @@ TEST_F(TestObjLock, out_trans_lock)
share::SCN min_commited_scn;
share::SCN flushed_scn;
ObTxIDSet conflict_tx_set;
unsigned char lock_mode_in_same_trans = 0x0;
uint64_t lock_mode_cnt_in_same_trans[TABLE_LOCK_MODE_COUNT] = {0, 0, 0, 0, 0};
ObLockParam param;
MyTxCtx default_ctx;
@ -200,7 +200,7 @@ TEST_F(TestObjLock, out_trans_lock)
ret = obj_lock_.lock(param,
store_ctx,
DEFAULT_OUT_TRANS_LOCK_OP,
lock_mode_in_same_trans,
lock_mode_cnt_in_same_trans,
allocator_,
conflict_tx_set);
ASSERT_EQ(OB_SUCCESS, ret);
@ -246,7 +246,7 @@ TEST_F(TestObjLock, out_trans_lock)
ret = obj_lock_.lock(param,
store_ctx,
DEFAULT_OUT_TRANS_LOCK_OP,
lock_mode_in_same_trans,
lock_mode_cnt_in_same_trans,
allocator_,
conflict_tx_set);
ASSERT_EQ(OB_SUCCESS, ret);
@ -302,7 +302,7 @@ TEST_F(TestObjLock, out_trans_unlock_twice)
share::SCN commit_version;
share::SCN commit_scn;
ObTxIDSet conflict_tx_set;
unsigned char lock_mode_in_same_trans = 0x0;
uint64_t lock_mode_cnt_in_same_trans[TABLE_LOCK_MODE_COUNT] = {0, 0, 0, 0, 0};
ObLockParam param;
MyTxCtx default_ctx;
@ -325,7 +325,7 @@ TEST_F(TestObjLock, out_trans_unlock_twice)
ret = obj_lock_.lock(param,
store_ctx,
DEFAULT_OUT_TRANS_LOCK_OP,
lock_mode_in_same_trans,
lock_mode_cnt_in_same_trans,
allocator_,
conflict_tx_set);
ASSERT_EQ(OB_SUCCESS, ret);
@ -512,7 +512,7 @@ TEST_F(TestObjLock, lock_conflict_in_in)
int64_t expired_time = 0;
ObTxIDSet conflict_tx_set;
int64_t conflict_modes;
unsigned char lock_mode_in_same_trans = 0x0;
uint64_t lock_mode_cnt_in_same_trans[TABLE_LOCK_MODE_COUNT] = {0, 0, 0, 0, 0};
ObLockParam param;
MyTxCtx default_ctx;
@ -533,14 +533,14 @@ TEST_F(TestObjLock, lock_conflict_in_in)
ret = obj_lock_.lock(param,
store_ctx,
DEFAULT_IN_TRANS_LOCK_OP,
lock_mode_in_same_trans,
lock_mode_cnt_in_same_trans,
allocator_,
conflict_tx_set);
ASSERT_EQ(OB_SUCCESS, ret);
ret = obj_lock_.lock(param,
store_ctx,
DEFAULT_CONFLICT_OUT_TRANS_LOCK_OP,
lock_mode_in_same_trans,
lock_mode_cnt_in_same_trans,
allocator_,
conflict_tx_set);
ASSERT_EQ(OB_TRY_LOCK_ROW_CONFLICT, ret);
@ -552,7 +552,7 @@ TEST_F(TestObjLock, lock_conflict_in_in)
ret = obj_lock_.lock(param,
store_ctx,
DEFAULT_CONFLICT_OUT_TRANS_LOCK_OP,
lock_mode_in_same_trans,
lock_mode_cnt_in_same_trans,
allocator_,
conflict_tx_set);
if (DEFAULT_CONFLICT_OUT_TRANS_LOCK_OP.is_dml_lock_op()) {
@ -579,26 +579,26 @@ TEST_F(TestObjLock, lock_conflict_in_in)
lock_second.create_trans_id_ = TRANS_ID2;
for (int first = 0; first < TABLE_LOCK_MODE_COUNT; first++) {
// lock the first lock
printf("first lock index:%d\n", first);
lock_first.lock_mode_ = LOCK_MODE_ARRAY[first];
lock_first.lock_mode_ = get_lock_mode_by_index(first);
printf("first lock index: %d, mode: %d\n", first, lock_first.lock_mode_);
ret = obj_lock_.lock(param,
store_ctx,
lock_first,
lock_mode_in_same_trans,
lock_mode_cnt_in_same_trans,
allocator_,
conflict_tx_set);
ASSERT_EQ(OB_SUCCESS, ret);
// lock the second lock and check conflict
for (int second = 0; second < TABLE_LOCK_MODE_COUNT; second++) {
printf("second lock index:%d\n", second);
lock_second.lock_mode_ = LOCK_MODE_ARRAY[second];
lock_second.lock_mode_ = get_lock_mode_by_index(second);
printf("second lock index: %d, mode: %d\n", second, lock_second.lock_mode_);
is_conflict = !request_lock(lock_first.lock_mode_,
lock_second.lock_mode_,
conflict_modes);
ret = obj_lock_.lock(param,
store_ctx2,
lock_second,
lock_mode_in_same_trans,
lock_mode_cnt_in_same_trans,
allocator_,
conflict_tx_set);
if (is_conflict) {
@ -627,7 +627,7 @@ TEST_F(TestObjLock, lock_conflict_in_out)
int64_t expired_time = 0;
ObTxIDSet conflict_tx_set;
int64_t conflict_modes = 0;
unsigned char lock_mode_in_same_trans = 0x0;
uint64_t lock_mode_cnt_in_same_trans[TABLE_LOCK_MODE_COUNT] = {0, 0, 0, 0, 0};
ObLockParam param;
MyTxCtx default_ctx;
@ -655,26 +655,26 @@ TEST_F(TestObjLock, lock_conflict_in_out)
param.expired_time_ = expired_time;
for (int first = 0; first < TABLE_LOCK_MODE_COUNT; first++) {
// lock the first lock
printf("first lock index:%d\n", first);
lock_first.lock_mode_ = LOCK_MODE_ARRAY[first];
lock_first.lock_mode_ = get_lock_mode_by_index(first);
printf("first lock index: %d, mode: %d\n", first, lock_first.lock_mode_);
ret = obj_lock_.lock(param,
store_ctx,
lock_first,
lock_mode_in_same_trans,
lock_mode_cnt_in_same_trans,
allocator_,
conflict_tx_set);
ASSERT_EQ(OB_SUCCESS, ret);
// lock the second lock and check conflict
for (int second = 0; second < TABLE_LOCK_MODE_COUNT; second++) {
printf("second lock index:%d\n", second);
lock_second.lock_mode_ = LOCK_MODE_ARRAY[second];
lock_second.lock_mode_ = get_lock_mode_by_index(second);
printf("second lock index: %d, mode: %d\n", second, lock_second.lock_mode_);
is_conflict = !request_lock(lock_first.lock_mode_,
lock_second.lock_mode_,
conflict_modes);
ret = obj_lock_.lock(param,
store_ctx2,
lock_second,
lock_mode_in_same_trans,
lock_mode_cnt_in_same_trans,
allocator_,
conflict_tx_set);
if (is_conflict) {
@ -702,7 +702,7 @@ TEST_F(TestObjLock, lock_conflict_out_in)
int64_t expired_time = 0;
ObTxIDSet conflict_tx_set;
int64_t conflict_modes = 0;
unsigned char lock_mode_in_same_trans = 0x0;
uint64_t lock_mode_cnt_in_same_trans[TABLE_LOCK_MODE_COUNT] = {0, 0, 0, 0, 0};
ObLockParam param;
MyTxCtx default_ctx;
@ -729,26 +729,26 @@ TEST_F(TestObjLock, lock_conflict_out_in)
param.expired_time_ = expired_time;
for (int first = 0; first < TABLE_LOCK_MODE_COUNT; first++) {
// lock the first lock
printf("first lock index:%d\n", first);
lock_first.lock_mode_ = LOCK_MODE_ARRAY[first];
lock_first.lock_mode_ = get_lock_mode_by_index(first);
printf("first lock index: %d, mode: %d\n", first, lock_first.lock_mode_);
ret = obj_lock_.lock(param,
store_ctx,
lock_first,
lock_mode_in_same_trans,
lock_mode_cnt_in_same_trans,
allocator_,
conflict_tx_set);
ASSERT_EQ(OB_SUCCESS, ret);
// lock the second lock and check conflict
for (int second = 0; second < TABLE_LOCK_MODE_COUNT; second++) {
printf("second lock index:%d\n", second);
lock_second.lock_mode_ = LOCK_MODE_ARRAY[second];
lock_second.lock_mode_ = get_lock_mode_by_index(second);
printf("second lock index: %d, mode: %d\n", second, lock_second.lock_mode_);
is_conflict = !request_lock(lock_first.lock_mode_,
lock_second.lock_mode_,
conflict_modes);
ret = obj_lock_.lock(param,
store_ctx2,
lock_second,
lock_mode_in_same_trans,
lock_mode_cnt_in_same_trans,
allocator_,
conflict_tx_set);
if (is_conflict) {
@ -776,7 +776,7 @@ TEST_F(TestObjLock, lock_conflict_out_out)
int64_t expired_time = 0;
ObTxIDSet conflict_tx_set;
int64_t conflict_modes = 0;
unsigned char lock_mode_in_same_trans = 0x0;
uint64_t lock_mode_cnt_in_same_trans[TABLE_LOCK_MODE_COUNT] = {0, 0, 0, 0, 0};
ObLockParam param;
MyTxCtx default_ctx;
@ -803,26 +803,26 @@ TEST_F(TestObjLock, lock_conflict_out_out)
param.expired_time_ = expired_time;
for (int first = 0; first < TABLE_LOCK_MODE_COUNT; first++) {
// lock the first lock
printf("first lock index:%d\n", first);
lock_first.lock_mode_ = LOCK_MODE_ARRAY[first];
lock_first.lock_mode_ = get_lock_mode_by_index(first);
printf("first lock index: %d, mode: %d\n", first, lock_first.lock_mode_);
ret = obj_lock_.lock(param,
store_ctx,
lock_first,
lock_mode_in_same_trans,
lock_mode_cnt_in_same_trans,
allocator_,
conflict_tx_set);
ASSERT_EQ(OB_SUCCESS, ret);
// lock the second lock and check conflict
for (int second = 0; second < TABLE_LOCK_MODE_COUNT; second++) {
printf("second lock index:%d\n", second);
lock_second.lock_mode_ = LOCK_MODE_ARRAY[second];
lock_second.lock_mode_ = get_lock_mode_by_index(second);
printf("second lock index: %d, mode: %d\n", second, lock_second.lock_mode_);
is_conflict = !request_lock(lock_first.lock_mode_,
lock_second.lock_mode_,
conflict_modes);
ret = obj_lock_.lock(param,
store_ctx2,
lock_second,
lock_mode_in_same_trans,
lock_mode_cnt_in_same_trans,
allocator_,
conflict_tx_set);
if (is_conflict) {

12
mittest/mtlenv/tablelock/test_obj_lock_map.cpp
View File

@ -107,7 +107,7 @@ TEST_F(TestObjLockMap, lock)
int64_t expired_time = 0;
ObOBJLock *obj_lock = NULL;
ObTxIDSet conflict_tx_set;
unsigned char lock_mode_in_same_trans = 0x0;
uint64_t lock_mode_cnt_in_same_trans[TABLE_LOCK_MODE_COUNT] = {0, 0, 0, 0, 0};
ObLockParam param;
MyTxCtx default_ctx;
@ -122,7 +122,7 @@ TEST_F(TestObjLockMap, lock)
ret = lock_map_.lock(param,
store_ctx,
DEFAULT_IN_TRANS_LOCK_OP,
lock_mode_in_same_trans,
lock_mode_cnt_in_same_trans,
conflict_tx_set);
ASSERT_EQ(OB_SUCCESS, ret);
ret = lock_map_.get_obj_lock_with_ref_(DEFAULT_TABLET_LOCK_ID,
@ -137,7 +137,7 @@ TEST_F(TestObjLockMap, lock)
ret = lock_map_.lock(param,
store_ctx,
DEFAULT_IN_TRANS_LOCK_OP,
lock_mode_in_same_trans,
lock_mode_cnt_in_same_trans,
conflict_tx_set);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(obj_lock->row_exclusive_, 2);
@ -149,7 +149,7 @@ TEST_F(TestObjLockMap, lock)
ret = lock_map_.lock(param,
store_ctx,
DEFAULT_IN_TRANS_LOCK_OP,
lock_mode_in_same_trans,
lock_mode_cnt_in_same_trans,
conflict_tx_set);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_EQ(obj_lock->row_exclusive_, 3);
@ -180,7 +180,7 @@ TEST_F(TestObjLockMap, unlock)
share::SCN min_commited_scn;
share::SCN flushed_scn;
ObTxIDSet conflict_tx_set;
unsigned char lock_mode_in_same_trans = 0x0;
uint64_t lock_mode_cnt_in_same_trans[TABLE_LOCK_MODE_COUNT] = {0, 0, 0, 0, 0};
ObLockParam param;
MyTxCtx default_ctx;
@ -201,7 +201,7 @@ TEST_F(TestObjLockMap, unlock)
ret = lock_map_.lock(param,
store_ctx,
DEFAULT_OUT_TRANS_LOCK_OP,
lock_mode_in_same_trans,
lock_mode_cnt_in_same_trans,
conflict_tx_set);
ASSERT_EQ(OB_SUCCESS, ret);
min_commited_scn = lock_map_.get_min_ddl_committed_scn(flushed_scn);

8
mittest/multi_replica/env/ob_multi_replica_test_base.cpp vendored
View File

@ -444,10 +444,9 @@ int ObMultiReplicaTestBase::init_replicas_()
for (int j = 0; j < MAX_ZONE_COUNT && OB_SUCC(ret); j++) {
std::string rpc_port_str = "";
if (OB_FAIL(wait_event_finish("ZONE" + std::to_string(j + 1) + "_RPC_PORT", rpc_port_str,
5000 /*5s*/, 100 /*100ms*/))) {
30000 /*30s*/, 100 /*100ms*/))) {
SERVER_LOG(ERROR, "read RPC_PORT event failed", K(ret), K(j), K(rpc_ports_[j]),
K(rpc_port_str.c_str()));
SERVER_LOG(ERROR, "read RPC_PORT event failed", K(ret), K(j), K(rpc_port_str.c_str()));
} else {
int tmp_rpc_port = std::stoi(rpc_port_str);
@ -462,8 +461,7 @@ int ObMultiReplicaTestBase::init_replicas_()
obrpc::ObServerInfo server_info;
std::string zone_dir = "zone" + std::to_string(j + 1);
server_info.zone_ = zone_dir.c_str();
server_info.server_ =
common::ObAddr(common::ObAddr::IPV4, local_ip_.c_str(), tmp_rpc_port);
server_info.server_ = common::ObAddr(common::ObAddr::IPV4, local_ip_.c_str(), tmp_rpc_port);
server_info.region_ = "sys_region";
server_list_.push_back(server_info);
}

2
mittest/multi_replica/test_mds_replay_from_ctx_table.cpp
View File

@ -76,7 +76,7 @@ void minor_freeze_tx_ctx_memtable(ObLS *ls)
checkpoint_executor->handlers_[logservice::TRANS_SERVICE_LOG_BASE_TYPE])
->common_checkpoints_[ObCommonCheckpointType::TX_CTX_MEMTABLE_TYPE]);
ASSERT_EQ(true, tx_ctx_memtable->is_active_memtable());
ASSERT_EQ(OB_SUCCESS, tx_ctx_memtable->flush(share::SCN::max_scn()));
ASSERT_EQ(OB_SUCCESS, tx_ctx_memtable->flush(share::SCN::max_scn(), -1));
// // TODO(handora.qc): use more graceful wait
// usleep(10 * 1000 * 1000);

2
mittest/simple_server/fast_commit_report.cpp
View File

@ -638,7 +638,7 @@ public:
ASSERT_EQ(OB_SUCCESS, ((ObTxDataMemtableMgr *)(memtable_mgr))->freeze());
share::SCN start_log_ts = tx_data_memtable->get_start_scn();
share::SCN end_log_ts = tx_data_memtable->get_end_scn();
ASSERT_EQ(OB_SUCCESS, tx_data_memtable->flush());
ASSERT_EQ(OB_SUCCESS, tx_data_memtable->flush(0));
wait_freeze_tx_table_finish(start_log_ts, end_log_ts);

2
mittest/simple_server/test_ls_recover.cpp
View File

@ -182,7 +182,7 @@ void ObLSBeforeRestartTest::minor_freeze_tx_ctx_table()
= dynamic_cast<ObTxCtxMemtable *>(dynamic_cast<ObLSTxService *>(checkpoint_executor
->handlers_[logservice::TRANS_SERVICE_LOG_BASE_TYPE])
->common_checkpoints_[ObCommonCheckpointType::TX_CTX_MEMTABLE_TYPE]);
ASSERT_EQ(OB_SUCCESS, tx_ctx_memtable->flush(share::SCN::max_scn()));
ASSERT_EQ(OB_SUCCESS, tx_ctx_memtable->flush(share::SCN::max_scn(), 0));
int retry_time = 0;
while (tx_ctx_memtable->is_frozen_memtable()) {
usleep(1000 * 1000);

2
mittest/simple_server/test_ob_minor_freeze.cpp
View File

@ -406,7 +406,7 @@ void ObMinorFreezeTest::batch_tablet_freeze()
const int64_t start = ObTimeUtility::current_time();
while (ObTimeUtility::current_time() - start <= freeze_duration_) {
ASSERT_EQ(OB_SUCCESS, ls_handles_.at(0).get_ls()->batch_tablet_freeze(tablet_ids_, (i % 2 == 0) ? true : false));
ASSERT_EQ(OB_SUCCESS, ls_handles_.at(0).get_ls()->batch_tablet_freeze(0, tablet_ids_, (i % 2 == 0) ? true : false));
i = i + 1;
}
}

4
mittest/simple_server/test_tx_recovery.cpp
View File

@ -112,7 +112,7 @@ public:
void minor_freeze_data_memtable(ObLS *ls)
{
TRANS_LOG(INFO, "minor_freeze_data_memtable begin");
ASSERT_EQ(OB_SUCCESS, ls->ls_freezer_.logstream_freeze());
ASSERT_EQ(OB_SUCCESS, ls->ls_freezer_.logstream_freeze(0));
// TODO(handora.qc): use more graceful wait
usleep(10 * 1000 * 1000);
@ -131,7 +131,7 @@ public:
dynamic_cast<ObLSTxService *>(
checkpoint_executor->handlers_[logservice::TRANS_SERVICE_LOG_BASE_TYPE])
->common_checkpoints_[ObCommonCheckpointType::TX_CTX_MEMTABLE_TYPE]);
ASSERT_EQ(OB_SUCCESS, tx_ctx_memtable->flush(share::SCN::max_scn()));
ASSERT_EQ(OB_SUCCESS, tx_ctx_memtable->flush(share::SCN::max_scn(), 0));
// TODO(handora.qc): use more graceful wait
usleep(10 * 1000 * 1000);

4
mittest/simple_server/test_tx_recovery2.cpp
View File

@ -112,7 +112,7 @@ public:
void minor_freeze_data_memtable(ObLS *ls)
{
TRANS_LOG(INFO, "minor_freeze_data_memtable begin");
ASSERT_EQ(OB_SUCCESS, ls->ls_freezer_.logstream_freeze());
ASSERT_EQ(OB_SUCCESS, ls->ls_freezer_.logstream_freeze(0));
// TODO(handora.qc): use more graceful wait
usleep(10 * 1000 * 1000);
@ -131,7 +131,7 @@ public:
dynamic_cast<ObLSTxService *>(
checkpoint_executor->handlers_[logservice::TRANS_SERVICE_LOG_BASE_TYPE])
->common_checkpoints_[ObCommonCheckpointType::TX_CTX_MEMTABLE_TYPE]);
ASSERT_EQ(OB_SUCCESS, tx_ctx_memtable->flush(share::SCN::max_scn()));
ASSERT_EQ(OB_SUCCESS, tx_ctx_memtable->flush(share::SCN::max_scn(), 0));
// TODO(handora.qc): use more graceful wait
usleep(10 * 1000 * 1000);

4
mittest/simple_server/test_tx_recovery3.cpp
View File

@ -112,7 +112,7 @@ public:
void minor_freeze_data_memtable(ObLS *ls)
{
TRANS_LOG(INFO, "minor_freeze_data_memtable begin");
ASSERT_EQ(OB_SUCCESS, ls->ls_freezer_.logstream_freeze());
ASSERT_EQ(OB_SUCCESS, ls->ls_freezer_.logstream_freeze(0));
// TODO(handora.qc): use more graceful wait
usleep(10 * 1000 * 1000);
@ -131,7 +131,7 @@ public:
dynamic_cast<ObLSTxService *>(
checkpoint_executor->handlers_[logservice::TRANS_SERVICE_LOG_BASE_TYPE])
->common_checkpoints_[ObCommonCheckpointType::TX_CTX_MEMTABLE_TYPE]);
ASSERT_EQ(OB_SUCCESS, tx_ctx_memtable->flush(share::SCN::max_scn()));
ASSERT_EQ(OB_SUCCESS, tx_ctx_memtable->flush(share::SCN::max_scn(), 0));
// TODO(handora.qc): use more graceful wait
usleep(10 * 1000 * 1000);

2
rpm/oceanbase.spec
View File

@ -1,4 +1,4 @@
Name: %NAME
Version:4.3.0.1
Version:4.3.1.0
Release: %RELEASE
BuildRequires: binutils = 2.30

2
src/diagnose/lua/ob_lua_api.cpp
View File

@ -1217,7 +1217,7 @@ int select_sql_workarea_active(lua_State *L)
lua_pushnil(L);
} else {
ObArray<uint64_t> ids;
common::ObSEArray<sql::ObSqlWorkareaProfileInfo, 32> wa_actives;
common::ObSEArray<sql::ObSqlWorkareaProfileInfo, 20> wa_actives;
std::vector<const char*> columns = {
"plan_id",
"sql_id",

7
src/logservice/cdcservice/ob_cdc_req.cpp
View File

@ -340,6 +340,7 @@ void ObCdcLSFetchLogReq::reset()
progress_ = OB_INVALID_TIMESTAMP;
flag_ = 0;
tenant_id_ = OB_INVALID_TENANT_ID;
compressor_type_ = ObCompressorType::INVALID_COMPRESSOR;
}
ObCdcLSFetchLogReq& ObCdcLSFetchLogReq::operator=(const ObCdcLSFetchLogReq &other)
@ -353,7 +354,7 @@ ObCdcLSFetchLogReq& ObCdcLSFetchLogReq::operator=(const ObCdcLSFetchLogReq &othe
progress_ = other.progress_;
flag_ = other.flag_;
tenant_id_ = other.tenant_id_;
compressor_type_ = other.compressor_type_;
return *this;
}
@ -367,7 +368,8 @@ bool ObCdcLSFetchLogReq::operator==(const ObCdcLSFetchLogReq &that) const
&& client_id_ == that.client_id_
&& progress_ == that.progress_
&& flag_ == that.flag_
&& tenant_id_ == that.tenant_id_;
&& tenant_id_ == that.tenant_id_
&& compressor_type_ == that.compressor_type_;
}
bool ObCdcLSFetchLogReq::operator!=(const ObCdcLSFetchLogReq &that) const
@ -497,6 +499,7 @@ void ObCdcLSFetchMissLogReq::reset()
client_id_.reset();
flag_ = 0;
tenant_id_ = OB_INVALID_TENANT_ID;
compressor_type_ = common::ObCompressorType::INVALID_COMPRESSOR;
}
int ObCdcLSFetchMissLogReq::append_miss_log(const MissLogParam &param)

4
src/logservice/libobcdc/src/ob_log_config.cpp
View File

@ -176,7 +176,9 @@ int ObLogConfig::load_from_map(const ConfigMap &configs,
} else {
(*pp_item)->set_value(iter->second.c_str());
(*pp_item)->set_version(version);
_LOG_INFO("load config succ, %s=%s", iter->first.c_str(), iter->second.c_str());
if (need_print_config(iter->first)) {
_LOG_INFO("load config succ, %s=%s", iter->first.c_str(), iter->second.c_str());
}
}
}
}

2
src/logservice/logfetcher/ob_log_fetch_log_rpc.cpp
View File

@ -740,7 +740,7 @@ int FetchLogARpc::handle_rpc_response(RpcRequest &rpc_req,
if (IDLE == state_) {
host_.switch_state(FetchStream::State::IDLE);
}
if (OB_TMP_FAIL(stream_worker_->dispatch_stream_task(host_, "FailPostProcess"))) {
if (OB_TMP_FAIL(stream_worker_->dispatch_stream_task(host_, "FailPostProcess", true))) {
LOG_ERROR_RET(tmp_ret, "dispatch stream task fail", KR(ret));
}
}

1
src/logservice/logfetcher/ob_log_fetcher_idle_pool.cpp
View File

@ -157,6 +157,7 @@ void ObLogFetcherIdlePool::mark_stop_flag()
}
}
// TODO: consider retry on failure
void ObLogFetcherIdlePool::handle(void *data, volatile bool &stop_flag)
{
int ret = OB_SUCCESS;

19
src/logservice/logfetcher/ob_log_ls_fetch_stream.cpp
View File

@ -246,9 +246,23 @@ int FetchStream::handle(volatile bool &stop_flag)
return ret;
}
#ifdef ERRSIM
ERRSIM_POINT_DEF(FAILED_TO_SCHEDULE_FETCH_STREAM);
#endif
int FetchStream::schedule(int timer_id)
{
return TG_SCHEDULE(timer_id, *this, g_schedule_time, false);
int ret = OB_SUCCESS;
#ifdef ERRSIM
if (OB_FAIL(FAILED_TO_SCHEDULE_FETCH_STREAM)) {
LOG_ERROR("ERRSIM: failed to schedule fetch stream");
} else {
#endif
ret = TG_SCHEDULE(timer_id, *this, g_schedule_time, false);
#ifdef ERRSIM
}
#endif
return ret;
}
// The purpose of a timed task is to assign itself to a worker thread
@ -264,7 +278,8 @@ void FetchStream::runTimerTask()
if (OB_ISNULL(stream_worker_)) {
LOG_ERROR("invalid stream worker", K(stream_worker_));
ret = OB_INVALID_ERROR;
} else if (OB_FAIL(stream_worker_->dispatch_stream_task(*this, "TimerWakeUp"))) {
// should never fail
} else if (OB_FAIL(stream_worker_->dispatch_stream_task(*this, "TimerWakeUp", true))) {
LOG_ERROR("dispatch stream task fail", KR(ret), K(this));
} else {
ATOMIC_STORE(&end_time, get_timestamp());

37
src/logservice/logfetcher/ob_ls_worker.cpp
View File

@ -271,9 +271,10 @@ int ObLSWorker::dispatch_fetch_task(LSFetchCtx &task, const char *dispatch_reaso
return ret;
}
int ObLSWorker::dispatch_stream_task(FetchStream &task, const char *from_mod)
int ObLSWorker::dispatch_stream_task(FetchStream &task, const char *from_mod, const bool retry_until_succ)
{
int ret = OB_SUCCESS;
const int64_t RETRY_INTERVAL = 100L * 1000; // 100ms
if (OB_UNLIKELY(! inited_)) {
LOG_ERROR("not init", K(inited_));
@ -291,16 +292,22 @@ int ObLSWorker::dispatch_stream_task(FetchStream &task, const char *from_mod)
}
// Rotating the task of fetching log streams to work threads
if (OB_FAIL(TG_PUSH_TASK(tg_id_, &task, hash_val))) {
LOG_ERROR("push stream task into thread queue fail", KR(ret));
}
do {
if (OB_FAIL(TG_PUSH_TASK(tg_id_, &task, hash_val))) {
LOG_ERROR("push stream task into thread queue fail", KR(ret));
if (retry_until_succ) {
ob_usleep(RETRY_INTERVAL);
}
}
} while (OB_FAIL(ret) && retry_until_succ);
}
return ret;
}
int ObLSWorker::hibernate_stream_task(FetchStream &task, const char *from_mod)
int ObLSWorker::hibernate_stream_task(FetchStream &task, const char *from_mod, const bool retry_until_succ)
{
int ret = OB_SUCCESS;
const int64_t RETRY_INTERVAL = 100L * 1000; // 100ms
bool print_stream_dispatch_info = ATOMIC_LOAD(&g_print_stream_dispatch_info);
if (print_stream_dispatch_info) {
@ -310,12 +317,16 @@ int ObLSWorker::hibernate_stream_task(FetchStream &task, const char *from_mod)
LOG_TRACE("[STAT] [STREAM_WORKER] [HIBERNATE_STREAM_TASK]",
"task", &task, K(from_mod), K(task));
}
if (OB_FAIL(task.schedule(timer_id_))) {
LOG_ERROR("schedule timer task fail", KR(ret));
} else {
// success
}
do {
if (OB_FAIL(task.schedule(timer_id_))) {
LOG_ERROR("schedule timer task fail", KR(ret));
if (retry_until_succ) {
ob_usleep(RETRY_INTERVAL);
}
} else {
// success
}
} while (OB_FAIL(ret) && retry_until_succ);
return ret;
}
@ -341,7 +352,7 @@ void ObLSWorker::handle(void *data, volatile bool &stop_flag)
else if (OB_UNLIKELY(is_paused) && ! (task->is_sys_log_stream() || task->is_rpc_ready())) {
LOG_TRACE("[STAT] [STREAM_WORKER] [HIBERNATE_STREAM_TASK_ON_PAUSE]", K(task));
if (OB_FAIL(hibernate_stream_task(*task, "PausedFetcher"))) {
if (OB_FAIL(hibernate_stream_task(*task, "PausedFetcher", true))) {
LOG_ERROR("hibernate_stream_task on pause fail", KR(ret), K(task), KPC(task));
}
} else if (OB_FAIL(task->handle(stop_flag))) {
@ -362,7 +373,7 @@ void ObLSWorker::handle(void *data, volatile bool &stop_flag)
// 4. the retry mechanism relys on that if LSWorker consume all the result and need to exit, the exit process
// couldn't fail, otherwise the same FetchStream would be scheduled multiple times.
task->switch_state(FetchStream::State::IDLE);
if (OB_TMP_FAIL(hibernate_stream_task(*task, "HandleTaskErr"))) {
if (OB_TMP_FAIL(hibernate_stream_task(*task, "HandleTaskErr", true))) {
LOG_ERROR_RET(tmp_ret, "hibernate_stream_task on handle task failure", K(task), KPC(task));
}
} else {

8
src/logservice/logfetcher/ob_ls_worker.h
View File

@ -48,10 +48,10 @@ public:
virtual int dispatch_fetch_task(LSFetchCtx &task, const char *dispatch_reason) = 0;
// Putting the fetch log stream task into the work thread task pool
virtual int dispatch_stream_task(FetchStream &task, const char *from_mod) = 0;
virtual int dispatch_stream_task(FetchStream &task, const char *from_mod, const bool retry_until_succ = false) = 0;
// Hibernate fetch log stream task
virtual int hibernate_stream_task(FetchStream &task, const char *from_mod) = 0;
virtual int hibernate_stream_task(FetchStream &task, const char *from_mod, const bool retry_until_succ = false) = 0;
};
//////////////////////////////////////////// ObLSWorker ////////////////////////////////////////////
@ -93,8 +93,8 @@ public:
int64_t get_fetcher_resume_tstamp();
int dispatch_fetch_task(LSFetchCtx &task, const char *dispatch_reason);
int dispatch_stream_task(FetchStream &task, const char *from_mod);
int hibernate_stream_task(FetchStream &task, const char *from_mod);
int dispatch_stream_task(FetchStream &task, const char *from_mod, const bool retry_until_succ = false);
int hibernate_stream_task(FetchStream &task, const char *from_mod, const bool retry_until_succ = false);
public:
virtual void handle(void *data) {}

5
src/logservice/palf/log_meta_info.cpp
View File

@ -230,6 +230,11 @@ bool LogConfigVersion::is_valid() const
return proposal_id_ != INVALID_PROPOSAL_ID && config_seq_ >= 0;
}
bool LogConfigVersion::is_initial_version() const
{
return config_seq_ <= 0;
}
int LogConfigVersion::generate(const int64_t proposal_id, const int64_t config_seq)
{
int ret = OB_SUCCESS;

1
src/logservice/palf/log_meta_info.h
View File

@ -86,6 +86,7 @@ public:
int generate(const int64_t proposal_id, const int64_t config_seq);
int inc_update_version(const int64_t proposal_id);
bool is_valid() const;
bool is_initial_version() const;
void reset();
int64_t to_string(char *buf, const int64_t buf_len);
bool operator==(const LogConfigVersion &config_version) const;

8
src/logservice/palf/log_sliding_window.cpp
View File

@ -1500,10 +1500,18 @@ int LogSlidingWindow::after_flush_log(const FlushLogCbCtx &flush_cb_ctx)
// follower need send ack to leader
const ObAddr &leader = (state_mgr_->get_leader().is_valid())? \
state_mgr_->get_leader(): state_mgr_->get_broadcast_leader();
LogConfigVersion config_version;
(void) mm_->get_config_version(config_version);
// flush op for different role
// migrating replicas do not send responses for reducing its impact on the leader
if (!leader.is_valid()) {
PALF_LOG(TRACE, "current leader is invalid, cannot send ack", K(ret), K_(palf_id), K_(self),
K(flush_cb_ctx), K(log_end_lsn), K(leader));
} else if (OB_UNLIKELY(config_version.is_initial_version())) {
if (REACH_TIME_INTERVAL(1 * 1000 * 1000)) {
PALF_LOG(INFO, "migrating replicas do not send responses", K(ret), K_(palf_id), K_(self),
K(log_end_lsn), K(leader));
}
} else if (OB_FAIL(submit_push_log_resp_(leader, flush_cb_ctx.curr_proposal_id_, log_end_lsn))) {
PALF_LOG(WARN, "submit_push_log_resp failed", K(ret), K_(palf_id), K_(self), K(leader), K(flush_cb_ctx));
} else {}

14
src/objit/include/objit/common/ob_item_type.h
View File

@ -867,6 +867,7 @@ typedef enum ObItemType
T_FUN_SYS_INNER_TABLE_OPTION_PRINTER = 1814,
T_FUN_SYS_INNER_TABLE_SEQUENCE_GETTER = 1815,
T_FUN_SYS_DECODE_TRACE_ID = 1816,
T_FUN_SYS_DOC_LENGTH = 1817,
T_FUN_SYS_END = 2000,
T_FUN_SYS_ALIGN_DATE4CMP = 2010,
T_FUN_SYS_INNER_ROW_CMP_VALUE = 2011,
@ -875,6 +876,8 @@ typedef enum ObItemType
T_FUN_SYS_GTID_SUBTRACT = 2013,
T_FUN_SYS_WAIT_FOR_EXECUTED_GTID_SET = 2014,
T_FUN_SYS_WAIT_UNTIL_SQL_THREAD_AFTER_GTIDS = 2015,
T_FUN_SYS_LAST_REFRESH_SCN = 2016,
T_FUN_SUM_OPNSIZE = 2017,
T_MAX_OP = 3000,
@ -2428,6 +2431,9 @@ typedef enum ObItemType
T_PARALLEL_DAS_DML,
T_DISABLE_PARALLEL_DAS_DML,
T_ENABLE_LOB_PREFETCH,
T_MV_OPTIONS,
T_MV_REWRITE,
T_MV_NO_REWRITE,
T_MAX //Attention: add a new type before T_MAX
} ObItemType;
@ -2460,6 +2466,14 @@ typedef enum ObContextType {
INITIALIZED_GLOBALLY = 3,
}ObContextType;
typedef enum ObOutlineType
{
OUTLINE_TYPE_INVALID = -1, //Attention: add a new type after CACHE_TYPE_INVALID
OUTLINE_TYPE_NORMAL,
OUTLINE_TYPE_FORMAT,
OUTLINE_TYPE_MAX //Attention: add a new type before CACHE_TYPE_MAX
} ObOutlineType;
#define IS_BOOL_OP(op) \
(((op) >= T_OP_EQ && (op) <= T_OP_NOT_IN && (op) != T_OP_POW) \
|| ((op) == T_OP_EXISTS) \

4
src/observer/CMakeLists.txt
View File

@ -38,6 +38,7 @@ ob_set_subtarget(ob_server common
ob_uniq_task_queue.cpp
ob_startup_accel_task_handler.cpp
ob_srv_rpc_handler.cpp
ob_check_params.cpp
)
ob_set_subtarget(ob_server common_mixed
@ -402,6 +403,9 @@ ob_set_subtarget(ob_server virtual_table
virtual_table/ob_all_virtual_cgroup_config.cpp
virtual_table/ob_virtual_flt_config.cpp
virtual_table/ob_all_virtual_kv_connection.cpp
virtual_table/ob_all_virtual_activity_metrics.cpp
virtual_table/ob_all_virtual_checkpoint_diagnose_info.cpp
virtual_table/ob_all_virtual_checkpoint_diagnose_memtable_info.cpp
)
ob_server_add_target(ob_server)

8
src/observer/mysql/ob_query_retry_ctrl.cpp
View File

@ -17,7 +17,6 @@
#include "sql/resolver/ob_stmt.h"
#include "pl/ob_pl.h"
#include "storage/tx/ob_trans_define.h"
#include "storage/memtable/ob_lock_wait_mgr.h"
#include "observer/mysql/ob_mysql_result_set.h"
#include "observer/ob_server_struct.h"
#include "observer/mysql/obmp_query.h"
@ -989,13 +988,6 @@ void ObQueryRetryCtrl::after_func(ObRetryParam &v)
if (OB_UNLIKELY(OB_SUCCESS == v.client_ret_)) {
LOG_ERROR_RET(OB_ERR_UNEXPECTED, "no matter need retry or not, v.client_ret_ should not be OB_SUCCESS", K(v));
}
// bug fix:
if (RETRY_TYPE_LOCAL == v.retry_type_) {
rpc::ObLockWaitNode* node = MTL(memtable::ObLockWaitMgr*)->get_thread_node();
if (NULL != node) {
node->reset_need_wait();
}
}
}
int ObQueryRetryCtrl::init()

4
src/observer/mysql/obmp_query.cpp
View File

@ -14,6 +14,7 @@
#include "observer/mysql/obmp_query.h"
#include "lib/allocator/ob_sql_mem_leak_checker.h"
#include "lib/utility/ob_macro_utils.h"
#include "lib/utility/ob_tracepoint.h"
#include "lib/worker.h"
@ -690,7 +691,7 @@ OB_INLINE int ObMPQuery::do_process(ObSQLSessionInfo &session,
ObTenantCachedSchemaGuardInfo &cached_schema_info = session.get_cached_schema_guard_info();
int64_t tenant_version = 0;
int64_t sys_version = 0;
common::ObSqlInfoGuard si_guard(sql);
SQL_INFO_GUARD(sql, session.get_cur_sql_id());
ObSqlFatalErrExtraInfoGuard extra_info_guard;
extra_info_guard.set_cur_sql(sql);
extra_info_guard.set_tenant_id(session.get_effective_tenant_id());
@ -1269,6 +1270,7 @@ OB_INLINE int ObMPQuery::response_result(ObMySQLResultSet &result,
// 通过判断 plan 是否为 null 来确定是 plan 还是 cmd
// 针对 plan 和 cmd 分开处理，逻辑会较为清晰。
if (OB_LIKELY(NULL != result.get_physical_plan())) {
ENABLE_SQL_MEMLEAK_GUARD;
if (need_trans_cb) {
ObAsyncPlanDriver drv(gctx_, ctx_, session, retry_ctrl_, *this);
// NOTE: sql_end_cb必须在drv.response_result()之前初始化好

1
src/observer/mysql/obmp_stmt_execute.cpp
View File

@ -1830,6 +1830,7 @@ int ObMPStmtExecute::process()
lib::CompatModeGuard g(sess->get_compatibility_mode() == ORACLE_MODE ?
lib::Worker::CompatMode::ORACLE : lib::Worker::CompatMode::MYSQL);
ObSQLSessionInfo::LockGuard lock_guard(session.get_query_lock());
SQL_INFO_GUARD(ctx_.cur_sql_, ObString(ctx_.sql_id_));
session.set_current_trace_id(ObCurTraceId::get_trace_id());
session.init_use_rich_format();
session.get_raw_audit_record().request_memory_used_ = 0;

2
src/observer/mysql/obmp_stmt_fetch.cpp
View File

@ -507,7 +507,7 @@ int ObMPStmtFetch::response_result(pl::ObPLCursorInfo &cursor,
if (OB_ITER_END == ret || OB_READ_NOTHING == ret) {
ret = OB_SUCCESS;
// need_fetch 为 true 并且 得到了 OB_ITER_END 错误码，说明正常找到了最后一行，要设置last_row
if (need_fetch) {
if (need_fetch || !cursor.is_scrollable()) {
last_row = true;
}
}

1
src/observer/mysql/obmp_stmt_prepare.cpp
View File

@ -179,6 +179,7 @@ int ObMPStmtPrepare::process()
lib::CompatModeGuard g(sess->get_compatibility_mode() == ORACLE_MODE ?
lib::Worker::CompatMode::ORACLE : lib::Worker::CompatMode::MYSQL);
ObSQLSessionInfo::LockGuard lock_guard(session.get_query_lock());
SQL_INFO_GUARD(ctx_.cur_sql_, ObString(ctx_.sql_id_));
session.set_current_trace_id(ObCurTraceId::get_trace_id());
session.init_use_rich_format();
session.get_raw_audit_record().request_memory_used_ = 0;

318
src/observer/ob_check_params.cpp Normal file
View File

@ -0,0 +1,318 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SERVER
#include "observer/ob_check_params.h"
namespace oceanbase
{
using namespace common;
namespace observer
{
int CheckAllParams::check_all_params(bool strict_check = true)
{
int ret = OB_SUCCESS;
if (OB_SUCC(ret) && OB_FAIL(check_vm_max_map_count(strict_check))) {}
if (OB_SUCC(ret) && OB_FAIL(check_vm_min_free_kbytes(strict_check))) {}
if (OB_SUCC(ret) && OB_FAIL(check_vm_overcommit_memory(strict_check))) {}
if (OB_SUCC(ret) && OB_FAIL(check_fs_file_max(strict_check))) {}
if (OB_SUCC(ret) && OB_FAIL(check_ulimit_open_files(strict_check))) {}
if (OB_SUCC(ret) && OB_FAIL(check_ulimit_max_user_processes(strict_check))) {}
if (OB_SUCC(ret) && OB_FAIL(check_ulimit_core_file_size(strict_check))) {}
if (OB_SUCC(ret) && OB_FAIL(check_ulimit_stack_size(strict_check))) {}
if (OB_SUCC(ret) && OB_FAIL(check_current_clocksource(strict_check))) {}
return ret;
}
int CheckAllParams::read_one_int(const char *file_name, int64_t &value)
{
int ret = OB_SUCCESS;
FILE *fp = fopen(file_name, "r");
if (fp != nullptr) {
if (1 != fscanf(fp, "%ld", &value)) {
ret = OB_IO_ERROR;
LOG_ERROR("Failed to read integer from file", K(ret));
}
fclose(fp);
} else {
ret = OB_FILE_NOT_EXIST;
LOG_WARN("File does not exist", K(ret));
}
return ret;
}
bool CheckAllParams::is_path_valid(const char *file_name)
{
return (access(file_name, F_OK) != -1);
}
int CheckAllParams::check_vm_max_map_count(bool strict_check)
{
int ret = OB_SUCCESS;
const char *file_path = "/proc/sys/vm/max_map_count";
if (is_path_valid(file_path)) {
int64_t max_map_count = 0;
if (OB_FAIL(read_one_int(file_path, max_map_count))) {
LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "msg", "[check OS params]:read file failed", K(file_path));
} else if (max_map_count >= 327600) {
LOG_INFO("[check OS params]:vm.max_map_count is within the range", K(max_map_count));
} else {
if (strict_check) {
ret = OB_IMPROPER_OS_PARAM;
LOG_DBA_ERROR(
OB_IMPROPER_OS_PARAM, "msg", "[check OS params]:vm.max_map_count is less than 327600", K(max_map_count));
} else {
LOG_WARN("[check OS params]:vm.max_map_count is less than 327600", K(max_map_count));
}
}
} else {
LOG_WARN("file path does not exist", K(file_path));
}
return ret;
}
int CheckAllParams::check_vm_min_free_kbytes(bool strict_check)
{
int ret = OB_SUCCESS;
const char *file_path = "/proc/sys/vm/min_free_kbytes";
if (is_path_valid(file_path)) {
int64_t vm_min_free_kbytes = 0;
if (OB_FAIL(read_one_int(file_path, vm_min_free_kbytes))) {
LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "msg", "[check OS params]:read file failed", K(file_path));
} else if (vm_min_free_kbytes >= 32768 && vm_min_free_kbytes <= 2097152) {
LOG_INFO("[check OS params]:vm.min_free_kbytes is within the range", K(vm_min_free_kbytes));
} else {
if (strict_check) {
ret = OB_IMPROPER_OS_PARAM;
LOG_DBA_ERROR(OB_IMPROPER_OS_PARAM,
"msg",
"[check OS params]:vm.min_free_kbytes is not within the allowed range [32768, 2097152]",
K(vm_min_free_kbytes));
} else {
LOG_WARN("[check OS params]:vm.min_free_kbytes is not within the allowed range [32768, 2097152]", K(vm_min_free_kbytes));
}
}
} else {
LOG_WARN("file path does not exist", K(file_path));
}
return ret;
}
int CheckAllParams::check_vm_overcommit_memory(bool strict_check)
{
int ret = OB_SUCCESS;
const char *file_path = "/proc/sys/vm/overcommit_memory";
if (is_path_valid(file_path)) {
int64_t vm_overcommit_memory = 0;
if (OB_FAIL(read_one_int(file_path, vm_overcommit_memory))) {
LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "msg", "[check OS params]:read file failed", K(file_path));
} else if (vm_overcommit_memory == 0) {
LOG_INFO("[check OS params]:vm.overcommit_memory is equal to 0", K(vm_overcommit_memory));
} else {
if (strict_check) {
ret = OB_IMPROPER_OS_PARAM;
LOG_DBA_ERROR(OB_IMPROPER_OS_PARAM, "msg", "[check OS params]:vm.overcommit_memory is not the value:0", K(vm_overcommit_memory));
} else {
LOG_WARN("[check OS params]:vm.overcommit_memory is not within the allowed value:0", K(vm_overcommit_memory));
}
}
} else {
LOG_WARN("file path does not exist", K(file_path));
}
return ret;
}
int CheckAllParams::check_fs_file_max(bool strict_check)
{
int ret = OB_SUCCESS;
const char *file_path = "/proc/sys/fs/file-max";
if (is_path_valid(file_path)) {
int64_t fs_file_max = 0;
if (OB_FAIL(read_one_int(file_path, fs_file_max))) {
LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "msg", "[check OS params]:read file failed", K(file_path));
} else if (fs_file_max >= 6573688) {
LOG_INFO("[check OS params]:fs.file-max is greater than or equal to 6573688", K(fs_file_max));
} else {
if (strict_check) {
ret = OB_IMPROPER_OS_PARAM;
LOG_DBA_ERROR(OB_IMPROPER_OS_PARAM, "msg", "[check OS params]:fs.file-max is less than 6573688", K(fs_file_max));
} else {
LOG_WARN("[check OS params]:fs.file-max is less than 6573688", K(fs_file_max));
}
}
} else {
LOG_WARN("file path does not exist", K(file_path));
}
return ret;
}
int CheckAllParams::check_ulimit_open_files(bool strict_check)
{
int ret = OB_SUCCESS;
// Check open files limit
struct rlimit rlim;
if (getrlimit(RLIMIT_NOFILE, &rlim) == 0) {
if (rlim.rlim_cur >= 655300 && rlim.rlim_max >= 655300) {
LOG_INFO("[check OS params]:open files limit is greater than or equal to 655300", K(rlim.rlim_cur), K(rlim.rlim_max));
} else {
if (strict_check) {
ret = OB_IMPROPER_OS_PARAM;
LOG_DBA_ERROR(OB_IMPROPER_OS_PARAM,
"msg",
"[check OS params]:open files limit's soft nofile or hard nofile is less than 655300, please run 'ulimit -n' to confirm the current limit and find a way to set it to the proper value",
K(rlim.rlim_cur),
K(rlim.rlim_max));
} else {
LOG_WARN(
"[check OS params]:open files limit's soft nofile or hard nofile is less than 655300, please run 'ulimit -n' to confirm the current limit and find a way to set it to the proper value", K(rlim.rlim_cur), K(rlim.rlim_max));
}
}
} else {
LOG_DBA_ERROR(OB_FILE_NOT_EXIST, "msg", "read ulimit.open_file_limit file failed");
}
return ret;
}
int CheckAllParams::check_ulimit_max_user_processes(bool strict_check)
{
int ret = OB_SUCCESS;
struct rlimit rlim;
// Check process limit
if (getrlimit(RLIMIT_NPROC, &rlim) == 0) {
if (rlim.rlim_cur >= 655300 && rlim.rlim_max >= 655300) {
LOG_INFO("[check OS params]:ulimit.max_user_processes is greater than or equal to 655300", K(rlim.rlim_cur), K(rlim.rlim_max));
} else {
if (strict_check) {
ret = OB_IMPROPER_OS_PARAM;
LOG_DBA_ERROR(OB_IMPROPER_OS_PARAM,
"msg",
"[check OS params]:ulimit.max_user_processes's soft nofile or hard nofile is less than 655300, please run 'ulimit -u' to confirm the current limit and find a way to set it to the proper value",
K(rlim.rlim_cur),
K(rlim.rlim_max));
} else {
LOG_WARN("[check OS params]:ulimit.max_user_processes's soft nofile or hard nofile is less than 655300, please run 'ulimit -u' to confirm the current limit and find a way to set it to the proper value",
K(rlim.rlim_cur),
K(rlim.rlim_max));
}
}
} else {
LOG_DBA_ERROR(OB_FILE_NOT_EXIST, "msg", "read ulimit.max_user_processes file failed");
}
return ret;
}
int CheckAllParams::check_ulimit_core_file_size(bool strict_check)
{
int ret = OB_SUCCESS;
struct rlimit rlim;
// Check core file size limit
if (getrlimit(RLIMIT_CORE, &rlim) == 0) {
if (rlim.rlim_cur == RLIM_INFINITY && rlim.rlim_max == RLIM_INFINITY) {
LOG_INFO("[check OS params]:core file size limit is unlimited");
} else {
// Violations of the recommended range will only trigger a warning, regardless of strict or non-strict checking.
LOG_WARN("[check OS params]:ulimit.core_file_size limit's soft nofile or hard nofile is limited", K(rlim.rlim_cur), K(rlim.rlim_max));
}
} else {
LOG_DBA_ERROR(OB_FILE_NOT_EXIST, "msg", "read ulimit.core_file_size file failed");
}
return ret;
}
int CheckAllParams::check_ulimit_stack_size(bool strict_check)
{
int ret = OB_SUCCESS;
struct rlimit rlim;
// Check stack size limit
if (getrlimit(RLIMIT_STACK, &rlim) == 0) {
if (rlim.rlim_cur >= (1 << 20) && rlim.rlim_max >= (1 << 20)) {
LOG_INFO("[check OS params]:stack size limit is larger than 1M", K(rlim.rlim_cur), K(rlim.rlim_max));
} else {
if (strict_check) {
ret = OB_IMPROPER_OS_PARAM;
LOG_DBA_ERROR(OB_IMPROPER_OS_PARAM,
"msg",
"[check OS params]: ulimit.stack_size limit's soft nofile or hard nofile is smaller than 1M, please run 'ulimit -s' to confirm the current limit and find a way to set it to the proper value",
K(rlim.rlim_cur),
K(rlim.rlim_max));
} else {
LOG_WARN(
"[check OS params]: ulimit.stack_size limit's soft nofile or hard nofile is smaller than 1M, please run 'ulimit -s' to confirm the current limit and find a way to set it to the proper value", K(rlim.rlim_cur), K(rlim.rlim_max));
}
}
} else {
LOG_DBA_ERROR(OB_FILE_NOT_EXIST, "msg", "read ulimit.stack_size file failed");
}
return ret;
}
int CheckAllParams::read_one_line(const char *file_path, char *buffer, size_t buffer_size)
{
int ret = OB_SUCCESS;
FILE *file = fopen(file_path, "r");
if (file != nullptr) {
if (fgets(buffer, buffer_size, file) == nullptr) {
ret = OB_IO_ERROR;
LOG_ERROR("Failed to read line from file", K(file_path), K(ret));
}
fclose(file);
} else {
ret = OB_FILE_NOT_EXIST;
LOG_WARN("File does not exist", K(ret));
}
return ret;
}
int CheckAllParams::check_current_clocksource(bool strict_check)
{
int ret = OB_SUCCESS;
const char *file_path = "/sys/devices/system/clocksource/clocksource0/current_clocksource";
if (is_path_valid(file_path)) {
char clocksource[256];
if (OB_FAIL(read_one_line(file_path, clocksource, sizeof(clocksource)))) {
LOG_DBA_ERROR(OB_ERR_UNEXPECTED, "msg", "[check OS params]:read file failed", K(file_path));
} else {
if (clocksource[strlen(clocksource) - 1] == '\n') {
clocksource[strlen(clocksource) - 1] = '\0';
}
if (strcmp(clocksource, "tsc") == 0 || strcmp(clocksource, "kvm-clock") == 0 || strcmp(clocksource, "arch_sys_counter") == 0) {
LOG_INFO("[check OS params]:current_clocksource is in proper range", K(clocksource), K(ret));
} else if (strict_check) {
ret = OB_IMPROPER_OS_PARAM;
LOG_DBA_ERROR(OB_IMPROPER_OS_PARAM, "msg", "[check OS params]:current_clocksource is not [tsc], [kvm-clock], [arch_sys_counter]", K(clocksource), K(ret));
} else {
LOG_WARN("[check OS params]:current_clocksource is not [tsc], [kvm-clock], [arch_sys_counter]", K(clocksource), K(ret));
}
}
} else {
LOG_WARN("file path does not exist", K(file_path));
}
return ret;
}
int check_os_params(bool strict_check_params = false)
{
int ret = OB_SUCCESS;
if (OB_FAIL(CheckAllParams::check_all_params(strict_check_params))) {
LOG_DBA_ERROR(OB_IMPROPER_OS_PARAM, "msg", "check os params failed");
}
return ret;
}
} // namespace observer
} // namespace oceanbase

52
src/observer/ob_check_params.h Normal file
View File

@ -0,0 +1,52 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#ifndef OCEANBASE_OBSERVER_OB_CHECK_PARAMS_
#define OCEANBASE_OBSERVER_OB_CHECK_PARAMS_
#include "deps/oblib/src/lib/file/file_directory_utils.h"
#include "share/ob_define.h"
#include "share/ob_errno.h"
#include <fstream>
#include <sys/resource.h>
namespace oceanbase
{
using namespace common;
namespace observer
{
using namespace common;
class CheckAllParams
{
public:
static int check_all_params(bool strict_check);
private:
static int read_one_int(const char *file_name, int64_t &value);
static int read_one_line(const char* file_path, char* buffer, size_t buffer_size);
static bool is_path_valid(const char *file_name);
static int check_vm_max_map_count(bool strict_check); // 1
static int check_vm_min_free_kbytes(bool strict_check); // 2
static int check_vm_overcommit_memory(bool strict_check); // 3
static int check_fs_file_max(bool strict_check);
static int check_ulimit_open_files(bool strict_check);
static int check_ulimit_max_user_processes(bool strict_check);
static int check_ulimit_core_file_size(bool strict_check);
static int check_ulimit_stack_size(bool strict_check); // 8
static int check_current_clocksource(bool strict_check); // 9
};
int check_os_params(bool strict_check_params);
} // namespace observer
} // namespace oceanbase
#endif

16
src/observer/ob_dump_task_generator.cpp
View File

@ -143,21 +143,7 @@ int ObDumpTaskGenerator::generate_task_from_file()
int ObDumpTaskGenerator::generate_mod_stat_task()
{
int ret = OB_SUCCESS;
auto &mem_dump = ObMemoryDump::get_instance();
ObMemoryDumpTask *task = mem_dump.alloc_task();
if (OB_ISNULL(task)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("alloc task failed");
} else {
task->type_ = STAT_LABEL;
COMMON_LOG(INFO, "task info", K(*task));
if (OB_FAIL(mem_dump.push(task))) {
LOG_WARN("push task failed", K(ret));
mem_dump.free_task(task);
}
}
return ret;
return ObMemoryDump::get_instance().generate_mod_stat_task();
}
void ObDumpTaskGenerator::dump_memory_leak()

4
src/observer/ob_inner_sql_connection.cpp
View File

@ -76,7 +76,7 @@ public:
virtual int execute(sql::ObSql &engine, sql::ObSqlCtx &ctx, sql::ObResultSet &res)
{
int ret = OB_SUCCESS;
common::ObSqlInfoGuard si_guard(sql_);
SQL_INFO_GUARD(sql_, ObString(OB_MAX_SQL_ID_LENGTH, ctx.sql_id_));
// Deep copy sql, because sql may be destroyed before result iteration.
const int64_t alloc_size = sizeof(ObString) + sql_.length() + 1; // 1 for C terminate char
void *mem = res.get_mem_pool().alloc(alloc_size);
@ -914,6 +914,7 @@ template <typename T>
int ObInnerSQLConnection::retry_while_no_tenant_resource(const int64_t cluster_id, const uint64_t &tenant_id, T function)
{
int ret = OB_SUCCESS;
DISABLE_SQL_MEMLEAK_GUARD;
share::ObLSID ls_id(share::ObLSID::SYS_LS_ID);
const int64_t max_retry_us = 128 * 1000;
int64_t retry_us = 2 * 1000;
@ -1827,6 +1828,7 @@ int ObInnerSQLConnection::execute(
const uint64_t tenant_id, sqlclient::ObIExecutor &executor)
{
int ret = OB_SUCCESS;
DISABLE_SQL_MEMLEAK_GUARD;
FLTSpanGuard(inner_execute);
SMART_VAR(ObInnerSQLResult, res, get_session()) {
if (OB_FAIL(res.init())) {

6
src/observer/ob_inner_sql_result.cpp
View File

@ -140,7 +140,7 @@ int ObInnerSQLResult::open()
LOG_WARN("switch tenant failed", K(ret), K(session_.get_effective_tenant_id()));
} else {
lib::CompatModeGuard g(compat_mode_);
common::ObSqlInfoGuard si_guard(session_.get_current_query_string());
SQL_INFO_GUARD(session_.get_current_query_string(), session_.get_cur_sql_id());
bool is_select = has_tenant_resource() ?
ObStmt::is_select_stmt(result_set_->get_stmt_type())
: ObStmt::is_select_stmt(remote_result_set_->get_stmt_type());
@ -205,7 +205,7 @@ int ObInnerSQLResult::inner_close()
{
int ret = OB_SUCCESS;
lib::CompatModeGuard g(compat_mode_);
common::ObSqlInfoGuard si_guard(session_.get_current_query_string());
SQL_INFO_GUARD(session_.get_current_query_string(), session_.get_cur_sql_id());
LOG_DEBUG("compat_mode_", K(ret), K(compat_mode_), K(lbt()));
MAKE_TENANT_SWITCH_SCOPE_GUARD(tenant_guard);
@ -244,7 +244,7 @@ int ObInnerSQLResult::next()
} else {
row_ = NULL;
lib::CompatModeGuard g(compat_mode_);
common::ObSqlInfoGuard si_guard(session_.get_current_query_string());
SQL_INFO_GUARD(session_.get_current_query_string(), session_.get_cur_sql_id());
WITH_CONTEXT(mem_context_) {
if (has_tenant_resource() && OB_FAIL(result_set_->get_next_row(row_))) {
if (OB_ITER_END != ret) {

6
src/observer/ob_server.cpp
View File

@ -122,6 +122,7 @@
#ifdef OB_BUILD_ORACLE_XML
#include "lib/xml/ob_libxml2_sax_handler.h"
#endif
#include "ob_check_params.h"
using namespace oceanbase::lib;
using namespace oceanbase::common;
@ -265,7 +266,10 @@ int ObServer::init(const ObServerOptions &opts, const ObPLogWriterCfg &log_cfg)
if (OB_FAIL(init_config())) {
LOG_ERROR("init config failed", KR(ret));
}
//check os params
if (OB_SUCC(ret) && OB_FAIL(check_os_params(GCONF.strict_check_os_params))) {
LOG_ERROR("check OS params failed", K(GCONF.strict_check_os_params));
}
// set large page param
ObLargePageHelper::set_param(config_.use_large_pages);

2
src/observer/ob_server.h
View File

@ -107,7 +107,7 @@ namespace observer
class ObServer
{
public:
static const int64_t DEFAULT_ETHERNET_SPEED = 1000 / 8 * 1024 * 1024; // default 125m/s 1000Mbit
static const int64_t DEFAULT_ETHERNET_SPEED = 10000 / 8 * 1024 * 1024; // change from default 125m/s 1000Mbit to 1250MBps 10000Mbit
static const int64_t DISK_USAGE_REPORT_INTERVAL = 1000L * 1000L * 300L; // 5min
static const uint64_t DEFAULT_CPU_FREQUENCY = 2500 * 1000; // 2500 * 1000 khz
static ObServer &get_instance();

12
src/observer/ob_server_reload_config.cpp
View File

@ -183,21 +183,15 @@ int ObServerReloadConfig::operator()()
reserve);
}
int64_t cache_size = GCONF.memory_chunk_cache_size;
if (0 == cache_size) {
bool use_large_chunk_cache = 1 != cache_size;
if (0 == cache_size || 1 == cache_size) {
cache_size = GMEMCONF.get_server_memory_limit();
if (cache_size >= (32L<<30)) {
cache_size -= (4L<<30);
}
}
int64_t large_cache_size = GCONF._memory_large_chunk_cache_size;
if (0 == large_cache_size) {
large_cache_size = lib::AChunkMgr::DEFAULT_LARGE_CHUNK_CACHE_SIZE;
}
lib::AChunkMgr::instance().set_max_chunk_cache_size(cache_size);
lib::AChunkMgr::instance().set_max_large_chunk_cache_size(large_cache_size);
lib::AChunkMgr::instance().set_max_chunk_cache_size(cache_size, use_large_chunk_cache);
if (!is_arbitration_mode) {
// Refresh cluster_id, cluster_name_hash for non arbitration mode

8
src/observer/ob_service.cpp
View File

@ -25,6 +25,7 @@
#include "lib/thread_local/ob_tsi_factory.h"
#include "lib/utility/utility.h"
#include "lib/time/ob_tsc_timestamp.h"
#include "lib/alloc/memory_dump.h"
#include "common/ob_member_list.h"
#include "common/ob_zone.h"
@ -2423,7 +2424,12 @@ int ObService::get_wrs_info(const obrpc::ObGetWRSArg &arg,
int ObService::refresh_memory_stat()
{
return ObDumpTaskGenerator::generate_mod_stat_task();
int ret = OB_SUCCESS;
if (OB_FAIL(ObMemoryDump::get_instance().check_sql_memory_leak())) {
ret = OB_ERR_UNEXPECTED;
LOG_USER_ERROR(OB_ERR_UNEXPECTED, "there has sql memory leak");
}
return ret;
}
int ObService::wash_memory_fragmentation()

24
src/observer/omt/ob_multi_tenant.cpp
View File

@ -130,6 +130,7 @@
#endif
#ifdef OB_BUILD_DBLINK
#include "lib/oracleclient/ob_oci_environment.h"
#include "lib/mysqlclient/ob_dblink_error_trans.h"
#endif
#include "lib/mysqlclient/ob_tenant_oci_envs.h"
#include "sql/udr/ob_udr_mgr.h"
@ -158,6 +159,7 @@
#include "storage/tenant_snapshot/ob_tenant_snapshot_service.h"
#include "share/index_usage/ob_index_usage_info_mgr.h"
#include "rootserver/mview/ob_mview_maintenance_service.h"
#include "storage/checkpoint/ob_checkpoint_diagnose.h"
using namespace oceanbase;
using namespace oceanbase::lib;
@ -571,6 +573,10 @@ int ObMultiTenant::init(ObAddr myaddr,
MTL_BIND2(mtl_new_default, ObIndexUsageInfoMgr::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, storage::ObTabletMemtableMgrPool::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
MTL_BIND2(mtl_new_default, rootserver::ObMViewMaintenanceService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObCheckpointDiagnoseMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
#ifdef OB_BUILD_DBLINK
MTL_BIND2(common::sqlclient::ObTenantDblinkKeeper::mtl_new, common::sqlclient::ObTenantDblinkKeeper::mtl_init, nullptr, nullptr, nullptr, common::sqlclient::ObTenantDblinkKeeper::mtl_destroy);
#endif
}
if (OB_SUCC(ret)) {
@ -1314,6 +1320,9 @@ int ObMultiTenant::update_tenant_config(uint64_t tenant_id)
if (OB_TMP_FAIL(update_throttle_config_(tenant_id))) {
LOG_WARN("update throttle config failed", K(ret), K(tenant_id));
}
if (OB_TMP_FAIL(update_checkpoint_diagnose_config())) {
LOG_WARN("failed to update tenant ddl config", K(tmp_ret), K(tenant_id));
}
}
}
LOG_INFO("update_tenant_config success", K(tenant_id));
@ -1365,6 +1374,21 @@ int ObMultiTenant::update_tenant_ddl_config()
return ret;
}
int ObMultiTenant::update_checkpoint_diagnose_config()
{
int ret = OB_SUCCESS;
ObCheckpointDiagnoseMgr *cdm = MTL(ObCheckpointDiagnoseMgr*);
omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID()));
const int64_t checkpoint_diagnose_preservation_count = tenant_config->_checkpoint_diagnose_preservation_count;
if (OB_ISNULL(cdm)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("cdm should not be null", K(ret));
} else if(OB_FAIL(cdm->update_max_trace_info_size(checkpoint_diagnose_preservation_count))) {
LOG_WARN("failed to update_max_trace_info_size", K(ret), K(checkpoint_diagnose_preservation_count));
}
return ret;
}
int ObMultiTenant::update_tenant_freezer_config_()
{
int ret = OB_SUCCESS;

1
src/observer/omt/ob_multi_tenant.h
View File

@ -118,6 +118,7 @@ public:
int update_palf_config();
int update_tenant_dag_scheduler_config();
int update_tenant_ddl_config();
int update_checkpoint_diagnose_config();
int get_tenant(const uint64_t tenant_id, ObTenant *&tenant) const;
int get_tenant_with_tenant_lock(const uint64_t tenant_id, common::ObLDHandle &handle, ObTenant *&tenant) const;
int get_active_tenant_with_tenant_lock(const uint64_t tenant_id, common::ObLDHandle &handle, ObTenant *&tenant) const;

2
src/observer/omt/ob_th_worker.cpp
View File

@ -19,6 +19,7 @@
#include "lib/stat/ob_diagnose_info.h"
#include "lib/stat/ob_session_stat.h"
#include "lib/allocator/ob_page_manager.h"
#include "lib/allocator/ob_sql_mem_leak_checker.h"
#include "lib/rc/context.h"
#include "lib/thread/ob_thread_name.h"
#include "ob_tenant.h"
@ -305,6 +306,7 @@ void ObThWorker::worker(int64_t &tenant_id, int64_t &req_recv_timestamp, int32_t
blocking_ts_ = &Thread::blocking_ts_;
ObTLTaGuard ta_guard(tenant_->id());
ObMemVersionNodeGuard mem_version_node_guard;
// Avoid adding and deleting entities from the root node for every request, the parameters are meaningless
CREATE_WITH_TEMP_ENTITY(RESOURCE_OWNER, OB_SERVER_TENANT_ID) {
auto *pm = common::ObPageManager::thread_local_instance();

Some files were not shown because too many files have changed in this diff Show More