hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge branch 'master' into develop

Browse Source
This commit is contained in:
王运来 2024-01-03 15:40:44 +08:00
commit 2b7308e37d
459 changed files with 13225 additions and 3932 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -231,6 +231,7 @@ tools/deploy/single-with-proxy.yaml
tools/deploy/single.yaml
tools/deploy/tpchobp_test.yaml
tools/deploy/bmsqlobp_test.yaml
tools/deploy/oci_test.yaml
tools/log_parser/slog_parser
tools/log_tool/log_statistics
tools/log_tool/log_tool
@ -286,6 +287,8 @@ tools/deploy/distributed.yaml
tools/deploy/obd_profile.sh
tools/deploy/single-with-proxy.yaml
tools/deploy/single.yaml
tools/docker/standalone/boot
tools/docker/standalone/init_store_for_fast_start.py
############# unittest #############
unittest/**/test_*

13
CMakeLists.txt
View File

@ -55,14 +55,17 @@ if(ENABLE_SMART_VAR_CHECK)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DENABLE_SMART_VAR_CHECK")
endif()
if(ENABLE_SERIALIZATION_CHECK)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DENABLE_SERIALIZATION_CHECK")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DENABLE_SERIALIZATION_CHECK")
endif()
if(ENABLE_DEBUG_LOG)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DENABLE_DEBUG_LOG")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DENABLE_DEBUG_LOG")
if(NOT DEFINED ENABLE_SERIALIZATION_CHECK)
set(ENABLE_SERIALIZATION_CHECK ${ENABLE_DEBUG_LOG})
endif()
endif()
if(ENABLE_SERIALIZATION_CHECK)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DENABLE_SERIALIZATION_CHECK")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DENABLE_SERIALIZATION_CHECK")
endif()
if (OB_GPERF_MODE)

3
cmake/Env.cmake
View File

@ -18,7 +18,6 @@ ob_define(ENABLE_MEMORY_DIAGNOSIS OFF)
ob_define(ENABLE_OBJ_LEAK_CHECK OFF)
ob_define(ENABLE_FATAL_ERROR_HANG ON)
ob_define(ENABLE_SMART_VAR_CHECK OFF)
ob_define(ENABLE_SERIALIZATION_CHECK OFF)
ob_define(ENABLE_COMPILE_DLL_MODE OFF)
ob_define(OB_CMAKE_RULES_CHECK ON)
ob_define(OB_STATIC_LINK_LGPL_DEPS ON)
@ -273,7 +272,7 @@ if( ${ARCHITECTURE} STREQUAL "x86_64" )
set(ARCH_LDFLAGS "")
set(OCI_DEVEL_INC "${DEP_3RD_DIR}/usr/include/oracle/12.2/client64")
else()
set(MARCH_CFLAGS "-march=armv8-a+crc" )
set(MARCH_CFLAGS "-march=armv8-a+crc+lse" )
set(MTUNE_CFLAGS "-mtune=generic" )
set(ARCH_LDFLAGS "-l:libatomic.a")
set(OCI_DEVEL_INC "${DEP_3RD_DIR}/usr/include/oracle/19.10/client64")

2
deps/init/oceanbase.el7.aarch64.deps vendored
View File

@ -28,7 +28,7 @@ devdeps-boost-1.74.0-22022110914.el7.aarch64.rpm
devdeps-s2geometry-0.9.0-12023092021.el7.aarch64.rpm
devdeps-icu-69.1-72022112416.el7.aarch64.rpm
devdeps-cos-c-sdk-5.0.16-52023070517.el7.aarch64.rpm
devdeps-s3-cpp-sdk-1.11.156-62023101011.el7.aarch64.rpm
devdeps-s3-cpp-sdk-1.11.156-102023122011.el7.aarch64.rpm
[tools]
obdevtools-binutils-2.30-12022100413.el7.aarch64.rpm

2
deps/init/oceanbase.el7.x86_64.deps vendored
View File

@ -31,7 +31,7 @@ devdeps-s2geometry-0.9.0-12023092021.el7.x86_64.rpm
devdeps-icu-69.1-72022112416.el7.x86_64.rpm
devdeps-cos-c-sdk-5.0.16-52023070517.el7.x86_64.rpm
devdeps-cloud-qpl-1.1.0-272023061419.el7.x86_64.rpm
devdeps-s3-cpp-sdk-1.11.156-62023101011.el7.x86_64.rpm
devdeps-s3-cpp-sdk-1.11.156-102023122011.el7.x86_64.rpm
[tools]
obdevtools-binutils-2.30-12022100413.el7.x86_64.rpm

2
deps/init/oceanbase.el8.aarch64.deps vendored
View File

@ -28,7 +28,7 @@ devdeps-boost-1.74.0-22022110914.el8.aarch64.rpm
devdeps-s2geometry-0.9.0-12023092021.el8.aarch64.rpm
devdeps-icu-69.1-72022112416.el8.aarch64.rpm
devdeps-cos-c-sdk-5.0.16-52023070517.el8.aarch64.rpm
devdeps-s3-cpp-sdk-1.11.156-62023101011.el8.aarch64.rpm
devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.aarch64.rpm
[tools]
obdevtools-binutils-2.30-12022100413.el8.aarch64.rpm

2
deps/init/oceanbase.el8.x86_64.deps vendored
View File

@ -30,7 +30,7 @@ devdeps-s2geometry-0.9.0-12023092021.el8.x86_64.rpm
devdeps-icu-69.1-72022112416.el8.x86_64.rpm
devdeps-cos-c-sdk-5.0.16-52023070517.el8.x86_64.rpm
devdeps-cloud-qpl-1.1.0-272023061419.el8.x86_64.rpm
devdeps-s3-cpp-sdk-1.11.156-62023101011.el8.x86_64.rpm
devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.x86_64.rpm
[tools]
obdevtools-binutils-2.30-12022100413.el8.x86_64.rpm

2
deps/init/oceanbase.el9.aarch64.deps vendored
View File

@ -32,7 +32,7 @@ devdeps-boost-1.74.0-22022110914.el8.aarch64.rpm
devdeps-s2geometry-0.9.0-12023092021.el8.aarch64.rpm
devdeps-icu-69.1-72022112416.el8.aarch64.rpm
devdeps-cos-c-sdk-5.0.16-52023070517.el8.aarch64.rpm
devdeps-s3-cpp-sdk-1.11.156-62023101011.el8.aarch64.rpm
devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.aarch64.rpm
[deps-el9]
devdeps-apr-1.6.5-232023090616.el9.aarch64.rpm target=el9

2
deps/init/oceanbase.el9.x86_64.deps vendored
View File

@ -34,7 +34,7 @@ devdeps-s2geometry-0.9.0-12023092021.el8.x86_64.rpm
devdeps-icu-69.1-72022112416.el8.x86_64.rpm
devdeps-cos-c-sdk-5.0.16-52023070517.el8.x86_64.rpm
devdeps-cloud-qpl-1.1.0-272023061419.el8.x86_64.rpm
devdeps-s3-cpp-sdk-1.11.156-62023101011.el8.x86_64.rpm
devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.x86_64.rpm
[deps-el9]
devdeps-apr-1.6.5-232023090616.el9.x86_64.rpm target=el9

41
deps/oblib/src/common/object/ob_obj_funcs.h vendored
View File

@ -2320,8 +2320,8 @@ template <>
inline int obj_val_serialize<ObExtendType>(const ObObj &obj, char* buf, const int64_t buf_len, int64_t& pos)
{
int ret = OB_SUCCESS;
OB_UNIS_ENCODE(obj.get_ext());
if (obj.is_pl_extend()) {
OB_UNIS_ENCODE(obj.get_ext());
COMMON_LOG(ERROR, "Unexpected serialize", K(OB_NOT_SUPPORTED), K(obj), K(obj.get_meta().get_extend_type()));
return OB_NOT_SUPPORTED; //TODO:@ryan.ly: close this feature before composite refactor
if (NULL == serialize_composite_callback) {
@ -2329,6 +2329,18 @@ inline int obj_val_serialize<ObExtendType>(const ObObj &obj, char* buf, const in
} else {
ret = serialize_composite_callback(obj, buf, buf_len, pos);
}
} else if (obj.is_ext_sql_array()) {
int64_t v = 0;
OB_UNIS_ENCODE(v);
const ObSqlArrayObj *array_obj = reinterpret_cast<const ObSqlArrayObj*>(obj.get_ext());
if (OB_SUCC(ret) && NULL != array_obj) {
int64_t len = array_obj->get_serialize_size();
int64_t tmp_pos = pos;
OB_UNIS_ENCODE(len);
OB_UNIS_ENCODE(*array_obj);
}
} else {
OB_UNIS_ENCODE(obj.get_ext());
}
return ret;
}
@ -2347,6 +2359,20 @@ inline int obj_val_deserialize<ObExtendType>(ObObj &obj, const char* buf, const
} else {
ret = deserialize_composite_callback(obj, buf, data_len, pos);
}
} else if (obj.is_ext_sql_array()) {
if (OB_UNLIKELY(v != 0)) {
ret = OB_NOT_SUPPORTED;
COMMON_LOG(WARN, "using such type in upgrade period", K(ret));
} else {
int64_t len = 0;
int64_t tmp_pos = pos;
OB_UNIS_DECODE(len);
/* record the buffer and delay it's deserialize.
* should call ObSqlArrayObj::do_real_deserialize which need an allocator
*/
obj.set_extend(reinterpret_cast<int64_t>(buf + pos), T_EXT_SQL_ARRAY, int32_t(len));
pos += len;
}
} else {
obj.set_obj_value(v);
}
@ -2357,8 +2383,8 @@ template <>
inline int64_t obj_val_get_serialize_size<ObExtendType>(const ObObj &obj)
{
int64_t len = 0;
OB_UNIS_ADD_LEN(obj.get_ext());
if (obj.is_pl_extend()) {
OB_UNIS_ADD_LEN(obj.get_ext());
COMMON_LOG_RET(ERROR, OB_NOT_SUPPORTED, "Unexpected serialize", K(OB_NOT_SUPPORTED), K(obj), K(obj.get_meta().get_extend_type()));
return len; //TODO:@ryan.ly: close this feature before composite refactor
if (NULL == composite_serialize_size_callback) {
@ -2366,6 +2392,17 @@ inline int64_t obj_val_get_serialize_size<ObExtendType>(const ObObj &obj)
} else {
len += composite_serialize_size_callback(obj);
}
} else if (obj.is_ext_sql_array()) {
int64_t v = 0;
OB_UNIS_ADD_LEN(v);
const ObSqlArrayObj *array_obj = reinterpret_cast<const ObSqlArrayObj*>(obj.get_ext());
if (NULL != array_obj) {
int64_t array_obj_len = array_obj->get_serialize_size();
OB_UNIS_ADD_LEN(array_obj_len);
OB_UNIS_ADD_LEN(*array_obj);
}
} else {
OB_UNIS_ADD_LEN(obj.get_ext());
}
return len;
}

18
deps/oblib/src/common/object/ob_obj_type.cpp vendored
View File

@ -650,24 +650,6 @@ bool is_match_alter_integer_column_online_ddl_rules(const common::ObObjMeta& src
|| (src_meta.is_unsigned_integer() && dst_meta.is_unsigned_integer())) // both are singed or unsigned integer
&& src_meta.get_type() <= dst_meta.get_type())) { // (unsigned) integer can be changed into larger by online ddl
is_online_ddl = true;
} else if (src_meta.is_unsigned_integer() && dst_meta.is_signed_integer()) {
if (src_meta.is_utinyint()) {
if (dst_meta.get_type() >= ObSmallIntType && dst_meta.get_type() <= ObIntType) { // unsigned tinyint -> smallint mediumint int bigint
is_online_ddl = true;
}
} else if (src_meta.is_usmallint()) {
if (dst_meta.get_type() >= ObMediumIntType && dst_meta.get_type() <= ObIntType) { // unsigned smallint -> mediumint int bigint
is_online_ddl = true;
}
} else if (src_meta.is_umediumint()) {
if (dst_meta.get_type() >= ObInt32Type && dst_meta.get_type() <= ObIntType) { // unsigned mediumint -> int bigint
is_online_ddl = true;
}
} else if (src_meta.is_uint32()) {
if (dst_meta.is_int()) { // unsigned int -> bigint
is_online_ddl = true;
}
}
}
return is_online_ddl;
}

120
deps/oblib/src/common/object/ob_object.cpp vendored
View File

@ -2227,35 +2227,6 @@ DEFINE_SERIALIZE(ObObjParam)
if (OB_SUCC(ret)) {
OB_UNIS_ENCODE(accuracy_);
OB_UNIS_ENCODE(res_flags_);
if (OB_SUCC(ret) && is_ext_sql_array()) {
const ObSqlArrayObj *array_obj = reinterpret_cast<const ObSqlArrayObj*>(get_ext());
int64_t n = sizeof(ObSqlArrayObj);
if (OB_ISNULL(array_obj)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected NULL ptr", K(ret), KP(array_obj));
} else if (buf_len - pos < n) {
ret = OB_BUF_NOT_ENOUGH;
LOG_WARN("serialize buf not enough", K(ret), "remain", buf_len - pos, "needed", n);
} else {
MEMCPY(buf + pos, array_obj, n);
pos += n;
if (array_obj->count_ == 0) {
/* do nothing */
} else if (OB_ISNULL(array_obj->data_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("data is NULL ptr", K(ret), KP(array_obj->data_));
} else {
n = sizeof(array_obj->data_[0]) * array_obj->count_;
if (buf_len - pos < n) {
ret = OB_BUF_NOT_ENOUGH;
LOG_WARN("serialize buf not enough", K(ret), "remain", buf_len - pos, "needed", n);
} else {
MEMCPY(buf + pos, static_cast<const void*>(array_obj->data_), n);
pos += n;
}
}
}
}
}
return ret;
}
@ -2266,30 +2237,6 @@ DEFINE_DESERIALIZE(ObObjParam)
if (OB_SUCC(ret)) {
OB_UNIS_DECODE(accuracy_);
OB_UNIS_DECODE(res_flags_);
if (OB_SUCC(ret) && is_ext_sql_array()) {
ObSqlArrayObj *array_obj = NULL;
int64_t n = sizeof(ObSqlArrayObj);
if (data_len - pos < n) {
ret = OB_BUF_NOT_ENOUGH;
LOG_WARN("deserialize buf not enough", K(ret), "remain", data_len - pos, "needed", n);
} else {
array_obj = reinterpret_cast<ObSqlArrayObj *>(const_cast<char *>(buf + pos));
pos += n;
}
if (OB_SUCC(ret) && array_obj->count_ > 0) {
n = sizeof(ObObjParam) * array_obj->count_;
if (data_len - pos < n) {
ret = OB_BUF_NOT_ENOUGH;
LOG_WARN("deserialize buf not enough", K(ret), "remain", data_len - pos, "needed", n);
} else {
array_obj->data_ = reinterpret_cast<ObObjParam *>(const_cast<char *>(buf + pos));
pos += n;
}
}
if (OB_SUCC(ret)) {
set_extend(reinterpret_cast<int64_t>(array_obj), T_EXT_SQL_ARRAY);
}
}
}
return ret;
}
@ -2299,18 +2246,6 @@ DEFINE_GET_SERIALIZE_SIZE(ObObjParam)
int64_t len = ObObj::get_serialize_size();
OB_UNIS_ADD_LEN(accuracy_);
OB_UNIS_ADD_LEN(res_flags_);
if (is_ext_sql_array()) {
len += sizeof(ObSqlArrayObj);
const ObSqlArrayObj *array_obj = reinterpret_cast<const ObSqlArrayObj*>(get_ext());
if (NULL != array_obj) {
len += sizeof(ObSqlArrayObj);
if (array_obj->count_ == 0) {
/* do nothing */
} else if (NULL != array_obj->data_) {
len += sizeof(array_obj->data_[0]) * array_obj->count_;
}
}
}
return len;
}
@ -2451,3 +2386,58 @@ int64_t ObHexEscapeSqlStr::get_extra_length() const
}
return ret_length;
}
int ObSqlArrayObj::do_real_deserialize(common::ObIAllocator &allocator, char *buf, int64_t data_len,
ObSqlArrayObj *&array_obj)
{
int ret = OB_SUCCESS;
int64_t n = sizeof(ObSqlArrayObj);
void *array_buf = allocator.alloc(n);
int64_t pos = 0;
if (OB_ISNULL(array_buf)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("allocate memory failed", K(ret));
} else {
array_obj = new (array_buf) ObSqlArrayObj();
if (OB_FAIL(array_obj->deserialize(allocator, buf, data_len, pos))) {
LOG_WARN("failed to deserialize ObSqlArrayObj", K(ret));
}
}
return ret;
}
DEFINE_SERIALIZE(ObSqlArrayObj)
{
int ret = OB_SUCCESS;
int64_t len = 0;
OB_UNIS_ENCODE(element_);
OB_UNIS_ENCODE_ARRAY(data_, count_);
return ret;
}
int ObSqlArrayObj::deserialize(ObIAllocator &allocator, const char* buf, const int64_t data_len,
int64_t& pos)
{
int ret = OB_SUCCESS;
OB_UNIS_DECODE(element_);
OB_UNIS_DECODE(count_);
if (OB_SUCC(ret) && count_ > 0) {
void *data_buf = allocator.alloc(sizeof(ObObjParam) * count_);
if (OB_ISNULL(data_buf)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("allocate memory failed", K(ret));
} else {
data_ = new (data_buf) common::ObObjParam[count_];
OB_UNIS_DECODE_ARRAY(data_, count_);
}
}
return ret;
}
DEFINE_GET_SERIALIZE_SIZE(ObSqlArrayObj)
{
int64_t len = 0;
OB_UNIS_ADD_LEN(element_);
OB_UNIS_ADD_LEN_ARRAY(data_, count_);
return len;
}

4
deps/oblib/src/common/object/ob_object.h vendored
View File

@ -4150,6 +4150,10 @@ struct ObSqlArrayObj
}
typedef common::ObArrayWrap<common::ObObjParam> DataArray;
static ObSqlArrayObj *alloc(common::ObIAllocator &allocator, int64_t count);
static int do_real_deserialize(common::ObIAllocator &allocator, char *buf, int64_t data_len, ObSqlArrayObj *&array_obj);
int serialize(char* buf, const int64_t buf_len, int64_t& pos) const;
int deserialize(common::ObIAllocator &allocator, const char* buf, const int64_t data_len, int64_t& pos);
int64_t get_serialize_size(void) const;
TO_STRING_KV("data", DataArray(data_, count_), K_(count), K_(element));
common::ObObjParam *data_;
int64_t count_;

81
deps/oblib/src/lib/compress/zlib_lite/zlib_lite_adaptor.cpp vendored
View File

@ -11,6 +11,9 @@
*/
#include "zlib_lite_adaptor.h"
#include <cpuid.h>
#include "codec_deflate_qpl.h"
#include "zlib_lite_src/deflate.h"
#include "zlib_lite_src/zlib.h"
@ -21,6 +24,47 @@ namespace common
{
namespace ZLIB_LITE
{
static bool check_support_qpl(void)
{
bool bret = false;
unsigned int eax, ebx, ecx, edx;
//LEVEL=0: Highest Function Parameter and Manufacturer ID
__cpuid(0, eax, ebx, ecx, edx);
//Determine whether it is an Intel manufacturer.
if (ebx == 0x756e6547 && edx == 0x49656e69 && ecx == 0x6c65746e) {
//LEVEL=1: Processor Info and Feature Bit
__cpuid(1, eax, ebx, ecx, edx);
unsigned int family, model;
/*
If the Family ID field is equal to 15, the family is equal to the sum of
the Extended Family ID and the Family ID fields.
Otherwise, the family is equal to the value of the Family ID field.
*/
family = (eax >> 8) & 0xF;
if (family == 0xF) {
family += (eax >> 20) & 0xF;
}
/*
If the Family ID field is either 6 or 15, the model is equal to the sum of
the Extended Model ID field shifted left by 4 bits and the Model field,
Otherwise, the model is equal to the value of the Model field.
*/
model = (eax >> 4) & 0xF;
if (family >= 0x6) {
model += ((eax >> 16) & 0xF) << 4;
}
// if Model less then 0X8F or family not equal 6, QPL compress and
// decompress is not supported, will use zlib instead.
if (family == 6 && model >= 0X8F) {
bret = true;
}
}
return bret;
}
/*zlib_lite supports two algorithms. On the platform that supports qpl, the qpl compression algorithm will be used,
otherwise the zlib algorithm will be used.*/
@ -37,12 +81,15 @@ int ObZlibLiteAdaptor::init(allocator alloc, deallocator dealloc, int32_t io_thr
int ret = 0;
#ifdef ENABLE_QPL_COMPRESSION
QplAllocator allocator;
allocator.allocate = alloc;
allocator.deallocate = dealloc;
int qpl_ret = qpl_init(allocator, io_thread_count);
if (0 != qpl_ret) {
ret = -1;
qpl_support_ = check_support_qpl();
if (qpl_support_) {
QplAllocator allocator;
allocator.allocate = alloc;
allocator.deallocate = dealloc;
int qpl_ret = qpl_init(allocator, io_thread_count);
if (0 != qpl_ret) {
ret = -1;
}
}
#endif
@ -52,7 +99,9 @@ int ObZlibLiteAdaptor::init(allocator alloc, deallocator dealloc, int32_t io_thr
void ObZlibLiteAdaptor::deinit()
{
#ifdef ENABLE_QPL_COMPRESSION
qpl_deinit();
if (qpl_support_) {
qpl_deinit();
}
#endif
}
@ -129,8 +178,10 @@ int64_t ObZlibLiteAdaptor::compress(const char* src_buffer,
{
#ifdef ENABLE_QPL_COMPRESSION
return qpl_compress(src_buffer, dst_buffer, static_cast<int>(src_data_size),
static_cast<int>(dst_buffer_size));
if (qpl_support_) {
return qpl_compress(src_buffer, dst_buffer, static_cast<int>(src_data_size),
static_cast<int>(dst_buffer_size));
}
#endif // ENABLE_QPL_COMPRESSION
@ -152,9 +203,11 @@ int64_t ObZlibLiteAdaptor::decompress(const char* src_buffer,
{
#ifdef ENABLE_QPL_COMPRESSION
return qpl_decompress(src_buffer, dst_buffer,
static_cast<int>(src_data_size),
static_cast<int>(dst_buffer_size));
if (qpl_support_) {
return qpl_decompress(src_buffer, dst_buffer,
static_cast<int>(src_data_size),
static_cast<int>(dst_buffer_size));
}
#endif
int64_t decompress_ret_size = dst_buffer_size;
@ -170,7 +223,9 @@ int64_t ObZlibLiteAdaptor::decompress(const char* src_buffer,
const char *ObZlibLiteAdaptor::compression_method() const
{
#ifdef ENABLE_QPL_COMPRESSION
return qpl_hardware_enabled() ? "qpl_hardware" : "qpl_software";
if (qpl_support_) {
return qpl_hardware_enabled() ? "qpl_hardware" : "qpl_software";
}
#endif
return "zlib_native";

2
deps/oblib/src/lib/compress/zlib_lite/zlib_lite_adaptor.h vendored
View File

@ -54,6 +54,8 @@ private:
int zlib_decompress(char *dest, int64_t *dest_len, const char *source, int64_t source_len);
private:
bool qpl_support_ = false;
//zlib compress level,default is 1.
static constexpr int compress_level = 1;

86
deps/oblib/src/lib/json_type/ob_json_base.cpp vendored
View File

@ -2147,53 +2147,63 @@ bool ObIJsonBase::is_real_json_null(const ObIJsonBase* ptr) const
return ret_bool;
}
int ObIJsonBase::trans_json_node(ObIAllocator* allocator, ObIJsonBase* &left, ObIJsonBase* &right) const
// left is scalar, right is ans of subpath
int ObIJsonBase::trans_json_node(ObIAllocator* allocator, ObIJsonBase* &scalar, ObIJsonBase* &path_res) const
{
INIT_SUCC(ret);
ObJsonNodeType left_type = left->json_type();
ObJsonNodeType right_type = right->json_type();
// 左边的需要根据右边的类型转换
ObJsonNodeType left_type = scalar->json_type();
ObJsonNodeType right_type = path_res->json_type();
if (left_type == ObJsonNodeType::J_STRING) {
ObString str(left->get_data_length(), left->get_data());
ObString str(scalar->get_data_length(), scalar->get_data());
if (is_json_number(right_type)) {
// fail is normal
ret = trans_to_json_number(allocator, str, left);
ret = trans_to_json_number(allocator, str, scalar);
} else if (right_type == ObJsonNodeType::J_DATE
|| right_type == ObJsonNodeType::J_DATETIME
|| right_type == ObJsonNodeType::J_TIME
|| right_type == ObJsonNodeType::J_ORACLEDATE) {
ret = trans_to_date_timestamp(allocator, str, left, true);
ret = trans_to_date_timestamp(allocator, str, scalar, true);
} else if (right_type == ObJsonNodeType::J_TIMESTAMP
|| right_type == ObJsonNodeType::J_OTIMESTAMP
|| right_type == ObJsonNodeType::J_OTIMESTAMPTZ) {
ret = trans_to_date_timestamp(allocator, str, left, false);
ret = trans_to_date_timestamp(allocator, str, scalar, false);
} else if (right_type == ObJsonNodeType::J_BOOLEAN) {
ret = trans_to_boolean(allocator, str, left);
// when scalar is string, path_res is boolean, case compare
if (str.case_compare("true") == 0 || str.case_compare("false") == 0) {
ret = trans_to_boolean(allocator, str, scalar);
} else {
ret = OB_NOT_SUPPORTED;
}
} else if (right_type != ObJsonNodeType::J_ARRAY && right_type != ObJsonNodeType::J_OBJECT) {
ret = ret = OB_INVALID_ARGUMENT;
LOG_WARN("CAN'T TRANS", K(ret));
}
// 右边需要根据左边的转换
} else if (left_type == ObJsonNodeType::J_NULL) {
// return error code, mean can't cast, return false ans directly
ret = OB_NOT_SUPPORTED;
} else if (right_type == ObJsonNodeType::J_STRING) {
ObString str(right->get_data_length(), right->get_data());
ObString str(path_res->get_data_length(), path_res->get_data());
if (is_json_number(left_type)) {
// fail is normal
ret = trans_to_json_number(allocator, str, right);
ret = trans_to_json_number(allocator, str, path_res);
} else if (left_type == ObJsonNodeType::J_DATE
|| left_type == ObJsonNodeType::J_DATETIME
|| left_type == ObJsonNodeType::J_TIME
|| left_type == ObJsonNodeType::J_ORACLEDATE) {
ret = trans_to_date_timestamp(allocator, str, right, true);
ret = trans_to_date_timestamp(allocator, str, path_res, true);
} else if (left_type == ObJsonNodeType::J_TIMESTAMP
|| left_type == ObJsonNodeType::J_OTIMESTAMP
|| left_type == ObJsonNodeType::J_OTIMESTAMPTZ) {
ret = trans_to_date_timestamp(allocator, str, right, false);
ret = trans_to_date_timestamp(allocator, str, path_res, false);
} else if (left_type == ObJsonNodeType::J_BOOLEAN) {
ret = trans_to_boolean(allocator, str, right);
ret = trans_to_boolean(allocator, str, path_res);
} else if (left_type != ObJsonNodeType::J_ARRAY && left_type != ObJsonNodeType::J_OBJECT) {
ret = ret = OB_INVALID_ARGUMENT;
ret = OB_INVALID_ARGUMENT;
LOG_WARN("CAN'T TRANS", K(ret));
}
} else if (left_type == ObJsonNodeType::J_BOOLEAN || is_json_number(left_type)) {
// scalar is boolean or number, and path_res is not string, return false
ret = OB_NOT_SUPPORTED;
} else {
// do nothing
LOG_WARN("CAN'T TRANS", K(ret));
@ -2247,7 +2257,7 @@ int ObIJsonBase::cmp_to_right_recursively(ObIAllocator* allocator, const ObJsonB
// 但只要有一个找到，且为123或"123"则为true
ObIJsonBase* left = jb_ptr;
ObIJsonBase* right = right_arg;
if (OB_FAIL(trans_json_node(allocator, left, right))) {
if (OB_FAIL(trans_json_node(allocator, right, left))) {
// fail is normal, it is not an error.
ret = OB_SUCCESS;
cmp_result = false;
@ -2270,7 +2280,7 @@ int ObIJsonBase::cmp_to_right_recursively(ObIAllocator* allocator, const ObJsonB
// 不相同的类型，同上
ObIJsonBase* left = hit[i];
ObIJsonBase* right = right_arg;
if (OB_FAIL(trans_json_node(allocator, left, right))) {
if (OB_FAIL(trans_json_node(allocator, right, left))) {
// fail is normal, it is not an error.
ret = OB_SUCCESS;
cmp_result = false;
@ -2291,7 +2301,6 @@ int ObIJsonBase::cmp_to_right_recursively(ObIAllocator* allocator, const ObJsonB
}
// for compare ——> ( scalar/sql_var, subpath)
// 左边调用compare，右边是数组时自动解包
// 只要有一个结果为true则返回true，找不到或结果为false均返回false
int ObIJsonBase::cmp_to_left_recursively(ObIAllocator* allocator, const ObJsonBaseVector& hit,
const ObJsonPathNodeType node_type,
@ -2310,44 +2319,7 @@ int ObIJsonBase::cmp_to_left_recursively(ObIAllocator* allocator, const ObJsonBa
} else if (hit[i]->json_type() == ObJsonNodeType::J_NULL && !is_real_json_null(hit[i])) {
cmp_result = false;
} else {
// error is ok
// if is array, compare with every node
// but only autowrap once
if (hit[i]->json_type() == ObJsonNodeType::J_ARRAY) {
uint64_t size = hit[i]->element_count();
ObIJsonBase *jb_ptr = NULL;
for (uint32_t array_i = 0; array_i < size && !cmp_result && OB_SUCC(ret); ++array_i) {
jb_ptr = NULL; // reset jb_ptr to NULL
ret = hit[i]->get_array_element(array_i, jb_ptr);
int cmp_res = -3;
// 类型相同可以直接用compare函数比较
if(OB_FAIL(ret) || OB_ISNULL(jb_ptr)) {
ret = OB_ERR_NULL_VALUE;
LOG_WARN("compare value is null.", K(ret));
} else if (is_same_type(left_arg, jb_ptr)) {
if (OB_SUCC(left_arg->compare((*jb_ptr), cmp_res, true))) {
cmp_based_on_node_type(node_type, cmp_res, cmp_result);
}
} else {
// 不相同的类型，oracle会将string类型转换为对应类型再进行比较
// 转换或比较失败也正常，并不报错
// 例如: [*].a == 123
// 里面可能有多个元素无法转换成数字或无法和数字比较甚至找不到.a
// 但只要有一个找到，且为123或"123"则为true
ObIJsonBase* left = left_arg;
ObIJsonBase* right = jb_ptr;
if (OB_FAIL(trans_json_node(allocator, left, right))) {
// fail is normal, it is not an error.
ret = OB_SUCCESS;
cmp_result = false;
} else if (OB_SUCC(left->compare((*right), cmp_res, true))) {
cmp_based_on_node_type(node_type, cmp_res, cmp_result);
} else {
cmp_result = false;
}
}
}
} else if (hit[i]->json_type() == ObJsonNodeType::J_OBJECT) {
if (hit[i]->json_type() == ObJsonNodeType::J_OBJECT || hit[i]->json_type() == ObJsonNodeType::J_ARRAY) {
cmp_result = false;
} else {
int cmp_res = -3;

271
deps/oblib/src/lib/json_type/ob_json_path.cpp vendored
View File

@ -786,13 +786,16 @@ int ObJsonPath::change_json_expr_res_type_if_need(common::ObIAllocator &allocato
switch (func_node->get_node_type()) {
case JPN_BOOLEAN :
case JPN_BOOL_ONLY : {
ret_node.type_ = T_CAST_ARGUMENT;
ret_node.value_ = 0;
ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] = T_VARCHAR;
ret_node.int16_values_[OB_NODE_CAST_COLL_IDX] = 0;
ret_node.int32_values_[OB_NODE_CAST_C_LEN_IDX] = 20;
ret_node.length_semantics_ = 0;
ret_node.is_hidden_const_ = 1;
if (json_expr_flag == OPT_JSON_QUERY && ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] == T_JSON) { // do nothing
} else {
ret_node.type_ = T_CAST_ARGUMENT;
ret_node.value_ = 0;
ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] = T_VARCHAR;
ret_node.int16_values_[OB_NODE_CAST_COLL_IDX] = 0;
ret_node.int32_values_[OB_NODE_CAST_C_LEN_IDX] = 20;
ret_node.length_semantics_ = 0;
ret_node.is_hidden_const_ = 1;
}
break;
}
case JPN_DATE : {
@ -824,14 +827,16 @@ int ObJsonPath::change_json_expr_res_type_if_need(common::ObIAllocator &allocato
case JPN_NUMBER :
case JPN_FLOOR :
case JPN_CEILING : {
ret_node.value_ = 0;
if (ret_node.type_ == T_NULL) {
if (ret_node.type_ == T_NULL
|| (json_expr_flag == OPT_JSON_QUERY && ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] == T_JSON)) {
ret_node.value_ = 0;
ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] = T_VARCHAR;
ret_node.int16_values_[OB_NODE_CAST_COLL_IDX] = 0;
ret_node.int32_values_[OB_NODE_CAST_C_LEN_IDX] = 4000;
ret_node.length_semantics_ = 0;
ret_node.is_hidden_const_ = 1;
} else {
ret_node.value_ = 0;
ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] = T_NUMBER;
ret_node.int16_values_[OB_NODE_CAST_N_PREC_IDX] = -1; /* precision */
ret_node.int16_values_[OB_NODE_CAST_N_SCALE_IDX] = -85; /* scale */
@ -855,24 +860,29 @@ int ObJsonPath::change_json_expr_res_type_if_need(common::ObIAllocator &allocato
case JPN_TYPE:
case JPN_STR_ONLY :
case JPN_STRING : {
ret_node.type_ = T_CAST_ARGUMENT;
ret_node.value_ = 0;
ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] = T_VARCHAR;
ret_node.int16_values_[OB_NODE_CAST_COLL_IDX] = 0;
ret_node.int32_values_[OB_NODE_CAST_C_LEN_IDX] = 4000;
ret_node.length_semantics_ = 0;
ret_node.is_hidden_const_ = 1;
if (json_expr_flag == OPT_JSON_QUERY && ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] == T_JSON) {
} else {
ret_node.type_ = T_CAST_ARGUMENT;
ret_node.value_ = 0;
ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] = T_VARCHAR;
ret_node.int16_values_[OB_NODE_CAST_COLL_IDX] = 0;
ret_node.int32_values_[OB_NODE_CAST_C_LEN_IDX] = 4000;
ret_node.length_semantics_ = 0;
ret_node.is_hidden_const_ = 1;
}
}
case JPN_UPPER:
case JPN_LOWER: {
ret_node.type_ = T_CAST_ARGUMENT;
ret_node.value_ = 0;
ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] = T_VARCHAR;
ret_node.int16_values_[OB_NODE_CAST_COLL_IDX] = 0;
ret_node.int32_values_[OB_NODE_CAST_C_LEN_IDX] = 75;
ret_node.length_semantics_ = 0;
ret_node.is_hidden_const_ = 1;
break;
if (json_expr_flag == OPT_JSON_QUERY && ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] == T_JSON) {
} else {
ret_node.type_ = T_CAST_ARGUMENT;
ret_node.value_ = 0;
ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] = T_VARCHAR;
ret_node.int16_values_[OB_NODE_CAST_COLL_IDX] = 0;
ret_node.int32_values_[OB_NODE_CAST_C_LEN_IDX] = 75;
ret_node.length_semantics_ = 0;
ret_node.is_hidden_const_ = 1;
}
break;
}
default : {
@ -2123,7 +2133,7 @@ int ObJsonPath::parse_name_with_rapidjson(char*& str, uint64_t& len)
}
// if keyname without double quote, end with ' ', '.', '[', '*'
bool ObJsonPathUtil::is_mysql_terminator(char ch)
bool ObJsonPathUtil::is_key_name_terminator(char ch)
{
bool ret_bool = false;
switch (ch) {
@ -2143,6 +2153,18 @@ bool ObJsonPathUtil::is_mysql_terminator(char ch)
ret_bool = true;
break;
}
case '(': {
ret_bool = true;
break;
}
case '?': {
ret_bool = true;
break;
}
case '-': {
ret_bool = true;
break;
}
default: {
break;
}
@ -2150,98 +2172,6 @@ bool ObJsonPathUtil::is_mysql_terminator(char ch)
return ret_bool;
}
// process JPN_MEMBER get keyname
// @param[in,out] name Keyname
// @param[in] is_quoted
// @return the error code.
int ObJsonPath::get_mysql_origin_key_name(char*& str, uint64_t& length, bool is_quoted)
{
INIT_SUCC(ret);
uint64_t start = 0;
uint64_t end = 0;
int len = expression_.length();
if (index_ < len) {
if (is_quoted) {
// with quote, check quote
if (expression_[index_] == ObJsonPathItem::DOUBLE_QUOTE) {
start = index_;
++index_;
while (index_ < len && end == 0) {
if (expression_[index_] == '\\') {
index_ += 2;
} else if (expression_[index_] == ObJsonPathItem::DOUBLE_QUOTE) {
end = index_;
++index_;
} else {
++index_;
}
}
if (end == 0 && index_ == len) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("should end with DOUBLE_QUOTE!", K(ret), K(index_), K(expression_));
}
} else {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("should start with DOUBLE_QUOTE!", K(ret), K(index_), K(expression_));
}
} else {
start = index_;
while (index_ < len && end == 0) {
if (ObJsonPathUtil::is_mysql_terminator(expression_[index_])) {
end = index_ - 1;
} else {
++index_;
}
}
if (index_ == len) {
end = index_ - 1;
}
}
} else {
ret = OB_ARRAY_OUT_OF_RANGE;
LOG_WARN("index out of range!", K(ret), K(index_), K(expression_));
}
if (OB_SUCC(ret)) {
if (end < start) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get keyname: end<start", K(ret), K(start), K(end), K(expression_));
} else {
len = end - start + 1;
char* start_ptr = expression_.ptr() + start;
if (!is_quoted) {
length = len + 2;
str = static_cast<char*> (allocator_->alloc(length));
if (OB_ISNULL(str)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to allocate memory for member_name.",
K(ret), K(len),KCSTRING(start_ptr));
} else {
str[0] = ObJsonPathItem::DOUBLE_QUOTE;
MEMCPY(str + 1, start_ptr, len);
str[len + 1] = ObJsonPathItem::DOUBLE_QUOTE;
}
} else {
length = len;
str = static_cast<char*> (allocator_->alloc(length));
if (OB_ISNULL(str)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to allocate memory for member_name.",
K(ret), K(len),KCSTRING(start_ptr));
} else {
MEMCPY(str, start_ptr, len);
}
}
}
}
return ret;
}
// parse JPN_MEMBER_WILDCARD
// @return the error code.
int ObJsonPath::parse_member_wildcard_node()
@ -2331,9 +2261,11 @@ int ObJsonPath::parse_mysql_member_node()
char* name = nullptr;
uint64_t name_len = 0;
bool is_func = false;
bool with_escape = false;
// get name
// add double quote for rapidjson requires
if (OB_FAIL(get_mysql_origin_key_name(name, name_len, is_quoted))) {
if (OB_FAIL(get_origin_key_name(name, name_len, is_quoted, is_func, with_escape))) {
LOG_WARN("fail to get keyname!", K(ret), K(index_), K(expression_));
} else {
if (OB_FAIL(parse_name_with_rapidjson(name, name_len))) {
@ -2431,6 +2363,31 @@ bool ObJsonPathUtil::is_scalar(const ObJsonPathNodeType node_type)
return ret_bool;
}
bool ObJsonPathUtil::is_escape(char ch)
{
return (('\n' == ch) || (ch == '\t') || (ch == '\r') || (ch == '\f') || (ch == '\e'));
}
int ObJsonPathUtil::append_character_of_escape(ObJsonBuffer& buf, char ch)
{
INIT_SUCC(ret);
if ('\n' == ch) {
ret = buf.append("n");
} else if ('\t' == ch) {
ret = buf.append("t");
} else if ('\r' == ch) {
ret = buf.append("r");
} else if ('\f' == ch) {
ret = buf.append("f");
} else if ('\e' == ch) {
ret = buf.append("e");
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("should be escape", K(ch), K(ret));
}
return ret;
}
void ObJsonPathUtil::skip_whitespace(const ObString &path, uint64_t& idx)
{
while (idx < path.length() && ObJsonPathUtil::is_whitespace(path[idx])) {
@ -3039,7 +2996,7 @@ bool ObJsonPathUtil::is_oracle_keyname(const char* name, uint64_t length)
// @param[in,out] name Keyname
// @param[in] is_quoted
// @return the error code.
int ObJsonPath::get_oracle_origin_key_name(char*& str, uint64_t& length, bool is_quoted, bool& is_func)
int ObJsonPath::get_origin_key_name(char*& str, uint64_t& length, bool is_quoted, bool& is_func, bool& with_escape)
{
INIT_SUCC(ret);
uint64_t start = 0;
@ -3060,6 +3017,9 @@ int ObJsonPath::get_oracle_origin_key_name(char*& str, uint64_t& length, bool is
} else if (expression_[index_] == ObJsonPathItem::DOUBLE_QUOTE) {
end = index_;
++index_;
} else if (ObJsonPathUtil::is_escape(expression_[index_])) {
with_escape = true;
++index_;
} else {
++index_;
}
@ -3074,35 +3034,28 @@ int ObJsonPath::get_oracle_origin_key_name(char*& str, uint64_t& length, bool is
LOG_WARN("should start with DOUBLE_QUOTE!", K(ret), K(index_), K(expression_));
}
} else {
if (ObJsonPathUtil::letter_or_not(expression_[index_]) || (expression_[index_] == '_')) {
// without '""'
start = index_;
++index_;
while (index_ < len) {
if (!(ObJsonPathUtil::letter_or_not(expression_[index_])
|| ObJsonPathUtil::is_digit(expression_[index_])
|| (expression_[index_] == '_'))) {
start = index_;
if (ObJsonPathUtil::is_digit(expression_[index_])) {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("shouldn't start with number!", K(ret), K(index_), K(expression_));
} else {
while (index_ < len && end == 0) {
if (ObJsonPathUtil::is_key_name_terminator(expression_[index_])) {
end = index_ - 1;
break;
} else {
++index_;
}
}
if (index_ == len) {
end = index_ - 1;
} else {
ObJsonPathUtil::skip_whitespace(expression_, index_);
// fun_name + ()
if (index_ < expression_.length()) {
if (expression_[index_] == '(') {
if (index_ < expression_.length() && expression_[index_] == '(') {
is_func = true;
}
}//now, index could equal to len
}
}
} else {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("wrong keyname!", K(ret), K(start), K(end), K(expression_));
}
}
} else {
@ -3115,11 +3068,11 @@ int ObJsonPath::get_oracle_origin_key_name(char*& str, uint64_t& length, bool is
ret = OB_INVALID_ARGUMENT;
LOG_WARN("get keyname: end<start", K(ret), K(start), K(end), K(expression_));
} else {
len = end-start + 1;
len = end - start + 1;
char* start_ptr = expression_.ptr() + start;
// no "", could be function name
if ((!is_quoted) && (!is_func)) {
length = len+2;
length = len + 2;
str = static_cast<char*> (allocator_->alloc(length));
if (OB_ISNULL(str)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
@ -3182,6 +3135,31 @@ int ObJsonPath::parse_func_node(char*& name, uint64_t& len)
return ret;
}
int ObJsonPath::deal_with_escape(char* &str, uint64_t& len)
{
INIT_SUCC(ret);
ObJsonBuffer buf(allocator_);
for (int i = 0; i < len && OB_SUCC(ret); ++i) {
char* tmp = str + i;
if (OB_ISNULL(tmp)) {
} else if (ObJsonPathUtil::is_escape(*tmp)) {
if (OB_FAIL(buf.append("\\"))) {
LOG_WARN("fail to append \\.", K(i), K(ret));
} else if (OB_FAIL(ObJsonPathUtil::append_character_of_escape(buf, *tmp))) {
LOG_WARN("fail to append_character_of_escape.", K(*tmp), K(i), K(ret));
}
} else {
ret = buf.append(tmp, 1);
}
}
if (OB_SUCC(ret)) {
str = buf.ptr();
len = buf.length();
}
return ret;
}
// parse JPN_ORACLE_MEMBER
// @return the error code.
int ObJsonPath::parse_oracle_member_node()
@ -3191,6 +3169,7 @@ int ObJsonPath::parse_oracle_member_node()
if (index_ < expression_.length()) {
bool is_quoted = false;
bool is_func = false;
bool with_escape = false;
// check double quote
if (expression_[index_] == ObJsonPathItem::DOUBLE_QUOTE) is_quoted = true;
@ -3199,12 +3178,19 @@ int ObJsonPath::parse_oracle_member_node()
uint64_t name_len = 0;
// get name
// add double quote for rapidjson requires
if (OB_FAIL(get_oracle_origin_key_name(name, name_len, is_quoted, is_func))) {
if (OB_FAIL(get_origin_key_name(name, name_len, is_quoted, is_func, with_escape))) {
LOG_WARN("fail to get keyname!", K(ret), K(index_), K(expression_));
} else if (is_quoted || (!is_func)) {
// with "", must be member node
if (OB_FAIL(parse_name_with_rapidjson(name, name_len))) {
LOG_WARN("fail to parse name with rapidjson",K(ret), K(index_), K(expression_),K(name));
if (lib::is_oracle_mode() && with_escape && OB_FAIL(ObJsonPath::deal_with_escape(name, name_len))) {
LOG_WARN("fail to deal escape!", K(ret), K(index_), K(expression_));
} else if (OB_FAIL(parse_name_with_rapidjson(name, name_len))) {
LOG_WARN("fail to parse name with rapidjson",
K(ret), K(index_), K(expression_),KCSTRING(name));
} else if (!is_quoted) {
if (!ObJsonPathUtil::is_ecmascript_identifier(name, name_len)) {
LOG_WARN("the key name isn't ECMAScript identifier!",
K(ret), KCSTRING(name));
}
}
if (OB_SUCC(ret)) {
@ -3932,7 +3918,8 @@ int ObJsonPath::parse_comp_string_num(ObJsonPathFilterNode* filter_comp_node, bo
char* str = nullptr;
uint64_t name_len = 0;
bool is_func = false;
if (OB_FAIL(get_oracle_origin_key_name(str, name_len, true, is_func))) {
bool with_escape = false;
if (OB_FAIL(get_origin_key_name(str, name_len, true, is_func, with_escape))) {
LOG_WARN("fail to get string scalar",K(ret), K(index_), K(expression_),K(str));
} else {
if (OB_ISNULL(str)) {

8
deps/oblib/src/lib/json_type/ob_json_path.h vendored
View File

@ -359,8 +359,8 @@ private:
bool& from_end1, bool& from_end2);
int add_single_array_node(bool is_cell_type, uint64_t& index1, uint64_t& index2,
bool& from_end1, bool& from_end2);
int get_mysql_origin_key_name(char* &str, uint64_t& length, bool is_quoted);
int get_oracle_origin_key_name(char* &str, uint64_t& length, bool is_quoted, bool& is_func);
int get_origin_key_name(char* &str, uint64_t& length, bool is_quoted, bool& is_func, bool& with_escape);
int deal_with_escape(char* &str, uint64_t& length);
int parse_name_with_rapidjson(char*& str, uint64_t& len);
int parse_func_node(char*& name, uint64_t& len);
int get_char_comparison_type(ObJsonPathFilterNode* filter_comp_node);
@ -455,7 +455,7 @@ public:
static bool is_oracle_keyname(const char* name, uint64_t length);
// add quote and
static int double_quote(ObString &name, ObJsonBuffer* tmp_name);
static bool is_mysql_terminator(char ch);
static bool is_key_name_terminator(char ch);
static bool is_begin_field_name(char ch);
static bool is_end_of_comparission(char ch);
static bool letter_or_not(char ch);
@ -480,6 +480,8 @@ public:
bool auto_wrap);
static bool is_letter(unsigned codepoint, const char* ori, uint64_t start, uint64_t end);
static bool is_escape(char ch);
static int append_character_of_escape(ObJsonBuffer& buf, char ch);
static bool is_connector_punctuation(unsigned codepoint);
static bool unicode_combining_mark(unsigned codepoint);
static bool is_utf8_unicode_charator(const char* ori, uint64_t& start, int64_t len);

3
deps/oblib/src/lib/mysqlclient/ob_tenant_oci_envs.h vendored
View File

@ -22,6 +22,7 @@ namespace sqlclient
class ObTenantOciEnvs
{
public:
static int mtl_new(ObTenantOciEnvs *&tenant_oci_envs) { return OB_SUCCESS; }
static int mtl_init(ObTenantOciEnvs *&tenant_oci_envs) { return OB_SUCCESS; }
static void mtl_destroy(ObTenantOciEnvs *&tenant_oci_envs) { }
};
@ -30,4 +31,4 @@ public:
} //sqlclient
} // namespace common
} // namespace oceanbase
#endif //OBTENANOCIENVS_H
#endif //OBTENANOCIENVS_H

2
deps/oblib/src/lib/ob_define.h vendored
View File

@ -1670,7 +1670,7 @@ OB_INLINE bool is_bootstrap_resource_pool(const uint64_t resource_pool_id)
const int64_t OB_MALLOC_NORMAL_BLOCK_SIZE = (1LL << 13) - 256; // 8KB
const int64_t OB_MALLOC_MIDDLE_BLOCK_SIZE = (1LL << 16) - 128; // 64KB
const int64_t OB_MALLOC_BIG_BLOCK_SIZE = (1LL << 21) - ACHUNK_PRESERVE_SIZE;// 2MB (-17KB)
const int64_t OB_MALLOC_REQ_NORMAL_BLOCK_SIZE = (256LL << 10); // 256KB
const int64_t OB_MALLOC_REQ_NORMAL_BLOCK_SIZE = (240LL << 10); // 240KB
const int64_t OB_MAX_MYSQL_RESPONSE_PACKET_SIZE = OB_MALLOC_BIG_BLOCK_SIZE;

2
deps/oblib/src/lib/oblog/ob_async_log_struct.h vendored
View File

@ -118,8 +118,6 @@ public:
int64_t file_size_;
struct stat stat_;
struct stat wf_stat_;
std::deque<std::string> file_list_;//to store the names of log-files
std::deque<std::string> wf_file_list_;//to store the names of warning log-files
};
} // common

42
deps/oblib/src/lib/oblog/ob_log.cpp vendored
View File

@ -743,7 +743,7 @@ void ObLogger::rotate_log(const int64_t size, const bool redirect_flag,
if (OB_LIKELY(size > 0) && max_file_size_ > 0 && log_struct.file_size_ >= max_file_size_) {
if (OB_LIKELY(0 == pthread_mutex_trylock(&file_size_mutex_))) {
rotate_log(log_struct.filename_, fd_type, redirect_flag, log_struct.fd_,
log_struct.wf_fd_, log_struct.file_list_, log_struct.wf_file_list_);
log_struct.wf_fd_, file_list_, wf_file_list_);
(void)ATOMIC_SET(&log_struct.file_size_, 0);
if (fd_type <= FD_ELEC_FILE) {
(void)log_new_file_info(log_struct);
@ -1255,17 +1255,17 @@ int ObLogger::record_old_log_file()
int ret = OB_SUCCESS;
if (max_file_index_ <= 0 || !rec_old_file_flag_) {
} else {
int tmp_ret = OB_SUCCESS;
ObSEArray<FileName, 4> files;
ObSEArray<FileName, 4> wf_files;
for (int type = FD_SVR_FILE; type < FD_AUDIT_FILE; ++type) {
ObSEArray<FileName, 4> files;
ObSEArray<FileName, 4> wf_files;
if (OB_FAIL(get_log_files_in_dir(log_file_[type].filename_, &files, &wf_files))) {
if (OB_TMP_FAIL(get_log_files_in_dir(log_file_[type].filename_, &files, &wf_files))) {
OB_LOG(WARN, "Get log files in log dir error", K(ret));
} else if (OB_FAIL(add_files_to_list(&files, &wf_files, log_file_[type].file_list_, log_file_[type].wf_file_list_))) {
OB_LOG(WARN, "Add files to list error", K(ret));
} else {
// do nothing
}
}
if (OB_FAIL(add_files_to_list(&files, &wf_files, file_list_, wf_file_list_))) {
OB_LOG(WARN, "Add files to list error", K(ret));
}
}
return ret;
}
@ -1348,6 +1348,28 @@ int ObLogger::get_log_files_in_dir(const char *filename, void *files, void *wf_f
return ret;
}
int compare_log_filename_by_date_suffix(const void *v1, const void *v2)
{
int ret = 0;
if (NULL == v1) {
ret = -1;
} else if (NULL == v2) {
ret = 1;
} else {
const int DATE_LENGTH = 17;
const char *str1 = static_cast<const char *>(v1);
const char *str2 = static_cast<const char *>(v2);
if (strlen(str1) < DATE_LENGTH) {
ret = -1;
} else if (strlen(str2) < DATE_LENGTH) {
ret = 1;
} else {
ret = str_cmp(str1 + strlen(str1) - DATE_LENGTH, str2 + strlen(str2) - DATE_LENGTH);
}
}
return ret;
}
int ObLogger::add_files_to_list(void *files,
void *wf_files,
std::deque<std::string> &file_list,
@ -1362,10 +1384,10 @@ int ObLogger::add_files_to_list(void *files,
ObIArray<FileName> *wf_files_arr = static_cast<ObIArray<FileName> *>(wf_files);
//sort files
if (files_arr->count() > 0) {
qsort(&files_arr->at(0), files_arr->count(), sizeof(FileName), str_cmp);
qsort(&files_arr->at(0), files_arr->count(), sizeof(FileName), compare_log_filename_by_date_suffix);
}
if (wf_files_arr->count() > 0) {
qsort(&wf_files_arr->at(0), wf_files_arr->count(), sizeof(FileName), str_cmp);
qsort(&wf_files_arr->at(0), wf_files_arr->count(), sizeof(FileName), compare_log_filename_by_date_suffix);
}
//Add to file_list

2
deps/oblib/src/lib/oblog/ob_log.h vendored
View File

@ -865,6 +865,8 @@ private:
// This info will be logged when log file created.
const char *new_file_info_;
bool info_as_wdiag_;
std::deque<std::string> file_list_;//to store the names of log-files
std::deque<std::string> wf_file_list_;//to store the names of warning log-files
};
inline ObLogger& ObLogger::get_logger()

127
deps/oblib/src/lib/restore/cos/ob_cos_wrapper.cpp vendored
View File

@ -23,6 +23,7 @@
#include "cos_auth.h"
#include "cos_sys_util.h"
#include "apr_errno.h"
#include "cos_crc64.h"
#include "ob_cos_wrapper.h"
@ -38,6 +39,7 @@ constexpr int OB_INVALID_ARGUMENT = -4002;
constexpr int OB_INIT_TWICE = -4005;
constexpr int OB_ALLOCATE_MEMORY_FAILED = -4013;
constexpr int OB_SIZE_OVERFLOW = -4019;
constexpr int OB_CHECKSUM_ERROR = -4103;
constexpr int OB_BACKUP_FILE_NOT_EXIST = -9011;
constexpr int OB_COS_ERROR = -9060;
constexpr int OB_IO_LIMIT = -9061;
@ -814,6 +816,7 @@ int ObCosWrapper::pread(
int64_t offset,
char *buf,
int64_t buf_size,
const bool is_range_read,
int64_t &read_size)
{
int ret = OB_SUCCESS;
@ -849,42 +852,57 @@ int ObCosWrapper::pread(
const char* const COS_RANGE_KEY = "Range";
char range_size[COS_RANGE_SIZE];
// Cos read range of [10, 100] include the start offset 10 and the end offset 100.
// But what we except is [10, 100) which does not include the end.
// So we subtract 1 from the end.
int n = snprintf(range_size, COS_RANGE_SIZE, "bytes=%ld-%ld", offset, offset + buf_size - 1);
if (0 >= n || COS_RANGE_SIZE <= n) {
ret = OB_SIZE_OVERFLOW;
cos_warn_log("[COS]fail to format range size,n=%d, ret=%d\n", n, ret);
} else if (NULL == (headers = cos_table_make(ctx->mem_pool, 1))) {
if (NULL == (headers = cos_table_make(ctx->mem_pool, 1))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
cos_warn_log("[COS]fail to make cos headers, ret=%d\n", ret);
} else {
apr_table_set(headers, COS_RANGE_KEY, range_size);
if (is_range_read) {
// Cos read range of [10, 100] include the start offset 10 and the end offset 100.
// But what we except is [10, 100) which does not include the end.
// So we subtract 1 from the end.
int n = snprintf(range_size, COS_RANGE_SIZE, "bytes=%ld-%ld", offset, offset + buf_size - 1);
if (0 >= n || COS_RANGE_SIZE <= n) {
ret = OB_SIZE_OVERFLOW;
cos_warn_log("[COS]fail to format range size,n=%d, ret=%d\n", n, ret);
} else {
apr_table_set(headers, COS_RANGE_KEY, range_size);
}
}
if (NULL == (cos_ret = cos_get_object_to_buffer(ctx->options, &bucket, &object, headers, NULL, &buffer, &resp_headers)) ||
!cos_status_is_ok(cos_ret)) {
if (OB_SUCCESS != ret) {
} else if (NULL == (cos_ret = cos_get_object_to_buffer(ctx->options, &bucket, &object, headers, NULL, &buffer, &resp_headers)) ||
!cos_status_is_ok(cos_ret)) {
convert_io_error(cos_ret, ret);
cos_warn_log("[COS]fail to get object to buffer, ret=%d\n", ret);
log_status(cos_ret);
} else {
read_size = 0;
int64_t size = 0;
int64_t buf_pos = 0;
cos_buf_t *content = NULL;
int64_t needed_size = -1;
cos_list_for_each_entry(cos_buf_t, content, &buffer, node) {
size = cos_buf_size(content);
if (buf_pos + size > buf_size) {
needed_size = size;
if (buf_size - buf_pos < size) {
needed_size = buf_size - buf_pos;
}
if (is_range_read && (buf_pos + size > buf_size)) {
ret = OB_COS_ERROR;
cos_warn_log("[COS]unexpected error, too much data returned, ret=%d, range_size=%s, buf_pos=%ld, size=%ld, req_id=%s.\n", ret, range_size, buf_pos, size, cos_ret->req_id);
log_status(cos_ret);
break;
} else {
// copy to buf
memcpy(buf + buf_pos, content->pos, (size_t)size);
buf_pos += size;
read_size += size;
memcpy(buf + buf_pos, content->pos, (size_t)needed_size);
buf_pos += needed_size;
read_size += needed_size;
if (buf_pos >= buf_size) {
break;
}
}
}
} // end cos_list_for_each_entry
}
}
}
@ -900,62 +918,8 @@ int ObCosWrapper::get_object(
int64_t buf_size,
int64_t &read_size)
{
int ret = OB_SUCCESS;
CosContext *ctx = reinterpret_cast<CosContext *>(h);
read_size = 0;
if (NULL == h) {
ret = OB_INVALID_ARGUMENT;
cos_warn_log("[COS]cos handle is null, ret=%d\n", ret);
} else if (bucket_name.empty()) {
ret = OB_INVALID_ARGUMENT;
cos_warn_log("[COS]bucket name is null, ret=%d\n", ret);
} else if (object_name.empty()) {
ret = OB_INVALID_ARGUMENT;
cos_warn_log("[COS]object name is null, ret=%d\n", ret);
} else if (NULL == buf || 0 >= buf_size) {
ret = OB_INVALID_ARGUMENT;
cos_warn_log("[COS]buffer is null, ret=%d\n", ret);
} else {
cos_string_t bucket;
cos_string_t object;
cos_str_set(&bucket, bucket_name.data_);
cos_str_set(&object, object_name.data_);
cos_table_t *headers = NULL;
cos_table_t *resp_headers = NULL;
cos_status_t *cos_ret = NULL;
cos_list_t buffer;
cos_list_init(&buffer);
if (NULL == (cos_ret = cos_get_object_to_buffer(ctx->options, &bucket, &object, headers, NULL, &buffer, &resp_headers)) ||
!cos_status_is_ok(cos_ret)) {
convert_io_error(cos_ret, ret);
cos_warn_log("[COS]fail to get object to buffer, ret=%d\n", ret);
log_status(cos_ret);
} else {
int64_t size = 0;
int64_t buf_pos = 0;
cos_buf_t *content = NULL;
cos_list_for_each_entry(cos_buf_t, content, &buffer, node) {
size = cos_buf_size(content);
if (buf_pos + size > buf_size) {
ret = OB_COS_ERROR;
cos_warn_log("[COS]unexpected error, too much data returned, ret=%d, buf_pos=%ld, size=%ld, buf_size=%ld, req_id=%s.\n", ret, buf_pos, size, buf_size, cos_ret->req_id);
log_status(cos_ret);
break;
} else {
// copy to buf
memcpy(buf + buf_pos, content->pos, (size_t)size);
buf_pos += size;
read_size += size;
}
}
}
}
return ret;
return ObCosWrapper::pread(h, bucket_name, object_name,
0, buf, buf_size, false/*is_range_read*/, read_size);
}
int ObCosWrapper::is_object_tagging(
@ -1444,7 +1408,8 @@ int ObCosWrapper::upload_part_from_buffer(
const CosStringBuffer &upload_id_str,
const int part_num,
const char *buf,
const int64_t buf_size)
const int64_t buf_size,
uint64_t &total_crc)
{
int ret = OB_SUCCESS;
CosContext *ctx = reinterpret_cast<CosContext *>(h);
@ -1488,6 +1453,9 @@ int ObCosWrapper::upload_part_from_buffer(
convert_io_error(cos_ret, ret);
cos_warn_log("[COS]fail to upload part to cos, ret=%d, part_num=%d\n", ret, part_num);
log_status(cos_ret);
} else {
// TODO @fangdan: supports parallel uploads
total_crc = cos_crc64(total_crc, (void *)buf, buf_size);
}
}
}
@ -1498,7 +1466,8 @@ int ObCosWrapper::complete_multipart_upload(
Handle *h,
const CosStringBuffer &bucket_name,
const CosStringBuffer &object_name,
const CosStringBuffer &upload_id_str)
const CosStringBuffer &upload_id_str,
const uint64_t total_crc)
{
int ret = OB_SUCCESS;
CosContext *ctx = reinterpret_cast<CosContext *>(h);
@ -1575,6 +1544,16 @@ int ObCosWrapper::complete_multipart_upload(
convert_io_error(cos_ret, ret);
cos_warn_log("[COS]fail to complete multipart upload, ret=%d\n", ret);
log_status(cos_ret);
} else {
const char *expected_crc_str = (const char *)(apr_table_get(resp_headers, COS_HASH_CRC64_ECMA));
if (NULL != expected_crc_str) {
uint64_t expected_crc = cos_atoui64(expected_crc_str);
if (expected_crc != total_crc) {
ret = OB_CHECKSUM_ERROR;
cos_warn_log("[COS]multipart object crc is not equal to returned crc, ret=%d, upload_id=%s, total_crc=%llu, expected_crc=%llu\n",
ret, upload_id_str.data_, total_crc, expected_crc);
}
}
}
}

7
deps/oblib/src/lib/restore/cos/ob_cos_wrapper.h vendored
View File

@ -269,6 +269,7 @@ public:
int64_t offset,
char *buf,
int64_t buf_size,
const bool is_range_read,
int64_t &read_size);
// Get whole object
@ -372,13 +373,15 @@ public:
const CosStringBuffer &upload_id_str,
const int part_num, /*the sequence number of this part, [1, 10000]*/
const char *buf,
const int64_t buf_size);
const int64_t buf_size,
uint64_t &total_crc);
static int complete_multipart_upload(
Handle *h,
const CosStringBuffer &bucket_name,
const CosStringBuffer &object_name,
const CosStringBuffer &upload_id_str);
const CosStringBuffer &upload_id_str,
const uint64_t total_crc);
static int abort_multipart_upload(
Handle *h,

47
deps/oblib/src/lib/restore/ob_storage_cos_base.cpp vendored
View File

@ -786,13 +786,38 @@ int ObStorageCosReader::pread(
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "invalid argument", K(ret), KP(buf), K(buf_size), K(offset));
} else {
qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer(
handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length());
qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer(
handle_.get_object_name().ptr(), handle_.get_object_name().length());
if (OB_FAIL(qcloud_cos::ObCosWrapper::pread(handle_.get_ptr(), bucket_name,
object_name, offset, buf, buf_size, read_size))) {
OB_LOG(WARN, "fail to read object from cos", K(ret), KP(buf), K(buf_size), K(offset));
// When is_range_read is true, it indicates that only a part of the data is read.
// When false, it indicates that the entire object is read
bool is_range_read = true;
int64_t get_data_size = buf_size;
if (has_meta_) {
if (file_length_ < offset) {
ret = OB_FILE_LENGTH_INVALID;
OB_LOG(WARN, "File lenth is invilid", K_(file_length), K(offset),
K(handle_.get_bucket_name()), K(handle_.get_object_name()), K(ret));
} else {
get_data_size = MIN(buf_size, file_length_ - offset);
if (get_data_size == file_length_) {
// read entire object
is_range_read = false;
}
}
}
if (OB_FAIL(ret)) {
} else if (get_data_size == 0) {
read_size = 0;
} else {
qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer(
handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length());
qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer(
handle_.get_object_name().ptr(), handle_.get_object_name().length());
if (OB_FAIL(qcloud_cos::ObCosWrapper::pread(handle_.get_ptr(), bucket_name,
object_name, offset, buf, get_data_size, is_range_read, read_size))) {
OB_LOG(WARN, "fail to read object from cos", K(ret), K(is_range_read),
KP(buf), K(buf_size), K(offset), K(get_data_size), K_(has_meta));
}
}
}
return ret;
@ -1026,7 +1051,8 @@ ObStorageCosMultiPartWriter::ObStorageCosMultiPartWriter()
base_buf_pos_(0),
upload_id_(NULL),
partnum_(0),
file_length_(-1)
file_length_(-1),
total_crc_(0)
{}
ObStorageCosMultiPartWriter::~ObStorageCosMultiPartWriter()
@ -1084,6 +1110,7 @@ int ObStorageCosMultiPartWriter::open(const ObString &uri, common::ObObjectStora
is_opened_ = true;
base_buf_pos_ = 0;
file_length_ = 0;
total_crc_ = 0;
}
}
return ret;
@ -1152,7 +1179,7 @@ int ObStorageCosMultiPartWriter::write_single_part()
qcloud_cos::CosStringBuffer upload_id_str = qcloud_cos::CosStringBuffer(
upload_id_, strlen(upload_id_));
if (OB_FAIL(qcloud_cos::ObCosWrapper::upload_part_from_buffer(handle_.get_ptr(), bucket_name,
object_name, upload_id_str, partnum_, base_buf_, base_buf_pos_))) {
object_name, upload_id_str, partnum_, base_buf_, base_buf_pos_, total_crc_))) {
OB_LOG(WARN, "fail to upload part to cos", K(ret), KP_(upload_id));
if (OB_TMP_FAIL(cleanup())) {
OB_LOG(WARN, "fail to abort multiupload", K(ret), K(tmp_ret), KP_(upload_id));
@ -1192,7 +1219,7 @@ int ObStorageCosMultiPartWriter::close()
upload_id_, strlen(upload_id_));
if (OB_FAIL(qcloud_cos::ObCosWrapper::complete_multipart_upload(handle_.get_ptr(), bucket_name,
object_name, upload_id_str))) {
object_name, upload_id_str, total_crc_))) {
OB_LOG(WARN, "fail to complete multipart upload", K(ret), KP_(upload_id));
if (OB_TMP_FAIL(cleanup())) {
OB_LOG(WARN, "fail to abort multiupload", K(ret), K(tmp_ret), KP_(upload_id));

1
deps/oblib/src/lib/restore/ob_storage_cos_base.h vendored
View File

@ -191,6 +191,7 @@ private:
char *upload_id_;
int partnum_;
int64_t file_length_;
uint64_t total_crc_;
DISALLOW_COPY_AND_ASSIGN(ObStorageCosMultiPartWriter);
};

100
deps/oblib/src/lib/restore/ob_storage_oss_base.cpp vendored
View File

@ -17,6 +17,7 @@
#include "common/ob_string_buf.h"
#include "apr_errno.h"
#include "ob_storage.h"
#include "aos_crc64.h"
#include "lib/hash/ob_hashset.h"
#include "lib/utility/ob_tracepoint.h"
@ -643,7 +644,8 @@ ObStorageOssMultiPartWriter::ObStorageOssMultiPartWriter()
object_(),
partnum_(0),
is_opened_(false),
file_length_(-1)
file_length_(-1),
total_crc_(0)
{
upload_id_.len = -1;
upload_id_.data = NULL;
@ -697,6 +699,7 @@ int ObStorageOssMultiPartWriter::open(const ObString &uri, common::ObObjectStora
is_opened_ = true;
base_buf_pos_ = 0;
file_length_ = 0;
total_crc_ = 0;
}
}
}
@ -790,6 +793,9 @@ int ObStorageOssMultiPartWriter::write_single_part()
K_(base_buf_pos), K_(bucket), K_(object), K(ret));
print_oss_info(resp_headers, aos_ret);
cleanup();
} else {
// TODO @fangdan: supports parallel uploads
total_crc_ = aos_crc64(total_crc_, base_buf_, base_buf_pos_);
}
bool is_slow = false;
print_access_storage_log("oss upload one part ", object_, start_time, base_buf_pos_, &is_slow);
@ -894,6 +900,16 @@ int ObStorageOssMultiPartWriter::close()
OB_LOG(WARN, "fail to complete multipart upload", K_(bucket), K_(object), K(ret));
print_oss_info(resp_headers, aos_ret);
cleanup();
} else {
const char *expected_crc_str = (const char *)(apr_table_get(resp_headers, OSS_HASH_CRC64_ECMA));
if (OB_NOT_NULL(expected_crc_str)) {
uint64_t expected_crc = aos_atoui64(expected_crc_str);
if (OB_UNLIKELY(expected_crc != total_crc_)) {
ret = OB_CHECKSUM_ERROR;
OB_LOG(WARN, "multipart object crc is not equal to returned crc",
K(ret), K_(total_crc), K(expected_crc));
}
}
}
}
}
@ -1011,15 +1027,29 @@ int ObStorageOssReader::pread(
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "aos pool or oss option is NULL", K(aos_pool), K(oss_option), K(ret));
} else {
if (has_meta_ && file_length_ == offset) {
// When is_range_read is true, it indicates that only a part of the data is read.
// When false, it indicates that the entire object is read
bool is_range_read = true;
int64_t get_data_size = buf_size;
if (has_meta_) {
if (file_length_ < offset) {
ret = OB_FILE_LENGTH_INVALID;
OB_LOG(WARN, "File lenth is invilid", K_(file_length),
K(offset), K_(bucket), K_(object), K(ret));
} else {
get_data_size = MIN(buf_size, file_length_ - offset);
if (get_data_size == file_length_) {
// read entire object
is_range_read = false;
}
}
}
if (OB_FAIL(ret)) {
} else if (get_data_size == 0) {
read_size = 0;
} else if (has_meta_ && file_length_ < offset) {
ret = OB_FILE_LENGTH_INVALID;
OB_LOG(WARN, "File lenth is invilid", K(file_length_), K(offset),
K(bucket_), K(object_), K(ret));
} else {
const int64_t start_time = ObTimeUtility::current_time();
int64_t get_data_size = buf_size;
aos_string_t bucket;
aos_string_t object;
aos_str_set(&bucket, bucket_.ptr());
@ -1037,49 +1067,61 @@ int ObStorageOssReader::pread(
const char *const OSS_RANGE_KEY = "Range";
char range_size[OSS_RANGE_SIZE];
//oss read size is [10, 100] include the 10 and 100 bytes
//we except is [10, 100) not include the end, so we subtraction 1 to the end
int n = snprintf(range_size, OSS_RANGE_SIZE, "bytes=%ld-%ld", offset, offset + get_data_size - 1);
if(n <=0 || n >= OSS_RANGE_SIZE) {
ret = OB_SIZE_OVERFLOW;
OB_LOG(WARN, "fail to get range size", K(n), K(OSS_RANGE_SIZE), K(ret));
} else if (NULL == (headers = aos_table_make(aos_pool, AOS_TABLE_INIT_SIZE))) {
if (NULL == (headers = aos_table_make(aos_pool, AOS_TABLE_INIT_SIZE))) {
ret = OB_OSS_ERROR;
OB_LOG(WARN, "fail to make oss headers", K(ret));
} else {
apr_table_set(headers, OSS_RANGE_KEY, range_size);
if (is_range_read) {
// oss read size is [10, 100] include the 10 and 100 bytes
// we except is [10, 100) not include the end, so we subtraction 1 to the end
if (OB_FAIL(databuff_printf(range_size, OSS_RANGE_SIZE, "bytes=%ld-%ld",
offset, offset + get_data_size - 1))) {
OB_LOG(WARN, "fail to get range size", K(ret),
K(offset), K(get_data_size), K(OSS_RANGE_SIZE));
} else {
apr_table_set(headers, OSS_RANGE_KEY, range_size);
}
}
if (NULL == (aos_ret = oss_get_object_to_buffer(oss_option, &bucket, &object, headers, params,
if (OB_FAIL(ret)) {
} else if (NULL == (aos_ret = oss_get_object_to_buffer(oss_option, &bucket, &object, headers, params,
&buffer, &resp_headers)) || !aos_status_is_ok(aos_ret)) {
convert_io_error(aos_ret, ret);
OB_LOG(WARN, "fail to get object to buffer", K(bucket_), K(object_), K(ret));
convert_io_error(aos_ret, ret);
OB_LOG(WARN, "fail to get object to buffer", K_(bucket), K_(object), K(ret));
print_oss_info(resp_headers, aos_ret);
} else {
//check date len
aos_list_for_each_entry(aos_buf_t, content, &buffer, node) {
len += aos_buf_size(content);
}
if(len > get_data_size) {
// For appendable file, there may be data written in between the time when the reader is opened and pread is called.
// At this point, the actual size of the file could be larger than the determined file length at the time of opening.
// Therefore, if it's not a range read, the data read could exceed the expected amount to be read
if (is_range_read && len > get_data_size) {
ret = OB_OSS_ERROR;
OB_LOG(WARN, "get data size error", K(get_data_size), K(len), K(bucket_),
K(object_), K(ret));
OB_LOG(WARN, "get data size error", K(get_data_size), K(len), K_(bucket),
K_(object), K(ret), K_(has_meta), K(offset));
} else {
//copy to buf
read_size = 0;
int64_t needed_size = -1;
aos_list_for_each_entry(aos_buf_t, content, &buffer, node) {
size = aos_buf_size(content);
if (buf_pos + size > get_data_size) {
needed_size = MIN(size, get_data_size - buf_pos);
if (is_range_read && (buf_pos + size > get_data_size)) {
ret = OB_SIZE_OVERFLOW;
OB_LOG(WARN, "the size is too long", K(buf_pos), K(size), K(get_data_size), K(ret));
break;
} else {
memcpy(buf + buf_pos, content->pos, (size_t)size);
buf_pos += size;
}
}
memcpy(buf + buf_pos, content->pos, (size_t)needed_size);
buf_pos += needed_size;
read_size += needed_size;
if (OB_SUCC(ret)) {
read_size = len;
}
if (buf_pos >= get_data_size) {
break;
}
}
} // end aos_list_for_each_entry
}
}
}

1
deps/oblib/src/lib/restore/ob_storage_oss_base.h vendored
View File

@ -186,6 +186,7 @@ private:
int partnum_;
bool is_opened_;
int64_t file_length_;
uint64_t total_crc_;
DISALLOW_COPY_AND_ASSIGN(ObStorageOssMultiPartWriter);
};

18
deps/oblib/src/lib/restore/ob_storage_s3_base.cpp vendored
View File

@ -1161,14 +1161,19 @@ int ObStorageS3Reader::pread_(char *buf,
} else {
Aws::S3::Model::GetObjectRequest request;
Aws::S3::Model::GetObjectOutcome outcome;
const bool check_crc = (default_s3_crc_algo == ObStorageCRCAlgorithm::OB_CRC32_ALGO);
if (check_crc) {
request.SetChecksumMode(Aws::S3::Model::ChecksumMode::ENABLED);
}
char range_read[64] = { 0 };
request.WithBucket(bucket_.ptr()).WithKey(object_.ptr());
request.SetAdditionalCustomHeaderValue("Connection", "keep-alive");
const int64_t get_data_size = has_meta_ ? MIN(buf_size, file_length_ - offset) : buf_size;
if (OB_FAIL(databuff_printf(range_read, sizeof(range_read),
"bytes=%ld-%ld", offset, offset + buf_size - 1))) {
OB_LOG(WARN, "fail to set range to read",
K(ret), K_(bucket), K_(object), K(offset), K(buf_size));
"bytes=%ld-%ld", offset, offset + get_data_size - 1))) {
OB_LOG(WARN, "fail to set range to read", K(ret),
K_(bucket), K_(object), K(offset), K(buf_size), K_(has_meta), K_(file_length));
} else if (FALSE_IT(request.SetRange(range_read))) {
} else if (OB_FAIL(s3_client_->get_object(request, outcome))) {
OB_LOG(WARN, "failed to get s3 object", K(ret), K(range_read));
@ -1952,6 +1957,7 @@ int ObStorageS3MultiPartWriter::close_()
}
// list parts
const bool check_crc = (default_s3_crc_algo == ObStorageCRCAlgorithm::OB_CRC32_ALGO);
Aws::S3::Model::CompletedMultipartUpload completedMultipartUpload;
int64_t part_num = 0;
if (OB_SUCC(ret)) {
@ -1972,6 +1978,9 @@ int ObStorageS3MultiPartWriter::close_()
for (int64_t i = 0; OB_SUCC(ret) && i < parts.size(); i++) {
Aws::S3::Model::CompletedPart tmp_part;
tmp_part.WithPartNumber(parts[i].GetPartNumber()).WithETag(parts[i].GetETag());
if (check_crc) {
tmp_part.SetChecksumCRC32(parts[i].GetChecksumCRC32());
}
completedMultipartUpload.AddParts(std::move(tmp_part));
}
}
@ -1984,7 +1993,7 @@ int ObStorageS3MultiPartWriter::close_()
Aws::S3::Model::CompleteMultipartUploadRequest request;
request.WithBucket(bucket_.ptr()).WithKey(object_.ptr());
request.WithUploadId(upload_id_).WithMultipartUpload(completedMultipartUpload);
const bool check_crc = (default_s3_crc_algo == ObStorageCRCAlgorithm::OB_CRC32_ALGO);
Aws::String complete_crc;
if (check_crc) {
if (OB_ISNULL(sum_hash_)) {
@ -2070,6 +2079,7 @@ int ObStorageS3MultiPartWriter::write_single_part_()
ret = OB_ERR_UNEXPECTED;
OB_LOG(WARN, "sum_hash should not be null", K(ret));
} else {
// TODO @fangdan: supports parallel uploads
Aws::Utils::Crypto::CRC32 part_hash;
request.SetChecksumAlgorithm(Aws::S3::Model::ChecksumAlgorithm::CRC32);
Aws::String part_str(base_buf_, base_buf_pos_);

2
deps/oblib/src/lib/restore/ob_storage_s3_base.h vendored
View File

@ -63,7 +63,7 @@ int init_s3_env();
void fin_s3_env();
// default s3 checksum algorithm
static ObStorageCRCAlgorithm default_s3_crc_algo = ObStorageCRCAlgorithm::OB_INVALID_CRC_ALGO;
static ObStorageCRCAlgorithm default_s3_crc_algo = ObStorageCRCAlgorithm::OB_CRC32_ALGO;
// set checksum algorithm for writing object into s3
void set_s3_checksum_algorithm(const ObStorageCRCAlgorithm crc_algo);
// get current checksum algorithm

14
deps/oblib/src/lib/thread/thread.cpp vendored
View File

@ -61,7 +61,8 @@ Thread::Thread(Threads *threads, int64_t idx, int64_t stack_size)
tid_before_stop_(0),
tid_(0),
thread_list_node_(this),
cpu_time_(0)
cpu_time_(0),
create_ret_(OB_NOT_RUNNING)
{}
Thread::~Thread()
@ -98,9 +99,14 @@ int Thread::start()
if (pret == 0) {
stop_ = false;
pret = pthread_create(&pth_, &attr, __th_start, this);
while (ATOMIC_LOAD(&create_ret_) == OB_NOT_RUNNING) {
sched_yield();
}
if (pret != 0) {
LOG_ERROR("pthread create failed", K(pret), K(errno));
pth_ = 0;
} else if (OB_FAIL(create_ret_)) {
LOG_ERROR("thread create failed", K(create_ret_));
}
}
if (0 != pret) {
@ -320,6 +326,7 @@ void* Thread::__th_start(void *arg)
WITH_CONTEXT(*mem_context) {
try {
in_try_stmt = true;
ATOMIC_STORE(&th->create_ret_, OB_SUCCESS);
th->run();
in_try_stmt = false;
} catch (OB_BASE_EXCEPTION &except) {
@ -327,6 +334,10 @@ void* Thread::__th_start(void *arg)
_LOG_ERROR("Exception caught!!! errno = %d, exception info = %s", except.get_errno(), except.what());
ret = OB_ERR_UNEXPECTED;
in_try_stmt = false;
if (1 == th->threads_->get_thread_count() && !th->has_set_stop()) {
LOG_WARN("thread exit by itself without set_stop", K(ret));
th->threads_->stop();
}
}
}
}
@ -335,7 +346,6 @@ void* Thread::__th_start(void *arg)
}
}
}
ATOMIC_FAA(&total_thread_count_, -1);
return nullptr;
}

1
deps/oblib/src/lib/thread/thread.h vendored
View File

@ -186,6 +186,7 @@ private:
int64_t tid_;
ThreadListNode thread_list_node_;
int64_t cpu_time_;
int create_ret_;
};
OB_INLINE bool Thread::has_set_stop() const

2
deps/oblib/src/lib/thread/threads.h vendored
View File

@ -103,8 +103,8 @@ public:
}
return pth;
}
protected:
int64_t get_thread_count() const { return n_threads_; }
protected:
uint64_t get_thread_idx() const { return thread_idx_; }
void set_thread_idx(int64_t idx) { thread_idx_ = idx; }

2
deps/oblib/src/lib/trace/ob_trace_event.h vendored
View File

@ -62,7 +62,7 @@ public:
int64_t total_time = 0;
const char* event_name = NULL;
const ObString nls_format;
for (int64_t i = 0; i < this->next_idx_; ++i) {
for (int64_t i = 0; i < MIN(EVENT_COUNT, this->next_idx_); ++i) {
const ObTraceEvent &ev = this->events_[i];
event_name = NAME(ev.id_);
if (prev_ts == 0) {

5
deps/oblib/src/lib/utility/ob_print_utils.cpp vendored
View File

@ -291,11 +291,6 @@ const char *to_cstring<int64_t>(const int64_t &v)
return to_cstring<int64_t>(v, BoolType<false>());
}
const char *to_cstring(const int64_t v)
{
return to_cstring<int64_t>(v, BoolType<false>());
}
////////////////////////////////////////////////////////////////
int databuff_printf(char *buf, const int64_t buf_len, const char *fmt, ...)

2
deps/oblib/src/lib/utility/ob_print_utils.h vendored
View File

@ -272,8 +272,6 @@ const char *to_cstring<const char *>(const char *const &str);
template <>
const char *to_cstring<int64_t>(const int64_t &v);
const char *to_cstring(const int64_t v);
template <typename T>
const char *to_cstring(T *obj)
{

3
deps/oblib/src/lib/utility/ob_tracepoint.h vendored
View File

@ -638,6 +638,8 @@ class EventTable
EN_ENABLE_AUTO_DOP_FORCE_PARALLEL_PLAN = 552,
EN_GENERATE_PLAN_WITH_RECONSTRUCT_SQL = 553,
EN_GENERATE_PLAN_WITH_NLJ = 554,
//EN_CHECK_OPERATOR_OUTPUT_ROWS = 555,
//EN_GENERATE_RANDOM_PLAN = 556,
// 600-700 For PX use
EN_PX_SQC_EXECUTE_FAILED = 600,
@ -804,6 +806,7 @@ class EventTable
EN_ENABLE_VECTOR_CAST = 2207,
EN_DISABLE_SORTKEY_SEPARATELY = 2208,
EN_ENABLE_VECTOR_IN = 2209,
EN_SQL_MEMORY_MRG_OPTION = 2210,
// WR && ASH
EN_CLOSE_ASH = 2301,
EN_DISABLE_HASH_BASE_DISTINCT = 2302,

9
deps/oblib/src/lib/utility/ob_unify_serialize.cpp vendored
View File

@ -22,19 +22,19 @@ void begin_record_serialization()
{
ser_diag_record.count = 0;
ser_diag_record.check_index = 0;
ser_diag_record.flag = 1;
ser_diag_record.flag = CHECK_STATUS_RECORDING;
}
void finish_record_serialization()
{
ser_diag_record.flag = 0;
ser_diag_record.flag = CHECK_STATUS_WATING;
}
void begin_check_serialization()
{
ser_diag_record.check_index = 0;
if (ser_diag_record.count > 0) {
ser_diag_record.flag = 2;
ser_diag_record.flag = CHECK_STATUS_COMPARING;
}
}
@ -42,8 +42,9 @@ void finish_check_serialization()
{
ser_diag_record.count = -1;
ser_diag_record.check_index = -1;
ser_diag_record.flag = 0;
ser_diag_record.flag = CHECK_STATUS_WATING;
}
} // namespace lib
} // namespace oceanbase
#endif

28
deps/oblib/src/lib/utility/ob_unify_serialize.h vendored
View File

@ -21,13 +21,19 @@ namespace lib
{
#ifdef ENABLE_SERIALIZATION_CHECK
enum ObSerializationCheckStatus
{
CHECK_STATUS_WATING = 0,
CHECK_STATUS_RECORDING = 1,
CHECK_STATUS_COMPARING = 2
};
static constexpr int MAX_SERIALIZE_RECORD_LENGTH = 256;
struct SerializeDiagnoseRecord
{
uint8_t encoded_lens[MAX_SERIALIZE_RECORD_LENGTH];
int count = -1;
int check_index = -1;
int flag = 0;
int flag = CHECK_STATUS_WATING;
};
RLOCAL_EXTERN(SerializeDiagnoseRecord, ser_diag_record);
void begin_record_serialization();
@ -78,21 +84,31 @@ void finish_check_serialization();
}
#ifdef ENABLE_SERIALIZATION_CHECK
#define IF_TYPE_MATCH(obj, type) \
std::is_same<type, decltype(obj)>::value || std::is_same<type &, decltype(obj)>::value
#define IF_NEED_TO_CHECK_SERIALIZATION(obj) \
!(std::is_const<decltype(obj)>::value || \
IF_TYPE_MATCH(obj, uint8_t) || \
IF_TYPE_MATCH(obj, int8_t) || \
IF_TYPE_MATCH(obj, bool) || \
IF_TYPE_MATCH(obj, char))
#define OB_UNIS_ADD_LEN(obj) \
{ \
int64_t this_len = NS_::encoded_length(obj); \
if (!(std::is_same<uint8_t, decltype(obj)>::value || std::is_same<int8_t, decltype(obj)>::value || \
std::is_same<bool, decltype(obj)>::value || std::is_same<char, decltype(obj)>::value)) { \
if (1 == oceanbase::lib::ser_diag_record.flag && \
if (IF_NEED_TO_CHECK_SERIALIZATION(obj)) { \
if (oceanbase::lib::CHECK_STATUS_RECORDING == oceanbase::lib::ser_diag_record.flag && \
oceanbase::lib::ser_diag_record.count < oceanbase::lib::MAX_SERIALIZE_RECORD_LENGTH) { \
oceanbase::lib::ser_diag_record.encoded_lens[oceanbase::lib::ser_diag_record.count++] = \
static_cast<uint8_t>(this_len); \
} else if (2 == oceanbase::lib::ser_diag_record.flag && \
} else if (oceanbase::lib::CHECK_STATUS_COMPARING == oceanbase::lib::ser_diag_record.flag && \
oceanbase::lib::ser_diag_record.check_index < oceanbase::lib::ser_diag_record.count) { \
int ret = OB_ERR_UNEXPECTED; \
int record_len = oceanbase::lib::ser_diag_record.encoded_lens[oceanbase::lib::ser_diag_record.check_index]; \
if (this_len != record_len) { \
OB_LOG(ERROR, "encoded length not match", "name", MSTR(obj), K(this_len), K(record_len), K(obj)); \
OB_LOG(ERROR, "encoded length not match", "name", MSTR(obj), K(this_len), K(record_len), "value", obj); \
} \
oceanbase::lib::ser_diag_record.check_index++; \
} \

1
deps/oblib/src/rpc/frame/ob_req_transport.cpp vendored
View File

@ -144,6 +144,7 @@ int async_cb(easy_request_t *r)
if (!OB_SUCC(ret)) {
LOG_WARN("process async request fail", K(r), K(ret), K(pcode));
}
THIS_WORKER.get_sql_arena_allocator().reset();
const int64_t cur_time = ObTimeUtility::current_time();
const int64_t total_time = cur_time - start_time;

1
deps/oblib/src/rpc/obmysql/ob_sql_sock_session.h vendored
View File

@ -30,6 +30,7 @@ public:
ObSqlSessionMemPool(): pool_() {}
virtual ~ObSqlSessionMemPool() {}
void* alloc(int64_t sz) { return pool_.alloc(sz); }
void set_tenant_id(int64_t tenant_id) { pool_.set_tenant_id(tenant_id); }
void reset() { pool_.destroy(); }
void reuse() { pool_.reuse(); }
private:

1
deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.cpp vendored
View File

@ -157,6 +157,7 @@ int ObAsyncRespCallback::handle_resp(int io_err, const char* buf, int64_t sz)
}
pool_.destroy();
ObCurTraceId::reset();
THIS_WORKER.get_sql_arena_allocator().reset();
const int64_t cur_time = ObTimeUtility::current_time();
const int64_t total_time = cur_time - start_time;
const int64_t decode_time = after_decode_time - start_time;

1
deps/oblib/src/rpc/obrpc/ob_rpc_mem_pool.h vendored
View File

@ -28,6 +28,7 @@ public:
~ObRpcMemPool() { destroy(); }
static ObRpcMemPool* create(int64_t tenant_id, const char* label, int64_t req_sz);
void* alloc(int64_t sz);
void set_tenant_id(int64_t tenant_id) { tenant_id_ = tenant_id; }
void reuse();
void destroy();
private:

1
deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h vendored
View File

@ -985,6 +985,7 @@ PCODE_DEF(OB_LOG_GET_ARB_MEMBER_INFO, 0x1522)
PCODE_DEF(OB_LOG_BATCH_FETCH_RESP, 0X1523)
PCODE_DEF(OB_LOG_GET_LS_CKPT, 0x1524)
PCODE_DEF(OB_LOG_ACQUIRE_REBUILD_INFO, 0x1525)
PCODE_DEF(OB_LOG_FORCE_SET_TENANT_LOG_DISK, 0x1526)
// 1531-1550 for obesi
// PCODE_DEF(OB_ESI_IS_EXIST, 0x1531)

61
deps/oblib/unittest/lib/restore/test_object_storage.cpp vendored
View File

@ -1240,6 +1240,66 @@ TEST_F(TestObjectStorage, test_cross_testing)
}
}
TEST_F(TestObjectStorage, test_read_single_file)
{
int ret = OB_SUCCESS;
if (enable_test) {
ObStorageUtil util;
ASSERT_EQ(OB_SUCCESS, util.open(&info_base));
const char *tmp_dir = "test_read_single_file";
const int64_t ts = ObTimeUtility::current_time();
ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld",
bucket, dir_name, tmp_dir, ts));
{
// normal
ObStorageWriter writer;
ObStorageReader reader;
ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_normal", dir_uri));
ASSERT_EQ(OB_SUCCESS, writer.open(uri, &info_base));
ASSERT_EQ(OB_SUCCESS, writer.write("123456", 6));
ASSERT_EQ(OB_SUCCESS, writer.close());
int64_t read_size = -1;
char read_buf[100];
memset(read_buf, 0, sizeof(read_buf));
ASSERT_EQ(OB_SUCCESS, reader.open(uri, &info_base));
ASSERT_EQ(OB_SUCCESS, reader.pread(read_buf, 100, 0, read_size));
ASSERT_EQ(6, read_size);
ASSERT_EQ(0, strncmp(read_buf, "123456", 6));
ASSERT_EQ(OB_SUCCESS, reader.close());
ASSERT_EQ(OB_SUCCESS, util.del_file(uri));
}
{
// appendable
ObStorageAppender appender;
ObStorageAdaptiveReader reader;
const char *write_content = "0123456789";
const int64_t first_content_len = 6;
ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_appendable", dir_uri));
ASSERT_EQ(OB_SUCCESS, appender.open(uri, &info_base));
ASSERT_EQ(OB_SUCCESS, appender.pwrite(write_content, first_content_len, 0));
int64_t read_size = -1;
char read_buf[100];
memset(read_buf, 0, sizeof(read_buf));
ASSERT_EQ(OB_SUCCESS, reader.open(uri, &info_base));
ASSERT_EQ(OB_SUCCESS, appender.pwrite(write_content, 1, first_content_len));
ASSERT_EQ(OB_SUCCESS, reader.pread(read_buf, 100, 0, read_size));
ASSERT_EQ(first_content_len, read_size);
ASSERT_EQ(0, strncmp(read_buf, write_content, first_content_len));
ASSERT_EQ(OB_SUCCESS, reader.close());
ASSERT_EQ(OB_SUCCESS, util.del_file(uri));
}
}
}
TEST_F(TestObjectStorage, test_multipart_write)
{
int ret = OB_SUCCESS;
@ -1265,6 +1325,7 @@ TEST_F(TestObjectStorage, test_multipart_write)
ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_multipart", dir_uri));
ObStorageMultiPartWriter writer;
// ObStorageWriter writer;
ASSERT_EQ(OB_SUCCESS, writer.open(uri, &info_base));
ASSERT_EQ(OB_SUCCESS, writer.write(write_buf, content_size));
ASSERT_EQ(content_size, writer.get_length());

8
deps/oblib/unittest/lib/utility/test_ob_unify_serialize.cpp vendored
View File

@ -16,6 +16,7 @@
#include <gtest/gtest.h>
#include <iostream>
#include <cstdlib>
#include "lib/utility/ob_print_utils.h" // TO_STRING_KV
#include "lib/utility/ob_unify_serialize.h"
#include "lib/random/ob_random.h"
@ -73,6 +74,7 @@ struct CVirtualTest {
};
struct CEmptyTest {
TO_STRING_KV(KP(this));
OB_UNIS_VERSION(1);
};
@ -165,6 +167,7 @@ public:
uint32_t vu32_;
int64_t vi64_;
uint64_t vu64_;
TO_STRING_KV(K_(b), K_(vi8), K_(vu8), K_(vi16), K_(vu16), K_(vi32), K_(vu32), K_(vi64), K_(vu64));
}; // end of class TestObUnifySerialize
OB_SERIALIZE_MEMBER(CIntTest, b_, vi8_, vu8_, vi16_, vu16_, vi32_, vu32_, vi64_, vu64_);
@ -220,7 +223,7 @@ public:
n.buf_[rlen] = 0;
return n;
}
TO_STRING_KV(K_(buf));
private:
char buf_[32];
};
@ -268,6 +271,7 @@ public:
}
return right;
}
TO_STRING_KV(K_(ia), K_(uia), K_(i64a), K_(ui64a), K_(ita));
static const CArrayTest RAND()
{
@ -318,7 +322,7 @@ public:
return et;
}
TO_STRING_KV(K(eval));
private:
enum Eval { E0, E1, E2, E3, EMAX } eval;
};

28
mittest/logservice/env/ob_simple_log_cluster_env.cpp vendored
View File

@ -349,13 +349,30 @@ int ObSimpleLogClusterTestEnv::create_paxos_group_with_arb(
int64_t &leader_idx,
const bool with_mock_election,
PalfHandleImplGuard &leader)
{
// if member_cnt_ is 3, arb_replica_idx should be 0,1,2
const ObMemberList member_list = get_member_list();
const int64_t member_cnt = get_member_cnt();
ObMember arb_member;
return create_paxos_group_with_arb(id, loc_cb, member_list, member_cnt, arb_member, arb_replica_idx, leader_idx, with_mock_election, leader);
}
// member_list and member_cnt contain arb member
int ObSimpleLogClusterTestEnv::create_paxos_group_with_arb(
const int64_t id,
palf::PalfLocationCacheCb *loc_cb,
ObMemberList member_list,
int64_t member_cnt,
ObMember arb_member,
int64_t &arb_replica_idx,
int64_t &leader_idx,
const bool with_mock_election,
PalfHandleImplGuard &leader)
{
int ret = OB_SUCCESS;
PalfBaseInfo palf_base_info;
palf_base_info.generate_by_default();
// if member_cnt_ is 3, arb_replica_idx should be 0,1,2
ObMemberList member_list = get_member_list();
ObMember arb_replica;
ObMember arb_replica = arb_member;
GlobalLearnerList learner_list;
arb_replica_idx = 2;
for (int i = 0; i < get_cluster().size(); i++) {
@ -367,7 +384,8 @@ int ObSimpleLogClusterTestEnv::create_paxos_group_with_arb(
if (-1 == arb_replica_idx) {
ret = OB_NOT_SUPPORTED;
PALF_LOG(ERROR, "there is not any arb server");
} else if (OB_FAIL(member_list.get_member_by_index(arb_replica_idx, arb_replica))) {
} else if (false == arb_replica.is_valid() &&
OB_FAIL(member_list.get_member_by_index(arb_replica_idx, arb_replica))) {
PALF_LOG(ERROR, "get_member_by_index failed", K(ret), K(arb_replica_idx));
} else {
member_list.remove_member(arb_replica);
@ -382,7 +400,7 @@ int ObSimpleLogClusterTestEnv::create_paxos_group_with_arb(
break;
} else if (OB_FAIL(svr->get_palf_env()->create_palf_handle_impl(id, palf::AccessMode::APPEND, palf_base_info, handle))) {
CLOG_LOG(WARN, "create_palf_handle_impl failed", K(ret), K(id), KPC(svr));
} else if (!svr->is_arb_server() && OB_FAIL(handle->set_initial_member_list(member_list, arb_replica, get_member_cnt()-1, learner_list))) {
} else if (!svr->is_arb_server() && OB_FAIL(handle->set_initial_member_list(member_list, arb_replica, member_cnt-1, learner_list))) {
CLOG_LOG(ERROR, "set_initial_member_list failed", K(ret), K(id), KPC(svr));
} else {
common::ObAddr leader_addr;

9
mittest/logservice/env/ob_simple_log_cluster_env.h vendored
View File

@ -166,6 +166,15 @@ public:
int64_t &leader_idx,
const bool with_mock_election,
PalfHandleImplGuard &leader);
int create_paxos_group_with_arb(const int64_t id,
palf::PalfLocationCacheCb *loc_cb,
ObMemberList member_list,
int64_t member_cnt,
ObMember arb_member,
int64_t &arb_replica_idx,
int64_t &leader_idx,
const bool with_mock_election,
PalfHandleImplGuard &leader);
int create_paxos_group_with_mock_election(const int64_t id,
int64_t &leader_idx,
PalfHandleImplGuard &leader);

32
mittest/logservice/test_ob_simple_log_arb.cpp
View File

@ -767,6 +767,38 @@ TEST_F(TestObSimpleLogClusterArbService, test_2f1a_upgrade_when_no_leader)
PALF_LOG(INFO, "end test_2f1a_upgrade_when_no_leader", K(id));
}
TEST_F(TestObSimpleLogClusterArbService, test_1f1a_create_palf_group)
{
oceanbase::common::ObClusterVersion::get_instance().cluster_version_ = CLUSTER_VERSION_4_1_0_0;
SET_CASE_LOG_FILE(TEST_NAME, "test_1f1a_create_palf_group");
PALF_LOG(INFO, "begin test_1f1a_create_palf_group");
OB_LOGGER.set_log_level("TRACE");
const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s
MockLocCB loc_cb;
int ret = OB_SUCCESS;
int64_t leader_idx = 0;
int64_t arb_replica_idx = -1;
PalfHandleImplGuard leader;
const int64_t id = ATOMIC_AAF(&palf_id_, 1);
common::ObMemberList member_list = get_member_list();
member_list.remove_server(get_cluster()[1]->get_addr());
const int64_t member_cnt = 2;
const common::ObMember &arb_member = get_arb_member();
EXPECT_EQ(OB_SUCCESS, create_paxos_group_with_arb(id, &loc_cb, member_list, member_cnt, arb_member, arb_replica_idx, leader_idx, false, leader));
EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
LogConfigVersion config_version;
EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(ObMember(get_cluster()[1]->get_addr(), 1), 2, config_version, CONFIG_CHANGE_TIMEOUT));
EXPECT_EQ(2, leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_replica_num_);
EXPECT_EQ(2, leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.get_member_number());
leader.reset();
delete_paxos_group(id);
PALF_LOG(INFO, "end test_2f1a_degrade_upgrade", K(id));
}
} // end unittest
} // end oceanbase

16
mittest/mtlenv/mock_tenant_module_env.h
View File

@ -171,6 +171,18 @@ public:
UNUSED(tenant_id);
return source_.get_gts(stc, task, gts, receive_gts_ts);
}
virtual int get_gts_sync(const uint64_t tenant_id,
const MonotonicTs stc,
int64_t timeout_us,
share::SCN &gts,
MonotonicTs &receive_gts_ts)
{
UNUSED(tenant_id);
UNUSED(timeout_us);
return source_.get_gts(stc, NULL, gts, receive_gts_ts);
}
virtual int get_gts(const uint64_t tenant_id, ObTsCbTask *task, share::SCN &gts)
{
UNUSED(tenant_id);
@ -666,7 +678,7 @@ int MockTenantModuleEnv::init()
STORAGE_LOG(ERROR, "init_before_start_mtl failed", K(ret));
} else {
oceanbase::ObClusterVersion::get_instance().update_data_version(DATA_CURRENT_VERSION);
MTL_BIND(ObTenantIOManager::mtl_init, ObTenantIOManager::mtl_destroy);
MTL_BIND2(nullptr, ObTenantIOManager::mtl_init, nullptr, nullptr, nullptr, ObTenantIOManager::mtl_destroy);
MTL_BIND2(mtl_new_default, omt::ObSharedTimer::mtl_init, omt::ObSharedTimer::mtl_start, omt::ObSharedTimer::mtl_stop, omt::ObSharedTimer::mtl_wait, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObTenantSchemaService::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObStorageLogger::mtl_init, ObStorageLogger::mtl_start, ObStorageLogger::mtl_stop, ObStorageLogger::mtl_wait, mtl_destroy_default);
@ -705,7 +717,7 @@ int MockTenantModuleEnv::init()
MTL_BIND2(mtl_new_default, ObTableLockService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(server_obj_pool_mtl_new<transaction::ObPartTransCtx>, nullptr, nullptr, nullptr, nullptr, server_obj_pool_mtl_destroy<transaction::ObPartTransCtx>);
MTL_BIND2(server_obj_pool_mtl_new<ObTableScanIterator>, nullptr, nullptr, nullptr, nullptr, server_obj_pool_mtl_destroy<ObTableScanIterator>);
MTL_BIND(ObTenantSQLSessionMgr::mtl_init, ObTenantSQLSessionMgr::mtl_destroy);
MTL_BIND2(ObTenantSQLSessionMgr::mtl_new, ObTenantSQLSessionMgr::mtl_init, nullptr, nullptr, nullptr, ObTenantSQLSessionMgr::mtl_destroy);
MTL_BIND2(mtl_new_default, ObTenantCGReadInfoMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObDecodeResourcePool::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObTenantDirectLoadMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);

67
mittest/mtlenv/storage/blocksstable/ob_index_block_data_prepare.h
View File

@ -1,5 +1,5 @@
/**
* Copyright (c) 2021 OceanBase
* Copyright (c) 2023 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
@ -65,6 +65,7 @@ public:
virtual void prepare_schema();
virtual void prepare_data(const int64_t micro_block_size = 0);
virtual void prepare_partial_ddl_data();
virtual void prepare_partial_cg_data();
virtual void prepare_cg_data();
virtual void insert_data(ObMacroBlockWriter &data_writer); // override to define data in sstable
virtual void insert_cg_data(ObMacroBlockWriter &data_writer); // override to define data in sstable
@ -136,6 +137,8 @@ protected:
int64_t mirco_blocks_per_macro_block_;
bool is_cg_data_;
bool is_ddl_merge_data_;
ObGetTableParam get_table_param_;
};
ObArenaAllocator TestIndexBlockDataPrepare::allocator_;
@ -296,13 +299,18 @@ void TestIndexBlockDataPrepare::SetUp()
ASSERT_EQ(OB_SUCCESS, TestTabletHelper::create_tablet(ls_handle, tablet_id, table_schema_, allocator_));
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle_));
get_table_param_.tablet_iter_.set_tablet_handle(tablet_handle_);
iter_param_.set_table_param(&get_table_param_);
sstable_.key_.table_type_ = ObITable::TableType::COLUMN_ORIENTED_SSTABLE;
partial_sstable_.key_.table_type_ = ObITable::TableType::DDL_MERGE_CO_SSTABLE;
if (is_cg_data_) {
if (is_cg_data_ && is_ddl_merge_data_) {
prepare_partial_cg_data();
partial_sstable_.key_.table_type_ = ObITable::TableType::DDL_MERGE_CG_SSTABLE;
} else if (is_cg_data_) {
prepare_cg_data();
} else if (is_ddl_merge_data_) {
prepare_partial_ddl_data();
partial_sstable_.key_.table_type_ = ObITable::TableType::DDL_MERGE_CO_SSTABLE;
} else {
prepare_data();
}
@ -575,8 +583,8 @@ void TestIndexBlockDataPrepare::close_builder_and_prepare_sstable(const int64_t
param.nested_offset_ = res.nested_offset_;
param.nested_size_ = res.nested_size_;
param.compressor_type_ = ObCompressorType::NONE_COMPRESSOR;
param.data_block_ids_ = res.data_block_ids_;
param.other_block_ids_ = res.other_block_ids_;
ASSERT_EQ(OB_SUCCESS, param.data_block_ids_.assign(res.data_block_ids_));
ASSERT_EQ(OB_SUCCESS, param.other_block_ids_.assign(res.other_block_ids_));
param.ddl_scn_.set_min();
param.filled_tx_scn_.set_min();
param.contain_uncommitted_row_ = false;
@ -888,6 +896,51 @@ void TestIndexBlockDataPrepare::prepare_partial_ddl_data()
ObTabletObjLoadHelper::free(allocator_, storage_schema);
}
void TestIndexBlockDataPrepare::prepare_partial_cg_data()
{
prepare_contrastive_sstable();
ObMacroBlockWriter writer;
ObMacroDataSeq start_seq(0);
start_seq.set_data_block();
row_generate_.reset();
ObWholeDataStoreDesc desc(true/*is ddl*/);
share::SCN end_scn;
end_scn.convert_from_ts(ObTimeUtility::current_time());
ASSERT_EQ(OB_SUCCESS, desc.init(table_schema_, ObLSID(ls_id_), ObTabletID(tablet_id_), merge_type_, SNAPSHOT_VERSION, CLUSTER_CURRENT_VERSION, end_scn));
void *builder_buf = allocator_.alloc(sizeof(ObSSTableIndexBuilder));
merge_root_index_builder_ = new (builder_buf) ObSSTableIndexBuilder();
ASSERT_NE(nullptr, merge_root_index_builder_);
desc.get_desc().sstable_index_builder_ = merge_root_index_builder_;
ASSERT_TRUE(desc.is_valid());
if (need_agg_data_) {
ASSERT_EQ(OB_SUCCESS, desc.get_desc().col_desc_->agg_meta_array_.assign(agg_col_metas_));
}
ASSERT_EQ(OB_SUCCESS, merge_root_index_builder_->init(desc.get_desc()));
ASSERT_EQ(OB_SUCCESS, writer.open(desc.get_desc(), start_seq));
ASSERT_EQ(OB_SUCCESS, row_generate_.init(table_schema_, &allocator_));
const int64_t partial_row_cnt = max_partial_row_cnt_;
insert_partial_data(writer, partial_row_cnt);
ASSERT_EQ(OB_SUCCESS, writer.close());
// data write ctx has been moved to merge_root_index_builder_
ASSERT_EQ(writer.get_macro_block_write_ctx().get_macro_block_count(), 0);
data_macro_block_cnt_ = merge_root_index_builder_->roots_[0]->macro_metas_->count();
ASSERT_GE(data_macro_block_cnt_, 0);
int64_t column_cnt = 0;
ObTabletID tablet_id(TestIndexBlockDataPrepare::tablet_id_);
ObLSID ls_id(ls_id_);
ObLSHandle ls_handle;
ObTabletHandle tablet_handle;
ObLSService *ls_svr = MTL(ObLSService*);
ObStorageSchema *storage_schema = nullptr;
ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD));
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle));
ASSERT_EQ(OB_SUCCESS, tablet_handle.get_obj()->load_storage_schema(allocator_, storage_schema));
ASSERT_EQ(OB_SUCCESS, storage_schema->get_stored_column_count_in_sstable(column_cnt));
prepare_partial_sstable(column_cnt);
prepare_merge_ddl_kvs();
ObTabletObjLoadHelper::free(allocator_, storage_schema);
}
void TestIndexBlockDataPrepare::prepare_partial_sstable(const int64_t column_cnt)
{
ObSSTableMergeRes res;
@ -987,13 +1040,13 @@ void TestIndexBlockDataPrepare::prepare_partial_sstable(const int64_t column_cnt
param.nested_offset_ = res.nested_offset_;
param.nested_size_ = res.nested_size_;
param.compressor_type_ = ObCompressorType::NONE_COMPRESSOR;
param.data_block_ids_ = res.data_block_ids_;
param.other_block_ids_ = res.other_block_ids_;
param.ddl_scn_.convert_from_ts(ObTimeUtility::current_time());
param.filled_tx_scn_.set_min();
param.contain_uncommitted_row_ = false;
param.encrypt_id_ = res.encrypt_id_;
param.master_key_id_ = res.master_key_id_;
ASSERT_EQ(OB_SUCCESS, param.data_block_ids_.assign(res.data_block_ids_));
ASSERT_EQ(OB_SUCCESS, param.other_block_ids_.assign(res.other_block_ids_));
if (param.table_key_.is_co_sstable() && param.column_group_cnt_ <= 1) {
param.column_group_cnt_ = column_cnt + 2; /* set column group_cnt to avoid return err, cnt is calculated as each + all + default*/
}

4
mittest/mtlenv/storage/blocksstable/test_block_cache.cpp
View File

@ -114,13 +114,9 @@ TEST_F(TestObMicroBlockCache, test_block_cache)
ObIndexBlockRowScanner idx_row_scanner;
ObMicroBlockData root_block;
ObMicroIndexInfo micro_idx_info;
ObArray<int32_t> agg_projector;
ObArray<ObColumnSchemaV2> agg_column_schema;
ObArray<ObMicroIndexInfo> micro_idx_infos;
sstable_.get_index_tree_root(root_block);
ASSERT_EQ(OB_SUCCESS, idx_row_scanner.init(
agg_projector,
agg_column_schema,
tablet_handle_.get_obj()->get_rowkey_read_info().get_datum_utils(),
allocator_,
context_.query_flag_,

6
mittest/mtlenv/storage/blocksstable/test_index_block_row_scanner.cpp
View File

@ -139,8 +139,6 @@ TEST_F(TestIndexBlockRowScanner, prefetch_and_scan)
ObIndexBlockRowParser idx_row_parser;
ObIndexBlockRowScanner idx_scanner;
ObIndexBlockRowScanner raw_idx_scanner;
ObArray<int32_t> agg_projector;
ObArray<ObColumnSchemaV2> agg_column_schema;
ObQueryFlag query_flag;
query_flag.set_use_block_cache();
ObIndexBlockDataTransformer transformer;
@ -154,9 +152,9 @@ TEST_F(TestIndexBlockRowScanner, prefetch_and_scan)
ASSERT_TRUE(root_blk_header->is_valid());
ASSERT_EQ(OB_SUCCESS, idx_scanner.init(
agg_projector, agg_column_schema, tablet_handle_.get_obj()->get_rowkey_read_info().get_datum_utils(), allocator_, query_flag, 0));
tablet_handle_.get_obj()->get_rowkey_read_info().get_datum_utils(), allocator_, query_flag, 0));
ASSERT_EQ(OB_SUCCESS, raw_idx_scanner.init(
agg_projector, agg_column_schema, tablet_handle_.get_obj()->get_rowkey_read_info().get_datum_utils(), allocator_, query_flag, 0));
tablet_handle_.get_obj()->get_rowkey_read_info().get_datum_utils(), allocator_, query_flag, 0));
ObMacroBlockHandle macro_handle;
const ObIndexBlockRowHeader *idx_row_header = nullptr;

402
mittest/mtlenv/storage/blocksstable/test_merge_cg_scanner.cpp Normal file
View File

@ -0,0 +1,402 @@
/**
* Copyright (c) 2023 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#define private public
#define protected public
#include "lib/random/ob_random.h"
#include "sql/engine/basic/ob_pushdown_filter.h"
#include "storage/tablet/ob_tablet.h"
#include "storage/column_store/ob_cg_prefetcher.h"
#include "storage/column_store/ob_cg_scanner.h"
#include "ob_index_block_data_prepare.h"
namespace oceanbase
{
using namespace storage;
using namespace common;
namespace blocksstable
{
class TestCGScanner : public TestIndexBlockDataPrepare
{
public:
TestCGScanner();
virtual ~TestCGScanner();
static void SetUpTestCase();
static void TearDownTestCase();
virtual void SetUp();
virtual void TearDown();
void prepare_cg_query_param(const bool is_reverse_scan, const ObVersionRange &scan_version, const uint32_t cg_idx);
void destroy_cg_query_param();
void check_data(ObCGRowScanner *cg_scanner, int64_t start, int64_t locate_count, sql::ObDatum *datums, const bool is_reverse);
void test_border(const bool is_reverse);
void test_random(const bool is_reverse);
public:
ObArenaAllocator allocator_;
ObDatumRow start_row_;
ObDatumRow end_row_;
ObTableAccessParam access_param_;
};
TestCGScanner::TestCGScanner()
: TestIndexBlockDataPrepare("Test cg sstable row scanner", ObMergeType::MAJOR_MERGE)
{
is_cg_data_ = true;
is_ddl_merge_data_ = true;
max_row_cnt_ = 150000;
max_partial_row_cnt_ = 78881;
partial_kv_start_idx_ = 3;
}
TestCGScanner::~TestCGScanner()
{
}
void TestCGScanner::SetUpTestCase()
{
TestIndexBlockDataPrepare::SetUpTestCase();
}
void TestCGScanner::TearDownTestCase()
{
TestIndexBlockDataPrepare::TearDownTestCase();
}
void TestCGScanner::prepare_cg_query_param(const bool is_reverse_scan, const ObVersionRange &scan_version, const uint32_t cg_idx)
{
schema_cols_.set_allocator(&allocator_);
schema_cols_.init(table_schema_.get_column_count());
ASSERT_EQ(OB_SUCCESS, table_schema_.get_column_ids(schema_cols_));
access_param_.iter_param_.table_id_ = table_schema_.get_table_id();
access_param_.iter_param_.tablet_id_ = table_schema_.get_table_id();
access_param_.iter_param_.is_same_schema_column_ = true;
access_param_.iter_param_.has_virtual_columns_ = false;
access_param_.iter_param_.vectorized_enabled_ = true;
access_param_.iter_param_.pd_storage_flag_.pd_blockscan_ = true;
access_param_.iter_param_.pd_storage_flag_.pd_filter_ = true;
access_param_.iter_param_.pd_storage_flag_.use_iter_pool_ = true;
access_param_.iter_param_.pd_storage_flag_.use_column_store_ = true;
/*
ObSEArray<ObColDesc, 1> cg_col_descs;
ObTableReadInfo *cg_read_info = nullptr;
ASSERT_EQ(OB_SUCCESS, tablet_handle_.get_obj()->get_cg_col_descs(cg_idx, cg_col_descs));
ASSERT_EQ(OB_SUCCESS, MTL(ObTenantCGReadInfoMgr *)->get_cg_read_info(cg_col_descs.at(0),
ObTabletID(tablet_id_),
cg_read_info));
*/
access_param_.iter_param_.read_info_ = cg_read_info_handle_.get_read_info();
//jsut for test
ObQueryFlag query_flag(ObQueryFlag::Forward,
true, /*is daily merge scan*/
true, /*is read multiple macro block*/
true, /*sys task scan, read one macro block in single io*/
false /*full row scan flag, obsoleted*/,
false,/*index back*/
false); /*query_stat*/
query_flag.set_not_use_row_cache();
query_flag.set_use_block_cache();
if (is_reverse_scan) {
query_flag.scan_order_ = ObQueryFlag::ScanOrder::Reverse;
} else {
query_flag.scan_order_ = ObQueryFlag::ScanOrder::Forward;
}
ASSERT_EQ(OB_SUCCESS, context_.init(query_flag,
store_ctx_,
allocator_,
allocator_,
scan_version));
context_.ls_id_ = ls_id_;
context_.limit_param_ = nullptr;
context_.is_inited_ = true;
}
void TestCGScanner::destroy_cg_query_param()
{
access_param_.reset();
schema_cols_.reset();
context_.reset();
read_info_.reset();
}
void TestCGScanner::SetUp()
{
TestIndexBlockDataPrepare::SetUp();
ASSERT_EQ(OB_SUCCESS, start_row_.init(allocator_, TEST_COLUMN_CNT));
ASSERT_EQ(OB_SUCCESS, end_row_.init(allocator_, TEST_COLUMN_CNT));
ObLSID ls_id(ls_id_);
ObTabletID tablet_id(tablet_id_);
ObLSHandle ls_handle;
ObLSService *ls_svr = MTL(ObLSService*);
ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD));
ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle_));
sstable_.key_.table_type_ = ObITable::TableType::NORMAL_COLUMN_GROUP_SSTABLE;
}
void TestCGScanner::TearDown()
{
tablet_handle_.reset();
TestIndexBlockDataPrepare::TearDown();
}
void TestCGScanner::check_data(
ObCGRowScanner *cg_scanner,
int64_t start,
int64_t locate_count,
sql::ObDatum *datums,
const bool is_reverse)
{
int ret = OB_SUCCESS;
uint64_t count = 0;
int64_t get_count = 0;
int64_t data_count = MIN(row_cnt_ - start, locate_count);
int64_t end = MIN(row_cnt_ - 1, start + locate_count - 1);
int64_t sql_batch_size = 256;
while (OB_SUCC(ret)) {
if (OB_FAIL(cg_scanner->get_next_rows(count, sql_batch_size))) {
if (OB_ITER_END != ret) {
STORAGE_LOG(WARN, "Fail to get next rows", K(ret));
ASSERT_EQ(OB_ITER_END, ret);
} else {
for (int64_t i = 0; i < count; i++) {
//STORAGE_LOG(INFO, "get next row info", K(get_count), K(i), K(datums[i].get_int()));
if (!is_reverse) {
ASSERT_EQ(start + get_count, datums[i].get_int()) << i << " start " << start << " locate_count " << locate_count;
} else {
ASSERT_EQ(end - get_count, datums[i].get_int()) << i << " start " << start << " locate_count " << locate_count;
}
get_count++;
}
}
} else {
for (int64_t i = 0; i < count; i++) {
//STORAGE_LOG(INFO, "get next row info", K(get_count), K(i), K(datums[i].get_int()));
if (!is_reverse) {
ASSERT_EQ(start + get_count, datums[i].get_int()) << i << " start " << start << " locate_count " << locate_count;
} else {
ASSERT_EQ(end - get_count, datums[i].get_int()) << i << " start " << start << " locate_count " << locate_count;
}
get_count++;
}
}
}
}
void TestCGScanner::test_random(const bool is_reverse)
{
ObVersionRange scan_version;
scan_version.base_version_ = 1;
scan_version.multi_version_start_ = 1;
scan_version.snapshot_version_ = INT64_MAX;
// prepare query param
prepare_cg_query_param(is_reverse, scan_version, 0);
sql::ObExecContext exec_ctx(allocator_);
sql::ObEvalCtx eval_ctx(exec_ctx);
sql::ObPushdownExprSpec expr_spec(allocator_);
static int64_t array_cnt = 1024;
eval_ctx.batch_idx_ = 0;
eval_ctx.batch_size_ = 256;
expr_spec.max_batch_size_ = 256;
void *datum_buf = allocator_.alloc((sizeof(sql::ObDatum) + sizeof(int64_t)) * array_cnt * 2);
sql::ObDatum *datums = new (datum_buf) sql::ObDatum[array_cnt];
eval_ctx.frames_ = (char **)(&datum_buf);
sql::ObPushdownOperator op(eval_ctx, expr_spec);
ObFixedArray<int32_t, ObIAllocator> out_cols_project;
out_cols_project.set_allocator(&allocator_);
out_cols_project.init(1);
out_cols_project.push_back(0);
sql::ExprFixedArray exprs;
exprs.set_allocator(&allocator_);
void *expr_buf = allocator_.alloc(sizeof(sql::ObExpr));
ASSERT_NE(nullptr, expr_buf);
sql::ObExpr *expr = reinterpret_cast<sql::ObExpr *>(expr_buf);
expr->reset();
expr->res_buf_off_ = sizeof(sql::ObDatum) * array_cnt;
expr->res_buf_len_ = sizeof(int64_t);
char *ptr = (char *)datum_buf + expr->res_buf_off_;
for (int64_t i = 0; i < array_cnt; i++) {
datums[i].ptr_ = ptr;
ptr += expr->res_buf_len_;
}
exprs.init(1);
exprs.push_back(expr);
access_param_.iter_param_.out_cols_project_ = &out_cols_project;
access_param_.iter_param_.output_exprs_ = &exprs;
access_param_.iter_param_.op_ = &op;
void *buf = allocator_.alloc(sizeof(ObBlockRowStore));
ASSERT_NE(nullptr, buf);
ObBlockRowStore *block_row_store = new (buf) ObBlockRowStore(context_);
ASSERT_EQ(OB_SUCCESS, block_row_store->init(access_param_));
context_.block_row_store_ = block_row_store;
buf = allocator_.alloc(sizeof(ObCGRowScanner));
ASSERT_NE(nullptr, buf);
ObCGRowScanner *cg_scanner = new (buf) ObCGRowScanner();
ObSSTableWrapper wrapper;
wrapper.sstable_ = &sstable_;
ASSERT_EQ(OB_SUCCESS, cg_scanner->init(access_param_.iter_param_, context_, wrapper));
int retry_cnt = 15;
while (retry_cnt > 0) {
int64_t start = ObRandom::rand(0, row_cnt_ - 1);
int64_t locate_count = ObRandom::rand(1, row_cnt_);
STORAGE_LOG(INFO, "start to locate random range", K(retry_cnt), K(start), K(locate_count), K(row_cnt_));
ASSERT_EQ(OB_SUCCESS, cg_scanner->locate(ObCSRange(start, locate_count)));
check_data(cg_scanner, start, locate_count, datums, is_reverse);
retry_cnt--;
}
STORAGE_LOG(INFO, "test random finished");
allocator_.free(datum_buf);
datum_buf = nullptr;
expr->reset();
allocator_.free(expr);
expr = nullptr;
out_cols_project.reset();
allocator_.free(block_row_store);
block_row_store = nullptr;
cg_scanner->reset();
allocator_.free(cg_scanner);
cg_scanner = nullptr;
destroy_cg_query_param();
}
void TestCGScanner::test_border(const bool is_reverse)
{
ObVersionRange scan_version;
scan_version.base_version_ = 1;
scan_version.multi_version_start_ = 1;
scan_version.snapshot_version_ = INT64_MAX;
// prepare query param
prepare_cg_query_param(is_reverse, scan_version, 0);
sql::ObExecContext exec_ctx(allocator_);
sql::ObEvalCtx eval_ctx(exec_ctx);
sql::ObPushdownExprSpec expr_spec(allocator_);
static int64_t array_cnt = 1024;
eval_ctx.batch_idx_ = 0;
eval_ctx.batch_size_ = 256;
expr_spec.max_batch_size_ = 256;
void *datum_buf = allocator_.alloc((sizeof(sql::ObDatum) + sizeof(int64_t)) * array_cnt * 2);
sql::ObDatum *datums = new (datum_buf) sql::ObDatum[array_cnt];
eval_ctx.frames_ = (char **)(&datum_buf);
sql::ObPushdownOperator op(eval_ctx, expr_spec);
ObFixedArray<int32_t, ObIAllocator> out_cols_project;
out_cols_project.set_allocator(&allocator_);
out_cols_project.init(1);
out_cols_project.push_back(0);
sql::ExprFixedArray exprs;
exprs.set_allocator(&allocator_);
void *expr_buf = allocator_.alloc(sizeof(sql::ObExpr));
ASSERT_NE(nullptr, expr_buf);
sql::ObExpr *expr = reinterpret_cast<sql::ObExpr *>(expr_buf);
expr->reset();
expr->res_buf_off_ = sizeof(sql::ObDatum) * array_cnt;
expr->res_buf_len_ = sizeof(int64_t);
char *ptr = (char *)datum_buf + expr->res_buf_off_;
for (int64_t i = 0; i < array_cnt; i++) {
datums[i].ptr_ = ptr;
ptr += expr->res_buf_len_;
}
exprs.init(1);
exprs.push_back(expr);
access_param_.iter_param_.out_cols_project_ = &out_cols_project;
access_param_.iter_param_.output_exprs_ = &exprs;
access_param_.iter_param_.op_ = &op;
void *buf = allocator_.alloc(sizeof(ObBlockRowStore));
ASSERT_NE(nullptr, buf);
ObBlockRowStore *block_row_store = new (buf) ObBlockRowStore(context_);
ASSERT_EQ(OB_SUCCESS, block_row_store->init(access_param_));
context_.block_row_store_ = block_row_store;
buf = allocator_.alloc(sizeof(ObCGRowScanner));
ASSERT_NE(nullptr, buf);
ObCGRowScanner *cg_scanner = new (buf) ObCGRowScanner();
ObSSTableWrapper wrapper;
wrapper.sstable_ = &sstable_;
ASSERT_EQ(OB_SUCCESS, cg_scanner->init(access_param_.iter_param_, context_, wrapper));
int64_t start = 0;
int64_t locate_count = 0;
ASSERT_EQ(OB_INVALID_ARGUMENT, cg_scanner->locate(ObCSRange(start, locate_count)));
start = row_cnt_;
locate_count = 1;
ASSERT_EQ(OB_ITER_END, cg_scanner->locate(ObCSRange(start, locate_count)));
start = row_cnt_ - 99;
locate_count = 100;
ASSERT_EQ(OB_SUCCESS, cg_scanner->locate(ObCSRange(start, locate_count)));
check_data(cg_scanner, start, locate_count, datums, is_reverse);
start = 0;
locate_count = row_cnt_ + 1;
ASSERT_EQ(OB_SUCCESS, cg_scanner->locate(ObCSRange(start, locate_count)));
check_data(cg_scanner, start, locate_count, datums, is_reverse);
STORAGE_LOG(INFO, "test border finished");
allocator_.free(datum_buf);
datum_buf = nullptr;
expr->reset();
allocator_.free(expr);
expr = nullptr;
out_cols_project.reset();
allocator_.free(block_row_store);
block_row_store = nullptr;
cg_scanner->reset();
allocator_.free(cg_scanner);
cg_scanner = nullptr;
destroy_cg_query_param();
}
TEST_F(TestCGScanner, test_border)
{
test_border(false);
}
TEST_F(TestCGScanner, test_random)
{
test_random(false);
}
TEST_F(TestCGScanner, test_border_reverse)
{
test_border(true);
}
TEST_F(TestCGScanner, test_random_reverse)
{
test_random(true);
}
}
}
int main(int argc, char **argv)
{
system("rm -f test_merge_cg_scanner.log*");
OB_LOGGER.set_file_name("test_merge_cg_scanner.log", true, true);
oceanbase::common::ObLogger::get_logger().set_log_level("INFO");
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

4
mittest/mtlenv/storage/test_co_merge.cpp
View File

@ -138,8 +138,8 @@ void close_builder_and_prepare_sstable(
param.master_key_id_ = res.master_key_id_;
param.nested_size_ = res.nested_size_;
param.nested_offset_ = res.nested_offset_;
param.data_block_ids_ = res.data_block_ids_;
param.other_block_ids_ = res.other_block_ids_;
ASSERT_EQ(OB_SUCCESS, param.data_block_ids_.assign(res.data_block_ids_));
ASSERT_EQ(OB_SUCCESS, param.other_block_ids_.assign(res.other_block_ids_));
param.nested_size_ = res.nested_size_;
param.nested_offset_ = res.nested_offset_;
if (is_major_merge_type(data_store_desc.get_merge_type())) {

1
mittest/simple_server/CMakeLists.txt
View File

@ -105,6 +105,7 @@ ob_unittest_observer(test_ls_replica test_ls_replica.cpp)
ob_unittest_observer(test_create_clone_tenant_resource_pool test_create_clone_tenant_resource_pool.cpp)
ob_unittest_observer(test_tablet_autoinc_mgr test_tablet_autoinc_mgr.cpp)
ob_unittest_observer(test_tenant_snapshot_service test_tenant_snapshot_service.cpp)
ob_unittest_observer(test_callbacks_with_reverse_order test_callbacks_with_reverse_order.cpp)
# TODO(muwei.ym): open later
ob_ha_unittest_observer(test_transfer_handler storage_ha/test_transfer_handler.cpp)
ob_ha_unittest_observer(test_transfer_and_restart_basic storage_ha/test_transfer_and_restart_basic.cpp)

336
mittest/simple_server/test_callbacks_with_reverse_order.cpp Normal file
View File

@ -0,0 +1,336 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include <gtest/gtest.h>
#include <thread>
#include <iostream>
#define protected public
#define private public
#include "env/ob_simple_cluster_test_base.h"
#include "storage/compaction/ob_compaction_diagnose.h"
#include "storage/compaction/ob_schedule_dag_func.h"
#include "storage/ls/ob_ls.h"
#include "storage/tx_storage/ob_ls_handle.h"
#include "storage/tx_storage/ob_ls_service.h"
#include "storage/tx/ob_tx_data_functor.h"
#include "storage/tablet/ob_tablet.h"
#include "storage/ob_relative_table.h"
#include "storage/ob_dml_running_ctx.h"
#include "storage/access/ob_rows_info.h"
static int qcc = 0;
static int qcc2 = 0;
static int qcc3 = 0;
namespace oceanbase
{
namespace storage
{
int ObLSTabletService::insert_tablet_rows(
const int64_t row_count,
ObTabletHandle &tablet_handle,
ObDMLRunningCtx &run_ctx,
ObStoreRow *rows,
ObRowsInfo &rows_info)
{
int ret = OB_SUCCESS;
ObRelativeTable &table = run_ctx.relative_table_;
const bool check_exists = !table.is_storage_index_table() || table.is_unique_index();
bool exists = false;
// // 1. Defensive checking of new rows.
// if (GCONF.enable_defensive_check()) {
// for (int64_t i = 0; OB_SUCC(ret) && i < row_count; i++) {
// ObStoreRow &tbl_row = rows[i];
// if (OB_FAIL(check_new_row_legitimacy(run_ctx, tbl_row.row_val_))) {
// LOG_WARN("Failed to check new row legitimacy", K(ret), K_(tbl_row.row_val));
// }
// }
// }
// 2. Check uniqueness constraint in memetable only(active + frozen).
// It would be more efficient and elegant to completely merge the uniqueness constraint
// and write conflict checking, but the implementation currently is to minimize intrusion
// into the memtable.
// if (check_exists && OB_FAIL(tablet_handle.get_obj()->rowkeys_exists(run_ctx.store_ctx_, table,
// rows_info, exists))) {
// LOG_WARN("Failed to check the uniqueness constraint", K(ret), K(rows_info));
// } else if (exists) {
// ret = OB_ERR_PRIMARY_KEY_DUPLICATE;
// blocksstable::ObDatumRowkey &duplicate_rowkey = rows_info.get_conflict_rowkey();
// LOG_WARN("Rowkey already exist", K(ret), K(table), K(duplicate_rowkey));
// }
// 3. Insert rows with uniqueness constraint and write conflict checking.
// Check write conflict in memtable + sstable.
// Check uniqueness constraint in sstable only.
if (OB_SUCC(ret)) {
if (OB_FAIL(tablet_handle.get_obj()->insert_rows(table, run_ctx.store_ctx_, rows, rows_info,
check_exists, *run_ctx.col_descs_, row_count, run_ctx.dml_param_.encrypt_meta_))) {
if (OB_ERR_PRIMARY_KEY_DUPLICATE == ret) {
blocksstable::ObDatumRowkey &duplicate_rowkey = rows_info.get_conflict_rowkey();
TRANS_LOG(WARN, "Rowkey already exist", K(ret), K(table), K(duplicate_rowkey),
K(rows_info.get_conflict_idx()));
} else if (OB_TRY_LOCK_ROW_CONFLICT != ret) {
TRANS_LOG(WARN, "Failed to insert rows to tablet", K(ret), K(rows_info));
}
}
}
// 4. Log user error message if rowkey is duplicate.
if (OB_ERR_PRIMARY_KEY_DUPLICATE == ret && !run_ctx.dml_param_.is_ignore_) {
int tmp_ret = OB_SUCCESS;
char rowkey_buffer[OB_TMP_BUF_SIZE_256];
ObString index_name = "PRIMARY";
if (OB_TMP_FAIL(extract_rowkey(table, rows_info.get_conflict_rowkey(),
rowkey_buffer, OB_TMP_BUF_SIZE_256, run_ctx.dml_param_.tz_info_))) {
TRANS_LOG(WARN, "Failed to extract rowkey", K(ret), K(tmp_ret));
}
if (table.is_index_table()) {
if (OB_TMP_FAIL(table.get_index_name(index_name))) {
TRANS_LOG(WARN, "Failed to get index name", K(ret), K(tmp_ret));
}
} else if (lib::is_oracle_mode() && OB_TMP_FAIL(table.get_primary_key_name(index_name))) {
TRANS_LOG(WARN, "Failed to get pk name", K(ret), K(tmp_ret));
}
LOG_USER_ERROR(OB_ERR_PRIMARY_KEY_DUPLICATE, rowkey_buffer, index_name.length(), index_name.ptr());
}
return ret;
}
int ObStorageTableGuard::refresh_and_protect_table(ObRelativeTable &relative_table)
{
int ret = OB_SUCCESS;
ObTabletTableIterator &iter = relative_table.tablet_iter_;
const share::ObLSID &ls_id = tablet_->get_tablet_meta().ls_id_;
const common::ObTabletID &tablet_id = tablet_->get_tablet_meta().tablet_id_;
bool need_print = false;
if (tablet_id.id() == 200001 && store_ctx_.mvcc_acc_ctx_.tx_id_.get_id() % 2 == 0 && qcc2 == 0) {
need_print = true;
qcc2++;
TRANS_LOG(INFO, "qc debug", K(store_ctx_.mvcc_acc_ctx_.tx_id_), KPC(iter.table_iter()->get_last_memtable()));
usleep(1 * 1000 * 1000);
TRANS_LOG(INFO, "qc debug", K(store_ctx_.mvcc_acc_ctx_.tx_id_), KPC(iter.table_iter()->get_last_memtable()));
}
if (tablet_id.id() == 200001 && store_ctx_.mvcc_acc_ctx_.tx_id_.get_id() % 2 == 1 && qcc3 == 0) {
while (qcc2 == 0) {
usleep(1000);
}
}
while (OB_SUCC(ret) && need_to_refresh_table(*iter.table_iter())) {
if (OB_FAIL(store_ctx_.ls_->get_tablet_svr()->get_read_tables(
tablet_id,
ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US,
store_ctx_.mvcc_acc_ctx_.get_snapshot_version().get_val_for_tx(),
iter,
relative_table.allow_not_ready()))) {
TRANS_LOG(WARN, "fail to get", K(store_ctx_.mvcc_acc_ctx_.tx_id_), K(ret));
} else {
// no worry. iter will hold tablet reference and its life cycle is longer than guard
tablet_ = iter.get_tablet();
if (store_ctx_.timeout_ > 0) {
const int64_t query_left_time = store_ctx_.timeout_ - ObTimeUtility::current_time();
if (query_left_time <= 0) {
ret = OB_TRANS_STMT_TIMEOUT;
}
}
}
}
if (need_print) {
TRANS_LOG(INFO, "qc debug", K(store_ctx_.mvcc_acc_ctx_.tx_id_), KPC(iter.table_iter()->get_last_memtable()));
}
if (OB_SUCC(ret)) {
if (tablet_id.id() == 200001 && store_ctx_.mvcc_acc_ctx_.tx_id_.get_id() % 2 == 1 && qcc3 == 0) {
qcc++;
qcc3++;
TRANS_LOG(INFO, "qc debug2", K(store_ctx_.mvcc_acc_ctx_.tx_id_), KPC(iter.table_iter()->get_last_memtable()));
usleep(2 * 1000 * 1000);
TRANS_LOG(INFO, "qc debug2", K(store_ctx_.mvcc_acc_ctx_.tx_id_), KPC(iter.table_iter()->get_last_memtable()));
}
}
return ret;
}
}
namespace unittest
{
using namespace oceanbase::transaction;
using namespace oceanbase::storage;
using namespace oceanbase::memtable;
using namespace oceanbase::storage::checkpoint;
#define EXE_SQL(sql_str) \
ASSERT_EQ(OB_SUCCESS, sql.assign(sql_str)); \
ASSERT_EQ(OB_SUCCESS, sql_proxy.write(sql.ptr(), affected_rows));
#define EXE_SQL_FMT(...) \
ASSERT_EQ(OB_SUCCESS, sql.assign_fmt(__VA_ARGS__)); \
ASSERT_EQ(OB_SUCCESS, sql_proxy.write(sql.ptr(), affected_rows));
#define WRITE_SQL_BY_CONN(conn, sql_str) \
ASSERT_EQ(OB_SUCCESS, sql.assign(sql_str)); \
ASSERT_EQ(OB_SUCCESS, conn->execute_write(OB_SYS_TENANT_ID, sql.ptr(), affected_rows));
#define WRITE_SQL_FMT_BY_CONN(conn, ...) \
ASSERT_EQ(OB_SUCCESS, sql.assign_fmt(__VA_ARGS__)); \
ASSERT_EQ(OB_SUCCESS, conn->execute_write(OB_SYS_TENANT_ID, sql.ptr(), affected_rows));
#define READ_SQL_BY_CONN(conn, sql_str) \
ASSERT_EQ(OB_SUCCESS, sql.assign(sql_str)); \
ASSERT_EQ(OB_SUCCESS, conn->execute_read(OB_SYS_TENANT_ID, sql.ptr(), read_res));
class ObCallbackReverseTest : public ObSimpleClusterTestBase
{
public:
ObCallbackReverseTest() : ObSimpleClusterTestBase("callbacks_with_reverse_order", "200G", "40G") {}
void prepare_tenant_env()
{
common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2();
int64_t affected_rows = 0;
ObSqlString sql;
sqlclient::ObISQLConnection *connection = nullptr;
ASSERT_EQ(OB_SUCCESS, sql_proxy.acquire(connection));
ASSERT_NE(nullptr, connection);
WRITE_SQL_BY_CONN(connection, "set GLOBAL ob_trx_timeout = 10000000000");
WRITE_SQL_BY_CONN(connection, "set GLOBAL ob_trx_idle_timeout = 10000000000");
WRITE_SQL_BY_CONN(connection, "set GLOBAL ob_query_timeout = 10000000000");
WRITE_SQL_BY_CONN(connection, "alter system set enable_early_lock_release = False;");
WRITE_SQL_BY_CONN(connection, "alter system set undo_retention = 1800;");
sleep(5);
}
void create_test_tenant(uint64_t &tenant_id)
{
TRANS_LOG(INFO, "create_tenant start");
ASSERT_EQ(OB_SUCCESS, create_tenant("tt1", "20G", "100G"));
ASSERT_EQ(OB_SUCCESS, get_tenant_id(tenant_id));
ASSERT_EQ(OB_SUCCESS, get_curr_simple_server().init_sql_proxy2());
TRANS_LOG(INFO, "create_tenant end", K(tenant_id));
}
// you should use single partition when using it
void get_tablet_id_with_table_name(const char *name,
ObTabletID &tablet)
{
common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy();
int ret = OB_SUCCESS;
ObSqlString sql;
int64_t affected_rows = 0;
int64_t tablet_id = 0;
ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("select tablet_id from oceanbase.__all_virtual_table where table_name=%s", name));
SMART_VAR(ObMySQLProxy::MySQLResult, res) {
ASSERT_EQ(OB_SUCCESS, sql_proxy.read(res, sql.ptr()));
sqlclient::ObMySQLResult *result = res.get_result();
ASSERT_NE(nullptr, result);
ASSERT_EQ(OB_SUCCESS, result->next());
ASSERT_EQ(OB_SUCCESS, result->get_int("tablet_id", tablet_id));
}
tablet = (uint64_t)tablet_id;
}
void minor_freeze_data()
{
common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2();
sqlclient::ObISQLConnection *connection = nullptr;
ASSERT_EQ(OB_SUCCESS, sql_proxy.acquire(connection));
int ret = OB_SUCCESS;
ObSqlString sql;
int64_t affected_rows = 0;
WRITE_SQL_BY_CONN(connection, "alter system minor freeze;");
}
void get_ls(uint64_t tenant_id, ObLS *&ls)
{
ls = nullptr;
share::ObTenantSwitchGuard tenant_guard;
ASSERT_EQ(OB_SUCCESS, tenant_guard.switch_to(tenant_id));
ObLSService *ls_svr = MTL(ObLSService*);
ASSERT_NE(nullptr, ls_svr);
ObLSHandle handle;
share::ObLSID ls_id(1001);
ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, handle, ObLSGetMod::STORAGE_MOD));
ASSERT_NE(nullptr, ls = handle.get_ls());
}
void get_memtable(const ObTabletID tablet_id,
ObTableHandleV2 &handle)
{
ObLS *ls = NULL;
get_ls(1002, ls);
ObTabletHandle tablet_handle;
ObTablet *tablet = nullptr;
ASSERT_EQ(OB_SUCCESS, ls->get_tablet_svr()->get_tablet(tablet_id, tablet_handle));
tablet = tablet_handle.get_obj();
ASSERT_EQ(OB_SUCCESS, tablet->get_active_memtable(handle));
}
private:
};
TEST_F(ObCallbackReverseTest, callback_reverse_test)
{
ObSqlString sql;
int64_t affected_rows = 0;
// ============================== Phase1. create tenant and table ==============================
TRANS_LOG(INFO, "create tenant start");
uint64_t tenant_id = 0;
create_test_tenant(tenant_id);
TRANS_LOG(INFO, "create tenant end");
share::ObTenantSwitchGuard tenant_guard;
ASSERT_EQ(OB_SUCCESS, tenant_guard.switch_to(tenant_id));
TRANS_LOG(INFO, "create table start");
common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2();
EXE_SQL("create table qcc (a int primary key)");
usleep(10 * 1000 * 1000);
TRANS_LOG(INFO, "create_table end");
prepare_tenant_env();
std::thread t1(
[this]() {
ObSqlString sql;
int64_t affected_rows = 0;
common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2();
sqlclient::ObISQLConnection *connection = nullptr;
ASSERT_EQ(OB_SUCCESS, sql_proxy.acquire(connection));
ASSERT_NE(nullptr, connection);
WRITE_SQL_BY_CONN(connection, "set SESSION ob_trx_timeout = 10000000000");
WRITE_SQL_BY_CONN(connection, "set SESSION ob_trx_idle_timeout = 10000000000");
WRITE_SQL_BY_CONN(connection, "set SESSION ob_query_timeout = 10000000000");
WRITE_SQL_BY_CONN(connection, "set SESSION ob_trx_lock_timeout = 0");
TRANS_LOG(INFO, "insert data start1");
WRITE_SQL_BY_CONN(connection, "begin;");
WRITE_SQL_FMT_BY_CONN(connection, "insert into qcc values(1);");
WRITE_SQL_BY_CONN(connection, "commit;");
TRANS_LOG(INFO, "insert data end1");
});
std::thread t2(
[this]() {
ObSqlString sql;
int64_t affected_rows = 0;
common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2();
sqlclient::ObISQLConnection *connection = nullptr;
ASSERT_EQ(OB_SUCCESS, sql_proxy.acquire(connection));
ASSERT_NE(nullptr, connection);
WRITE_SQL_BY_CONN(connection, "set SESSION ob_trx_timeout = 10000000000");
WRITE_SQL_BY_CONN(connection, "set SESSION ob_trx_idle_timeout = 10000000000");
WRITE_SQL_BY_CONN(connection, "set SESSION ob_query_timeout = 10000000000");
WRITE_SQL_BY_CONN(connection, "set SESSION ob_trx_lock_timeout = 0");
TRANS_LOG(INFO, "insert data start2");
WRITE_SQL_BY_CONN(connection, "begin;");
WRITE_SQL_FMT_BY_CONN(connection, "insert into qcc values(1);");
WRITE_SQL_BY_CONN(connection, "commit;");
TRANS_LOG(INFO, "insert data end2");
});
std::thread t3(
[this]() {
while (qcc == 0) {
TRANS_LOG(INFO, "qcc is not increased", K(qcc));
usleep(100 * 1000);
}
minor_freeze_data();
});
t1.join();
t2.join();
t3.join();
ASSERT_EQ(1, qcc);
}
} // namespace unittest
} // namespace oceanbase
int main(int argc, char **argv)
{
using namespace oceanbase::unittest;
oceanbase::unittest::init_log_and_gtest(argc, argv);
OB_LOGGER.set_log_level("info");
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

12
src/libtable/test/ob_batch_execute_test.cpp
View File

@ -9903,7 +9903,17 @@ TEST_F(TestBatchExecute, table_query_with_filter)
ASSERT_EQ(OB_SUCCESS, query.set_scan_index(ObString::make_string("primary")));
ASSERT_EQ(OB_SUCCESS, query.set_filter(ObString::make_string("TableCompareFilter(<, 'C2:50')")));
int ret = the_table->execute_query(query, iter);
ASSERT_NE(OB_SUCCESS, ret);
ASSERT_EQ(OB_SUCCESS, ret);
int64_t result_cnt = 0;
while (OB_SUCC(iter->get_next_entity(result_entity))) {
result_cnt++;
ObObj v1, v3;
ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C1, v1));
ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, v3));
ASSERT_LE(v1.get_int(), 50);
// fprintf(stderr, "(%ld,%ld,%s)\n", v1.get_int(), v2.get_int(), S(v3));
}
ASSERT_EQ(10, result_cnt);
// fprintf(stderr, "query ret=%d\n", ret);
} // end case 6
{

16
src/logservice/data_dictionary/ob_data_dict_service.cpp
View File

@ -415,6 +415,7 @@ int ObDataDictService::generate_dict_and_dump_(const share::SCN &snapshot_scn)
int64_t schema_version = OB_INVALID_VERSION;
ObArray<uint64_t> database_ids;
ObArray<uint64_t> table_ids;
int64_t filter_table_count = 0;
if (OB_FAIL(sql_client_.get_schema_version(tenant_id_, snapshot_scn, schema_version))) {
ret = OB_SCHEMA_EAGAIN;
@ -424,7 +425,7 @@ int ObDataDictService::generate_dict_and_dump_(const share::SCN &snapshot_scn)
DDLOG(WARN, "handle_tenant_meta_ failed", KR(ret), K(snapshot_scn));
} else if (OB_FAIL(handle_database_metas_(schema_version, database_ids))) {
DDLOG(WARN, "handle_database_metas_ failed", KR(ret), K(snapshot_scn));
} else if (OB_FAIL(handle_table_metas_(schema_version, table_ids))) {
} else if (OB_FAIL(handle_table_metas_(schema_version, table_ids, filter_table_count))) {
DDLOG(WARN, "handle_table_metas_ failed", KR(ret), K(snapshot_scn), K(schema_version));
}
@ -433,7 +434,8 @@ int ObDataDictService::generate_dict_and_dump_(const share::SCN &snapshot_scn)
K(snapshot_scn),
K(schema_version),
"database_count", database_ids.count(),
"table_count", table_ids.count());
"table_count", table_ids.count(),
K(filter_table_count));
return ret;
}
@ -602,12 +604,15 @@ int ObDataDictService::handle_database_metas_(
int ObDataDictService::handle_table_metas_(
const int64_t schema_version,
const ObIArray<uint64_t> &table_ids)
const ObIArray<uint64_t> &table_ids,
int64_t &filter_table_count)
{
int ret = OB_SUCCESS;
ObArenaAllocator tb_meta_allocator("ObDatDictTbMeta");
schema::ObSchemaGetterGuard schema_guard; // will reset while getting schem_guard
lib::ObMemAttr mem_attr(tenant_id_, "ObDatDictTbMeta");
ObArenaAllocator tb_meta_allocator(mem_attr);
static const int64_t batch_table_meta_size = 200;
filter_table_count = 0;
schema::ObSchemaGetterGuard schema_guard; // will reset while getting schem_guard
for (int i = 0; OB_SUCC(ret) && ! stop_flag_ && i < table_ids.count(); i++) {
const ObTableSchema *table_schema = NULL;
@ -633,6 +638,7 @@ int ObDataDictService::handle_table_metas_(
} else if (OB_FAIL(filter_table_(*table_schema, is_filtered))) {
DDLOG(WARN, "filter_table_ failed", KR(ret), K(is_filtered), KPC(table_schema));
} else if (is_filtered) {
filter_table_count++;
DDLOG(DEBUG, "filter_table_",
K(schema_version),
"table_id", table_schema->get_table_id(),

3
src/logservice/data_dictionary/ob_data_dict_service.h
View File

@ -104,7 +104,8 @@ private:
const ObIArray<uint64_t> &database_ids);
int handle_table_metas_(
const int64_t schema_version,
const ObIArray<uint64_t> &table_ids);
const ObIArray<uint64_t> &table_ids,
int64_t &filter_table_count);
int filter_table_(const share::schema::ObTableSchema &table_schema, bool &is_filtered);
private:
static const int64_t TIMER_TASK_INTERVAL;

3
src/logservice/libobcdc/src/ob_log_fetcher.cpp
View File

@ -132,7 +132,8 @@ int ObLogFetcher::init(
cfg.blacklist_history_overdue_time_min,
cfg.blacklist_history_clear_interval_min,
is_tenant_mode,
TCTX.tenant_id_))) {
TCTX.tenant_id_,
OB_SERVER_TENANT_ID))) {
LOG_ERROR("ObLogRouterService init failer", KR(ret), K(prefer_region), K(cluster_id));
} else if (OB_FAIL(progress_controller_.init(cfg.ls_count_upper_limit))) {
LOG_ERROR("init progress controller fail", KR(ret));

3
src/logservice/logfetcher/ob_log_fetcher.cpp
View File

@ -126,7 +126,8 @@ int ObLogFetcher::init(
cfg.blacklist_history_overdue_time_min,
cfg.blacklist_history_clear_interval_min,
true/*is_tenant_mode*/,
source_tenant_id))) {
source_tenant_id,
self_tenant_id))) {
LOG_ERROR("ObLogRouterService init failer", KR(ret), K(region), K(cluster_id), K(source_tenant_id));
} else if (OB_FAIL(progress_controller_.init(cfg.ls_count_upper_limit))) {
LOG_ERROR("init progress controller fail", KR(ret));

18
src/logservice/logrouteservice/ob_log_route_service.cpp
View File

@ -30,6 +30,7 @@ ObLogRouteService::ObLogRouteService() :
cluster_id_(OB_INVALID_CLUSTER_ID),
is_tenant_mode_(false),
source_tenant_id_(OB_INVALID_TENANT_ID),
self_tenant_id_(OB_INVALID_TENANT_ID),
is_stopped_(true),
ls_route_key_set_(),
ls_router_map_(),
@ -72,13 +73,14 @@ int ObLogRouteService::init(ObISQLClient *proxy,
const int64_t blacklist_history_overdue_time_min,
const int64_t blacklist_history_clear_interval_min,
const bool is_tenant_mode,
const uint64_t tenant_id)
const uint64_t source_tenant_id,
const uint64_t self_tenant_id)
{
int ret = OB_SUCCESS;
int tmp_ret = OB_SUCCESS;
const int64_t size = sizeof(ObLSRouterValue);
lib::ObMemAttr log_router_mem_attr(OB_SYS_TENANT_ID, "LogRouter");
lib::ObMemAttr asyn_task_mem_attr(OB_SYS_TENANT_ID, "RouterAsynTask");
lib::ObMemAttr log_router_mem_attr(self_tenant_id, "LogRouter");
lib::ObMemAttr asyn_task_mem_attr(self_tenant_id, "RouterAsynTask");
timer_.set_run_wrapper(MTL_CTX());
if (IS_INIT) {
@ -102,7 +104,7 @@ int ObLogRouteService::init(ObISQLClient *proxy,
K(external_server_blacklist));
} else if (OB_FAIL(systable_queryer_.init(cluster_id, is_across_cluster, *proxy, err_handler))) {
LOG_WARN("systable_queryer_ init failed", KR(ret), K(cluster_id), K(is_across_cluster));
} else if (OB_FAIL(all_svr_cache_.init(systable_queryer_, is_tenant_mode, tenant_id, prefer_region,
} else if (OB_FAIL(all_svr_cache_.init(systable_queryer_, is_tenant_mode, source_tenant_id, prefer_region,
all_server_cache_update_interval_sec, all_zone_cache_update_interval_sec))) {
LOG_WARN("all_svr_cache_ init failed", KR(ret), K(is_tenant_mode), K(prefer_region),
K(all_server_cache_update_interval_sec), K(all_zone_cache_update_interval_sec));
@ -114,7 +116,8 @@ int ObLogRouteService::init(ObISQLClient *proxy,
LOG_WARN("TG_SET_HANDLER_AND_START failed", KR(ret), K(tg_id_));
} else {
cluster_id_ = cluster_id;
source_tenant_id_ = tenant_id;
source_tenant_id_ = source_tenant_id;
self_tenant_id_ = self_tenant_id;
log_router_allocator_.set_nway(NWAY);
asyn_task_allocator_.set_nway(NWAY);
timer_id_ = lib::TGDefIDs::LogRouterTimer;
@ -201,9 +204,14 @@ void ObLogRouteService::destroy()
systable_queryer_.destroy();
all_svr_cache_.destroy();
svr_blacklist_.destroy();
log_router_allocator_.destroy();
asyn_task_allocator_.destroy();
err_handler_ = NULL;
cluster_id_ = OB_INVALID_CLUSTER_ID;
self_tenant_id_ = OB_INVALID_TENANT_ID;
source_tenant_id_ = OB_INVALID_TENANT_ID;
background_refresh_time_sec_ = 0;
blacklist_survival_time_sec_ = 0;

4
src/logservice/logrouteservice/ob_log_route_service.h
View File

@ -86,7 +86,8 @@ public:
const int64_t blacklist_history_overdue_time_min = 30,
const int64_t blacklist_history_clear_interval_min = 20,
const bool is_tenant_mode = false,
const uint64_t tenant_id = OB_INVALID_TENANT_ID);
const uint64_t tenant_id = OB_INVALID_TENANT_ID,
const uint64_t self_tenant_id = OB_SERVER_TENANT_ID);
int start();
void stop();
void wait();
@ -375,6 +376,7 @@ private:
int64_t cluster_id_;
bool is_tenant_mode_;
int64_t source_tenant_id_;
uint64_t self_tenant_id_;
volatile bool is_stopped_ CACHE_ALIGNED;
LSRouteKeySet ls_route_key_set_;
LSRouterMap ls_router_map_;

36
src/logservice/palf/log_config_mgr.cpp
View File

@ -59,6 +59,7 @@ LogConfigMgr::LogConfigMgr()
last_wait_barrier_time_us_(OB_INVALID_TIMESTAMP),
last_wait_committed_end_lsn_(),
last_sync_meta_for_arb_election_leader_time_us_(OB_INVALID_TIMESTAMP),
forwarding_config_proposal_id_(INVALID_PROPOSAL_ID),
parent_lock_(common::ObLatchIds::PALF_CM_PARENT_LOCK),
register_time_us_(OB_INVALID_TIMESTAMP),
parent_(),
@ -172,6 +173,7 @@ void LogConfigMgr::destroy()
all_learnerlist_.reset();
paxos_member_region_map_.destroy();
last_sync_meta_for_arb_election_leader_time_us_ = OB_INVALID_TIMESTAMP;
forwarding_config_proposal_id_ = INVALID_PROPOSAL_ID;
region_ = DEFAULT_REGION_NAME;
state_ = ConfigChangeState::INIT;
reconfig_barrier_.reset();
@ -229,7 +231,6 @@ int LogConfigMgr::set_initial_member_list(const common::ObMemberList &member_lis
PALF_LOG(WARN, "LogConfigMgr not init", KR(ret));
} else if (!member_list.is_valid() ||
!arb_member.is_valid() ||
(replica_num & 1) != 0 ||
replica_num <= 0 ||
replica_num > OB_MAX_MEMBER_NUMBER ||
INVALID_PROPOSAL_ID == proposal_id ||
@ -245,6 +246,7 @@ int LogConfigMgr::set_initial_member_list(const common::ObMemberList &member_lis
if (OB_FAIL(set_initial_config_info_(config_info, proposal_id, init_config_version))) {
PALF_LOG(WARN, "set_initial_config_info failed", K(ret), K_(palf_id), K_(self), K(config_info), K(proposal_id));
} else {
forwarding_config_proposal_id_ = proposal_id;
PALF_LOG(INFO, "set_initial_member_list success", K(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(member_list), K(arb_member), K(replica_num), K(proposal_id));
}
}
@ -2459,6 +2461,38 @@ int LogConfigMgr::sync_get_committed_end_lsn_(const LogConfigChangeArgs &args,
return ret;
}
// need rlock of PalfHandleImpl
// The arb server don't support set_initial_member_list,
// so we need to forward LogConfigMeta to arb member. otherwise,
// if only 1F1A are created successfully when creating PALF group,
// A will not vote for the F because of empty config meta.
int LogConfigMgr::forward_initial_config_meta_to_arb()
{
int ret = OB_SUCCESS;
const common::ObMember &arb_member = log_ms_meta_.curr_.config_.arbitration_member_;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
} else if (INVALID_PROPOSAL_ID == forwarding_config_proposal_id_ ||
false == arb_member.is_valid()) {
// skip
} else {
common::ObMemberList forward_list;
if (forwarding_config_proposal_id_ != state_mgr_->get_proposal_id() ||
forwarding_config_proposal_id_ != log_ms_meta_.proposal_id_) {
forwarding_config_proposal_id_ = INVALID_PROPOSAL_ID;
PALF_LOG(INFO, "stop forward_initial_config_meta_to_arb", KR(ret), K_(palf_id), K_(self));
} else if (OB_FAIL(forward_list.add_member(arb_member))) {
PALF_LOG(WARN, "add_member failed", KR(ret), K_(palf_id), K_(self), K(forward_list), K(arb_member));
} else if (OB_FAIL(log_engine_->submit_change_config_meta_req(forward_list,
log_ms_meta_.proposal_id_, log_ms_meta_.prev_log_proposal_id_,
log_ms_meta_.prev_lsn_, log_ms_meta_.prev_mode_pid_, log_ms_meta_))) {
PALF_LOG(WARN, "submit_change_config_meta_req failed", KR(ret), K_(palf_id), K_(self),
K(arb_member), K_(log_ms_meta));
}
}
return ret;
}
//================================ Config Change ================================
//================================ Child ================================

2
src/logservice/palf/log_config_mgr.h
View File

@ -477,6 +477,7 @@ public:
int sync_meta_for_arb_election_leader();
void set_sync_to_degraded_learners();
bool is_sync_to_degraded_learners() const;
int forward_initial_config_meta_to_arb();
// ================ Config Change ==================
// ==================== Child ========================
virtual int register_parent();
@ -680,6 +681,7 @@ private:
mutable int64_t last_wait_barrier_time_us_;
mutable LSN last_wait_committed_end_lsn_;
int64_t last_sync_meta_for_arb_election_leader_time_us_;
int64_t forwarding_config_proposal_id_;
// ================= Config Change =================
// ==================== Child ========================
mutable common::ObSpinLock parent_lock_;

1
src/logservice/palf/log_io_task.cpp
View File

@ -696,7 +696,6 @@ int LogIOFlashbackTask::do_task_(int tg_id, IPalfHandleImplGuard &guard)
} else if (OB_FAIL(push_task_into_cb_thread_pool_(tg_id, this))) {
PALF_LOG(WARN, "push_flush_cb_to_thread_pool_ failed", K(ret));
} else {
PALF_LOG(INFO, "LogIOFlashbackTask do_task success", K(ret), K(palf_id_));
}
return ret;
}

1
src/logservice/palf/palf_handle_impl.cpp
View File

@ -2594,6 +2594,7 @@ int PalfHandleImpl::check_and_switch_state()
} else if (OB_FAIL(mode_mgr_.leader_do_loop_work())) {
PALF_LOG(WARN, "LogModeMgr::leader_do_loop_work failed", KR(ret), K_(self), K_(palf_id));
}
(void) config_mgr_.forward_initial_config_meta_to_arb();
} while (0);
if (OB_UNLIKELY(config_state_changed)) {
WLockGuard guard(lock_);

8
src/logservice/restoreservice/ob_log_restore_net_driver.cpp
View File

@ -381,13 +381,11 @@ int ObLogRestoreNetDriver::add_ls_if_needed_with_lock_(const share::ObLSID &id,
int ObLogRestoreNetDriver::init_fetcher_if_needed_(const int64_t cluster_id, const uint64_t tenant_id)
{
int ret = OB_SUCCESS;
void *buffer = NULL;
if (NULL != fetcher_) {
// fetcher already exist
} else if (OB_ISNULL(buffer = mtl_malloc(sizeof(logfetcher::ObLogFetcher), "LogFetcher"))) {
} else if (OB_ISNULL(fetcher_ = MTL_NEW(logfetcher::ObLogFetcher, "LogFetcher"))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
} else {
fetcher_ = new (buffer) logfetcher::ObLogFetcher();
const logfetcher::LogFetcherUser log_fetcher_user = logfetcher::LogFetcherUser::STANDBY;
const bool is_loading_data_dict_baseline_data = false;
const logfetcher::ClientFetchingMode fetching_mode = logfetcher::ClientFetchingMode::FETCHING_MODE_INTEGRATED;
@ -516,7 +514,7 @@ void ObLogRestoreNetDriver::destroy_fetcher_()
} else {
fetcher_->stop();
fetcher_->destroy();
mtl_free(fetcher_);
MTL_DELETE(ObLogFetcher, "LogFetcher", fetcher_);
fetcher_ = NULL;
}
}
@ -532,7 +530,7 @@ void ObLogRestoreNetDriver::destroy_fetcher_forcedly_()
CLOG_LOG(INFO, "destroy_fetcher forcedly");
fetcher_->stop();
fetcher_->destroy();
mtl_free(fetcher_);
MTL_DELETE(ObLogFetcher, "LogFetcher", fetcher_);
fetcher_ = NULL;
}

9
src/objit/include/objit/common/ob_item_type.h
View File

@ -843,6 +843,9 @@ typedef enum ObItemType
T_FUN_SYS_ST_SYMDIFFERENCE = 1729,
T_FUN_SYS_PRIV_ST_ASMVTGEOM = 1730,
T_FUN_SYS_PRIV_ST_MAKE_VALID = 1731,
T_FUN_SYS_XML_CONCAT = 1732,
T_FUN_SYS_XML_FOREST = 1733,
T_FUN_SYS_XML_EXISTSNODE = 1734,
///< @note add new oracle only function type before this line
T_FUN_SYS_TABLET_AUTOINC_NEXTVAL = 1801, // add only for heap table
@ -1246,7 +1249,6 @@ typedef enum ObItemType
T_ROLLBACK,
T_ANY_HOST_NAME,
T_USER_WITH_HOST_NAME,
T_FLUSH_PRIVILEGES,
T_INDEX_COLUMN_LIST,
T_INDEX_USING_ALGORITHM,
@ -2381,6 +2383,11 @@ typedef enum ObItemType
T_COLUMN_GROUP_DROP,
T_ALTER_COLUMN_GROUP_OPTION,
T_BLOCKING,
T_PQ_SUBQUERY,
T_FLUSH_PRIVILEGES,
T_SCHEMA_ID,
T_CANCEL_TRANSFER_PARTITION,
T_CANCEL_BALANCE_JOB,
T_MAX //Attention: add a new type before T_MAX
} ObItemType;

2
src/observer/mysql/obmp_stmt_execute.cpp
View File

@ -2544,6 +2544,7 @@ int ObMPStmtExecute::parse_param_value(ObIAllocator &allocator,
LOG_WARN("failed to parse basic param value", K(ret));
} else {
param.set_param_meta();
param.set_length(param.get_val_len());
}
}
} else if (!support_send_long_data(type)) {
@ -2640,6 +2641,7 @@ int ObMPStmtExecute::parse_param_value(ObIAllocator &allocator,
LOG_WARN("failed to parse basic param value", K(ret));
} else {
param.set_param_meta();
param.set_length(param.get_val_len());
}
}
piece->get_allocator()->free(tmp);

23
src/observer/ob_rpc_processor_simple.cpp
View File

@ -72,6 +72,7 @@
#include "sql/spm/ob_spm_controller.h"
#endif
#include "sql/plan_cache/ob_ps_cache.h"
#include "pl/pl_cache/ob_pl_cache_mgr.h"
#include "rootserver/ob_primary_ls_service.h" // for ObPrimaryLSService
#include "rootserver/ob_root_utils.h"
#include "sql/session/ob_sql_session_info.h"
@ -1348,7 +1349,27 @@ int ObFlushCacheP::process()
break;
}
case CACHE_TYPE_PL_OBJ: {
if (arg_.is_all_tenant_) {
if (arg_.is_fine_grained_) { // fine-grained plan cache evict
bool is_evict_by_schema_id = common::OB_INVALID_ID != arg_.schema_id_;
MTL_SWITCH(arg_.tenant_id_) {
ObPlanCache* plan_cache = MTL(ObPlanCache*);
if (arg_.db_ids_.count() == 0) {
if (is_evict_by_schema_id) {
ret = plan_cache->flush_pl_cache_single_cache_obj<pl::ObGetPLKVEntryBySchemaIdOp, uint64_t>(OB_INVALID_ID, arg_.schema_id_);
} else {
ret = plan_cache->flush_pl_cache_single_cache_obj<pl::ObGetPLKVEntryBySQLIDOp, ObString>(OB_INVALID_ID, arg_.sql_id_);
}
} else {
for (uint64_t i=0; i<arg_.db_ids_.count(); i++) {
if (is_evict_by_schema_id) {
ret = plan_cache->flush_pl_cache_single_cache_obj<pl::ObGetPLKVEntryBySchemaIdOp, uint64_t>(arg_.db_ids_.at(i), arg_.schema_id_);
} else {
ret = plan_cache->flush_pl_cache_single_cache_obj<pl::ObGetPLKVEntryBySQLIDOp, ObString>(arg_.db_ids_.at(i), arg_.sql_id_);
}
}
}
}
} else if (arg_.is_all_tenant_) {
common::ObArray<uint64_t> tenant_ids;
if (OB_ISNULL(GCTX.omt_)) {
ret = OB_ERR_UNEXPECTED;

12
src/observer/ob_server.cpp
View File

@ -877,6 +877,12 @@ int ObServer::start()
FLOG_INFO("success to start multi tenant");
}
if (FAILEDx(wr_service_.start())) {
LOG_ERROR("failed to start wr service", K(ret));
} else {
LOG_INFO("success to start wr service");
}
if (FAILEDx(ObServerCheckpointSlogHandler::get_instance().start())) {
LOG_ERROR("fail to start server checkpoint slog handler", KR(ret));
} else {
@ -889,12 +895,6 @@ int ObServer::start()
FLOG_INFO("success to start log pool");
}
if (FAILEDx(wr_service_.start())) {
LOG_ERROR("failed to start wr service", K(ret));
} else {
LOG_INFO("success to start wr service");
}
if (FAILEDx(try_update_hidden_sys())) {
LOG_ERROR("fail to update hidden sys tenant", KR(ret));
} else {

8
src/observer/ob_srv_deliver.cpp
View File

@ -237,6 +237,13 @@ int get_user_tenant(ObRequest &req, char *user_name_buf, char *tenant_name_buf)
return ret;
}
static void set_sql_sock_mem_pool_tenant_id(ObRequest &req, int64_t tenant_id)
{
if (req.get_nio_protocol() == ObRequest::TRANSPORT_PROTO_POC) {
obmysql::ObSqlSockSession* sess = (obmysql::ObSqlSockSession*)req.get_server_handle_context();
sess->pool_.set_tenant_id(tenant_id);
}
}
int dispatch_req(const uint64_t tenant_id, ObRequest &req, QueueThread *global_mysql_queue)
{
int ret = OB_SUCCESS;
@ -259,6 +266,7 @@ int dispatch_req(const uint64_t tenant_id, ObRequest &req, QueueThread *global_m
} else if (OB_ISNULL(mysql_queue)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("mysql_queue is NULL", K(ret), K(tenant_id));
} else if (FALSE_IT(set_sql_sock_mem_pool_tenant_id(req, tenant_id))) {
} else if (!mysql_queue->queue_.push(&req, MAX_QUEUE_LEN)) { // MAX_QUEUE_LEN = 10000;
ret = OB_QUEUE_OVERFLOW;
EVENT_INC(MYSQL_DELIVER_FAIL);

24
src/observer/omt/ob_multi_tenant.cpp
View File

@ -438,7 +438,7 @@ int ObMultiTenant::init(ObAddr myaddr,
}
if (OB_SUCC(ret) && mtl_bind_flag) {
MTL_BIND(ObTenantIOManager::mtl_init, ObTenantIOManager::mtl_destroy);
MTL_BIND2(nullptr, ObTenantIOManager::mtl_init, nullptr, nullptr, nullptr, ObTenantIOManager::mtl_destroy);
// base mtl
MTL_BIND2(mtl_new_default, storage::mds::ObTenantMdsService::mtl_init, storage::mds::ObTenantMdsService::mtl_start, storage::mds::ObTenantMdsService::mtl_stop, storage::mds::ObTenantMdsService::mtl_wait, mtl_destroy_default);
@ -522,29 +522,29 @@ int ObMultiTenant::init(ObAddr myaddr,
MTL_BIND2(mtl_new_default, ObUDRMgr::mtl_init, nullptr, ObUDRMgr::mtl_stop, nullptr, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObTenantCGReadInfoMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObDecodeResourcePool::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
MTL_BIND2(nullptr, ObPxPools::mtl_init, nullptr, ObPxPools::mtl_stop, nullptr, ObPxPools::mtl_destroy);
MTL_BIND(ObTenantDfc::mtl_init, ObTenantDfc::mtl_destroy);
MTL_BIND(init_compat_mode, nullptr);
MTL_BIND2(mtl_new_default, ObPxPools::mtl_init, nullptr, ObPxPools::mtl_stop, nullptr, ObPxPools::mtl_destroy);
MTL_BIND2(ObTenantDfc::mtl_new, ObTenantDfc::mtl_init, nullptr, nullptr, nullptr, ObTenantDfc::mtl_destroy);
MTL_BIND2(nullptr, init_compat_mode, nullptr, nullptr, nullptr, nullptr);
MTL_BIND2(ObMySQLRequestManager::mtl_new, ObMySQLRequestManager::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, ObMySQLRequestManager::mtl_destroy);
MTL_BIND2(mtl_new_default, ObTenantWeakReadService::mtl_init, mtl_start_default,
mtl_stop_default,
mtl_wait_default,
mtl_destroy_default);
//MTL_BIND(ObTransAuditRecordMgr::mtl_init, ObTransAuditRecordMgr::mtl_destroy);
MTL_BIND(ObTenantSqlMemoryManager::mtl_init, ObTenantSqlMemoryManager::mtl_destroy);
MTL_BIND(ObPlanMonitorNodeList::mtl_init, ObPlanMonitorNodeList::mtl_destroy);
//MTL_BIND2(ObTransAuditRecordMgr::mtl_init, ObTransAuditRecordMgr::mtl_destroy);
MTL_BIND2(ObTenantSqlMemoryManager::mtl_new, ObTenantSqlMemoryManager::mtl_init, nullptr, nullptr, nullptr, ObTenantSqlMemoryManager::mtl_destroy);
MTL_BIND2(mtl_new_default, ObPlanMonitorNodeList::mtl_init, nullptr, nullptr, nullptr, ObPlanMonitorNodeList::mtl_destroy);
MTL_BIND2(mtl_new_default, ObTableLoadService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObSharedMacroBlockMgr::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND(ObFLTSpanMgr::mtl_init, ObFLTSpanMgr::mtl_destroy);
MTL_BIND(common::sqlclient::ObTenantOciEnvs::mtl_init, common::sqlclient::ObTenantOciEnvs::mtl_destroy);
MTL_BIND2(mtl_new_default, ObFLTSpanMgr::mtl_init, nullptr, nullptr, nullptr, ObFLTSpanMgr::mtl_destroy);
MTL_BIND2(common::sqlclient::ObTenantOciEnvs::mtl_new, common::sqlclient::ObTenantOciEnvs::mtl_init,
nullptr, nullptr, nullptr, common::sqlclient::ObTenantOciEnvs::mtl_destroy);
MTL_BIND2(mtl_new_default, ObPlanCache::mtl_init, nullptr, ObPlanCache::mtl_stop, nullptr, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObPsCache::mtl_init, nullptr, ObPsCache::mtl_stop, nullptr, mtl_destroy_default);
MTL_BIND2(server_obj_pool_mtl_new<ObPartTransCtx>, nullptr, nullptr, nullptr, nullptr, server_obj_pool_mtl_destroy<ObPartTransCtx>);
MTL_BIND2(server_obj_pool_mtl_new<ObTableScanIterator>, nullptr, nullptr, nullptr, nullptr, server_obj_pool_mtl_destroy<ObTableScanIterator>);
MTL_BIND2(mtl_new_default, ObTenantDirectLoadMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
MTL_BIND(ObDetectManager::mtl_init, ObDetectManager::mtl_destroy);
MTL_BIND(ObTenantSQLSessionMgr::mtl_init, ObTenantSQLSessionMgr::mtl_destroy);
MTL_BIND2(ObDetectManager::mtl_new, ObDetectManager::mtl_init, nullptr, nullptr, nullptr, ObDetectManager::mtl_destroy);
MTL_BIND2(ObTenantSQLSessionMgr::mtl_new, ObTenantSQLSessionMgr::mtl_init, nullptr, nullptr, nullptr, ObTenantSQLSessionMgr::mtl_destroy);
MTL_BIND2(mtl_new_default, ObDTLIntermResultManager::mtl_init, ObDTLIntermResultManager::mtl_start,
ObDTLIntermResultManager::mtl_stop, ObDTLIntermResultManager::mtl_wait, ObDTLIntermResultManager::mtl_destroy);
if (GCONF._enable_new_sql_nio && GCONF._enable_tenant_sql_net_thread) {

5
src/observer/omt/ob_tenant.h
View File

@ -138,10 +138,7 @@ public:
{
int ret = common::OB_SUCCESS;
uint64_t tenant_id = MTL_ID();
pools = OB_NEW(ObPxPools, ObMemAttr(tenant_id, "PxPools"));
if (OB_ISNULL(pools)) {
ret = common::OB_ALLOCATE_MEMORY_FAILED;
} else if (OB_FAIL(pools->init(tenant_id))) {
if (OB_FAIL(pools->init(tenant_id))) {
}
return ret;
}

2
src/observer/omt/ob_th_worker.cpp
View File

@ -224,7 +224,7 @@ ObThWorker::Status ObThWorker::check_wait()
} else if (curr_time > last_check_time_ + WORKER_CHECK_PERIOD) {
st = check_throttle();
if (st != WS_OUT_OF_THROTTLE) {
if (OB_UNLIKELY(curr_time > get_query_start_time() + threshold)) {
if (OB_UNLIKELY(0 != threshold && curr_time > get_query_start_time() + threshold)) {
tenant_->lq_yield(*this);
}
}

1
src/observer/table/ob_table_cg_service.cpp
View File

@ -2004,6 +2004,7 @@ int ObTableDmlCgService::generate_das_base_ctdef(uint64_t index_tid,
base_ctdef.index_tid_ = index_tid;
base_ctdef.is_ignore_ = false; // insert ignore
base_ctdef.is_batch_stmt_ = false;
base_ctdef.is_table_api_ = true;
int64_t binlog_row_image = share::ObBinlogRowImage::FULL;
ObSQLSessionInfo &session = ctx.get_session_info();

6
src/observer/table/ob_table_context.cpp
View File

@ -545,7 +545,7 @@ int ObTableCtx::adjust_column_type(const ObExprResType &column_type,
if (!is_autoincrement) {
ret = OB_BAD_NULL_ERROR;
}
} else if (obj.is_null()) {
} else if (obj.is_null() || is_inc()) {
// continue
} else if (column_type.get_type() != obj.get_type()
&& !(ob_is_string_type(column_type.get_type()) && ob_is_string_type(obj.get_type()))) {
@ -876,7 +876,9 @@ int ObTableCtx::init_scan(const ObTableQuery &query,
int ret = OB_SUCCESS;
const ObString &index_name = query.get_index_name();
const ObIArray<ObString> &select_columns = query.get_select_columns();
const bool select_all_columns = select_columns.empty() || query.is_aggregate_query() || is_ttl_table_;
bool has_filter = (query.get_htable_filter().is_valid() || query.get_filter_string().length() > 0);
const bool select_all_columns = select_columns.empty() || query.is_aggregate_query() || is_ttl_table_
|| (has_filter && !is_htable());
const ObColumnSchemaV2 *column_schema = nullptr;
operation_type_ = ObTableOperationType::Type::SCAN;
// init is_weak_read_,scan_order_

8
src/observer/table/ob_table_context.h
View File

@ -247,6 +247,10 @@ public:
return ObTableOperationType::Type::APPEND == operation_type_
|| ObTableOperationType::Type::INCREMENT == operation_type_;
}
OB_INLINE bool is_inc() const
{
return ObTableOperationType::Type::INCREMENT == operation_type_;
}
OB_INLINE bool is_dml() const
{
return ObTableOperationType::Type::GET != operation_type_ && !is_scan_;
@ -556,7 +560,6 @@ public:
column_infos_(alloc),
alloc_(alloc)
{
das_ctdef_.is_table_api_ = true;
}
TO_STRING_KV(K_(das_ctdef),
K_(related_ctdefs));
@ -596,7 +599,6 @@ public:
related_ins_ctdefs_(alloc),
alloc_(alloc)
{
das_ctdef_.is_table_api_ = true;
}
TO_STRING_KV(K_(full_row),
K_(delta_row),
@ -653,7 +655,6 @@ public:
related_ctdefs_(alloc),
alloc_(alloc)
{
das_ctdef_.is_table_api_ = true;
}
TO_STRING_KV(K_(das_ctdef),
K_(related_ctdefs));
@ -745,7 +746,6 @@ public:
das_ctdef_(alloc),
alloc_(alloc)
{
das_ctdef_.is_table_api_ = true;
}
TO_STRING_KV(K_(das_ctdef));
ObDASLockCtDef das_ctdef_;

1
src/observer/table/ob_table_execute_processor.cpp
View File

@ -114,6 +114,7 @@ int ObTableApiExecuteP::init_tb_ctx()
ObTableOperationType::Type op_type = arg_.table_operation_.type();
tb_ctx_.set_entity(&arg_.table_operation_.entity());
tb_ctx_.set_operation_type(op_type);
tb_ctx_.set_entity_type(arg_.entity_type_);
if (tb_ctx_.is_init()) {
LOG_INFO("tb ctx has been inited", K_(tb_ctx));

49
src/observer/table/ob_table_filter.cpp
View File

@ -350,6 +350,49 @@ int ObTableFilterOperator::check_limit_param()
return ret;
}
int ObTableFilterOperator::init_full_column_name(const ObIArray<ObString>& col_arr)
{
int ret = OB_SUCCESS;
bool is_select_column_empty = query_->get_select_columns().empty(); // query select column is empty when do queryAndMutate
if (is_aggregate_query()) {
// do nothing
} else if (OB_FAIL(full_column_name_.assign(col_arr))) {
LOG_WARN("fail to assign full column name", K(ret));
} else if (!is_select_column_empty && OB_FAIL(one_result_->assign_property_names(query_->get_select_columns()))) { // normal query should reset select column
LOG_WARN("fail to assign query column name", K(ret));
}
return ret;
}
int ObTableFilterOperator::add_row(table::ObTableQueryResult *next_result, ObNewRow *row)
{
int ret = OB_SUCCESS;
ObNewRow new_row;
const ObIArray<ObString> &select_columns = query_->get_select_columns();
if (!select_columns.empty()) {
size_t new_size = select_columns.count();
size_t old_size = full_column_name_.count();
ObObj cell_arr[new_size];
new_row.assign(cell_arr, new_size);
for (size_t i = 0; i < old_size; i ++) {
int64_t idx = -1;
if (!has_exist_in_array(select_columns, full_column_name_.at(i), &idx)) {
// do nothing
} else {
cell_arr[idx] = row->get_cell(i);
}
}
if (OB_FAIL(next_result->add_row(new_row))) {
LOG_WARN("failed to add row", K(ret));
}
} else { // query select column is empty when do queryAndMutate
if (OB_FAIL(next_result->add_row(*row))) {
LOG_WARN("failed to add row", K(ret));
}
}
return ret;
}
int ObTableFilterOperator::get_next_result(ObTableQueryResult *&next_result)
{
int ret = OB_SUCCESS;
@ -453,7 +496,7 @@ int ObTableFilterOperator::get_normal_result(table::ObTableQueryResult *&next_re
if (OB_SUCC(ret)) {
if (NULL != last_row_) {
if (OB_FAIL(one_result_->add_row(*last_row_))) {
if (OB_FAIL(add_row(one_result_, last_row_))) {
LOG_WARN("failed to add row", K(ret));
} else {
row_idx_++;
@ -469,7 +512,7 @@ int ObTableFilterOperator::get_normal_result(table::ObTableQueryResult *&next_re
bool has_reach_limit = (row_idx_ >= offset + limit);
next_result = one_result_;
ObNewRow *row = nullptr;
const ObIArray<ObString> &select_columns = one_result_->get_select_columns();
const ObIArray<ObString> &select_columns = full_column_name_;
const int64_t N = select_columns.count();
while (OB_SUCC(ret) && (!has_limit || !has_reach_limit) &&
@ -490,7 +533,7 @@ int ObTableFilterOperator::get_normal_result(table::ObTableQueryResult *&next_re
if (has_limit && row_idx_ < offset) {
row_idx_++;
} else if (OB_FAIL(one_result_->add_row(*row))) {
} else if (OB_FAIL(add_row(one_result_, row))) {
if (OB_BUF_NOT_ENOUGH == ret) {
ret = OB_SUCCESS;
last_row_ = row;

10
src/observer/table/ob_table_filter.h
View File

@ -207,9 +207,12 @@ public:
is_first_result_(true),
has_more_rows_(true),
row_idx_(0),
agg_calculator_(query)
agg_calculator_(query),
full_column_name_()
{}
virtual ~ObTableFilterOperator() {}
virtual ~ObTableFilterOperator() {
full_column_name_.reset();
}
virtual int get_next_result(ObTableQueryResult *&next_result) override;
virtual bool has_more_result() const override { return has_more_rows_; }
virtual void set_one_result(ObTableQueryResult *result) override { one_result_ = result; }
@ -220,6 +223,8 @@ public:
int get_aggregate_result(table::ObTableQueryResult *&next_result);
int get_normal_result(table::ObTableQueryResult *&next_result);
bool is_aggregate_query() { return agg_calculator_.is_exist(); }
int add_row(table::ObTableQueryResult *next_result, ObNewRow *row);
int init_full_column_name(const ObIArray<ObString>& col_arr);
private:
int check_limit_param();
private:
@ -234,6 +239,7 @@ private:
bool has_more_rows_;
int64_t row_idx_; // not filtered row index
ObTableAggCalculator agg_calculator_;
ObSEArray<ObString, 64> full_column_name_;
};
} // end namespace table

1
src/observer/table/ob_table_query_and_mutate_processor.cpp
View File

@ -220,6 +220,7 @@ int ObTableQueryAndMutateP::init_scan_tb_ctx(ObTableApiCacheGuard &cache_guard)
const ObTableQuery &query = arg_.query_and_mutate_.get_query();
bool is_weak_read = false;
tb_ctx_.set_scan(true);
tb_ctx_.set_entity_type(arg_.entity_type_);
if (tb_ctx_.is_init()) {
LOG_INFO("tb ctx has been inited", K_(tb_ctx));

6
src/observer/table/ob_table_query_common.cpp
View File

@ -68,8 +68,8 @@ int ObTableQueryUtils::generate_query_result_iterator(ObIAllocator &allocator,
bool has_filter = (query.get_htable_filter().is_valid() || query.get_filter_string().length() > 0);
const ObString &kv_attributes = tb_ctx.get_table_schema()->get_kv_attributes();
if (OB_FAIL(one_result.assign_property_names(tb_ctx.get_query_col_names()))) {
LOG_WARN("fail to assign property names to one result", K(ret), K(tb_ctx));
if (OB_FAIL(one_result.deep_copy_property_names(tb_ctx.get_query_col_names()))) {
LOG_WARN("fail to deep copy property names to one result", K(ret), K(tb_ctx));
} else if (has_filter) {
if (is_hkv) {
ObHTableFilterOperator *htable_result_iter = nullptr;
@ -105,6 +105,8 @@ int ObTableQueryUtils::generate_query_result_iterator(ObIAllocator &allocator,
one_result))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to alloc table query result iterator", K(ret));
} else if (OB_FAIL(table_result_iter->init_full_column_name(tb_ctx.get_query_col_names()))) {
LOG_WARN("fail to int full column name", K(ret));
} else if (OB_FAIL(table_result_iter->parse_filter_string(&allocator))) {
LOG_WARN("fail to parse table filter string", K(ret));
} else {

1
src/observer/table/ob_table_query_processor.cpp
View File

@ -127,6 +127,7 @@ int ObTableQueryP::init_tb_ctx(ObTableApiCacheGuard &cache_guard)
ObExprFrameInfo *expr_frame_info = nullptr;
bool is_weak_read = arg_.consistency_level_ == ObTableConsistencyLevel::EVENTUAL;
tb_ctx_.set_scan(true);
tb_ctx_.set_entity_type(arg_.entity_type_);
if (tb_ctx_.is_init()) {
LOG_INFO("tb ctx has been inited", K_(tb_ctx));

3
src/observer/table/ob_table_query_sync_processor.cpp
View File

@ -395,6 +395,7 @@ int ObTableQuerySyncP::init_tb_ctx(ObTableCtx &ctx)
ObExprFrameInfo &expr_frame_info = query_ctx.expr_frame_info_;
bool is_weak_read = arg_.consistency_level_ == ObTableConsistencyLevel::EVENTUAL;
ctx.set_scan(true);
ctx.set_entity_type(arg_.entity_type_);
if (ctx.is_init()) {
LOG_INFO("tb ctx has been inited", K(ctx));
@ -541,7 +542,7 @@ int ObTableQuerySyncP::query_scan_without_init()
if (OB_ISNULL(result_iter)) {
ret = OB_ERR_NULL_VALUE;
LOG_WARN("unexpected null result iterator", K(ret));
} else if (OB_FAIL(result_.assign_property_names(tb_ctx.get_query_col_names()))) {
} else if (OB_FAIL(result_.deep_copy_property_names(tb_ctx.get_query_col_names()))) {
LOG_WARN("fail to assign property names to one result", K(ret), K(tb_ctx));
} else {
ObTableQueryResult *query_result = nullptr;

1
src/observer/table_load/client/ob_table_direct_load_rpc_executor.cpp
View File

@ -221,6 +221,7 @@ int ObTableDirectLoadBeginExecutor::create_table_ctx()
start_arg.tenant_id_ = tenant_id;
start_arg.table_id_ = table_id;
start_arg.parallelism_ = arg_.parallel_;
start_arg.is_load_data_ = true;
if (OB_FAIL(ObTableLoadRedefTable::start(start_arg, start_res,
*client_task_->get_session_info()))) {
LOG_WARN("fail to start redef table", KR(ret), K(start_arg));

9
src/observer/table_load/ob_table_load_mem_compactor.cpp
View File

@ -575,9 +575,6 @@ int ObTableLoadMemCompactor::finish()
LOG_WARN("fail to start parallel merge", KR(ret));
}
}
if (OB_SUCC(ret)) {
mem_ctx_.reset(); // mem_ctx的tables已经copy，需要提前释放
}
return ret;
}
@ -608,6 +605,9 @@ int ObTableLoadMemCompactor::build_result_for_heap_table()
}
}
}
if (OB_SUCC(ret)) {
mem_ctx_.reset();
}
return ret;
}
@ -625,6 +625,9 @@ int ObTableLoadMemCompactor::add_table_to_parallel_merge_ctx()
LOG_WARN("fail to add tablet sstable", KR(ret));
}
}
if (OB_SUCC(ret)) {
mem_ctx_.reset(); // mem_ctx的tables已经copy，需要提前释放
}
return ret;
}

5
src/observer/table_load/ob_table_load_merger.cpp
View File

@ -656,6 +656,11 @@ int ObTableLoadMerger::handle_merge_thread_finish(int ret_code)
if (OB_UNLIKELY(is_stop_ || has_error_)) {
} else {
LOG_INFO("LOAD MERGE COMPLETED");
// release tmpfile
// TODO(suzhi.yt) release all tables and merge tasks
if (!store_ctx_->is_fast_heap_table_) {
table_compact_ctx_.result_.release_all_table_data();
}
if (store_ctx_->ctx_->schema_.is_column_store_) {
if (OB_FAIL(build_rescan_ctx())) {
LOG_WARN("fail to build rescan ctx", KR(ret));

6
src/observer/table_load/ob_table_load_multiple_heap_table_compactor.cpp
View File

@ -496,9 +496,6 @@ int ObTableLoadMultipleHeapTableCompactor::finish()
} else if (OB_FAIL(compact_ctx_->handle_table_compact_success())) {
LOG_WARN("fail to handle_table_compact_success", KR(ret));
}
if (OB_SUCC(ret)) {
mem_ctx_.reset(); // mem_ctx的tables已经copy，需要提前释放
}
return ret;
}
@ -529,6 +526,9 @@ int ObTableLoadMultipleHeapTableCompactor::build_result_for_heap_table()
}
}
}
if (OB_SUCC(ret)) {
mem_ctx_.reset(); // mem_ctx的tables已经copy，需要提前释放
}
return ret;
}

31
src/observer/table_load/ob_table_load_service.cpp
View File

@ -407,17 +407,22 @@ int ObTableLoadService::check_support_direct_load(uint64_t table_id)
ObTableLoadSchema::get_table_schema(tenant_id, table_id, schema_guard, table_schema))) {
LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(table_id));
}
// check if it is an oracle temporary table
else if (lib::is_oracle_mode() && table_schema->is_tmp_table()) {
// check if it is a user table
else if (!table_schema->is_user_table()) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("direct-load does not support oracle temporary table", KR(ret));
FORWARD_USER_ERROR_MSG(ret, "direct-load does not support oracle temporary table");
}
// check if it is a view
else if (table_schema->is_view_table()) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("direct-load does not support view table", KR(ret));
FORWARD_USER_ERROR_MSG(ret, "direct-load does not support view table");
if (lib::is_oracle_mode() && table_schema->is_tmp_table()) {
LOG_WARN("direct-load does not support oracle temporary table", KR(ret));
FORWARD_USER_ERROR_MSG(ret, "direct-load does not support oracle temporary table");
} else if (table_schema->is_view_table()) {
LOG_WARN("direct-load does not support view table", KR(ret));
FORWARD_USER_ERROR_MSG(ret, "direct-load does not support view table");
} else if (table_schema->is_mlog_table()) {
LOG_WARN("direct-load does not support materialized view log table", KR(ret));
FORWARD_USER_ERROR_MSG(ret, "direct-load does not support materialized view log table");
} else {
LOG_WARN("direct-load does not support non-user table", KR(ret));
FORWARD_USER_ERROR_MSG(ret, "direct-load does not support non-user table");
}
}
// check if exists generated column
else if (OB_UNLIKELY(table_schema->has_generated_column())) {
@ -441,6 +446,12 @@ int ObTableLoadService::check_support_direct_load(uint64_t table_id)
LOG_WARN("direct-load does not support table has udt column", KR(ret));
FORWARD_USER_ERROR_MSG(ret, "direct-load does not support table has udt column");
}
// check if table has mlog
else if (table_schema->has_mlog_table()) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("direct-load does not support table with materialized view log", KR(ret));
FORWARD_USER_ERROR_MSG(ret, "direct-load does not support table with materialized view log");
}
}
return ret;
}

23
src/observer/table_load/ob_table_load_store.cpp
View File

@ -961,6 +961,29 @@ int ObTableLoadStore::px_finish_trans(const ObTableLoadTransId &trans_id)
return ret;
}
int ObTableLoadStore::px_check_for_write(const ObTabletID &tablet_id)
{
int ret = OB_SUCCESS;
if (IS_NOT_INIT) {
ret = OB_NOT_INIT;
LOG_WARN("ObTableLoadStore not init", KR(ret), KP(this));
} else {
bool is_exist = false;
for (int64_t i = 0; i < store_ctx_->ls_partition_ids_.count(); ++i) {
const ObTableLoadLSIdAndPartitionId &ls_part_id = store_ctx_->ls_partition_ids_.at(i);
if (ls_part_id.part_tablet_id_.tablet_id_ == tablet_id) {
is_exist = true;
break;
}
}
if (OB_UNLIKELY(!is_exist)) {
ret = OB_NOT_MASTER;
LOG_WARN("not partition master", KR(ret), K(tablet_id), K(store_ctx_->ls_partition_ids_));
}
}
return ret;
}
int ObTableLoadStore::px_write(const ObTableLoadTransId &trans_id,
const ObTabletID &tablet_id, const ObIArray<ObNewRow> &row_array)
{

1
src/observer/table_load/ob_table_load_store.h
View File

@ -85,6 +85,7 @@ private:
public:
int px_start_trans(const table::ObTableLoadTransId &trans_id);
int px_finish_trans(const table::ObTableLoadTransId &trans_id);
int px_check_for_write(const ObTabletID &tablet_id);
int px_write(const table::ObTableLoadTransId &trans_id,
const ObTabletID &tablet_id,
const common::ObIArray<common::ObNewRow> &row_array);

77
src/observer/table_load/ob_table_load_table_compactor.cpp
View File

@ -97,22 +97,26 @@ int ObTableLoadTableCompactResult::add_table(ObIDirectLoadPartitionTable *table)
return ret;
}
void ObTableLoadTableCompactResult::release_all_table_data()
{
for (int64_t i = 0; i < all_table_array_.count(); ++i) {
ObIDirectLoadPartitionTable *table = all_table_array_.at(i);
table->release_data();
}
}
/**
* ObTableLoadTableCompactCtx
*/
ObTableLoadTableCompactCtx::ObTableLoadTableCompactCtx()
: allocator_("TLD_TCCtx"), store_ctx_(nullptr), merger_(nullptr), compactor_(nullptr)
: store_ctx_(nullptr), merger_(nullptr), compactor_(nullptr)
{
}
ObTableLoadTableCompactCtx::~ObTableLoadTableCompactCtx()
{
if (nullptr != compactor_) {
compactor_->~ObTableLoadTableCompactor();
allocator_.free(compactor_);
compactor_ = nullptr;
}
release_compactor();
}
int ObTableLoadTableCompactCtx::init(ObTableLoadStoreCtx *store_ctx, ObTableLoadMerger &merger)
@ -125,7 +129,6 @@ int ObTableLoadTableCompactCtx::init(ObTableLoadStoreCtx *store_ctx, ObTableLoad
if (OB_FAIL(result_.init())) {
LOG_WARN("fail to init result", KR(ret));
} else {
allocator_.set_tenant_id(MTL_ID());
store_ctx_ = store_ctx;
merger_ = &merger;
}
@ -138,37 +141,61 @@ bool ObTableLoadTableCompactCtx::is_valid() const
return nullptr != store_ctx_ && nullptr != merger_;
}
ObTableLoadTableCompactor *ObTableLoadTableCompactCtx::new_compactor()
int ObTableLoadTableCompactCtx::new_compactor()
{
ObTableLoadTableCompactor *ret = nullptr;
if (store_ctx_->is_multiple_mode_) {
if (store_ctx_->table_data_desc_.is_heap_table_) {
ret = OB_NEWx(ObTableLoadMultipleHeapTableCompactor, (&allocator_));
} else {
ret = OB_NEWx(ObTableLoadMemCompactor, (&allocator_));
}
int ret = OB_SUCCESS;
obsys::ObWLockGuard guard(rwlock_);
if (OB_NOT_NULL(compactor_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected not null compactor", KR(ret), KP(compactor_));
} else {
ret = OB_NEWx(ObTableLoadGeneralTableCompactor, (&allocator_));
ObMemAttr attr(MTL_ID(), "TLD_Compactor");
if (store_ctx_->is_multiple_mode_) {
if (store_ctx_->table_data_desc_.is_heap_table_) {
compactor_ = OB_NEW(ObTableLoadMultipleHeapTableCompactor, attr);
} else {
compactor_ = OB_NEW(ObTableLoadMemCompactor, attr);
}
} else {
compactor_ = OB_NEW(ObTableLoadGeneralTableCompactor, attr);
}
if (OB_ISNULL(compactor_)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObTableLoadTableCompactor", KR(ret));
}
}
return ret;
}
void ObTableLoadTableCompactCtx::release_compactor()
{
obsys::ObWLockGuard guard(rwlock_);
if (nullptr != compactor_) {
ObMemAttr attr(MTL_ID(), "TLD_Compactor");
OB_DELETE(ObTableLoadTableCompactor, attr, compactor_);
compactor_ = nullptr;
}
}
int ObTableLoadTableCompactCtx::start()
{
int ret = OB_SUCCESS;
if (OB_ISNULL(compactor_ = new_compactor())) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("fail to new ObTableLoadGeneralTableCompactor", KR(ret));
} else if (OB_FAIL(compactor_->init(this))) {
LOG_WARN("fail to init compactor", KR(ret));
} else if (OB_FAIL(compactor_->start())) {
LOG_WARN("fail to start compactor", KR(ret));
if (OB_FAIL(new_compactor())) {
LOG_WARN("fail to new compactor", KR(ret));
} else {
obsys::ObRLockGuard guard(rwlock_);
if (OB_FAIL(compactor_->init(this))) {
LOG_WARN("fail to init compactor", KR(ret));
} else if (OB_FAIL(compactor_->start())) {
LOG_WARN("fail to start compactor", KR(ret));
}
}
return ret;
}
void ObTableLoadTableCompactCtx::stop()
{
obsys::ObRLockGuard guard(rwlock_);
if (OB_NOT_NULL(compactor_)) {
compactor_->stop();
}
@ -177,9 +204,7 @@ void ObTableLoadTableCompactCtx::stop()
int ObTableLoadTableCompactCtx::handle_table_compact_success()
{
// release compactor
compactor_->~ObTableLoadTableCompactor();
allocator_.free(compactor_);
compactor_ = nullptr;
release_compactor();
// notify merger
return merger_->handle_table_compact_success();
}

6
src/observer/table_load/ob_table_load_table_compactor.h
View File

@ -39,6 +39,7 @@ public:
void reset();
int init();
int add_table(storage::ObIDirectLoadPartitionTable *table);
void release_all_table_data();
public:
typedef common::ObLinkHashMap<common::ObTabletID, ObTableLoadTableCompactTabletResult>
TabletResultMap;
@ -59,12 +60,13 @@ public:
int handle_table_compact_success();
TO_STRING_KV(KP_(store_ctx), KP_(merger), KP_(compactor));
private:
ObTableLoadTableCompactor *new_compactor();
int new_compactor();
void release_compactor();
public:
common::ObArenaAllocator allocator_;
ObTableLoadStoreCtx *store_ctx_;
ObTableLoadMerger *merger_;
mutable obsys::ObRWLock rwlock_;
ObTableLoadTableCompactor *compactor_;
ObTableLoadTableCompactResult result_;
};

29
src/observer/virtual_table/ob_all_virtual_tenant_snapshot_ls_replica.cpp
View File

@ -110,13 +110,14 @@ int ObAllVirtualTenantSnapshotLSReplica::process_curr_tenant(ObNewRow *&row)
SERVER_LOG(WARN, "row is null or column count mismatch", KR(ret), KP(r), K(cur_row_.count_));
} else {
ObString ls_meta_package_str;
HEAP_VAR(ObLSMetaPackage, ls_meta_package) {
for (int64_t i = 0; OB_SUCC(ret) && i < cur_row_.count_; ++i) {
uint64_t col_id = output_column_ids_.at(i);
if (LS_META_PACKAGE == col_id) { // decode ls_meta_package column
int64_t length = 0;
EXTRACT_VARCHAR_FIELD_MYSQL(*result_, "ls_meta_package", ls_meta_package_str);
if (ls_meta_package_str.empty()) {
for (int64_t i = 0; OB_SUCC(ret) && i < cur_row_.count_; ++i) {
uint64_t col_id = output_column_ids_.at(i);
if (LS_META_PACKAGE == col_id) { // decode ls_meta_package column
int64_t length = 0;
HEAP_VAR(ObLSMetaPackage, ls_meta_package) {
EXTRACT_VARCHAR_FIELD_MYSQL_SKIP_RET(*result_, "ls_meta_package", ls_meta_package_str);
if (OB_FAIL(ret)) {
} else if (ls_meta_package_str.empty()) {
cur_row_.cells_[i].reset();
} else if (OB_ISNULL(ls_meta_buf_)) {
ret = OB_ERR_UNEXPECTED;
@ -128,17 +129,17 @@ int ObAllVirtualTenantSnapshotLSReplica::process_curr_tenant(ObNewRow *&row)
SERVER_LOG(WARN, "fail to get ls_meta_package string", KR(ret), K(length));
} else {
cur_row_.cells_[i].set_lob_value(ObObjType::ObLongTextType, ls_meta_buf_,
static_cast<int32_t>(length));
static_cast<int32_t>(length));
cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation(
ObCharset::get_default_charset()));
}
} else if (col_id - OB_APP_MIN_COLUMN_ID >= 0 && col_id - OB_APP_MIN_COLUMN_ID < r->count_) {
// direct copy other columns
cur_row_.cells_[i] = r->get_cell(col_id - OB_APP_MIN_COLUMN_ID);
} else {
ret = OB_ERR_UNEXPECTED;
SERVER_LOG(WARN, "unexpected column id", KR(ret), K(col_id), K(output_column_ids_));
}
} else if (col_id - OB_APP_MIN_COLUMN_ID >= 0 && col_id - OB_APP_MIN_COLUMN_ID < r->count_) {
// direct copy other columns
cur_row_.cells_[i] = r->get_cell(col_id - OB_APP_MIN_COLUMN_ID);
} else {
ret = OB_ERR_UNEXPECTED;
SERVER_LOG(WARN, "unexpected column id", KR(ret), K(col_id), K(output_column_ids_));
}
}
}

29
src/observer/virtual_table/ob_all_virtual_tenant_snapshot_ls_replica_history.cpp
View File

@ -110,13 +110,14 @@ int ObAllVirtualTenantSnapshotLSReplicaHistory::process_curr_tenant(ObNewRow *&r
SERVER_LOG(WARN, "row is null or column count mismatch", KR(ret), KP(r), K(cur_row_.count_));
} else {
ObString ls_meta_package_str;
HEAP_VAR(ObLSMetaPackage, ls_meta_package) {
for (int64_t i = 0; OB_SUCC(ret) && i < cur_row_.count_; ++i) {
uint64_t col_id = output_column_ids_.at(i);
if (LS_META_PACKAGE == col_id) { // decode ls_meta_package column
int64_t length = 0;
EXTRACT_VARCHAR_FIELD_MYSQL(*result_, "ls_meta_package", ls_meta_package_str);
if (ls_meta_package_str.empty()) {
for (int64_t i = 0; OB_SUCC(ret) && i < cur_row_.count_; ++i) {
uint64_t col_id = output_column_ids_.at(i);
if (LS_META_PACKAGE == col_id) { // decode ls_meta_package column
int64_t length = 0;
HEAP_VAR(ObLSMetaPackage, ls_meta_package) {
EXTRACT_VARCHAR_FIELD_MYSQL_SKIP_RET(*result_, "ls_meta_package", ls_meta_package_str);
if (OB_FAIL(ret)) {
} else if (ls_meta_package_str.empty()) {
cur_row_.cells_[i].reset();
} else if (OB_ISNULL(ls_meta_buf_)) {
ret = OB_ERR_UNEXPECTED;
@ -128,17 +129,17 @@ int ObAllVirtualTenantSnapshotLSReplicaHistory::process_curr_tenant(ObNewRow *&r
SERVER_LOG(WARN, "fail to get ls_meta_package string", KR(ret), K(length));
} else {
cur_row_.cells_[i].set_lob_value(ObObjType::ObLongTextType, ls_meta_buf_,
static_cast<int32_t>(length));
static_cast<int32_t>(length));
cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation(
ObCharset::get_default_charset()));
}
} else if (col_id - OB_APP_MIN_COLUMN_ID >= 0 && col_id - OB_APP_MIN_COLUMN_ID < r->count_) {
// direct copy other columns
cur_row_.cells_[i] = r->get_cell(col_id - OB_APP_MIN_COLUMN_ID);
} else {
ret = OB_ERR_UNEXPECTED;
SERVER_LOG(WARN, "unexpected column id", KR(ret), K(col_id), K(output_column_ids_));
}
} else if (col_id - OB_APP_MIN_COLUMN_ID >= 0 && col_id - OB_APP_MIN_COLUMN_ID < r->count_) {
// direct copy other columns
cur_row_.cells_[i] = r->get_cell(col_id - OB_APP_MIN_COLUMN_ID);
} else {
ret = OB_ERR_UNEXPECTED;
SERVER_LOG(WARN, "unexpected column id", KR(ret), K(col_id), K(output_column_ids_));
}
}
}

12
src/observer/virtual_table/ob_gv_sql.cpp
View File

@ -206,7 +206,17 @@ int ObGVSql::fill_cells(const ObILibCacheObject *cache_obj, const ObPlanCache &p
//sql_id
case share::ALL_VIRTUAL_PLAN_STAT_CDE::SQL_ID: {
ObString sql_id;
if (!cache_obj->is_sql_crsr()) {
if (OB_NOT_NULL(pl_func)) {
if (OB_FAIL(ob_write_string(*allocator_,
pl_func->get_stat().sql_id_,
sql_id))) {
SERVER_LOG(ERROR, "copy sql_id failed", K(ret));
} else {
cells[i].set_varchar(sql_id);
cells[i].set_collation_type(ObCharset::get_default_collation(
ObCharset::get_default_charset()));
}
} else if (!cache_obj->is_sql_crsr()) {
cells[i].set_null();
} else if (OB_FAIL(ob_write_string(*allocator_,
plan->stat_.sql_id_,

Some files were not shown because too many files have changed in this diff Show More