hcq/oceanbase
hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge branch 'master' into develop

Browse Source
This commit is contained in:
wangyunlai 2024-08-28 13:47:52 +08:00
commit 4a48d6789c
1392 changed files with 136631 additions and 53347 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.gitignore vendored
View File

@ -152,11 +152,11 @@ src/sql/parser/sql_parser_oracle_utf8_mode_lex.c
src/sql/parser/sql_parser_oracle_utf8_mode_lex.h
src/sql/parser/sql_parser_oracle_utf8_mode_tab.c
src/sql/parser/sql_parser_oracle_utf8_mode_tab.h
src/sql/parser/sql_parser_oracle_latin1_mode_lex.c
src/sql/parser/sql_parser_oracle_latin1_mode_lex.h
src/sql/parser/sql_parser_oracle_latin1_mode_tab.c
src/sql/parser/sql_parser_oracle_latin1_mode_tab.h
src/sql/parser/non_reserved_keywords_oracle_latin1_mode.c
src/sql/parser/sql_parser_oracle_single_byte_mode_lex.c
src/sql/parser/sql_parser_oracle_single_byte_mode_lex.h
src/sql/parser/sql_parser_oracle_single_byte_mode_tab.c
src/sql/parser/sql_parser_oracle_single_byte_mode_tab.h
src/sql/parser/non_reserved_keywords_oracle_single_byte_mode.c
src/sql/parser/_gen_parser.output
src/sql/parser/_gen_parser.error
src/pl/parser/pl_parser_mysql_mode_lex.c
@ -171,6 +171,7 @@ src/pl/parser/_gen_parser.error
src/pl/parser/_gen_pl_parser.output
src/pl/parser/pl_parser_mysql_mode.output
src/pl/parser/pl_parser_oracle_mode.output
src/share/inner_table/sys_package/system_package.cpp
############# close_modules #############
close_modules/oracle_pl/pl/parser/*.output

6
cmake/Env.cmake
View File

@ -117,6 +117,8 @@ if(OB_BUILD_CLOSE_MODULES)
ob_define(OB_BUILD_ORACLE_PL ON)
# dblink
ob_define(OB_BUILD_DBLINK ON)
# odps
ob_define(OB_BUILD_CPP_ODPS ON)
# 仲裁功能
ob_define(OB_BUILD_ARBITRATION ON)
@ -172,6 +174,10 @@ if(OB_BUILD_DBLINK)
add_definitions(-DOB_BUILD_DBLINK)
endif()
if(OB_BUILD_CPP_ODPS)
add_definitions(-DOB_BUILD_CPP_ODPS)
endif()
# should not use initial-exec for tls-model if building OBCDC.
if(NOT OB_BUILD_CDC)
add_definitions(-DENABLE_INITIAL_EXEC_TLS_MODEL)

1
deps/init/oceanbase.el7.aarch64.deps vendored
View File

@ -33,6 +33,7 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el7.aarch64.rpm
devdeps-protobuf-c-1.4.1-100000072023102410.el7.aarch64.rpm
devdeps-roaringbitmap-croaring-3.0.0-42024042816.el7.aarch64.rpm
devdeps-apache-arrow-9.0.0-302024052920.el7.aarch64.rpm
# devdeps-apache-orc-1.8.3-202024072510.el7.aarch64.rpm
[tools]
obdevtools-binutils-2.30-12022100413.el7.aarch64.rpm

1
deps/init/oceanbase.el7.x86_64.deps vendored
View File

@ -36,6 +36,7 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el7.x86_64.rpm
devdeps-protobuf-c-1.4.1-100000062023102016.el7.x86_64.rpm
devdeps-roaringbitmap-croaring-3.0.0-42024042816.el7.x86_64.rpm
devdeps-apache-arrow-9.0.0-222024052223.el7.x86_64.rpm
# devdeps-apache-orc-1.8.3-202024072510.el7.x86_64.rpm
[tools]
obdevtools-binutils-2.30-12022100413.el7.x86_64.rpm

1
deps/init/oceanbase.el8.aarch64.deps vendored
View File

@ -33,6 +33,7 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.aarch64.rpm
devdeps-protobuf-c-1.4.1-100000072023102410.el8.aarch64.rpm
devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.aarch64.rpm
devdeps-apache-arrow-9.0.0-322024052923.el8.aarch64.rpm
# devdeps-apache-orc-1.8.3-202024072510.el8.aarch64.rpm
[tools]
obdevtools-binutils-2.30-12022100413.el8.aarch64.rpm

2
deps/init/oceanbase.el8.x86_64.deps vendored
View File

@ -35,6 +35,8 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.x86_64.rpm
devdeps-protobuf-c-1.4.1-100000062023102016.el8.x86_64.rpm
devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.x86_64.rpm
devdeps-apache-arrow-9.0.0-172024052218.el8.x86_64.rpm
devdeps-odps-cpp-sdk-1.0.0-482024080517.el8.x86_64.rpm
# devdeps-apache-orc-1.8.3-202024072510.el8.x86_64.rpm
[tools]
obdevtools-binutils-2.30-12022100413.el8.x86_64.rpm

1
deps/init/oceanbase.el9.aarch64.deps vendored
View File

@ -37,6 +37,7 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.aarch64.rpm
devdeps-protobuf-c-1.4.1-100000072023102410.el8.aarch64.rpm
devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.aarch64.rpm
devdeps-apache-arrow-9.0.0-322024052923.el8.aarch64.rpm
# devdeps-apache-orc-1.8.3-202024072510.el8.aarch64.rpm
[deps-el9]
devdeps-apr-1.6.5-232023090616.el9.aarch64.rpm target=el9

1
deps/init/oceanbase.el9.x86_64.deps vendored
View File

@ -39,6 +39,7 @@ devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.x86_64.rpm
devdeps-protobuf-c-1.4.1-100000062023102016.el8.x86_64.rpm
devdeps-apache-arrow-9.0.0-172024052218.el8.x86_64.rpm
devdeps-roaringbitmap-croaring-3.0.0-42024042816.el8.x86_64.rpm
# devdeps-apache-orc-1.8.3-202024072510.el8.x86_64.rpm
[deps-el9]
devdeps-apr-1.6.5-232023090616.el9.x86_64.rpm target=el9

13
deps/oblib/src/CMakeLists.txt vendored
View File

@ -21,6 +21,7 @@ target_include_directories(
${DEP_DIR}/include/apr-1/
${DEP_DIR}/include/icu/common
${DEP_DIR}/include/apache-arrow
# ${DEP_DIR}/include/apache-orc
${USSL_INCLUDE_DIRS}
)
@ -208,6 +209,12 @@ target_link_libraries(oblib_base_base_base
${DEP_DIR}/lib64/libarrow.a
${DEP_DIR}/lib64/libparquet.a
${DEP_DIR}/lib64/libarrow_bundled_dependencies.a
# ${DEP_DIR}/lib64/liborc.a
# ${DEP_DIR}/lib64/libsnappy.a
# ${DEP_DIR}/lib64/libprotoc.a
# ${DEP_DIR}/lib64/libprotobuf.a
# ${DEP_DIR}/lib64/liblz4.a
# ${DEP_DIR}/lib64/libzstd.a
-L${DEP_DIR}/var/usr/lib64
-L${DEP_DIR}/var/usr/lib
-L${DEP_3RD_DIR}/usr/lib
@ -237,6 +244,12 @@ target_link_libraries(oblib_base_base_base
${DEP_DIR}/lib64/libarrow.a
${DEP_DIR}/lib64/libparquet.a
${DEP_DIR}/lib64/libarrow_bundled_dependencies.a
# ${DEP_DIR}/lib64/liborc.a
# ${DEP_DIR}/lib64/libsnappy.a
# ${DEP_DIR}/lib64/libprotoc.a
# ${DEP_DIR}/lib64/libprotobuf.a
# ${DEP_DIR}/lib64/liblz4.a
# ${DEP_DIR}/lib64/libzstd.a
-L${DEP_DIR}/var/usr/lib64
-L${DEP_DIR}/var/usr/lib
-L${DEP_3RD_DIR}/usr/lib

9
deps/oblib/src/common/ob_field.cpp vendored
View File

@ -31,10 +31,13 @@ int ObParamedSelectItemCtx::deep_copy(const ObParamedSelectItemCtx &other, ObIAl
LOG_WARN("invalid null allocator", K(ret));
} else if (OB_FAIL(ob_write_string(*allocator, other.paramed_cname_, paramed_cname_))) {
LOG_WARN("failed to write stirng", K(ret));
} else if (OB_FAIL(param_str_offsets_.assign(other.param_str_offsets_))) {
LOG_WARN("failed to deep copy param string offsets", K(ret));
} else if (OB_FAIL(param_idxs_.assign(other.param_idxs_))) {
LOG_WARN("failed to deep copy param idxs", K(ret));
} else if (OB_FAIL(neg_param_idxs_.assign(other.neg_param_idxs_))) {
LOG_WARN("failed to deep copy neg param idxs", K(ret));
} else {
param_str_offsets_ = other.param_str_offsets_;
param_idxs_ = other.param_idxs_;
neg_param_idxs_ = other.neg_param_idxs_;
esc_str_flag_ = other.esc_str_flag_;
need_check_dup_name_ = other.need_check_dup_name_;
is_column_field_ = other.is_column_field_;

103
deps/oblib/src/common/ob_member.cpp vendored
View File

@ -86,6 +86,21 @@ void ObMember::reset_migrating()
flag_ &= ~(1UL << MIGRATING_FLAG_BIT);
}
bool ObMember::is_columnstore() const
{
return (flag_ >> COLUMNSTORE_FLAG_BIT) & 1U;
}
void ObMember::set_columnstore()
{
flag_ |= (1UL << COLUMNSTORE_FLAG_BIT);
}
void ObMember::reset_columnstore()
{
flag_ &= ~(1UL << COLUMNSTORE_FLAG_BIT);
}
OB_SERIALIZE_MEMBER(ObMember, server_, timestamp_, flag_);
bool ObReplicaMember::is_readonly_replica() const
@ -93,19 +108,63 @@ bool ObReplicaMember::is_readonly_replica() const
return REPLICA_TYPE_READONLY == replica_type_;
}
int ObReplicaMember::init(
const common::ObAddr &server,
const int64_t timestamp,
const common::ObReplicaType replica_type)
{
int ret = OB_SUCCESS;
reset();
if (OB_UNLIKELY(!server.is_valid()
|| !ObReplicaTypeCheck::is_replica_type_valid(replica_type))) {
ret = OB_INVALID_ARGUMENT;
COMMON_LOG(WARN, "invalid argument", K(ret), K(server), K(replica_type));
} else {
server_ = server;
timestamp_ = timestamp;
replica_type_ = replica_type;
if (REPLICA_TYPE_COLUMNSTORE == replica_type) {
ObMember::set_columnstore();
}
}
return ret;
}
int ObReplicaMember::init(
const ObMember &member,
const common::ObReplicaType replica_type)
{
int ret = OB_SUCCESS;
reset();
if (OB_UNLIKELY(!member.is_valid()
|| !ObReplicaTypeCheck::is_replica_type_valid(replica_type))) {
ret = OB_INVALID_ARGUMENT;
COMMON_LOG(WARN, "invalid argument", K(ret), K(member), K(replica_type));
} else if (OB_FAIL(ObMember::assign(member))) {
COMMON_LOG(WARN, "failed to assign member", K(ret), K(member));
} else if (OB_FALSE_IT(replica_type_ = replica_type)) {
// should never be here
} else if (OB_UNLIKELY(! is_valid())) { // check flag_ and replica_type_ correct
ret = OB_INVALID_ARGUMENT;
COMMON_LOG(WARN, "invalid argument", K(ret), K(member), K(replica_type), KPC(this));
}
return ret;
}
void ObReplicaMember::reset()
{
ObMember::reset();
replica_type_ = REPLICA_TYPE_FULL;
region_ = DEFAULT_REGION_NAME;
memstore_percent_ = 100;
}
bool ObReplicaMember::is_valid() const
{
// columnstore bit is 1 if and only if replica_type is C
bool is_flag_valid = (is_columnstore() == (REPLICA_TYPE_COLUMNSTORE == replica_type_));
return ObMember::is_valid()
&& ObReplicaTypeCheck::is_replica_type_valid(replica_type_)
&& !region_.is_empty()
&& is_flag_valid
&& memstore_percent_ <= 100
&& memstore_percent_ >= 0;
}
@ -115,46 +174,6 @@ common::ObReplicaType ObReplicaMember::get_replica_type() const
return replica_type_;
}
int ObReplicaMember::set_replica_type(const common::ObReplicaType replica_type)
{
int ret = OB_SUCCESS;
if (!ObReplicaTypeCheck::is_replica_type_valid(replica_type)) {
ret = OB_INVALID_ARGUMENT;
} else {
replica_type_ = replica_type;
}
return ret;
}
const common::ObRegion &ObReplicaMember::get_region() const
{
return region_;
}
int ObReplicaMember::set_member(const ObMember &member)
{
int ret = OB_SUCCESS;
if (!member.is_valid()) {
ret = OB_INVALID_ARGUMENT;
COMMON_LOG(WARN, "invalid args", K(ret), K(member));
} else if (OB_FAIL(ObMember::assign(member))) {
COMMON_LOG(WARN, "failed to assign member", K(ret), K(member));
}
return ret;
}
int ObReplicaMember::set_region(const common::ObRegion &region)
{
int ret = OB_SUCCESS;
if (region.is_empty()) {
ret = OB_INVALID_ARGUMENT;
} else {
region_ = region;
}
return ret;
}
ObReplicaMember &ObReplicaMember::operator=(const ObReplicaMember &rhs)
{
server_ = rhs.server_;

68
deps/oblib/src/common/ob_member.h vendored
View File

@ -49,11 +49,16 @@ public:
void set_migrating();
void reset_migrating();
bool is_columnstore() const;
void set_columnstore();
void reset_columnstore();
TO_STRING_KV(K_(server), K_(timestamp), K_(flag));
TO_YSON_KV(OB_Y_(server), OB_ID(t), timestamp_, OB_Y_(flag));
OB_UNIS_VERSION(1);
protected:
static const int64_t MIGRATING_FLAG_BIT = 1;
static const int64_t COLUMNSTORE_FLAG_BIT = 0;
common::ObAddr server_;
int64_t timestamp_;
int64_t flag_;
@ -90,61 +95,46 @@ inline int member_to_string(const common::ObMember &member, ObSqlString &member_
class ObReplicaMember : public ObMember
{
public:
// default constructor
ObReplicaMember()
: ObMember(),
replica_type_(REPLICA_TYPE_FULL),
region_(DEFAULT_REGION_NAME),
memstore_percent_(100)
{}
// construct with only server and timestamp, when we don't know or care about replica_type
// TODO(cangming.zl): remove this constructor when DRTask do not use it.
ObReplicaMember(const common::ObAddr &server,
const int64_t timestamp)
: ObMember(server, timestamp),
: ObMember(ObMember(server, timestamp)),
replica_type_(REPLICA_TYPE_FULL),
region_(DEFAULT_REGION_NAME),
memstore_percent_(100)
{}
ObReplicaMember(const ObMember &member)
: ObMember(member),
replica_type_(REPLICA_TYPE_FULL),
region_(DEFAULT_REGION_NAME),
memstore_percent_(100)
{}
/* After the subsequent type conversion code is completed, remove the constructor */
ObReplicaMember(const common::ObAddr &server,
const int64_t timestamp,
const common::ObReplicaType replica_type)
: ObMember(server, timestamp),
replica_type_(replica_type),
region_(DEFAULT_REGION_NAME),
memstore_percent_(100)
{}
// construct with server, timestamp and replica_type,
// this func will set columnstore flag if replica_type is C.
ObReplicaMember(const common::ObAddr &server,
const int64_t timestamp,
const common::ObReplicaType replica_type,
const int64_t memstore_percent)
: ObMember(server, timestamp),
const int64_t memstore_percent = 100)
: ObMember(ObMember(server, timestamp)),
replica_type_(replica_type),
region_(DEFAULT_REGION_NAME),
memstore_percent_(memstore_percent)
{}
ObReplicaMember(const common::ObAddr &server,
const int64_t timestamp,
const common::ObReplicaType replica_type,
const common::ObRegion &region,
const int64_t memstore_percent)
: ObMember(server, timestamp),
replica_type_(replica_type),
region_(region),
memstore_percent_(memstore_percent)
{}
{
if (REPLICA_TYPE_COLUMNSTORE == replica_type) {
ObMember::set_columnstore();
}
}
public:
// init with server, timestamp, replica_type.
// this func will set columnstore flag if replica_type is C.
int init(const common::ObAddr &server,
const int64_t timestamp,
const common::ObReplicaType replica_type);
// init with existing member whose flag_ may have been set.
// this function will check whether flag_ is consistent with replica_type.
int init(const ObMember &member,
const common::ObReplicaType replica_type);
common::ObReplicaType get_replica_type() const;
int set_replica_type(const common::ObReplicaType replica_type);
const common::ObRegion &get_region() const;
int set_region(const common::ObRegion &region);
int set_member(const ObMember &member);
int64_t get_memstore_percent() const { return memstore_percent_; }
void set_memstore_percent(const int64_t memstore_percent) { memstore_percent_ = memstore_percent; }
virtual void reset();
virtual bool is_valid() const;
virtual bool is_readonly_replica() const;
@ -154,8 +144,8 @@ public:
OB_UNIS_VERSION(1);
private:
common::ObReplicaType replica_type_;
common::ObRegion region_;
int64_t memstore_percent_;
int64_t memstore_percent_; // obsolate, only as placeholder
common::ObRegion region_ = DEFAULT_REGION_NAME; // obsolate, only as placeholder
};
} // namespace common
} // namespace oceanbase

27
deps/oblib/src/common/object/ob_obj_type.cpp vendored
View File

@ -698,6 +698,33 @@ bool is_match_alter_string_column_online_ddl_rules(const common::ObObjMeta& src_
return is_online_ddl;
}
int ob_sql_type_str_with_coll(char *buff,
int64_t buff_length,
int64_t &pos,
ObObjType type,
int64_t length,
int64_t precision,
int64_t scale,
ObCollationType coll_type,
const uint64_t sub_type/* common::ObGeoType::GEOTYPEMAX */)
{
int ret = OB_SUCCESS;
if (OB_FAIL(ob_sql_type_str(buff, buff_length, pos, type, length, precision, scale, coll_type, sub_type))) {
LOG_WARN("fail to get data type str", K(ret), K(sub_type), K(buff), K(buff_length), K(pos));
} else if (lib::is_mysql_mode() && ob_is_string_type(type) && CS_TYPE_BINARY != coll_type) {
if (ObCharset::is_default_collation(coll_type)) {
if (OB_FAIL(databuff_printf(buff, buff_length, pos, " CHARSET %s", ObCharset::charset_name(coll_type)))) {
LOG_WARN("fail to concat charset str", K(ret), K(sub_type), K(buff), K(buff_length), K(pos));
}
} else {
if (OB_FAIL(databuff_printf(buff, buff_length, pos, " CHARSET %s COLLATE %s", ObCharset::charset_name(coll_type), ObCharset::collation_name(coll_type)))) {
LOG_WARN("fail to concat charset and coll_type str", K(ret), K(sub_type), K(buff), K(buff_length), K(pos));
}
}
}
return ret;
}
int ob_sql_type_str(char *buff,
int64_t buff_length,
int64_t &pos,

11
deps/oblib/src/common/object/ob_obj_type.h vendored
View File

@ -1333,6 +1333,17 @@ const char *ob_obj_type_str(ObObjType type);
const char *inner_obj_type_str(ObObjType type);
const char *ob_sql_type_str(ObObjType type);
//such as “char(20) CHARSET utf8mb4" or "char(20) CHARSET utf8mb4 COLLATE utf8mb4_bin". with charset and collate
int ob_sql_type_str_with_coll(char *buff,
int64_t buff_length,
int64_t &pos,
ObObjType type,
int64_t length,
int64_t precision,
int64_t scale,
ObCollationType coll_type,
const uint64_t sub_type = static_cast<uint64_t>(common::ObGeoType::GEOTYPEMAX));
//such as "double(10,7)". with accuracy
int ob_sql_type_str(char *buff,
int64_t buff_length,

2
deps/oblib/src/lib/CMakeLists.txt vendored
View File

@ -22,6 +22,8 @@ ob_set_subtarget(oblib_lib charset
charset/ob_ctype_gb18030.cc
charset/ob_ctype_gbk.cc
charset/ob_ctype_latin1.cc
charset/ob_ctype_ascii.cc
charset/ob_ctype_tis620.cc
charset/ob_ctype_mb.cc
charset/ob_ctype_simple.cc
charset/ob_ctype_uca.cc

8
deps/oblib/src/lib/alloc/ob_malloc_sample_struct.h vendored
View File

@ -87,9 +87,10 @@ struct ObMallocSamplePairCmp
}
};
inline uint64_t ob_malloc_sample_hash(const char* data)
inline uint64_t ob_malloc_sample_hash(uint64_t v1, uint64_t v2)
{
return (uint64_t)data * 0xdeece66d + 0xb;
uint64_t data = (v1<<32) | ((v2<<32)>>32);
return data * 0xdeece66d + 0xb;
}
inline ObMallocSampleLimiter::ObMallocSampleLimiter()
@ -124,7 +125,8 @@ inline bool ObMallocSampleLimiter::malloc_sample_allowed(const int64_t size, con
// Full sample when size is bigger than 16M.
ret = true;
} else {
uint64_t hash_val = ob_malloc_sample_hash(attr.label_.str_);
int64_t tid = ob_gettid();
uint64_t hash_val = ob_malloc_sample_hash((uint64_t)attr.label_.str_, tid);
if (rate_limiters[hash_val & MAX_MALLOC_SAMPLER_NUM].try_acquire(size)) {
ret = true;
}

6
deps/oblib/src/lib/alloc/ob_malloc_time_monitor.cpp vendored
View File

@ -15,8 +15,6 @@
#include "lib/utility/ob_print_utils.h"
using namespace oceanbase::lib;
using namespace oceanbase::common;
volatile int64_t ObMallocTimeMonitor::WARN_THRESHOLD = 100000;
void ObMallocTimeMonitor::print()
{
char buf[1024] = {'\0'};
@ -29,8 +27,8 @@ void ObMallocTimeMonitor::print()
int64_t avg_cost_time = (0 == delta_count ? 0 : delta_total_cost_time / delta_count);
last_total_cost_times_[i] = total_cost_time;
last_counts_[i] = count;
int64_t left = (0 == i ? 0 : TIME_SLOT[i-1]);
int64_t right = TIME_SLOT[i];
int64_t left = TIME_SLOT[i];
int64_t right = TIME_SLOT[i + 1];
databuff_printf(buf, sizeof(buf), pos, "[MALLOC_TIME_MONITOR] [%8ld,%20ld): delta_total_cost_time=%15ld, delta_count=%15ld, avg_cost_time=%8ld\n",
left, right, delta_total_cost_time, delta_count, avg_cost_time);

31
deps/oblib/src/lib/alloc/ob_malloc_time_monitor.h vendored
View File

@ -22,9 +22,10 @@ namespace lib
class ObMallocTimeMonitor
{
public:
static volatile int64_t WARN_THRESHOLD;
static constexpr const int64_t TIME_SLOT[] = {10, 100, 1000, 10000, 100000, 1000000, INT64_MAX};
static const int64_t TIME_SLOT_NUM = ARRAYSIZEOF(TIME_SLOT);
static const int64_t WARN_THRESHOLD = 100000;
static const int64_t RECORD_THRESHOLD = 1000;
static constexpr const int64_t TIME_SLOT[] = {1000, 10000, 100000, 1000000, INT64_MAX};
static const int64_t TIME_SLOT_NUM = ARRAYSIZEOF(TIME_SLOT) - 1;
ObMallocTimeMonitor()
{
MEMSET(this, 0, sizeof(*this));
@ -35,28 +36,32 @@ public:
static ObMallocTimeMonitor instance;
return instance;
}
void inc(int64_t cost_time)
{
for (int i = 0; i < TIME_SLOT_NUM; ++i) {
if (cost_time < TIME_SLOT[i]) {
if (cost_time < TIME_SLOT[i + 1]) {
total_cost_times_[i].inc(cost_time);
counts_[i].inc(1);
break;
}
}
}
void record_malloc_time(ObBasicTimeGuard& time_guard, const int64_t size, const ObMemAttr& attr)
{
const int64_t cost_time = time_guard.get_diff();
inc(cost_time);
if (OB_UNLIKELY(cost_time > WARN_THRESHOLD)) {
const int64_t buf_len = 1024;
char buf[buf_len] = {'\0'};
int64_t pos = attr.to_string(buf, buf_len);
(void)logdata_printf(buf, buf_len, pos, ", size=%ld, ", size);
pos += time_guard.to_string(buf + pos, buf_len - pos);
int64_t tid = GETTID();
fprintf(stderr, "[%ld]OB_MALLOC COST TOO MUCH TIME, cost_time=%ld, %.*s\n", tid, cost_time, static_cast<int>(pos), buf);
if (OB_UNLIKELY(cost_time >= RECORD_THRESHOLD)) {
inc(cost_time);
if (OB_UNLIKELY(cost_time > WARN_THRESHOLD)) {
const int64_t buf_len = 1024;
char buf[buf_len] = {'\0'};
int64_t pos = attr.to_string(buf, buf_len);
(void)logdata_printf(buf, buf_len, pos, ", size=%ld, ", size);
pos += time_guard.to_string(buf + pos, buf_len - pos);
int64_t tid = GETTID();
fprintf(stderr, "[%ld]OB_MALLOC COST TOO MUCH TIME, cost_time=%ld, %.*s\n", tid, cost_time, static_cast<int>(pos), buf);
}
}
}
void print();

2
deps/oblib/src/lib/allocator/ob_slice_alloc.h vendored
View File

@ -293,7 +293,7 @@ private:
class ObSliceAlloc
{
public:
enum { MAX_ARENA_NUM = 32, MAX_REF_NUM = 4096, DEFAULT_BLOCK_SIZE = OB_MALLOC_NORMAL_BLOCK_SIZE };
enum { MAX_ARENA_NUM = 32, MAX_REF_NUM = 128, DEFAULT_BLOCK_SIZE = OB_MALLOC_NORMAL_BLOCK_SIZE };
typedef ObSimpleSync Sync;
typedef ObBlockSlicer Block;
typedef ObBlockAllocMgr BlockAlloc;

486
deps/oblib/src/lib/charset/ob_charset.cpp vendored
View File

@ -282,6 +282,8 @@ const ObCharsetWrapper ObCharset::charset_wrap_arr_[ObCharset::VALID_CHARSET_TYP
{CHARSET_GB18030, "GB18030 charset", CS_TYPE_GB18030_CHINESE_CI, 4},
{CHARSET_LATIN1, "cp1252 West European", CS_TYPE_LATIN1_SWEDISH_CI, 1},
{CHARSET_GB18030_2022, "GB18030-2022 charset", CS_TYPE_GB18030_2022_PINYIN_CI, 4},
{CHARSET_ASCII, "US ASCII", CS_TYPE_ASCII_GENERAL_CI, 1},
{CHARSET_TIS620, "TIS620 Thai", CS_TYPE_TIS620_THAI_CI, 1},
};
const ObCollationWrapper ObCharset::collation_wrap_arr_[ObCharset::VALID_COLLATION_TYPES] =
@ -294,9 +296,9 @@ const ObCollationWrapper ObCharset::collation_wrap_arr_[ObCharset::VALID_COLLATI
{CS_TYPE_UTF16_GENERAL_CI, CHARSET_UTF16, CS_TYPE_UTF16_GENERAL_CI, true, true, 1},
{CS_TYPE_UTF16_BIN, CHARSET_UTF16, CS_TYPE_UTF16_BIN, false, true, 1},
//{CS_TYPE_UTF8MB4_ZH_0900_AS_CS, CHARSET_UTF8MB4, CS_TYPE_UTF8MB4_ZH_0900_AS_CS, false, true, 0},
{CS_TYPE_UTF8MB4_UNICODE_CI, CHARSET_UTF8MB4, CS_TYPE_UTF8MB4_UNICODE_CI, false, true, 1},
{CS_TYPE_UTF16_UNICODE_CI, CHARSET_UTF16, CS_TYPE_UTF16_UNICODE_CI, false, true, 1},
{CS_TYPE_GB18030_CHINESE_CI, CHARSET_GB18030, CS_TYPE_GB18030_CHINESE_CI, true, true, 1},
{CS_TYPE_UTF8MB4_UNICODE_CI, CHARSET_UTF8MB4, CS_TYPE_UTF8MB4_UNICODE_CI, false, true, 8},
{CS_TYPE_UTF16_UNICODE_CI, CHARSET_UTF16, CS_TYPE_UTF16_UNICODE_CI, false, true, 8},
{CS_TYPE_GB18030_CHINESE_CI, CHARSET_GB18030, CS_TYPE_GB18030_CHINESE_CI, true, true, 2},
{CS_TYPE_GB18030_BIN, CHARSET_GB18030, CS_TYPE_GB18030_BIN, false, true, 1},
{CS_TYPE_LATIN1_SWEDISH_CI, CHARSET_LATIN1, CS_TYPE_LATIN1_SWEDISH_CI,true, true, 1},
{CS_TYPE_LATIN1_BIN, CHARSET_LATIN1, CS_TYPE_LATIN1_BIN,false, true, 1},
@ -307,7 +309,15 @@ const ObCollationWrapper ObCharset::collation_wrap_arr_[ObCharset::VALID_COLLATI
{CS_TYPE_GB18030_2022_RADICAL_CS, CHARSET_GB18030_2022, CS_TYPE_GB18030_2022_RADICAL_CS, false, true, 1},
{CS_TYPE_GB18030_2022_STROKE_CI, CHARSET_GB18030_2022, CS_TYPE_GB18030_2022_STROKE_CI, false, true, 1},
{CS_TYPE_GB18030_2022_STROKE_CS, CHARSET_GB18030_2022, CS_TYPE_GB18030_2022_STROKE_CS, false, true, 1},
};
{CS_TYPE_UTF8MB4_CROATIAN_CI, CHARSET_UTF8MB4, CS_TYPE_UTF8MB4_CROATIAN_CI, false, true, 8},
{CS_TYPE_UTF8MB4_UNICODE_520_CI, CHARSET_UTF8MB4, CS_TYPE_UTF8MB4_UNICODE_520_CI, false, true, 8},
{CS_TYPE_UTF8MB4_CZECH_CI, CHARSET_UTF8MB4, CS_TYPE_UTF8MB4_CZECH_CI, false, true, 8},
{CS_TYPE_ASCII_GENERAL_CI, CHARSET_ASCII, CS_TYPE_ASCII_GENERAL_CI,true, true, 1},
{CS_TYPE_ASCII_BIN, CHARSET_ASCII, CS_TYPE_ASCII_BIN,false, true, 1},
{CS_TYPE_TIS620_THAI_CI, CHARSET_TIS620, CS_TYPE_TIS620_THAI_CI,true, true, 1},
{CS_TYPE_TIS620_BIN, CHARSET_TIS620, CS_TYPE_TIS620_BIN,false, true, 1},
{CS_TYPE_UTF8MB4_0900_AI_CI, CHARSET_UTF8MB4, CS_TYPE_UTF8MB4_0900_AI_CI, false, true, 1},
};
ObCharsetInfo *ObCharset::charset_arr[CS_TYPE_MAX] = {
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, // 0 ~ 7
@ -353,8 +363,11 @@ ObCharsetInfo *ObCharset::charset_arr[CS_TYPE_MAX] = {
&ob_charset_gb18030_2022_stroke_cs, NULL, // 222
&ob_charset_utf8mb4_unicode_ci, // 224
NULL, NULL, NULL, NULL, NULL, NULL, NULL, // 225
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, // 232
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, // 240
NULL, NULL, &ob_charset_utf8mb4_czech_uca_ci, // 232
NULL, NULL, NULL, NULL, NULL, // 235
NULL, NULL, NULL, NULL, NULL, &ob_charset_utf8mb4_croatian_uca_ci, //240
&ob_charset_utf8mb4_unicode_520_ci, //246
NULL, // 247
&ob_charset_gb18030_chinese_ci, // 248
&ob_charset_gb18030_bin, // 249
NULL, &ob_charset_gb18030_chinese_cs, // 250
@ -1396,6 +1409,14 @@ const char *ObCharset::charset_name(ObCharsetType charset_type)
ret_name = "gb18030_2022";
break;
}
case CHARSET_ASCII: {
ret_name = "ascii";
break;
}
case CHARSET_TIS620: {
ret_name = "tis620";
break;
}
default: {
break;
}
@ -1493,7 +1514,7 @@ const char* ObCharset::collation_level(const ObCollationLevel cs_level)
ObCharsetType ObCharset::charset_type(const ObString &cs_name)
{
ObCharsetType charset_type = CHARSET_INVALID;
if (0 == cs_name.case_compare("utf8")) {
if (0 == cs_name.case_compare("utf8") || 0 == cs_name.case_compare("utf8mb3")) {
// utf8是utf8mb4的别名
charset_type = CHARSET_UTF8MB4;
} else if (0 == cs_name.case_compare(ob_charset_utf8mb4_bin.csname)) {
@ -1510,6 +1531,10 @@ ObCharsetType ObCharset::charset_type(const ObString &cs_name)
charset_type = CHARSET_LATIN1;
} else if (0 == cs_name.case_compare(ob_charset_gb18030_2022_bin.csname)) {
charset_type = CHARSET_GB18030_2022;
} else if (0 == cs_name.case_compare(ob_charset_ascii_bin.csname)) {
charset_type = CHARSET_ASCII;
} else if (0 == cs_name.case_compare(ob_charset_tis620_bin.csname)) {
charset_type = CHARSET_TIS620;
}
return charset_type;
}
@ -1530,6 +1555,10 @@ ObCharsetType ObCharset::charset_type_by_name_oracle(const ObString &cs_name)
charset_type = CHARSET_LATIN1;
} else if (0 == cs_name.case_compare("ZHS32GB18030_2022")) {
charset_type = CHARSET_GB18030_2022;
} else if (0 == cs_name.case_compare("US7ASCII")) {
charset_type = CHARSET_ASCII;
} else if (0 == cs_name.case_compare("TH8TISASCII")) {
charset_type = CHARSET_TIS620;
}
return charset_type;
}
@ -1550,9 +1579,11 @@ ObCharsetType ObCharset::charset_type(const char *cs_name)
ObCollationType ObCharset::collation_type(const ObString &cs_name)
{
ObCollationType collation_type = CS_TYPE_INVALID;
if (0 == cs_name.case_compare("utf8_bin")) {
if (0 == cs_name.case_compare("utf8_bin") ||
0 == cs_name.case_compare("utf8mb3_bin")) {
collation_type = CS_TYPE_UTF8MB4_BIN;
} else if (0 == cs_name.case_compare("utf8_general_ci")) {
} else if (0 == cs_name.case_compare("utf8_general_ci") ||
0 == cs_name.case_compare("utf8mb3_general_ci")) {
collation_type = CS_TYPE_UTF8MB4_GENERAL_CI;
} else if (0 == cs_name.case_compare(ob_charset_utf8mb4_bin.name)) {
collation_type = CS_TYPE_UTF8MB4_BIN;
@ -1570,6 +1601,8 @@ ObCollationType ObCharset::collation_type(const ObString &cs_name)
collation_type = CS_TYPE_UTF16_BIN;
} else if (0 == cs_name.case_compare("utf8_unicode_ci")) {
collation_type = CS_TYPE_UTF8MB4_UNICODE_CI;
} else if (0 == cs_name.case_compare("utf8mb4_0900_ai_ci")) {
collation_type = CS_TYPE_UTF8MB4_0900_AI_CI;
} else if (0 == cs_name.case_compare(ob_charset_utf16_unicode_ci.name)) {
collation_type = CS_TYPE_UTF16_UNICODE_CI;
} else if (0 == cs_name.case_compare(ob_charset_utf8mb4_unicode_ci.name)) {
@ -1600,6 +1633,26 @@ ObCollationType ObCharset::collation_type(const ObString &cs_name)
collation_type = CS_TYPE_GB18030_2022_STROKE_CI;
} else if (0 == cs_name.case_compare(ob_charset_gb18030_2022_stroke_cs.name)) {
collation_type = CS_TYPE_GB18030_2022_STROKE_CS;
} else if (0 == cs_name.case_compare("utf8_croatian_ci")) {
collation_type = CS_TYPE_UTF8MB4_CROATIAN_CI;
} else if (0 == cs_name.case_compare(ob_charset_utf8mb4_croatian_uca_ci.name)) {
collation_type = CS_TYPE_UTF8MB4_CROATIAN_CI;
} else if (0 == cs_name.case_compare("utf8_unicode_520_ci")) {
collation_type = CS_TYPE_UTF8MB4_UNICODE_520_CI;
} else if (0 == cs_name.case_compare(ob_charset_utf8mb4_unicode_520_ci.name)) {
collation_type = CS_TYPE_UTF8MB4_UNICODE_520_CI;
} else if (0 == cs_name.case_compare("utf8_czech_ci")) {
collation_type = CS_TYPE_UTF8MB4_CZECH_CI;
} else if (0 == cs_name.case_compare(ob_charset_utf8mb4_czech_uca_ci.name)) {
collation_type = CS_TYPE_UTF8MB4_CZECH_CI;
} else if (0 == cs_name.case_compare(ob_charset_ascii_bin.name)) {
collation_type = CS_TYPE_ASCII_BIN;
} else if (0 == cs_name.case_compare(ob_charset_ascii.name)) {
collation_type = CS_TYPE_ASCII_GENERAL_CI;
} else if (0 == cs_name.case_compare(ob_charset_tis620_bin.name)) {
collation_type = CS_TYPE_TIS620_BIN;
} else if (0 == cs_name.case_compare(ob_charset_tis620_thai_ci.name)) {
collation_type = CS_TYPE_TIS620_THAI_CI;
}
return collation_type;
}
@ -1617,6 +1670,10 @@ bool ObCharset::is_valid_collation(ObCharsetType charset_type, ObCollationType c
if (CS_TYPE_UTF8MB4_BIN == collation_type
|| CS_TYPE_UTF8MB4_GENERAL_CI == collation_type
|| CS_TYPE_UTF8MB4_UNICODE_CI == collation_type
|| CS_TYPE_UTF8MB4_CROATIAN_CI == collation_type
|| CS_TYPE_UTF8MB4_UNICODE_520_CI == collation_type
|| CS_TYPE_UTF8MB4_CZECH_CI == collation_type
|| CS_TYPE_UTF8MB4_0900_AI_CI == collation_type
) {
ret = true;
}
@ -1645,9 +1702,18 @@ bool ObCharset::is_valid_collation(ObCharsetType charset_type, ObCollationType c
}
} else if (CHARSET_GB18030_2022 == charset_type) {
ret = is_gb18030_2022(collation_type);
} else if (CHARSET_ASCII == charset_type) {
if (CS_TYPE_ASCII_GENERAL_CI == collation_type || CS_TYPE_ASCII_BIN == collation_type) {
ret = true;
}
} else if (CHARSET_TIS620 == charset_type) {
if (CS_TYPE_TIS620_THAI_CI == collation_type || CS_TYPE_TIS620_BIN == collation_type) {
ret = true;
}
}
return ret;
}
ObCollationType ObCharset::get_coll_type_by_nlssort_param(ObCharsetType charset_type,
const ObString &nlssort_param)
{
@ -1662,6 +1728,8 @@ ObCollationType ObCharset::get_coll_type_by_nlssort_param(ObCharsetType charset_
CS_TYPE_GB18030_BIN,
CS_TYPE_LATIN1_BIN,
CS_TYPE_GB18030_2022_BIN,
CS_TYPE_ASCII_BIN,
CS_TYPE_TIS620_BIN,
};
static ObCollationType non_bin_coll_marks[NLS_COLLATION_MAX] = {
CS_TYPE_INVALID,
@ -1698,10 +1766,7 @@ ObCollationType ObCharset::get_coll_type_by_nlssort_param(ObCharsetType charset_
} else if (nls_coll_type == NLS_COLLATION_SCHINESE_STROKE2_M) {
coll_type = CS_TYPE_GB18030_2022_STROKE_CS;
} else {
if (charset_type != CHARSET_LATIN1) {
coll_type = static_cast<ObCollationType>(
non_bin_coll_marks[nls_coll_type] + (charset_type - CHARSET_BINARY));
}
coll_type = static_cast<ObCollationType>(non_bin_coll_marks[nls_coll_type] + (charset_type - CHARSET_BINARY));
}
}
return coll_type;
@ -1724,7 +1789,15 @@ bool ObCharset::is_valid_collation(int64_t collation_type_int)
|| CS_TYPE_LATIN1_BIN == collation_type
|| is_gb18030_2022(collation_type)
|| CS_TYPE_UTF8MB4_UNICODE_CI == collation_type
|| CS_TYPE_UTF8MB4_0900_AI_CI == collation_type
|| CS_TYPE_UTF16_UNICODE_CI == collation_type
|| CS_TYPE_UTF8MB4_CROATIAN_CI == collation_type
|| CS_TYPE_UTF8MB4_UNICODE_520_CI == collation_type
|| CS_TYPE_UTF8MB4_CZECH_CI == collation_type
|| CS_TYPE_ASCII_GENERAL_CI == collation_type
|| CS_TYPE_ASCII_BIN == collation_type
|| CS_TYPE_TIS620_THAI_CI == collation_type
|| CS_TYPE_TIS620_BIN == collation_type
|| (CS_TYPE_EXTENDED_MARK < collation_type && collation_type < CS_TYPE_MAX)
;
}
@ -1739,6 +1812,10 @@ ObCharsetType ObCharset::charset_type_by_coll(ObCollationType collation_type)
case CS_TYPE_UTF8MB4_ZH_0900_AS_CS:
case CS_TYPE_UTF8MB4_ZH2_0900_AS_CS:
case CS_TYPE_UTF8MB4_ZH3_0900_AS_CS:
case CS_TYPE_UTF8MB4_CROATIAN_CI:
case CS_TYPE_UTF8MB4_UNICODE_520_CI:
case CS_TYPE_UTF8MB4_CZECH_CI:
case CS_TYPE_UTF8MB4_0900_AI_CI:
case CS_TYPE_UTF8MB4_UNICODE_CI: {
charset_type = CHARSET_UTF8MB4;
break;
@ -1774,7 +1851,10 @@ ObCharsetType ObCharset::charset_type_by_coll(ObCollationType collation_type)
break;
}
case CS_TYPE_LATIN1_SWEDISH_CI:
case CS_TYPE_LATIN1_BIN: {
case CS_TYPE_LATIN1_BIN:
case CS_TYPE_LATIN1_ZH_0900_AS_CS:
case CS_TYPE_LATIN1_ZH2_0900_AS_CS:
case CS_TYPE_LATIN1_ZH3_0900_AS_CS: {
charset_type = CHARSET_LATIN1;
break;
}
@ -1791,6 +1871,22 @@ ObCharsetType ObCharset::charset_type_by_coll(ObCollationType collation_type)
charset_type = CHARSET_GB18030_2022;
break;
}
case CS_TYPE_ASCII_GENERAL_CI:
case CS_TYPE_ASCII_BIN:
case CS_TYPE_ASCII_ZH_0900_AS_CS:
case CS_TYPE_ASCII_ZH2_0900_AS_CS:
case CS_TYPE_ASCII_ZH3_0900_AS_CS: {
charset_type = CHARSET_ASCII;
break;
}
case CS_TYPE_TIS620_THAI_CI:
case CS_TYPE_TIS620_BIN:
case CS_TYPE_TIS620_ZH_0900_AS_CS:
case CS_TYPE_TIS620_ZH2_0900_AS_CS:
case CS_TYPE_TIS620_ZH3_0900_AS_CS: {
charset_type = CHARSET_TIS620;
break;
}
default: {
break;
}
@ -1821,6 +1917,12 @@ ObNlsCharsetId ObCharset::charset_type_to_ora_charset_id(ObCharsetType cs_type)
case CHARSET_GB18030_2022:
cs_id = CHARSET_ZHS32GB18030_2022_ID;
break;
case CHARSET_ASCII:
cs_id = CHARSET_US7ASCII_ID;
break;
case CHARSET_TIS620:
cs_id = CHARSET_TH8TISASCII_ID;
break;
default:
break;
}
@ -1833,28 +1935,77 @@ ObCharsetType ObCharset::ora_charset_type_to_charset_type(ObNlsCharsetId charset
switch (charset_id)
{
case CHARSET_AL32UTF8_ID:
cs_type = CHARSET_UTF8MB4;
break;
case CHARSET_UTF8_ID:
cs_type = CHARSET_UTF8MB4;
break;
case CHARSET_ZHS16GBK_ID:
cs_type = CHARSET_GBK;
break;
cs_type = CHARSET_GBK;
break;
case CHARSET_ZHS32GB18030_ID:
cs_type = CHARSET_GB18030;
break;
cs_type = CHARSET_GB18030;
break;
case CHARSET_AL16UTF16_ID:
cs_type = CHARSET_UTF16;
break;
cs_type = CHARSET_UTF16;
break;
case CHARSET_WE8MSWIN1252_ID:
cs_type = CHARSET_LATIN1;
cs_type = CHARSET_LATIN1;
break;
case CHARSET_ZHS32GB18030_2022_ID:
cs_type = CHARSET_GB18030_2022;
break;
cs_type = CHARSET_GB18030_2022;
break;
case CHARSET_US7ASCII_ID:
cs_type = CHARSET_ASCII;
break;
case CHARSET_TH8TISASCII_ID:
cs_type = CHARSET_TIS620;
break;
default:
break;
break;
}
return cs_type;
}
ObCollationType ObCharset::ora_charset_type_to_coll_type(ObNlsCharsetId charset_id)
{
ObCollationType coll_type = CS_TYPE_INVALID;
switch (charset_id)
{
case CHARSET_AL32UTF8_ID:
case CHARSET_UTF8_ID:
coll_type = CS_TYPE_UTF8MB4_BIN;
break;
case CHARSET_ZHS16GBK_ID:
coll_type = CS_TYPE_GBK_BIN;
break;
case CHARSET_ZHS32GB18030_ID:
coll_type = CS_TYPE_GB18030_BIN;
break;
case CHARSET_AL16UTF16_ID:
coll_type = CS_TYPE_UTF16_BIN;
break;
case CHARSET_WE8MSWIN1252_ID:
coll_type = CS_TYPE_LATIN1_BIN;
break;
case CHARSET_ZHS32GB18030_2022_ID:
coll_type = CS_TYPE_GB18030_2022_BIN;
break;
case CHARSET_US7ASCII_ID:
coll_type = CS_TYPE_ASCII_BIN;
break;
case CHARSET_TH8TISASCII_ID:
coll_type = CS_TYPE_TIS620_BIN;
break;
default:
break;
}
return coll_type;
}
bool ObCharset::is_valid_ora_charset_id(ObNlsCharsetId charset_id)
{
return CS_TYPE_INVALID != ora_charset_type_to_coll_type(charset_id);
}
bool ObCharset::is_valid_nls_collation(ObNLSCollation nls_collation)
{
return nls_collation > NLS_COLLATION_INVALID && nls_collation < NLS_COLLATION_MAX;
@ -1946,7 +2097,7 @@ int ObCharset::result_collation(
return ret;
}
int ObCharset::aggregate_collation(
int ObCharset::aggregate_collation_old(
const ObCollationLevel collation_level1,
const ObCollationType collation_type1,
const ObCollationLevel collation_level2,
@ -2049,6 +2200,12 @@ int ObCharset::aggregate_collation(
} else if (charset_type_by_coll(collation_type1) == CHARSET_GB18030_2022) {
res_type = CS_TYPE_GB18030_2022_BIN;
res_level = (CS_TYPE_GB18030_2022_BIN == collation_type1) ? collation_level1 : collation_level2;
} else if (charset_type_by_coll(collation_type1) == CHARSET_ASCII) {
res_type = CS_TYPE_ASCII_BIN;
res_level = (CS_TYPE_ASCII_BIN == collation_type1) ? collation_level1 : collation_level2;
} else if (charset_type_by_coll(collation_type1) == CHARSET_TIS620) {
res_type = CS_TYPE_TIS620_BIN;
res_level = (CS_TYPE_TIS620_BIN == collation_type1) ? collation_level1 : collation_level2;
} else {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("Unexpected charset", K(ret), K(collation_type1), K(collation_type2), KCSTRING(lbt()));
@ -2079,6 +2236,180 @@ int ObCharset::aggregate_collation(
return ret;
}
int ObCharset::aggregate_collation_new(
const ObCollationLevel collation_level1,
const ObCollationType collation_type1,
const ObCollationLevel collation_level2,
const ObCollationType collation_type2,
ObCollationLevel &res_level,
ObCollationType &res_type,
uint32_t flags)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(
CS_LEVEL_INVALID == collation_level1
|| CS_LEVEL_INVALID == collation_level2
|| !is_valid_collation(collation_type1)
|| !is_valid_collation(collation_type2))) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN ("invalid collation level or type",
K(ret), K(collation_level1), K(collation_type1), K(collation_level2), K(collation_type2));
} else if (collation_type1 != collation_type2 &&
CS_LEVEL_EXPLICIT == collation_level1 &&
CS_LEVEL_EXPLICIT == collation_level2) {
ret = OB_CANT_AGGREGATE_2COLLATIONS;
} else {
ObCharsetInfo *cs1 = static_cast<ObCharsetInfo *>(ObCharset::charset_arr[collation_type1]);
ObCharsetInfo *cs2 = static_cast<ObCharsetInfo *>(ObCharset::charset_arr[collation_type2]);
ObCharsetType charset_type1 = charset_type_by_coll(collation_type1);
ObCharsetType charset_type2 = charset_type_by_coll(collation_type2);
if (charset_type1 != charset_type2) {
if (CS_TYPE_BINARY == collation_type1) {
if (collation_level1 <= collation_level2) {
res_type = collation_type1;
res_level = collation_level1;
} else {
res_type = collation_type2;
res_level = collation_level2;
}
} else if (CS_TYPE_BINARY == collation_type2) {
if (collation_level2 <= collation_level1) {
res_type = collation_type2;
res_level = collation_level2;
} else {
res_type = collation_type1;
res_level = collation_level1;
}
} else if ((flags & OB_COLL_ALLOW_SUPERSET_CONV) &&
left_is_superset(collation_level1,
collation_type1,
collation_level2,
collation_type2)) {
res_type = collation_type1;
res_level = collation_level1;
} else if ((flags & OB_COLL_ALLOW_SUPERSET_CONV) &&
left_is_superset(collation_level2,
collation_type2,
collation_level1,
collation_type1)) {
res_type = collation_type2;
res_level = collation_level2;
} else if ((flags & OB_COLL_ALLOW_COERCIBLE_CONV) &&
collation_level1 < collation_level2 &&
collation_level2 >= CS_LEVEL_SYSCONST) {
res_type = collation_type1;
res_level = collation_level1;
} else if ((flags & OB_COLL_ALLOW_COERCIBLE_CONV) &&
collation_level2 < collation_level1 &&
collation_level1 >= CS_LEVEL_SYSCONST) {
res_type = collation_type2;
res_level = collation_level2;
} else {
// Cannot apply conversion
res_type = CS_TYPE_BINARY;
res_level = CS_LEVEL_NONE;
}
if (lib::is_oracle_mode()) {
if (charset_type1 == CHARSET_UTF8MB4 && charset_type2 == CHARSET_UTF16) {
res_type = collation_type2;
res_level = collation_level2;
} else if (charset_type1 == CHARSET_UTF16 && charset_type2 == CHARSET_UTF8MB4) {
res_type = collation_type1;
res_level = collation_level1;
}
}
} else if (collation_level1 < collation_level2) {
res_type = collation_type1;
res_level = collation_level1;
} else if (collation_level2 < collation_level1) {
res_type = collation_type2;
res_level = collation_level2;
} else if (collation_type1 == collation_type2) {
res_type = collation_type1;
res_level = collation_level1;
} else if (CS_LEVEL_EXPLICIT == collation_level1) {
ret = OB_CANT_AGGREGATE_2COLLATIONS;
// ERROR 1267 (HY000): Illegal mix of collations (utf8_general_ci,EXPLICIT) and (utf8_bin,EXPLICIT) for operation '='
} else if ((cs1->state & OB_CS_BINSORT) && (cs2->state & OB_CS_BINSORT)) {
// If we have two different binary collations for the same character set,
// and none of them is explicit, we don't know which to choose. For
// example: utf8mb4_bin is a binary padding collation, utf8mb4_0900_bin is
// a binary non-padding collation. Cannot determine if the resulting
// collation should be padding or non-padding, unless they are also
// aggregated with a third explicit collation.
res_type = CS_TYPE_BINARY;
res_level = CS_LEVEL_NONE;
ret = OB_CANT_AGGREGATE_2COLLATIONS;
} else if (cs1->state & OB_CS_BINSORT) {
res_type = collation_type1;
res_level = collation_level1;
} else if (cs2->state & OB_CS_BINSORT) {
res_type = collation_type2;
res_level = collation_level2;
} else {
/*
test (c1 char(10) COLLATE utf8mb4_unicode_ci, c2 char(10) COLLATE utf8mb4_general_ci)
collation(concat(c1,c2)) = utf8mb4_bin;
*/
res_type = ObCharset::get_bin_collation(charset_type1);
res_level = CS_LEVEL_NONE;
}
if (OB_SUCC(ret)) {
ObCharsetType res_cs = charset_type_by_coll(res_type);
if (CHARSET_GB18030 == res_cs) {
if (CHARSET_GB18030_2022 == charset_type1 || CHARSET_GB18030_2022 == charset_type2) {
ret = OB_CANT_AGGREGATE_2COLLATIONS;
}
} else if (CHARSET_GB18030_2022 == res_cs) {
if (CHARSET_GB18030 == charset_type1 || CHARSET_GB18030 == charset_type2) {
ret = OB_CANT_AGGREGATE_2COLLATIONS;
}
}
}
}
if (OB_FAIL(ret)) {
LOG_WARN("Illegal mix of collations", K(ret),
"type1", ObCharset::collation_name(collation_type1),
"level1", ObCharset::collation_level(collation_level1),
"type2", ObCharset::collation_name(collation_type2),
"level2", ObCharset::collation_level(collation_level2));
}
return ret;
}
bool ObCharset::left_is_superset(const ObCollationLevel collation_level1,
const ObCollationType collation_type1,
const ObCollationLevel collation_level2,
const ObCollationType collation_type2)
{
ObCharsetInfo *cs1 = static_cast<ObCharsetInfo *>(ObCharset::charset_arr[collation_type1]);
ObCharsetInfo *cs2 = static_cast<ObCharsetInfo *>(ObCharset::charset_arr[collation_type2]);
bool bret = false;
if (cs1->state & OB_CS_UNICODE &&
(collation_level1 < collation_level2 ||
(collation_level1 == collation_level2 &&
(!(cs2->state & OB_CS_UNICODE) ||
/* The code below makes 4-byte utf8 a superset over 3-byte utf8 */
(cs1->state & OB_CS_UNICODE_SUPPLEMENT &&
!(cs2->state & OB_CS_UNICODE_SUPPLEMENT) &&
cs1->mbmaxlen > cs2->mbmaxlen &&
cs1->mbminlen == cs2->mbminlen))))) {
bret = true;
} else if (test_all_bits(cs1->state, OB_CS_UNICODE| OB_CS_UNICODE_SUPPLEMENT) &&
(cs2->state & OB_CS_UNICODE) &&
collation_level1 == collation_level2) {
/* Allow convert from any Unicode to utf32 or utf8mb4 */
bret = true;
} else if ((cs2->state & OB_CS_PUREASCII) &&
(collation_level1 < collation_level2 ||
(collation_level1 == collation_level2 && !(cs1->state & OB_CS_PUREASCII)))) {
/* Allow convert from ASCII */
bret = true;
}
return bret;
}
bool ObCharset::is_bin_sort(ObCollationType collation_type)
{
bool ret = false;
@ -2094,6 +2425,7 @@ bool ObCharset::is_bin_sort(ObCollationType collation_type)
return ret;
}
bool ObCharset::is_ci_collate(ObCollationType collation_type)
{
bool ret = false;
@ -2149,6 +2481,14 @@ ObCollationType ObCharset::get_default_collation(ObCharsetType charset_type)
collation_type = CS_TYPE_GB18030_2022_PINYIN_CI;
break;
}
case CHARSET_ASCII: {
collation_type = CS_TYPE_ASCII_GENERAL_CI;
break;
}
case CHARSET_TIS620: {
collation_type = CS_TYPE_TIS620_THAI_CI;
break;
}
default: {
break;
}
@ -2195,6 +2535,14 @@ ObCollationType ObCharset::get_default_collation_oracle(ObCharsetType charset_ty
collation_type = CS_TYPE_GB18030_2022_BIN;
break;
}
case CHARSET_ASCII: {
collation_type = CS_TYPE_ASCII_BIN;
break;
}
case CHARSET_TIS620: {
collation_type = CS_TYPE_TIS620_BIN;
break;
}
default: {
break;
}
@ -2234,6 +2582,14 @@ int ObCharset::get_default_collation(ObCharsetType charset_type, ObCollationType
collation_type = CS_TYPE_GB18030_2022_PINYIN_CI;
break;
}
case CHARSET_ASCII: {
collation_type = CS_TYPE_ASCII_GENERAL_CI;
break;
}
case CHARSET_TIS620: {
collation_type = CS_TYPE_TIS620_THAI_CI;
break;
}
default: {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid charset type", K(ret), K(charset_type));
@ -2275,6 +2631,14 @@ ObCollationType ObCharset::get_bin_collation(ObCharsetType charset_type)
collation_type = CS_TYPE_GB18030_2022_BIN;
break;
}
case CHARSET_ASCII: {
collation_type = CS_TYPE_ASCII_BIN;
break;
}
case CHARSET_TIS620: {
collation_type = CS_TYPE_TIS620_BIN;
break;
}
default: {
break;
}
@ -2407,6 +2771,8 @@ bool ObCharset::is_default_collation(ObCollationType collation_type)
case CS_TYPE_UTF16_GENERAL_CI:
case CS_TYPE_GB18030_CHINESE_CI:
case CS_TYPE_LATIN1_SWEDISH_CI:
case CS_TYPE_ASCII_GENERAL_CI:
case CS_TYPE_TIS620_THAI_CI:
case CS_TYPE_GB18030_2022_PINYIN_CI:
case CS_TYPE_BINARY: {
ret = true;
@ -2847,6 +3213,8 @@ int ObCharset::get_aggregate_len_unit(const ObCollationType collation_type, bool
ObCharsetType res_charset = ObCharset::charset_type_by_coll(collation_type);
if (CHARSET_UTF8MB4 == res_charset
|| CHARSET_LATIN1 == res_charset
|| CHARSET_ASCII == res_charset
|| CHARSET_TIS620 == res_charset
|| CHARSET_UTF16 == res_charset
|| CHARSET_GBK == res_charset
|| CHARSET_GB18030 == res_charset
@ -2969,7 +3337,7 @@ int ObCharset::charset_convert(ObIAllocator &alloc,
char *res_buf = static_cast<char *>(alloc.alloc(res_buf_len));
if (OB_ISNULL(res_buf)) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_WARN("alloc memory failed", K(ret));
LOG_WARN("alloc memory failed", K(ret), K(lbt()));
} else {
if (OB_SUCC(charset_convert(src_cs_type, in.ptr(), in.length(),
dst_cs_type, res_buf, res_buf_len, res_len))) {
@ -3114,6 +3482,8 @@ bool ObCharset::is_valid_connection_collation(ObCollationType collation_type)
ObCharsetType cs_type = ObCharset::charset_type_by_coll(collation_type);
return cs_type == CHARSET_UTF8MB4
|| cs_type == CHARSET_LATIN1
|| cs_type == CHARSET_ASCII
|| cs_type == CHARSET_TIS620
|| cs_type == CHARSET_GBK
|| cs_type == CHARSET_GB18030
|| cs_type == CHARSET_GB18030_2022
@ -3142,6 +3512,12 @@ const char *ObCharset::get_oracle_charset_name_by_charset_type(ObCharsetType cha
case CHARSET_LATIN1:
ret = "WE8MSWIN1252";
break;
case CHARSET_ASCII:
ret = "US7ASCII";
break;
case CHARSET_TIS620:
ret = "TH8TISASCII";
break;
default:
break;
}
@ -3170,6 +3546,12 @@ int ObCharset::get_nls_charset_id_by_charset_type(ObCharsetType charset_type)
case CHARSET_GB18030_2022:
ret_id = ObNlsCharsetId::CHARSET_ZHS32GB18030_2022_ID;
break;
case CHARSET_ASCII:
ret_id = ObNlsCharsetId::CHARSET_US7ASCII_ID;
break;
case CHARSET_TIS620:
ret_id = ObNlsCharsetId::CHARSET_TH8TISASCII_ID;
break;
default:
break;
}
@ -3278,6 +3660,37 @@ int ObCharset::init_charset()
ObCharsetLoader loader;
ob_charset_loader_init_mysys(&loader);
add_coll(CS_TYPE_UTF8MB4_0900_BIN, &ob_charset_utf8mb4_0900_bin);
add_coll(CS_TYPE_UTF8MB4_0900_AI_CI, &ob_charset_utf8mb4_0900_ai_ci);
add_coll(CS_TYPE_TIS620_BIN, &ob_charset_tis620_bin);
add_coll(CS_TYPE_TIS620_THAI_CI, &ob_charset_tis620_thai_ci);
//init charset_handler&collation_handler for some special charset
ObCharsetInfo *special_charset[] = {&ob_charset_ascii,&ob_charset_ascii_bin};
for (int i = 0; OB_SUCC(ret) && i < array_elements(special_charset); i++) {
ObCharsetInfo *cs = special_charset[i];
ObCharsetHandler *charset_handler = cs->cset;
ObCollationHandler *coll_handler = cs->coll;
if (OB_ISNULL(charset_handler) || OB_ISNULL(coll_handler)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null pointer", K(charset_handler), K(coll_handler), K(ret));
} else if (charset_handler->init &&
charset_handler->init(cs, &loader)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("fail to init charset handler", K(ret));
} else {
add_coll((ObCollationType)cs->number, cs);
/*debug
if (cs->tab_from_uni) {
OB_UNI_IDX test = cs->tab_from_uni[0];
//int size = sizeof(test->tab)/sizeof(test->tab[0]);
LOG_INFO("charset debug", K(cs->name),K(test.from), K(test.to));
for (int i=0;i<255;i++) {
LOG_INFO("charset debug",K(i),K(test.tab[i]));
}
}
*/
}
}
if (OB_SUCC(ret)) {
auto *utf8_pinyin = &ob_charset_utf8mb4_zh_0900_as_cs;
ObCollationHandler *pinyin_coll = ob_charset_utf8mb4_zh_0900_as_cs.coll;
@ -3289,7 +3702,8 @@ int ObCharset::init_charset()
ObCollationType pinyin_colls[] = {
CS_TYPE_GBK_ZH_0900_AS_CS, CS_TYPE_UTF8MB4_ZH_0900_AS_CS,
CS_TYPE_GB18030_ZH_0900_AS_CS, CS_TYPE_UTF16_ZH_0900_AS_CS,
CS_TYPE_GB18030_2022_ZH_0900_AS_CS
CS_TYPE_GB18030_2022_ZH_0900_AS_CS, CS_TYPE_LATIN1_ZH_0900_AS_CS,
CS_TYPE_ASCII_ZH_0900_AS_CS,CS_TYPE_TIS620_ZH_0900_AS_CS
};
add_coll(CS_TYPE_UTF8MB4_ZH_0900_AS_CS, utf8_pinyin);
@ -3316,7 +3730,8 @@ int ObCharset::init_charset()
ObCollationType radical_colls[] = {
CS_TYPE_GBK_ZH2_0900_AS_CS, CS_TYPE_UTF8MB4_ZH2_0900_AS_CS,
CS_TYPE_GB18030_ZH2_0900_AS_CS, CS_TYPE_UTF16_ZH2_0900_AS_CS,
CS_TYPE_GB18030_2022_ZH2_0900_AS_CS
CS_TYPE_GB18030_2022_ZH2_0900_AS_CS,CS_TYPE_LATIN1_ZH2_0900_AS_CS,
CS_TYPE_ASCII_ZH2_0900_AS_CS,CS_TYPE_TIS620_ZH2_0900_AS_CS
};
add_coll(CS_TYPE_UTF8MB4_ZH2_0900_AS_CS, utf8_radical);
@ -3343,7 +3758,8 @@ int ObCharset::init_charset()
ObCollationType stroke_colls[] = {
CS_TYPE_GBK_ZH3_0900_AS_CS, CS_TYPE_UTF8MB4_ZH3_0900_AS_CS,
CS_TYPE_GB18030_ZH3_0900_AS_CS, CS_TYPE_UTF16_ZH3_0900_AS_CS,
CS_TYPE_GB18030_2022_ZH3_0900_AS_CS
CS_TYPE_GB18030_2022_ZH3_0900_AS_CS, CS_TYPE_LATIN1_ZH3_0900_AS_CS,
CS_TYPE_ASCII_ZH3_0900_AS_CS,CS_TYPE_TIS620_ZH3_0900_AS_CS
};
add_coll(CS_TYPE_UTF8MB4_ZH3_0900_AS_CS, utf8_stroke);
@ -3363,6 +3779,14 @@ int ObCharset::init_charset()
//init utf8_0900_binary
add_coll(CS_TYPE_UTF8MB4_0900_BIN, &ob_charset_utf8mb4_0900_bin);
if (OB_SUCC(ret)) {
if (OB_FAIL(ob_charset_utf8mb4_croatian_uca_ci.coll->init(&ob_charset_utf8mb4_croatian_uca_ci, &loader))) {
LOG_WARN("fail to init collation", K(ret));
} else if (OB_FAIL(ob_charset_utf8mb4_czech_uca_ci.coll->init(&ob_charset_utf8mb4_czech_uca_ci, &loader))) {
LOG_WARN("fail to init collation", K(ret));
}
}
return ret;
}

80
deps/oblib/src/lib/charset/ob_charset.h vendored
View File

@ -38,6 +38,8 @@ enum ObCharsetType
CHARSET_GB18030 = 5,
CHARSET_LATIN1 = 6,
CHARSET_GB18030_2022 = 7,
CHARSET_ASCII = 8,
CHARSET_TIS620 = 9,
CHARSET_MAX,
};
@ -49,21 +51,25 @@ enum ObCharsetType
*there is no possibly to reach AGGREGATE_2CHARSET[CHARSET_UTF8MB4][CHARSET_UTF8MB4] and so on
*/
static const int AGGREGATE_2CHARSET[CHARSET_MAX][CHARSET_MAX] = {
//CHARSET_INVALI,CHARSET_UTF8MB4...
{0,0,0,0,0,0,0,0},//CHARSET_INVALI
{0,0,0,0,0,0,0,0},//CHARSET_BINARY
{0,0,0,1,2,1,1,1},//CHARSET_UTF8MB4
{0,0,2,0,2,0,1,0},//CHARSET_GBK
{0,0,1,1,0,1,1,1},//CHARSET_UTF16
{0,0,2,0,2,0,1,0},//CHARSET_GB18030
{0,0,2,2,2,2,0,2},//CHARSET_LATIN1
{0,0,2,0,2,0,1,0} //CHARSET_GB18030_2022
//CHARSET_INVALI,CHARSET_UTF8MB4...
{0,0,0,0,0,0,0,0,0,0},//CHARSET_INVALI
{0,0,0,0,0,0,0,0,0,0},//CHARSET_BINARY
{0,0,0,1,2,1,1,1,1,1},//CHARSET_UTF8MB4
{0,0,2,0,2,0,1,0,1,0},//CHARSET_GBK
{0,0,1,1,0,1,1,1,1,1},//CHARSET_UTF16
{0,0,2,0,2,0,1,0,1,0},//CHARSET_GB18030
{0,0,2,2,2,2,0,2,1,0},//CHARSET_LATIN1
{0,0,2,0,2,0,1,0,1,0}, //CHARSET_GB18030_2022
{0,0,2,2,2,2,2,2,0,2},//CHARSET_ASCII
{0,0,2,0,2,0,0,0,1,0},//CHARSET_TIS620
};
enum ObCollationType
{
CS_TYPE_INVALID = 0,
CS_TYPE_LATIN1_SWEDISH_CI = 8,
CS_TYPE_ASCII_GENERAL_CI = 11,
CS_TYPE_TIS620_THAI_CI = 18,
CS_TYPE_GBK_CHINESE_CI = 28,
CS_TYPE_UTF8MB4_GENERAL_CI = 45,
CS_TYPE_UTF8MB4_BIN = 46,
@ -71,7 +77,9 @@ enum ObCollationType
CS_TYPE_UTF16_GENERAL_CI = 54,
CS_TYPE_UTF16_BIN = 55,
CS_TYPE_BINARY = 63,
CS_TYPE_ASCII_BIN = 65,
CS_TYPE_GBK_BIN = 87,
CS_TYPE_TIS620_BIN = 89,
CS_TYPE_COLLATION_FREE = 100, // mysql中间没有使用这个
CS_TYPE_UTF16_UNICODE_CI = 101,
CS_TYPE_ANY = 125, // unused in mysql
@ -83,10 +91,14 @@ enum ObCollationType
CS_TYPE_GB18030_2022_STROKE_CI = 221, // unused in mysql
CS_TYPE_GB18030_2022_STROKE_CS = 222, // unused in mysql
CS_TYPE_UTF8MB4_UNICODE_CI = 224,
CS_TYPE_UTF8MB4_CZECH_CI = 234,
CS_TYPE_UTF8MB4_CROATIAN_CI = 245,
CS_TYPE_UTF8MB4_UNICODE_520_CI = 246,
CS_TYPE_GB18030_CHINESE_CI = 248,
CS_TYPE_GB18030_BIN = 249,
CS_TYPE_GB18030_CHINESE_CS = 251,
CS_TYPE_UTF8MB4_0900_AI_CI = 255,
CS_TYPE_EXTENDED_MARK = 256, //the cs types below can not used for storing
CS_TYPE_UTF8MB4_0900_BIN, //309 in mysql 8.0
@ -96,24 +108,33 @@ enum ObCollationType
CS_TYPE_GBK_ZH_0900_AS_CS,
CS_TYPE_UTF16_ZH_0900_AS_CS,
CS_TYPE_GB18030_ZH_0900_AS_CS,
CS_TYPE_latin1_ZH_0900_AS_CS, //invaid, not really used
CS_TYPE_LATIN1_ZH_0900_AS_CS,
CS_TYPE_GB18030_2022_ZH_0900_AS_CS,
CS_TYPE_ASCII_ZH_0900_AS_CS,
CS_TYPE_TIS620_ZH_0900_AS_CS,
//radical-stroke order
CS_TYPE_RADICAL_BEGIN_MARK,
CS_TYPE_UTF8MB4_ZH2_0900_AS_CS,
CS_TYPE_GBK_ZH2_0900_AS_CS,
CS_TYPE_UTF16_ZH2_0900_AS_CS,
CS_TYPE_GB18030_ZH2_0900_AS_CS,
CS_TYPE_latin1_ZH2_0900_AS_CS ,//invaid
CS_TYPE_LATIN1_ZH2_0900_AS_CS,
CS_TYPE_GB18030_2022_ZH2_0900_AS_CS,
CS_TYPE_ASCII_ZH2_0900_AS_CS,
CS_TYPE_TIS620_ZH2_0900_AS_CS,
//stroke order
CS_TYPE_STROKE_BEGIN_MARK,
CS_TYPE_UTF8MB4_ZH3_0900_AS_CS,
CS_TYPE_GBK_ZH3_0900_AS_CS,
CS_TYPE_UTF16_ZH3_0900_AS_CS,
CS_TYPE_GB18030_ZH3_0900_AS_CS,
CS_TYPE_latin1_ZH3_0900_AS_CS, //invaid
CS_TYPE_LATIN1_ZH3_0900_AS_CS,
CS_TYPE_GB18030_2022_ZH3_0900_AS_CS,
CS_TYPE_ASCII_ZH3_0900_AS_CS,
CS_TYPE_TIS620_ZH3_0900_AS_CS,
CS_TYPE_MAX
};
@ -122,7 +143,9 @@ enum ObCollationType
enum ObNlsCharsetId
{
CHARSET_INVALID_ID = 0,
CHARSET_US7ASCII_ID = 1,
CHARSET_WE8MSWIN1252_ID=31,
CHARSET_TH8TISASCII_ID = 41,
CHARSET_ZHS16GBK_ID = 852,
CHARSET_ZHS32GB18030_ID = 854,
CHARSET_ZHS32GB18030_2022_ID = 859, // not used in oracle
@ -211,8 +234,8 @@ public:
//比如latin1 1byte ,utf8mb4 4byte,转换因子为4，也可以理解为最多使用4字节存储一个字符
static const int32_t CharConvertFactorNum = 4;
static const int64_t VALID_CHARSET_TYPES = 7;
static const int64_t VALID_COLLATION_TYPES = 20;
static const int64_t VALID_CHARSET_TYPES = 9;
static const int64_t VALID_COLLATION_TYPES = 31;
static int init_charset();
// strntodv2 is an enhanced version of strntod,
@ -393,7 +416,9 @@ public:
|| CHARSET_UTF16 == charset_type
|| CHARSET_GB18030 == charset_type
|| CHARSET_GB18030_2022 == charset_type
|| CHARSET_LATIN1 == charset_type;
|| CHARSET_LATIN1 == charset_type
|| CHARSET_ASCII == charset_type
|| CHARSET_TIS620 == charset_type;
}
static bool is_gb18030_2022(int64_t coll_type_int) {
ObCollationType coll_type = static_cast<ObCollationType>(coll_type_int);
@ -422,12 +447,24 @@ public:
const ObCollationType type2,
ObCollationLevel &res_level,
ObCollationType &res_type);
static int aggregate_collation(const ObCollationLevel level1,
static int aggregate_collation_old(const ObCollationLevel level1,
const ObCollationType type1,
const ObCollationLevel level2,
const ObCollationType type2,
ObCollationLevel &res_level,
ObCollationType &res_type);
static int aggregate_collation_new(
const ObCollationLevel collation_level1,
const ObCollationType collation_type1,
const ObCollationLevel collation_level2,
const ObCollationType collation_type2,
ObCollationLevel &res_level,
ObCollationType &res_type,
uint32_t flags);
static bool left_is_superset(const ObCollationLevel collation_level1,
const ObCollationType collation_type1,
const ObCollationLevel collation_level2,
const ObCollationType collation_type2);
static bool is_bin_sort(ObCollationType collation_type);
static bool is_ci_collate(ObCollationType collation_type);
@ -552,6 +589,8 @@ public:
static ObNlsCharsetId charset_type_to_ora_charset_id(ObCharsetType cs_type);
static ObCharsetType ora_charset_type_to_charset_type(ObNlsCharsetId charset_id);
static bool is_valid_nls_collation(ObNLSCollation nls_collation);
static bool is_valid_ora_charset_id(ObNlsCharsetId charset_id);
static ObCollationType ora_charset_type_to_coll_type(ObNlsCharsetId charset_id);
static ObCollationType get_coll_type_by_nlssort_param(ObCharsetType charset_type,
const ObString &nlssort_param);
private:
@ -599,12 +638,9 @@ public:
if (OB_ERR_INCORRECT_STRING_VALUE == ret && ignore_invalid_character) {
ret = common::OB_SUCCESS;
wchar = INT32_MAX;
length = ObCharset::is_mbchar(collation_type, temp_str.ptr(), temp_str.ptr() + temp_str.length());
if (length <= 0) {
int64_t min_len = 0;
ObCharset::get_mbminlen_by_coll(collation_type, min_len);
length = static_cast<int32_t>(min_len);
}
int64_t min_len = 0;
ObCharset::get_mbminlen_by_coll(collation_type, min_len);
length = static_cast<int32_t>(min_len);
}
}
if (OB_SUCC(ret)) {

10
deps/oblib/src/lib/charset/ob_charset_string_helper.h vendored
View File

@ -705,6 +705,16 @@ public:
foreach_char_prototype<CHARSET_LATIN1, HANDLE_FUNC, true>(str, func, ignore_convert_failed, stop_when_truncated, truncated_len)
: foreach_char_prototype<CHARSET_LATIN1, HANDLE_FUNC, false>(str, func, ignore_convert_failed, stop_when_truncated, truncated_len);
break;
case CHARSET_ASCII:
ret = convert_unicode ?
foreach_char_prototype<CHARSET_ASCII, HANDLE_FUNC, true>(str, func, ignore_convert_failed, stop_when_truncated, truncated_len)
: foreach_char_prototype<CHARSET_ASCII, HANDLE_FUNC, false>(str, func, ignore_convert_failed, stop_when_truncated, truncated_len);
break;
case CHARSET_TIS620:
ret = convert_unicode ?
foreach_char_prototype<CHARSET_TIS620, HANDLE_FUNC, true>(str, func, ignore_convert_failed, stop_when_truncated, truncated_len)
: foreach_char_prototype<CHARSET_TIS620, HANDLE_FUNC, false>(str, func, ignore_convert_failed, stop_when_truncated, truncated_len);
break;
case CHARSET_BINARY:
ret = convert_unicode ?
foreach_char_prototype<CHARSET_BINARY, HANDLE_FUNC, true>(str, func, ignore_convert_failed, stop_when_truncated, truncated_len)

62
deps/oblib/src/lib/charset/ob_ctype.h vendored
View File

@ -23,6 +23,7 @@
#define OB_UTF8MB4_GENERAL_CS OB_UTF8MB4 "_general_cs"
#define OB_UTF8MB4_BIN OB_UTF8MB4 "_bin"
#define OB_UTF8MB4_UNICODE_CI OB_UTF8MB4 "_unicode_ci"
#define OB_UTF8MB4_0900_AI_CI OB_UTF8MB4 "_0900_ai_ci"
#define OB_UTF16 "utf16"
@ -195,7 +196,7 @@ typedef struct ObUnicaseInfo
typedef struct ObCharsetHandler
{
//my_bool (*init)(struct ObCharsetInfo *, MY_CHARSET_LOADER *loader);
bool (*init)(struct ObCharsetInfo *, ObCharsetLoader *loader);
/* Multibyte routines */
unsigned int (*ismbchar)(const struct ObCharsetInfo *, const char *,
const char *);
@ -320,6 +321,17 @@ typedef struct ObCollationHandler
size_t len);
} ObCollationHandler;
typedef struct OB_UNI_IDX {
uint16 from;
uint16 to;
const uchar *tab;
} OB_UNI_IDX;
typedef struct {
int nchars;
OB_UNI_IDX uidx;
} uni_idx;
struct ObCharsetInfo
{
unsigned int number;
@ -336,8 +348,8 @@ struct ObCharsetInfo
unsigned char *to_upper;
unsigned char *sort_order;
ObUCAInfo *uca;
//uint16 *tab_to_uni;
//MY_UNI_IDX *tab_from_uni;
uint16 *tab_to_uni;
OB_UNI_IDX *tab_from_uni;
ObUnicaseInfo *caseinfo;
unsigned char *state_map;
unsigned char *ident_map;
@ -407,6 +419,17 @@ static inline unsigned int ob_ismbchar(const ObCharsetInfo *cs, const unsigned c
#define ob_mbcharlen_2(s, a, b) ((s)->cset->mbcharlen((s), ((((a)&0xFF) << 8) + ((b)&0xFF))))
#define ob_mbmaxlenlen(s) ((s)->mbmaxlenlen)
#define test_all_bits(a, b) (((a) & (b)) == (b))
#define OB_COLL_ALLOW_SUPERSET_CONV 1
#define OB_COLL_ALLOW_COERCIBLE_CONV 2
#define OB_COLL_DISALLOW_NONE 4
#define OB_COLL_ALLOW_NUMERIC_CONV 8
#define OB_COLL_ALLOW_NEW_CONV 16 //allow new rule set for charset aggregation
#define OB_COLL_ALLOW_CONV \
(OB_COLL_ALLOW_SUPERSET_CONV | OB_COLL_ALLOW_COERCIBLE_CONV | OB_COLL_ALLOW_NEW_CONV)
#define OB_COLL_CMP_CONV (OB_COLL_ALLOW_CONV | OB_COLL_DISALLOW_NONE)
typedef struct ob_uni_ctype
{
@ -426,7 +449,6 @@ extern ObUnicaseInfo ob_unicase_unicode520;
extern ObCharsetInfo ob_charset_bin;
extern ObCharsetInfo ob_charset_utf8mb4_bin;
extern ObCharsetInfo ob_charset_utf8mb4_general_ci;
extern ObCharsetInfo ob_charset_latin1;
extern ObCharsetInfo ob_charset_gbk_chinese_ci;
extern ObCharsetInfo ob_charset_gbk_bin;
extern ObCharsetInfo ob_charset_utf16_general_ci;
@ -441,15 +463,22 @@ extern ObCharsetInfo ob_charset_gb18030_2022_radical_cs;
extern ObCharsetInfo ob_charset_gb18030_2022_stroke_ci;
extern ObCharsetInfo ob_charset_gb18030_2022_stroke_cs;
extern ObCharsetInfo ob_charset_gb18030_2022_bin;
extern ObCharsetInfo ob_charset_utf8mb4_unicode_ci;
extern ObCharsetInfo ob_charset_utf16_unicode_ci;
extern ObCharsetInfo ob_charset_utf8mb4_zh_0900_as_cs;
extern ObCharsetInfo ob_charset_utf8mb4_zh2_0900_as_cs;
extern ObCharsetInfo ob_charset_utf8mb4_zh3_0900_as_cs;
extern ObCharsetInfo ob_charset_utf8mb4_unicode_ci;
extern ObCharsetInfo ob_charset_utf8mb4_0900_bin;
extern ObCharsetInfo ob_charset_utf8mb4_0900_ai_ci;
extern ObCharsetInfo ob_charset_utf16_unicode_ci;
extern ObCharsetInfo ob_charset_latin1;
extern ObCharsetInfo ob_charset_latin1_bin;
extern ObCharsetInfo ob_charset_utf8mb4_croatian_uca_ci;
extern ObCharsetInfo ob_charset_utf8mb4_unicode_520_ci;
extern ObCharsetInfo ob_charset_utf8mb4_czech_uca_ci;
extern ObCharsetInfo ob_charset_ascii;
extern ObCharsetInfo ob_charset_ascii_bin;
extern ObCharsetInfo ob_charset_tis620_thai_ci;
extern ObCharsetInfo ob_charset_tis620_bin;
extern ObCollationHandler ob_collation_mb_bin_handler;
extern ObCharsetHandler ob_charset_utf8mb4_handler;
extern ObCharsetHandler ob_charset_utf16_handler;
@ -630,7 +659,10 @@ size_t ob_strxfrm_pad(const ObCharsetInfo *cs, unsigned char *str, unsigned char
unsigned char *strend, unsigned int nweights, unsigned int flags);
size_t ob_strnxfrmlen_simple(const struct ObCharsetInfo *, size_t);
int ob_wildcmp_8bit(const ObCharsetInfo* cs, const char* str, const char* str_end, const char* wildstr,
const char* wildend, int escape, int w_one, int w_many);
uint32_t ob_instr_simple(const ObCharsetInfo* cs , const char* b, size_t b_length,
const char* s, size_t s_length, ob_match_t* match, unsigned int nmatch);
size_t ob_strnxfrmlen_unicode_full_bin(const struct ObCharsetInfo *, size_t);
size_t ob_strnxfrmlen_utf8mb4(const struct ObCharsetInfo *, size_t);
@ -667,11 +699,23 @@ char *strmake(char *, const char *, size_t);
size_t ob_casedn_8bit(const ObCharsetInfo *cs __attribute__((unused)),
char* str __attribute__((unused)), size_t srclen __attribute__((unused)),
char* dst __attribute__((unused)), size_t dstlen __attribute__((unused)));
int ob_strcasecmp_8bit(const ObCharsetInfo *cs, const char *s, const char *t);
size_t ob_caseup_8bit(const ObCharsetInfo *cs __attribute__((unused)),
char* str __attribute__((unused)), size_t srclen __attribute__((unused)),
char* dst __attribute__((unused)), size_t dstlen __attribute__((unused)));
bool ob_cset_init_8bit(ObCharsetInfo *cs, ObCharsetLoader *loader);
bool ob_coll_init_simple(ObCharsetInfo *cs, ObCharsetLoader *loader);
size_t ob_well_formed_len_ascii(const ObCharsetInfo *cs,
const char *start, const char *end,
size_t nchars,
int *error);
int ob_8bit(const ObCharsetInfo *cs, ob_wc_t *wc, const uchar *str,
const uchar *end);
int ob_wc_mb_8bit(const ObCharsetInfo *cs, ob_wc_t wc, uchar *str, uchar *end);
int ob_mb_wc_8bit(const ObCharsetInfo *cs, ob_wc_t *wc, const uchar *str,const uchar *end);
unsigned int ob_ismbchar_8bit(const ObCharsetInfo *cs __attribute__((unused)), const char *p, const char *e);
extern "C" void right_to_die_or_duty_to_live_c();

114
deps/oblib/src/lib/charset/ob_ctype_ascii.cc vendored Normal file
View File

@ -0,0 +1,114 @@
/**
* Copyright (code) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "lib/charset/ob_mysql_global.h"
#include "lib/charset/ob_ctype.h"
#include "lib/utility/ob_macro_utils.h"
#include "lib/charset/ob_ctype_ascii_tab.h"
static ObCharsetHandler ob_charset_ascii_handler = {
ob_cset_init_8bit,
ob_ismbchar_8bit,
ob_mbcharlen_8bit,
ob_numchars_8bit,
ob_charpos_8bit,
ob_max_bytes_charpos_8bit,
ob_well_formed_len_ascii,
ob_lengthsp_8bit,
//ob_numcells_8bit,
ob_mb_wc_8bit,
ob_wc_mb_8bit,
ob_mb_ctype_8bit,
//ob_caseup_str_8bit,
//ob_casedn_str_8bit,
ob_caseup_8bit,
ob_casedn_8bit,
//ob_snprintf_8bit,
//ob_long10_to_str_8bit,
//ob_longlong10_to_str_8bit,
ob_fill_8bit,
ob_strntol_8bit,
ob_strntoul_8bit,
ob_strntoll_8bit,
ob_strntoull_8bit,
ob_strntod_8bit,
//ob_strtoll10_8bit,
ob_strntoull10rnd_8bit,
ob_scan_8bit};
ObCharsetInfo ob_charset_ascii = {
11,0,0,
OB_CS_COMPILED | OB_CS_PRIMARY | OB_CS_PUREASCII,
"ascii",
"ascii_general_ci",
"US ASCII",
NULL,
NULL,
ctype_ascii_general_ci,
to_lower_ascii_general_ci,
to_upper_ascii_general_ci,
sort_order_ascii_general_ci,
NULL,
to_uni_ascii_general_ci,
NULL,
&ob_unicase_default,
NULL,
NULL,
1,
1,
1,
1,
1,
1,
0,
255,
' ',
false,
1,
1,
&ob_charset_ascii_handler,
&ob_collation_8bit_simple_ci_handler,
PAD_SPACE};
ObCharsetInfo ob_charset_ascii_bin = {
65,0,0,
OB_CS_COMPILED | OB_CS_BINSORT | OB_CS_PUREASCII,
"ascii",
"ascii_bin",
"US ASCII",
NULL,
NULL,
ctype_ascii_bin,
to_lower_ascii_bin,
to_upper_ascii_bin,
NULL,
NULL,
to_uni_ascii_bin,
nullptr,
&ob_unicase_default,
NULL,
NULL,
1,
1,
1,
1,
1,
1,
0,
255,
' ',
false,
1,
1,
&ob_charset_ascii_handler,
&ob_collation_8bit_bin_handler,
PAD_SPACE};

242
deps/oblib/src/lib/charset/ob_ctype_ascii_tab.h vendored Normal file
View File

@ -0,0 +1,242 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
static unsigned char ctype_ascii_general_ci[] = {
0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x28, 0x28,
0x28, 0x28, 0x28, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x48, 0x10, 0x10,
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10,
0x10, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x10,
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81,
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x10, 0x10, 0x10, 0x10,
0x10, 0x10, 0x82, 0x82, 0x82, 0x82, 0x82, 0x82, 0x02, 0x02, 0x02, 0x02,
0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
0x02, 0x02, 0x02, 0x02, 0x10, 0x10, 0x10, 0x10, 0x20, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00};
static unsigned char to_lower_ascii_general_ci[] = {
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B,
0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23,
0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F,
0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B,
0x3C, 0x3D, 0x3E, 0x3F, 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73,
0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F,
0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B,
0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, 0x80, 0x81, 0x82, 0x83,
0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F,
0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B,
0x9C, 0x9D, 0x9E, 0x9F, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7,
0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3,
0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF,
0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB,
0xCC, 0xCD, 0xCE, 0xCF, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3,
0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB,
0xFC, 0xFD, 0xFE, 0xFF};
static unsigned char to_upper_ascii_general_ci[] = {
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B,
0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23,
0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F,
0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B,
0x3C, 0x3D, 0x3E, 0x3F, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53,
0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F,
0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B,
0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57,
0x58, 0x59, 0x5A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, 0x80, 0x81, 0x82, 0x83,
0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F,
0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B,
0x9C, 0x9D, 0x9E, 0x9F, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7,
0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3,
0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF,
0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB,
0xCC, 0xCD, 0xCE, 0xCF, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3,
0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB,
0xFC, 0xFD, 0xFE, 0xFF};
static unsigned char sort_order_ascii_general_ci[] = {
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B,
0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23,
0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F,
0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B,
0x3C, 0x3D, 0x3E, 0x3F, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53,
0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F,
0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B,
0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57,
0x58, 0x59, 0x5A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, 0x80, 0x81, 0x82, 0x83,
0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F,
0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B,
0x9C, 0x9D, 0x9E, 0x9F, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7,
0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3,
0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF,
0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB,
0xCC, 0xCD, 0xCE, 0xCF, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3,
0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB,
0xFC, 0xFD, 0xFE, 0xFF};
static uint16 to_uni_ascii_general_ci[] = {
0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008,
0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F, 0x0010, 0x0011,
0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001A,
0x001B, 0x001C, 0x001D, 0x001E, 0x001F, 0x0020, 0x0021, 0x0022, 0x0023,
0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002A, 0x002B, 0x002C,
0x002D, 0x002E, 0x002F, 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035,
0x0036, 0x0037, 0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E,
0x003F, 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047,
0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F, 0x0050,
0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059,
0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F, 0x0060, 0x0061, 0x0062,
0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006A, 0x006B,
0x006C, 0x006D, 0x006E, 0x006F, 0x0070, 0x0071, 0x0072, 0x0073, 0x0074,
0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D,
0x007E, 0x007F, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000};
static unsigned char ctype_ascii_bin[] = {
0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x28, 0x28,
0x28, 0x28, 0x28, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x48, 0x10, 0x10,
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10,
0x10, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x10,
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81,
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x10, 0x10, 0x10, 0x10,
0x10, 0x10, 0x82, 0x82, 0x82, 0x82, 0x82, 0x82, 0x02, 0x02, 0x02, 0x02,
0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
0x02, 0x02, 0x02, 0x02, 0x10, 0x10, 0x10, 0x10, 0x20, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00};
static unsigned char to_lower_ascii_bin[] = {
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B,
0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23,
0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F,
0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B,
0x3C, 0x3D, 0x3E, 0x3F, 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73,
0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F,
0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B,
0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, 0x80, 0x81, 0x82, 0x83,
0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F,
0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B,
0x9C, 0x9D, 0x9E, 0x9F, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7,
0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3,
0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF,
0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB,
0xCC, 0xCD, 0xCE, 0xCF, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3,
0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB,
0xFC, 0xFD, 0xFE, 0xFF};
static unsigned char to_upper_ascii_bin[] = {
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B,
0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23,
0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F,
0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B,
0x3C, 0x3D, 0x3E, 0x3F, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53,
0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F,
0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B,
0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57,
0x58, 0x59, 0x5A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, 0x80, 0x81, 0x82, 0x83,
0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F,
0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B,
0x9C, 0x9D, 0x9E, 0x9F, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7,
0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3,
0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF,
0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB,
0xCC, 0xCD, 0xCE, 0xCF, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3,
0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB,
0xFC, 0xFD, 0xFE, 0xFF};
static uint16 to_uni_ascii_bin[] = {
0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008,
0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F, 0x0010, 0x0011,
0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001A,
0x001B, 0x001C, 0x001D, 0x001E, 0x001F, 0x0020, 0x0021, 0x0022, 0x0023,
0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002A, 0x002B, 0x002C,
0x002D, 0x002E, 0x002F, 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035,
0x0036, 0x0037, 0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E,
0x003F, 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047,
0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F, 0x0050,
0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059,
0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F, 0x0060, 0x0061, 0x0062,
0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006A, 0x006B,
0x006C, 0x006D, 0x006E, 0x006F, 0x0070, 0x0071, 0x0072, 0x0073, 0x0074,
0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D,
0x007E, 0x007F, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000};

12
deps/oblib/src/lib/charset/ob_ctype_bin.cc vendored
View File

@ -384,10 +384,16 @@ void ob_hash_sort_bin(const ObCharsetInfo *cs __attribute__((unused)),
}
}
extern "C" {
static bool ob_coll_init_8bit_bin(ObCharsetInfo *cs, ObCharsetLoader *loader) {
cs->max_sort_char = 255;
return false;
}
}
static ObCharsetHandler ob_charset_handler=
{
NULL,
NULL,
ob_mbcharlen_8bit,
ob_numchars_8bit,
@ -412,7 +418,7 @@ static ObCharsetHandler ob_charset_handler=
ObCollationHandler ob_collation_8bit_bin_handler =
{
NULL, /* init */
ob_coll_init_8bit_bin, /* init */
NULL, /* uninit */
ob_strnncoll_8bit_bin,
ob_strnncollsp_8bit_bin,
@ -459,6 +465,8 @@ ObCharsetInfo ob_charset_bin =
bin_char_array,
NULL,
NULL,
NULL,
NULL,
&ob_unicase_default,
NULL,
NULL,

28
deps/oblib/src/lib/charset/ob_ctype_gb18030.cc vendored
View File

@ -962,6 +962,7 @@ static ObCollationHandler ob_collation_cs_handler =
};
static ObCharsetHandler ob_charset_gb18030_handler = {
NULL,
ob_ismbchar_gb18030,
ob_mbcharlen_gb18030,
ob_numchars_mb,
@ -1005,8 +1006,8 @@ ObCharsetInfo ob_charset_gb18030_chinese_ci = {
to_upper_gb18030, /* UPPER */
sort_order_gb18030_ci, /* sort */
NULL, /* uca */
// NULL, /* tab_to_uni */
// NULL, /* tab_from_uni */
NULL, /* tab_to_uni */
NULL, /* tab_from_uni */
&ob_caseinfo_gb18030, /* caseinfo */
NULL, /* state_map */
NULL, /* ident_map */
@ -1041,8 +1042,8 @@ ObCharsetInfo ob_charset_gb18030_chinese_cs = {
to_upper_gb18030, /* UPPER */
sort_order_gb18030, /* sort */
NULL, /* uca */
// NULL, /* tab_to_uni */
// NULL, /* tab_from_uni */
NULL, /* tab_to_uni */
NULL, /* tab_from_uni */
&ob_caseinfo_gb18030, /* caseinfo */
NULL, /* state_map */
NULL, /* ident_map */
@ -1077,8 +1078,8 @@ ObCharsetInfo ob_charset_gb18030_bin = {
to_upper_gb18030,
NULL,
NULL,
NULL,
NULL,
&ob_caseinfo_gb18030,
NULL,
NULL,
@ -1599,6 +1600,7 @@ static ObCollationHandler ob_collation_2022_stroke_cs_handler =
static ObCharsetHandler ob_charset_gb18030_2022_handler =
{
NULL,
ob_ismbchar_gb18030,
ob_mbcharlen_gb18030,
ob_numchars_mb,
@ -1637,6 +1639,8 @@ ObCharsetInfo ob_charset_gb18030_2022_bin =
to_upper_gb18030, /* UPPER */
NULL, /* sort order */
NULL, /* uca */
NULL,
NULL,
&ob_caseinfo_gb18030, /* caseinfo */
NULL, /* state_map */
NULL, /* ident_map */
@ -1673,6 +1677,8 @@ ObCharsetInfo ob_charset_gb18030_2022_pinyin_ci =
to_upper_gb18030, /* UPPER */
sort_order_gb18030_ci, /* sort order */
NULL, /* uca */
NULL,
NULL,
&ob_caseinfo_gb18030, /* caseinfo */
NULL, /* state_map */
NULL, /* ident_map */
@ -1709,6 +1715,8 @@ ObCharsetInfo ob_charset_gb18030_2022_pinyin_cs =
to_upper_gb18030, /* UPPER */
sort_order_gb18030, /* sort order */
NULL, /* uca */
NULL,
NULL,
&ob_caseinfo_gb18030, /* caseinfo */
NULL, /* state_map */
NULL, /* ident_map */
@ -1745,6 +1753,8 @@ ObCharsetInfo ob_charset_gb18030_2022_radical_ci =
to_upper_gb18030, /* UPPER */
sort_order_gb18030_ci, /* sort order */
NULL, /* uca */
NULL,
NULL,
&ob_caseinfo_gb18030, /* caseinfo */
NULL, /* state_map */
NULL, /* ident_map */
@ -1781,6 +1791,8 @@ ObCharsetInfo ob_charset_gb18030_2022_radical_cs =
to_upper_gb18030, /* UPPER */
sort_order_gb18030, /* sort order */
NULL, /* uca */
NULL,
NULL,
&ob_caseinfo_gb18030, /* caseinfo */
NULL, /* state_map */
NULL, /* ident_map */
@ -1817,6 +1829,8 @@ ObCharsetInfo ob_charset_gb18030_2022_stroke_ci =
to_upper_gb18030, /* UPPER */
sort_order_gb18030_ci, /* sort order */
NULL, /* uca */
NULL,
NULL,
&ob_caseinfo_gb18030, /* caseinfo */
NULL, /* state_map */
NULL, /* ident_map */
@ -1853,6 +1867,8 @@ ObCharsetInfo ob_charset_gb18030_2022_stroke_cs =
to_upper_gb18030, /* UPPER */
sort_order_gb18030, /* sort order */
NULL, /* uca */
NULL,
NULL,
&ob_caseinfo_gb18030, /* caseinfo */
NULL, /* state_map */
NULL, /* ident_map */

5
deps/oblib/src/lib/charset/ob_ctype_gbk.cc vendored
View File

@ -345,6 +345,7 @@ static ObCollationHandler ob_collation_gbk_ci_handler =
static ObCharsetHandler ob_charset_gbk_handler=
{
NULL,
ismbchar_gbk,
mbcharlen_gbk,
ob_numchars_mb,
@ -385,6 +386,8 @@ ObCharsetInfo ob_charset_gbk_chinese_ci=
to_upper_gbk,
sort_order_gbk,
NULL,
NULL,
NULL,
&ob_caseinfo_gbk,
NULL,
NULL,
@ -418,6 +421,8 @@ ObCharsetInfo ob_charset_gbk_bin=
to_upper_gbk,
NULL,
NULL,
NULL,
NULL,
&ob_caseinfo_gbk,
NULL,
NULL,

14
deps/oblib/src/lib/charset/ob_ctype_latin1.cc vendored
View File

@ -40,14 +40,14 @@ static int ob_wc_mb_latin1(const ObCharsetInfo *cs __attribute__((unused)),
static ObCharsetHandler ob_charset_latin1_handler=
{
//NULL,
NULL,
ob_ismbchar_8bit,
ob_mbcharlen_8bit,
ob_numchars_8bit,
ob_charpos_8bit,
ob_max_bytes_charpos_8bit,
ob_well_formed_len_8bit,
ob_lengthsp_binary,
ob_lengthsp_8bit,
//ob_numcells_8bit,
ob_mb_wc_latin1,
ob_wc_mb_latin1,
@ -83,8 +83,8 @@ ObCharsetInfo ob_charset_latin1 = {
to_upper_latin1,
sort_order_latin1,
NULL,
//NULL,
//NULL,
NULL,
NULL,
&ob_unicase_default,
NULL,
NULL,
@ -107,7 +107,7 @@ ObCharsetInfo ob_charset_latin1 = {
ObCharsetInfo ob_charset_latin1_bin = {
47,0,0,
OB_CS_COMPILED | OB_CS_BINSORT,
OB_LATIN1,
OB_LATIN1,
OB_LATIN1_BIN,
"cp1252 West European",
NULL,
@ -117,8 +117,8 @@ ObCharsetInfo ob_charset_latin1_bin = {
to_upper_latin1,
NULL,
NULL,
//NULL,
//NULL,
NULL,
NULL,
&ob_unicase_default,
NULL,
NULL,

180
deps/oblib/src/lib/charset/ob_ctype_simple.cc vendored
View File

@ -26,6 +26,9 @@
#define CUTLIM (UINT64_MAX % 10)
#define SPACE_INT 0x20202020
#define DIGITS_IN_ULONGLONG 20
#define PLANE_SIZE 0x100
#define PLANE_NUM 0x100
#define PLANE_NUMBER(x) (((x) >> 8) % PLANE_NUM)
static ulonglong d10[DIGITS_IN_ULONGLONG]=
{
@ -848,6 +851,13 @@ size_t ob_casedn_8bit(const ObCharsetInfo *cs __attribute__((unused)),
return srclen;
}
int ob_strcasecmp_8bit(const ObCharsetInfo *cs, const char *s, const char *t) {
const uchar *map = cs->to_upper;
while (map[(uchar)*s] == map[(uchar)*t++])
if (!*s++) return 0;
return ((int)map[(uchar)s[0]] - (int)map[(uchar)t[-1]]);
}
int ob_strnncoll_simple(const ObCharsetInfo *cs __attribute__((unused)),
const unsigned char *s, size_t slen,
const unsigned char *t, size_t tlen,
@ -1073,8 +1083,176 @@ uint32_t ob_instr_simple(const ObCharsetInfo* cs , const char* b, size_t b_lengt
return 0;
}
static int pcmp(const void *f, const void *s) {
const uni_idx *F = (const uni_idx *)f;
const uni_idx *S = (const uni_idx *)s;
int res;
if (!(res = ((S->nchars) - (F->nchars))))
res = ((F->uidx.from) - (S->uidx.to));
return res;
}
static bool create_fromuni(ObCharsetInfo *cs, ObCharsetLoader *loader)
{
uni_idx idx[PLANE_NUM];
int i, n;
OB_UNI_IDX *tab_from_uni;
/*
Check that Unicode map is loaded.
It can be not loaded when the collation is
listed in Index.xml but not specified
in the character set specific XML file.
*/
if (!cs->tab_to_uni) return true;
/* Clear plane statistics */
memset(idx, 0, sizeof(idx));
/* Count number of characters in each plane */
for (i = 0; i < 0x100; i++) {
uint16 wc = cs->tab_to_uni[i];
int pl = PLANE_NUMBER(wc);
if (wc || !i) {
if (!idx[pl].nchars) {
idx[pl].uidx.from = wc;
idx[pl].uidx.to = wc;
} else {
idx[pl].uidx.from = wc < idx[pl].uidx.from ? wc : idx[pl].uidx.from;
idx[pl].uidx.to = wc > idx[pl].uidx.to ? wc : idx[pl].uidx.to;
}
idx[pl].nchars++;
}
}
/* Sort planes in descending order */
qsort(&idx, PLANE_NUM, sizeof(uni_idx), &pcmp);
for (i = 0; i < PLANE_NUM; i++) {
int ch, numchars;
uchar *tab;
/* Skip empty plane */
if (!idx[i].nchars) break;
numchars = idx[i].uidx.to - idx[i].uidx.from + 1;
if (!(idx[i].uidx.tab = tab = (uchar *)(loader->once_alloc)(
numchars * sizeof(*idx[i].uidx.tab))))
return true;
memset(tab, 0, numchars * sizeof(*idx[i].uidx.tab));
for (ch = 1; ch < PLANE_SIZE; ch++) {
uint16 wc = cs->tab_to_uni[ch];
if (wc >= idx[i].uidx.from && wc <= idx[i].uidx.to && wc) {
int ofs = wc - idx[i].uidx.from;
/*
Character sets like armscii8 may have two code points for
one character. When converting from UNICODE back to
armscii8, select the lowest one, which is in the ASCII
range.
*/
if (tab[ofs] == '\0') tab[ofs] = ch;
}
}
}
/* Allocate and fill reverse table for each plane */
n = i;
if (!(cs->tab_from_uni = tab_from_uni =
(OB_UNI_IDX *)(loader->once_alloc)(sizeof(OB_UNI_IDX) * (n + 1))))
return true;
for (i = 0; i < n; i++) tab_from_uni[i] = idx[i].uidx;
/* Set end-of-list marker */
memset(&tab_from_uni[i], 0, sizeof(OB_UNI_IDX));
return false;
}
bool ob_cset_init_8bit(ObCharsetInfo *cs, ObCharsetLoader *loader)
{
cs->caseup_multiply = 1;
cs->casedn_multiply = 1;
cs->pad_char = ' ';
return create_fromuni(cs, loader);
}
static void set_max_sort_char(ObCharsetInfo *cs) {
uchar max_char;
uint i;
if (!cs->sort_order) return;
max_char = cs->sort_order[(uchar)cs->max_sort_char];
for (i = 0; i < 256; i++) {
if ((uchar)cs->sort_order[i] > max_char) {
max_char = (uchar)cs->sort_order[i];
cs->max_sort_char = i;
}
}
}
bool ob_coll_init_simple(ObCharsetInfo *cs, ObCharsetLoader *loader) {
set_max_sort_char(cs);
return false;
}
size_t ob_well_formed_len_ascii(const ObCharsetInfo *cs,
const char *start, const char *end,
size_t nchars,
int *error)
{
const char *oldstart = start;
*error = 0;
while (start < end) {
if ((*start & 0x80) != 0) {
*error = 1;
break;
}
start++;
}
return start - oldstart;
}
int ob_mb_wc_8bit(const ObCharsetInfo *cs, ob_wc_t *wc, const uchar *str,
const uchar *end) {
if (str >= end) return OB_CS_TOOSMALL;
*wc = cs->tab_to_uni[*str];
return (!wc[0] && str[0]) ? -1 : 1;
}
int ob_wc_mb_8bit(const ObCharsetInfo *cs, ob_wc_t wc, uchar *str, uchar *end) {
const OB_UNI_IDX *idx;
if (str >= end) return OB_CS_TOOSMALL;
for (idx = cs->tab_from_uni; idx->tab; idx++) {
if (idx->from <= wc && idx->to >= wc) {
str[0] = idx->tab[wc - idx->from];
return (!str[0] && wc) ? OB_CS_ILUNI : 1;
}
}
return OB_CS_ILUNI;
}
unsigned int ob_ismbchar_8bit(const ObCharsetInfo *cs __attribute__((unused)), const char *p, const char *e)
{
ob_charset_assert(e > p);
/*ob_ismbchar_8bit
*this is a mock interface, in mysql there is not is_mbchar for single byte charset
*/
if (0x00 <= (unsigned char)(*p) && (unsigned char)(*p) <= 0xFF) {
return 1;
}
return 0;
}
ObCollationHandler ob_collation_8bit_simple_ci_handler = {
NULL, /* init */
ob_coll_init_simple, /* init */
NULL,
ob_strnncoll_simple,
ob_strnncollsp_simple,

336
deps/oblib/src/lib/charset/ob_ctype_tis620.cc vendored Normal file
View File

@ -0,0 +1,336 @@
/**
* Copyright (code) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#include "lib/charset/ob_mysql_global.h"
#include "lib/charset/ob_ctype.h"
#include "lib/utility/ob_macro_utils.h"
#include "lib/charset/ob_ctype_tis620_tab.h"
#include "lib/allocator/ob_malloc.h"
/*
Convert thai string to "Standard C String Function" sortable string
SYNOPSIS
thai2sortable()
tstr String to convert. Does not have to end with \0
len Length of tstr
*/
static size_t thai2sortable(uchar *tstr, size_t len) {
uchar *p;
size_t tlen;
uchar l2bias;
tlen = len;
l2bias = 256 - 8;
for (p = tstr; tlen > 0; p++, tlen--) {
uchar c = *p;
if (isthai(c)) {
const int *t_ctype0 = t_ctype[c];
if (isconsnt(c)) l2bias -= 8;
if (isldvowel(c) && tlen != 1 && isconsnt(p[1])) {
/* simply swap between leading-vowel and consonant */
*p = p[1];
p[1] = c;
tlen--;
p++;
continue;
}
/* if found level 2 char (L2_GARAN,L2_TONE*,L2_TYKHU) move to last */
if (t_ctype0[1] >= L2_GARAN) {
/*
l2bias use to control position weight of l2char
example (*=l2char) XX*X must come before X*XX
*/
memmove((char *)p, (char *)(p + 1), tlen - 1);
tstr[len - 1] = l2bias + t_ctype0[1] - L2_GARAN + 1;
p--;
continue;
}
} else {
l2bias -= 8;
*p = to_lower_tis620[c];
}
}
return len;
}
/*
strncoll() replacement, compare 2 string, both are converted to sortable
string
NOTE:
We can't cut strings at end \0 as this would break comparison with
LIKE characters, where the min range is stored as end \0
Arg: 2 Strings and it compare length
Ret: strcmp result
*/
extern "C" {
static int ob_strnncoll_tis620(const ObCharsetInfo *cs [[maybe_unused]],
const uchar *s1, size_t len1, const uchar *s2,
size_t len2, bool s2_is_prefix) {
uchar buf[80];
uchar *tc1, *tc2;
int i;
if (s2_is_prefix && len1 > len2) len1 = len2;
tc1 = buf;
if ((len1 + len2 + 2) > (int)sizeof(buf))
tc1 = static_cast<uchar *>(oceanbase::ob_malloc(len1 + len2 + 2, "CharsetInit"));
tc2 = tc1 + len1 + 1;
memcpy(tc1, s1, len1);
tc1[len1] = 0; /* if length(s1)> len1, need to put 'end of string' */
memcpy(tc2, s2, len2);
tc2[len2] = 0; /* put end of string */
thai2sortable(tc1, len1);
thai2sortable(tc2, len2);
i = strcmp((char *)tc1, (char *)tc2);
if (tc1 != buf) oceanbase::ob_free(tc1);
return i;
}
static int ob_strnncollsp_tis620(const ObCharsetInfo *cs [[maybe_unused]],
const uchar *a0, size_t a_length,
const uchar *b0, size_t b_length,
bool diff_if_only_endspace_difference __attribute__((unused))) {
uchar buf[80], *end, *a, *b, *alloced = NULL;
size_t length;
int res = 0;
a = buf;
if ((a_length + b_length + 2) > (int)sizeof(buf))
alloced = a = (uchar *)oceanbase::ob_malloc(a_length + b_length + 2, "CharsetInit");
b = a + a_length + 1;
memcpy(a, a0, a_length);
a[a_length] = 0; /* if length(a0)> len1, need to put 'end of string' */
memcpy(b, b0, b_length);
b[b_length] = 0; /* put end of string */
a_length = thai2sortable(a, a_length);
b_length = thai2sortable(b, b_length);
end = a + (length = std::min(a_length, b_length));
while (a < end) {
if (*a++ != *b++) {
res = ((int)a[-1] - (int)b[-1]);
goto ret;
}
}
if (a_length != b_length) {
int swap = 1;
/*
Check the next not space character of the longer key. If it's < ' ',
then it's smaller than the other key.
*/
if (a_length < b_length) {
/* put shorter key in s */
a_length = b_length;
a = b;
swap = -1; /* swap sign of result */
res = -res;
}
for (end = a + a_length - length; a < end; a++) {
if (*a != ' ') {
res = (*a < ' ') ? -swap : swap;
goto ret;
}
}
}
ret:
if (alloced) oceanbase::ob_free(alloced);
return res;
}
/*
strnxfrm replacement, convert Thai string to sortable string
Arg: Destination buffer, source string, dest length and source length
Ret: Converted string size
*/
static size_t ob_strnxfrm_tis620(const ObCharsetInfo *cs, uchar *dst,
size_t dstlen, uint nweights, const uchar *src,
size_t srclen, uint flags,bool* is_valid_unicode) {
size_t dstlen0 = dstlen;
size_t min_len = std::min(dstlen, srclen);
size_t len = 0;
*is_valid_unicode = 1;
/*
We don't use strmake here, since it requires one more character for
the terminating '\0', while this function itself and the following calling
functions do not require it
*/
while (len < min_len) {
if (!(dst[len] = src[len])) break;
len++;
}
len = thai2sortable(dst, len);
dstlen = std::min(dstlen, size_t(nweights));
len = std::min(len, size_t(dstlen));
len = ob_strxfrm_pad(cs, dst, dst + len, dst + dstlen, (uint)(dstlen - len),
flags);
if ((flags & OB_STRXFRM_PAD_TO_MAXLEN) && len < dstlen0) {
size_t fill_length = dstlen0 - len;
cs->cset->fill(cs, (char *)dst + len, fill_length, cs->pad_char);
len = dstlen0;
}
return len;
}
} // extern "C"
extern "C" {
static int ob_mb_wc_tis620(const ObCharsetInfo *cs [[maybe_unused]], ob_wc_t *wc,
const uchar *str, const uchar *end) {
if (str >= end) return OB_CS_TOOSMALL;
*wc = cs_to_uni_tis620[*str];
return (!wc[0] && str[0]) ? -1 : 1;
}
static int ob_wc_mb_tis620(const ObCharsetInfo *cs [[maybe_unused]], ob_wc_t wc,
uchar *str, uchar *end) {
unsigned char *pl;
if (str >= end) return OB_CS_TOOSMALL;
pl = uni_to_cs_tis620[(wc >> 8) & 0xFF];
str[0] = pl ? pl[wc & 0xFF] : '\0';
return (!str[0] && wc) ? OB_CS_ILUNI : 1;
}
} // extern "C"
static ObCollationHandler ob_collation_tis620_handler = {
NULL, /* init */
NULL,
ob_strnncoll_tis620,
ob_strnncollsp_tis620,
ob_strnxfrm_tis620,
ob_strnxfrmlen_simple,
NULL,//varlen
ob_like_range_simple,
ob_wildcmp_8bit, /* wildcmp */
NULL,//ob_strcasecmp_8bit,
ob_instr_simple, /* QQ: To be fixed */
ob_hash_sort_simple,
ob_propagate_simple};
static ObCharsetHandler ob_charset_tis620_handler = {
NULL, /* init */
ob_ismbchar_8bit, /* ismbchar */
ob_mbcharlen_8bit, /* mbcharlen */
ob_numchars_8bit,
ob_charpos_8bit,
ob_max_bytes_charpos_8bit,
ob_well_formed_len_8bit,
ob_lengthsp_8bit,
//ob_numcells_8bit,
ob_mb_wc_tis620, /* mb_wc */
ob_wc_mb_tis620, /* wc_mb */
ob_mb_ctype_8bit,
//ob_caseup_str_8bit,
//ob_casedn_str_8bit,
ob_caseup_8bit,
ob_casedn_8bit,
//ob_snprintf_8bit,
//ob_long10_to_str_8bit,
//ob_longlong10_to_str_8bit,
ob_fill_8bit,
ob_strntol_8bit,
ob_strntoul_8bit,
ob_strntoll_8bit,
ob_strntoull_8bit,
ob_strntod_8bit,
//ob_strtoll10_8bit,
ob_strntoull10rnd_8bit,
ob_scan_8bit};
ObCharsetInfo ob_charset_tis620_thai_ci = {
18,
0,
0, /* number */
OB_CS_COMPILED | OB_CS_PRIMARY | OB_CS_STRNXFRM, /* state */
"tis620", /* cs name */
"tis620_thai_ci", /* m_coll_name */
"TIS620 Thai", /* comment */
NULL, /* tailoring */
NULL, /* coll_param */
ctype_tis620,
to_lower_tis620,
to_upper_tis620,
sort_order_tis620,
NULL, /* uca */
NULL, /* tab_to_uni */
NULL, /* tab_from_uni */
&ob_unicase_default, /* caseinfo */
NULL, /* state_map */
NULL, /* ident_map */
4, /* strxfrm_multiply */
1, /* caseup_multiply */
1, /* casedn_multiply */
1, /* mbminlen */
1, /* mbmaxlen */
1,
0, /* min_sort_char */
255, /* max_sort_char */
' ', /* pad char */
false, /* escape_with_backslash_is_dangerous */
1, /* levels_for_compare */
1,
&ob_charset_tis620_handler,
&ob_collation_tis620_handler,
PAD_SPACE};
ObCharsetInfo ob_charset_tis620_bin = {
89,
0,
0,
OB_CS_COMPILED | OB_CS_BINSORT,
"tis620",
"tis620_bin",
"TIS620 Thai",
NULL,
NULL,
ctype_tis620,
to_lower_tis620,
to_upper_tis620,
NULL,
NULL,
NULL,
NULL,
&ob_unicase_default,
NULL,
NULL,
1,
1,
1,
1,
1,
1,
0,
255,
' ',
false,
1,
1,
&ob_charset_tis620_handler,
&ob_collation_8bit_bin_handler,
PAD_SPACE};

830
deps/oblib/src/lib/charset/ob_ctype_tis620_tab.h vendored Normal file
View File

@ -0,0 +1,830 @@
/**
* Copyright (c) 2021 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define TOT_LEVELS 5
#define LAST_LEVEL 4 /* TOT_LEVELS - 1 */
/* level 1 symbols & order */
enum l1_symbols {
L1_08 = TOT_LEVELS,
L1_18,
L1_28,
L1_38,
L1_48,
L1_58,
L1_68,
L1_78,
L1_88,
L1_98,
L1_A8,
L1_B8,
L1_C8,
L1_D8,
L1_E8,
L1_F8,
L1_G8,
L1_H8,
L1_I8,
L1_J8,
L1_K8,
L1_L8,
L1_M8,
L1_N8,
L1_O8,
L1_P8,
L1_Q8,
L1_R8,
L1_S8,
L1_T8,
L1_U8,
L1_V8,
L1_W8,
L1_X8,
L1_Y8,
L1_Z8,
L1_KO_KAI,
L1_KHO_KHAI,
L1_KHO_KHUAT,
L1_KHO_KHWAI,
L1_KHO_KHON,
L1_KHO_RAKHANG,
L1_NGO_NGU,
L1_CHO_CHAN,
L1_CHO_CHING,
L1_CHO_CHANG,
L1_SO_SO,
L1_CHO_CHOE,
L1_YO_YING,
L1_DO_CHADA,
L1_TO_PATAK,
L1_THO_THAN,
L1_THO_NANGMONTHO,
L1_THO_PHUTHAO,
L1_NO_NEN,
L1_DO_DEK,
L1_TO_TAO,
L1_THO_THUNG,
L1_THO_THAHAN,
L1_THO_THONG,
L1_NO_NU,
L1_BO_BAIMAI,
L1_PO_PLA,
L1_PHO_PHUNG,
L1_FO_FA,
L1_PHO_PHAN,
L1_FO_FAN,
L1_PHO_SAMPHAO,
L1_MO_MA,
L1_YO_YAK,
L1_RO_RUA,
L1_RU,
L1_LO_LING,
L1_LU,
L1_WO_WAEN,
L1_SO_SALA,
L1_SO_RUSI,
L1_SO_SUA,
L1_HO_HIP,
L1_LO_CHULA,
L1_O_ANG,
L1_HO_NOKHUK,
L1_NKHIT,
L1_SARA_A,
L1_MAI_HAN_AKAT,
L1_SARA_AA,
L1_SARA_AM,
L1_SARA_I,
L1_SARA_II,
L1_SARA_UE,
L1_SARA_UEE,
L1_SARA_U,
L1_SARA_UU,
L1_SARA_E,
L1_SARA_AE,
L1_SARA_O,
L1_SARA_AI_MAIMUAN,
L1_SARA_AI_MAIMALAI
};
/* level 2 symbols & order */
enum l2_symbols {
L2_BLANK = TOT_LEVELS,
L2_THAII,
L2_YAMAK,
L2_PINTHU,
L2_GARAN,
L2_TYKHU,
L2_TONE1,
L2_TONE2,
L2_TONE3,
L2_TONE4
};
/* level 3 symbols & order */
enum l3_symbols {
L3_BLANK = TOT_LEVELS,
L3_SPACE,
L3_NB_SACE,
L3_LOW_LINE,
L3_HYPHEN,
L3_COMMA,
L3_SEMICOLON,
L3_COLON,
L3_EXCLAMATION,
L3_QUESTION,
L3_SOLIDUS,
L3_FULL_STOP,
L3_PAIYAN_NOI,
L3_MAI_YAMOK,
L3_GRAVE,
L3_CIRCUMFLEX,
L3_TILDE,
L3_APOSTROPHE,
L3_QUOTATION,
L3_L_PARANTHESIS,
L3_L_BRACKET,
L3_L_BRACE,
L3_R_BRACE,
L3_R_BRACKET,
L3_R_PARENTHESIS,
L3_AT,
L3_BAHT,
L3_DOLLAR,
L3_FONGMAN,
L3_ANGKHANKHU,
L3_KHOMUT,
L3_ASTERISK,
L3_BK_SOLIDUS,
L3_AMPERSAND,
L3_NUMBER,
L3_PERCENT,
L3_PLUS,
L3_LESS_THAN,
L3_EQUAL,
L3_GREATER_THAN,
L3_V_LINE
};
/* level 4 symbols & order */
enum l4_symbols { L4_BLANK = TOT_LEVELS, L4_MIN, L4_CAP, L4_EXT };
enum level_symbols { L_UPRUPR = TOT_LEVELS, L_UPPER, L_MIDDLE, L_LOWER };
#define _is(c) (t_ctype[(c)][LAST_LEVEL])
#define _level 8
#define _consnt 16
#define _ldvowel 32
#define _fllwvowel 64
#define _uprvowel 128
#define _lwrvowel 256
#define _tone 512
#define _diacrt1 1024
#define _diacrt2 2048
#define _combine 4096
#define _stone 8192
#define _tdig 16384
#define _rearvowel (_fllwvowel | _uprvowel | _lwrvowel)
#define _diacrt (_diacrt1 | _diacrt2)
#define levelof(c) (_is(c) & _level)
#define isthai(c) ((c) >= 128)
#define istalpha(c) \
(_is(c) & (_consnt | _ldvowel | _rearvowel | _tone | _diacrt1 | _diacrt2))
#define isconsnt(c) (_is(c) & _consnt)
#define isldvowel(c) (_is(c) & _ldvowel)
#define isfllwvowel(c) (_is(c) & _fllwvowel)
#define ismidvowel(c) (_is(c) & (_ldvowel | _fllwvowel))
#define isuprvowel(c) (_is(c) & _uprvowel)
#define islwrvowel(c) (_is(c) & _lwrvowel)
#define isuprlwrvowel(c) (_is(c) & (_lwrvowel | _uprvowel))
#define isrearvowel(c) (_is(c) & _rearvowel)
#define isvowel(c) (_is(c) & (_ldvowel | _rearvowel))
#define istone(c) (_is(c) & _tone)
#define isunldable(c) (_is(c) & (_rearvowel | _tone | _diacrt1 | _diacrt2))
#define iscombinable(c) (_is(c) & _combine)
#define istdigit(c) (_is(c) & _tdig)
#define isstone(c) (_is(c) & _stone)
#define isdiacrt1(c) (_is(c) & _diacrt1)
#define isdiacrt2(c) (_is(c) & _diacrt2)
#define isdiacrt(c) (_is(c) & _diacrt)
#define M L_MIDDLE
//#define U L_UPPER
#define L L_LOWER
#define UU L_UPRUPR
#define X L_MIDDLE
static const int t_ctype[][TOT_LEVELS] = {
/*0x00*/ {0, 0, 0, 0, X},
/*0x01*/ {0, 0, 0, 0, X},
/*0x02*/ {0, 0, 0, 0, X},
/*0x03*/ {0, 0, 0, 0, X},
/*0x04*/ {0, 0, 0, 0, X},
/*0x05*/ {0, 0, 0, 0, X},
/*0x06*/ {0, 0, 0, 0, X},
/*0x07*/ {0, 0, 0, 0, X},
/*0x08*/ {0, 0, 0, 0, X},
/*0x09*/ {0, 0, 0, 0, X},
/*0x0A*/ {0, 0, 0, 0, X},
/*0x0B*/ {0, 0, 0, 0, X},
/*0x0C*/ {0, 0, 0, 0, X},
/*0x0D*/ {0, 0, 0, 0, X},
/*0x0E*/ {0, 0, 0, 0, X},
/*0x0F*/ {0, 0, 0, 0, X},
/*0x10*/ {0, 0, 0, 0, X},
/*0x11*/ {0, 0, 0, 0, X},
/*0x12*/ {0, 0, 0, 0, X},
/*0x13*/ {0, 0, 0, 0, X},
/*0x14*/ {0, 0, 0, 0, X},
/*0x15*/ {0, 0, 0, 0, X},
/*0x16*/ {0, 0, 0, 0, X},
/*0x17*/ {0, 0, 0, 0, X},
/*0x18*/ {0, 0, 0, 0, X},
/*0x19*/ {0, 0, 0, 0, X},
/*0x1A*/ {0, 0, 0, 0, X},
/*0x1B*/ {0, 0, 0, 0, X},
/*0x1C*/ {0, 0, 0, 0, X},
/*0x1D*/ {0, 0, 0, 0, X},
/*0x1E*/ {0, 0, 0, 0, X},
/*0x1F*/ {0, 0, 0, 0, X},
/*0x20*/ {0, 0, L3_SPACE, 0, M},
/*0x21*/ {0, 0, L3_EXCLAMATION, 0, M},
/*0x22*/ {0, 0, L3_QUOTATION, 0, M},
/*0x23*/ {0, 0, L3_NUMBER, 0, M},
/*0x24*/ {0, 0, L3_DOLLAR, 0, M},
/*0x25*/ {0, 0, L3_PERCENT, 0, M},
/*0x26*/ {0, 0, L3_AMPERSAND, 0, M},
/*0x27*/ {0, 0, L3_APOSTROPHE, 0, M},
/*0x28*/ {0, 0, L3_L_PARANTHESIS, 0, M},
/*0x29*/ {0, 0, L3_R_PARENTHESIS, 0, M},
/*0x2A*/ {0, 0, L3_ASTERISK, 0, M},
/*0x2B*/ {0, 0, L3_PLUS, 0, M},
/*0x2C*/ {0, 0, L3_COMMA, 0, M},
/*0x2D*/ {0, 0, L3_HYPHEN, 0, M},
/*0x2E*/ {0, 0, L3_FULL_STOP, 0, M},
/*0x2F*/ {0, 0, L3_SOLIDUS, 0, M},
/*0x30*/ {L1_08, L2_BLANK, L3_BLANK, L4_BLANK, M},
/*0x31*/ {L1_18, L2_BLANK, L3_BLANK, L4_BLANK, M},
/*0x32*/ {L1_28, L2_BLANK, L3_BLANK, L4_BLANK, M},
/*0x33*/ {L1_38, L2_BLANK, L3_BLANK, L4_BLANK, M},
/*0x34*/ {L1_48, L2_BLANK, L3_BLANK, L4_BLANK, M},
/*0x35*/ {L1_58, L2_BLANK, L3_BLANK, L4_BLANK, M},
/*0x36*/ {L1_68, L2_BLANK, L3_BLANK, L4_BLANK, M},
/*0x37*/ {L1_78, L2_BLANK, L3_BLANK, L4_BLANK, M},
/*0x38*/ {L1_88, L2_BLANK, L3_BLANK, L4_BLANK, M},
/*0x39*/ {L1_98, L2_BLANK, L3_BLANK, L4_BLANK, M},
/*0x3A*/ {0, 0, L3_COLON, 0, M},
/*0x3B*/ {0, 0, L3_SEMICOLON, 0, M},
/*0x3C*/ {0, 0, L3_LESS_THAN, 0, M},
/*0x3D*/ {0, 0, L3_EQUAL, 0, M},
/*0x3E*/ {0, 0, L3_GREATER_THAN, 0, M},
/*0x3F*/ {0, 0, L3_QUESTION, 0, M},
/*0x40*/ {0, 0, L3_AT, 0, M},
/*0x41*/ {L1_A8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x42*/ {L1_B8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x43*/ {L1_C8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x44*/ {L1_D8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x45*/ {L1_E8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x46*/ {L1_F8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x47*/ {L1_G8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x48*/ {L1_H8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x49*/ {L1_I8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x4A*/ {L1_J8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x4B*/ {L1_K8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x4C*/ {L1_L8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x4D*/ {L1_M8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x4E*/ {L1_N8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x4F*/ {L1_O8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x50*/ {L1_P8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x51*/ {L1_Q8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x52*/ {L1_R8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x53*/ {L1_S8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x54*/ {L1_T8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x55*/ {L1_U8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x56*/ {L1_V8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x57*/ {L1_W8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x58*/ {L1_X8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x59*/ {L1_Y8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x5A*/ {L1_Z8, L2_BLANK, L3_BLANK, L4_CAP, M},
/*0x5B*/ {0, 0, L3_L_BRACKET, 0, M},
/*0x5C*/ {0, 0, L3_BK_SOLIDUS, 0, M},
/*0x5D*/ {0, 0, L3_R_BRACKET, 0, M},
/*0x5E*/ {0, 0, L3_CIRCUMFLEX, 0, M},
/*0x5F*/ {0, 0, L3_LOW_LINE, 0, M},
/*0x60*/ {0, 0, L3_GRAVE, 0, M},
/*0x61*/ {L1_A8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x62*/ {L1_B8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x63*/ {L1_C8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x64*/ {L1_D8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x65*/ {L1_E8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x66*/ {L1_F8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x67*/ {L1_G8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x68*/ {L1_H8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x69*/ {L1_I8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x6A*/ {L1_J8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x6B*/ {L1_K8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x6C*/ {L1_L8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x6D*/ {L1_M8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x6E*/ {L1_N8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x6F*/ {L1_O8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x70*/ {L1_P8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x71*/ {L1_Q8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x72*/ {L1_R8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x73*/ {L1_S8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x74*/ {L1_T8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x75*/ {L1_U8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x76*/ {L1_V8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x77*/ {L1_W8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x78*/ {L1_X8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x79*/ {L1_Y8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x7A*/ {L1_Z8, L2_BLANK, L3_BLANK, L4_MIN, M},
/*0x7B*/ {0, 0, L3_L_BRACE, 0, M},
/*0x7C*/ {0, 0, L3_V_LINE, 0, M},
/*0x7D*/ {0, 0, L3_R_BRACE, 0, M},
/*0x7E*/ {0, 0, L3_TILDE, 0, M},
/*0x7F*/ {0, 0, 0, 0, X},
/*0x80*/ {0, 0, 0, 0, X},
/*0x81*/ {0, 0, 0, 0, X},
/*0x82*/ {0, 0, 0, 0, X},
/*0x83*/ {0, 0, 0, 0, X},
/*0x84*/ {0, 0, 0, 0, X},
/*0x85*/ {0, 0, 0, 0, X},
/*0x86*/ {0, 0, 0, 0, X},
/*0x87*/ {0, 0, 0, 0, X},
/*0x88*/ {0, 0, 0, 0, X},
/*0x89*/ {0, 0, 0, 0, X},
/*0x8A*/ {0, 0, 0, 0, X},
/*0x8B*/ {0, 0, 0, 0, X},
/*0x8C*/ {0, 0, 0, 0, X},
/*0x8D*/ {0, 0, 0, 0, X},
/*0x8E*/ {0, 0, 0, 0, X},
/*0x8F*/ {0, 0, 0, 0, X},
/*0x90*/ {0, 0, 0, 0, X},
/*0x91*/ {0, 0, 0, 0, X},
/*0x92*/ {0, 0, 0, 0, X},
/*0x93*/ {0, 0, 0, 0, X},
/*0x94*/ {0, 0, 0, 0, X},
/*0x95*/ {0, 0, 0, 0, X},
/*0x96*/ {0, 0, 0, 0, X},
/*0x97*/ {0, 0, 0, 0, X},
/*0x98*/ {0, 0, 0, 0, X},
/*0x99*/ {0, 0, 0, 0, X},
/*0x9A*/ {0, 0, 0, 0, X},
/*0x9B*/ {0, 0, 0, 0, X},
/*0x9C*/ {0, 0, 0, 0, X},
/*0x9D*/ {0, 0, 0, 0, X},
/*0x9E*/ {0, 0, 0, 0, X},
/*0x9F*/ {0, 0, 0, 0, X},
/*0xA0*/ {0, 0, L3_NB_SACE, 0, X},
/*0xA1*/ {L1_KO_KAI, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xA2*/ {L1_KHO_KHAI, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xA3*/ {L1_KHO_KHUAT, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xA4*/ {L1_KHO_KHWAI, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xA5*/ {L1_KHO_KHON, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xA6*/ {L1_KHO_RAKHANG, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xA7*/ {L1_NGO_NGU, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xA8*/ {L1_CHO_CHAN, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xA9*/ {L1_CHO_CHING, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xAA*/ {L1_CHO_CHANG, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xAB*/ {L1_SO_SO, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xAC*/ {L1_CHO_CHOE, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xAD*/ {L1_YO_YING, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xAE*/ {L1_DO_CHADA, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xAF*/ {L1_TO_PATAK, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xB0*/ {L1_THO_THAN, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xB1*/ {L1_THO_NANGMONTHO, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xB2*/ {L1_THO_PHUTHAO, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xB3*/ {L1_NO_NEN, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xB4*/ {L1_DO_DEK, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xB5*/ {L1_TO_TAO, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xB6*/ {L1_THO_THUNG, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xB7*/ {L1_THO_THAHAN, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xB8*/ {L1_THO_THONG, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xB9*/ {L1_NO_NU, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xBA*/ {L1_BO_BAIMAI, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xBB*/ {L1_PO_PLA, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xBC*/ {L1_PHO_PHUNG, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xBD*/ {L1_FO_FA, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xBE*/ {L1_PHO_PHAN, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xBF*/ {L1_FO_FAN, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xC0*/ {L1_PHO_SAMPHAO, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xC1*/ {L1_MO_MA, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xC2*/ {L1_YO_YAK, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xC3*/ {L1_RO_RUA, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xC4*/ {L1_RU, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xC5*/ {L1_LO_LING, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xC6*/ {L1_LU, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xC7*/ {L1_WO_WAEN, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xC8*/ {L1_SO_SALA, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xC9*/ {L1_SO_RUSI, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xCA*/ {L1_SO_SUA, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xCB*/ {L1_HO_HIP, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xCC*/ {L1_LO_CHULA, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xCD*/ {L1_O_ANG, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xCE*/ {L1_HO_NOKHUK, L2_BLANK, L3_BLANK, L4_BLANK, M | _consnt},
/*0xCF*/ {0, 0, L3_PAIYAN_NOI, 0, M},
/*0xD0*/ {L1_SARA_A, L2_BLANK, L3_BLANK, L4_BLANK, M | _fllwvowel},
/*0xD1*/ {L1_MAI_HAN_AKAT, L2_BLANK, L3_BLANK, L4_BLANK, L_UPPER | _uprvowel},
/*0xD2*/ {L1_SARA_AA, L2_BLANK, L3_BLANK, L4_BLANK, M | _fllwvowel},
/*0xD3*/ {L1_SARA_AM, L2_BLANK, L3_BLANK, L4_BLANK, M | _fllwvowel},
/*0xD4*/ {L1_SARA_I, L2_BLANK, L3_BLANK, L4_BLANK, L_UPPER | _uprvowel},
/*0xD5*/ {L1_SARA_II, L2_BLANK, L3_BLANK, L4_BLANK, L_UPPER | _uprvowel},
/*0xD6*/ {L1_SARA_UE, L2_BLANK, L3_BLANK, L4_BLANK, L_UPPER | _uprvowel},
/*0xD7*/ {L1_SARA_UEE, L2_BLANK, L3_BLANK, L4_BLANK, L_UPPER | _uprvowel},
/*0xD8*/ {L1_SARA_U, L2_BLANK, L3_BLANK, L4_BLANK, L | _lwrvowel},
/*0xD9*/ {L1_SARA_UU, L2_BLANK, L3_BLANK, L4_BLANK, L | _lwrvowel},
/*0xDA*/ {0, L2_PINTHU, L3_BLANK, L4_BLANK, L},
/*0xDB*/ {0, 0, 0, 0, X},
/*0xDC*/ {0, 0, 0, 0, X},
/*0xDD*/ {0, 0, 0, 0, X},
/*0xDE*/ {0, 0, 0, 0, X},
/*0xDF*/ {0, 0, L3_BAHT, 0, M},
/*0xE0*/ {L1_SARA_E, L2_BLANK, L3_BLANK, L4_BLANK, M | _ldvowel},
/*0xE1*/ {L1_SARA_AE, L2_BLANK, L3_BLANK, L4_BLANK, M | _ldvowel},
/*0xE2*/ {L1_SARA_O, L2_BLANK, L3_BLANK, L4_BLANK, M | _ldvowel},
/*0xE3*/ {L1_SARA_AI_MAIMUAN, L2_BLANK, L3_BLANK, L4_BLANK, M | _ldvowel},
/*0xE4*/ {L1_SARA_AI_MAIMALAI, L2_BLANK, L3_BLANK, L4_BLANK, M | _ldvowel},
/*0xE5*/ {L1_SARA_AA, L2_BLANK, L3_BLANK, L4_EXT, M | _fllwvowel},
/*0xE6*/ {0, 0, L3_MAI_YAMOK, 0, M | _stone},
/*0xE7*/ {0, L2_TYKHU, L3_BLANK, L4_BLANK, L_UPPER | _diacrt1 | _stone},
/*0xE8*/
{0, L2_TONE1, L3_BLANK, L4_BLANK, UU | _tone | _combine | _stone},
/*0xE9*/
{0, L2_TONE2, L3_BLANK, L4_BLANK, UU | _tone | _combine | _stone},
/*0xEA*/
{0, L2_TONE3, L3_BLANK, L4_BLANK, UU | _tone | _combine | _stone},
/*0xEB*/
{0, L2_TONE4, L3_BLANK, L4_BLANK, UU | _tone | _combine | _stone},
/*0xEC*/
{0, L2_GARAN, L3_BLANK, L4_BLANK, UU | _diacrt2 | _combine | _stone},
/*0xED*/ {L1_NKHIT, L2_BLANK, L3_BLANK, L4_BLANK, L_UPPER | _diacrt1},
/*0xEE*/ {0, L2_YAMAK, L3_BLANK, L4_BLANK, L_UPPER | _diacrt1},
/*0xEF*/ {0, 0, L3_FONGMAN, 0, M},
/*0xF0*/ {L1_08, L2_THAII, L3_BLANK, L4_BLANK, M | _tdig},
/*0xF1*/ {L1_18, L2_THAII, L3_BLANK, L4_BLANK, M | _tdig},
/*0xF2*/ {L1_28, L2_THAII, L3_BLANK, L4_BLANK, M | _tdig},
/*0xF3*/ {L1_38, L2_THAII, L3_BLANK, L4_BLANK, M | _tdig},
/*0xF4*/ {L1_48, L2_THAII, L3_BLANK, L4_BLANK, M | _tdig},
/*0xF5*/ {L1_58, L2_THAII, L3_BLANK, L4_BLANK, M | _tdig},
/*0xF6*/ {L1_68, L2_THAII, L3_BLANK, L4_BLANK, M | _tdig},
/*0xF7*/ {L1_78, L2_THAII, L3_BLANK, L4_BLANK, M | _tdig},
/*0xF8*/ {L1_88, L2_THAII, L3_BLANK, L4_BLANK, M | _tdig},
/*0xF9*/ {L1_98, L2_THAII, L3_BLANK, L4_BLANK, M | _tdig},
/*0xFA*/ {0, 0, L3_ANGKHANKHU, 0, X},
/*0xFB*/ {0, 0, L3_KHOMUT, 0, X},
/*0xFC*/ {0, 0, 0, 0, X},
/*0xFD*/ {0, 0, 0, 0, X},
/*0xFE*/ {0, 0, 0, 0, X},
/* Utilize 0xFF for max_sort_chr */
/*0xFF*/ {255 /*0*/, 0, 0, 0, X},
};
static unsigned char ctype_tis620[257] = {
0, /* For standard library */
32, 32, 32, 32, 32, 32, 32, 32, 32, 40, 40, 40, 40, 40, 32, 32,
32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32,
72, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
132, 132, 132, 132, 132, 132, 132, 132, 132, 132, 16, 16, 16, 16, 16, 16,
16, 129, 129, 129, 129, 129, 129, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 16, 16, 16, 16, 16,
16, 130, 130, 130, 130, 130, 130, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 16, 16, 16, 16, 32,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
};
static unsigned char to_lower_tis620[] = {
'\000', '\001', '\002', '\003', '\004',
'\005', '\006', '\007', '\010', '\011',
'\012', '\013', '\014', '\015', '\016',
'\017', '\020', '\021', '\022', '\023',
'\024', '\025', '\026', '\027', '\030',
'\031', '\032', '\033', '\034', '\035',
'\036', '\037', ' ', '!', '"',
'#', '$', '%', '&', '\'',
'(', ')', '*', '+', ',',
'-', '.', '/', '0', '1',
'2', '3', '4', '5', '6',
'7', '8', '9', ':', ';',
'<', '=', '>', '?', '@',
'a', 'b', 'c', 'd', 'e',
'f', 'g', 'h', 'i', 'j',
'k', 'l', 'm', 'n', 'o',
'p', 'q', 'r', 's', 't',
'u', 'v', 'w', 'x', 'y',
'z', '[', '\\', ']', '^',
'_', '`', 'a', 'b', 'c',
'd', 'e', 'f', 'g', 'h',
'i', 'j', 'k', 'l', 'm',
'n', 'o', 'p', 'q', 'r',
's', 't', 'u', 'v', 'w',
'x', 'y', 'z', '{', '|',
'}', '~', '\177', (uchar)'\200', (uchar)'\201',
(uchar)'\202', (uchar)'\203', (uchar)'\204', (uchar)'\205', (uchar)'\206',
(uchar)'\207', (uchar)'\210', (uchar)'\211', (uchar)'\212', (uchar)'\213',
(uchar)'\214', (uchar)'\215', (uchar)'\216', (uchar)'\217', (uchar)'\220',
(uchar)'\221', (uchar)'\222', (uchar)'\223', (uchar)'\224', (uchar)'\225',
(uchar)'\226', (uchar)'\227', (uchar)'\230', (uchar)'\231', (uchar)'\232',
(uchar)'\233', (uchar)'\234', (uchar)'\235', (uchar)'\236', (uchar)'\237',
(uchar)'\240', (uchar)'\241', (uchar)'\242', (uchar)'\243', (uchar)'\244',
(uchar)'\245', (uchar)'\246', (uchar)'\247', (uchar)'\250', (uchar)'\251',
(uchar)'\252', (uchar)'\253', (uchar)'\254', (uchar)'\255', (uchar)'\256',
(uchar)'\257', (uchar)'\260', (uchar)'\261', (uchar)'\262', (uchar)'\263',
(uchar)'\264', (uchar)'\265', (uchar)'\266', (uchar)'\267', (uchar)'\270',
(uchar)'\271', (uchar)'\272', (uchar)'\273', (uchar)'\274', (uchar)'\275',
(uchar)'\276', (uchar)'\277', (uchar)'\300', (uchar)'\301', (uchar)'\302',
(uchar)'\303', (uchar)'\304', (uchar)'\305', (uchar)'\306', (uchar)'\307',
(uchar)'\310', (uchar)'\311', (uchar)'\312', (uchar)'\313', (uchar)'\314',
(uchar)'\315', (uchar)'\316', (uchar)'\317', (uchar)'\320', (uchar)'\321',
(uchar)'\322', (uchar)'\323', (uchar)'\324', (uchar)'\325', (uchar)'\326',
(uchar)'\327', (uchar)'\330', (uchar)'\331', (uchar)'\332', (uchar)'\333',
(uchar)'\334', (uchar)'\335', (uchar)'\336', (uchar)'\337', (uchar)'\340',
(uchar)'\341', (uchar)'\342', (uchar)'\343', (uchar)'\344', (uchar)'\345',
(uchar)'\346', (uchar)'\347', (uchar)'\350', (uchar)'\351', (uchar)'\352',
(uchar)'\353', (uchar)'\354', (uchar)'\355', (uchar)'\356', (uchar)'\357',
(uchar)'\360', (uchar)'\361', (uchar)'\362', (uchar)'\363', (uchar)'\364',
(uchar)'\365', (uchar)'\366', (uchar)'\367', (uchar)'\370', (uchar)'\371',
(uchar)'\372', (uchar)'\373', (uchar)'\374', (uchar)'\375', (uchar)'\376',
(uchar)'\377',
};
static unsigned char to_upper_tis620[] = {
'\000', '\001', '\002', '\003', '\004',
'\005', '\006', '\007', '\010', '\011',
'\012', '\013', '\014', '\015', '\016',
'\017', '\020', '\021', '\022', '\023',
'\024', '\025', '\026', '\027', '\030',
'\031', '\032', '\033', '\034', '\035',
'\036', '\037', ' ', '!', '"',
'#', '$', '%', '&', '\'',
'(', ')', '*', '+', ',',
'-', '.', '/', '0', '1',
'2', '3', '4', '5', '6',
'7', '8', '9', ':', ';',
'<', '=', '>', '?', '@',
'A', 'B', 'C', 'D', 'E',
'F', 'G', 'H', 'I', 'J',
'K', 'L', 'M', 'N', 'O',
'P', 'Q', 'R', 'S', 'T',
'U', 'V', 'W', 'X', 'Y',
'Z', '[', '\\', ']', '^',
'_', '`', 'A', 'B', 'C',
'D', 'E', 'F', 'G', 'H',
'I', 'J', 'K', 'L', 'M',
'N', 'O', 'P', 'Q', 'R',
'S', 'T', 'U', 'V', 'W',
'X', 'Y', 'Z', '{', '|',
'}', '~', '\177', (uchar)'\200', (uchar)'\201',
(uchar)'\202', (uchar)'\203', (uchar)'\204', (uchar)'\205', (uchar)'\206',
(uchar)'\207', (uchar)'\210', (uchar)'\211', (uchar)'\212', (uchar)'\213',
(uchar)'\214', (uchar)'\215', (uchar)'\216', (uchar)'\217', (uchar)'\220',
(uchar)'\221', (uchar)'\222', (uchar)'\223', (uchar)'\224', (uchar)'\225',
(uchar)'\226', (uchar)'\227', (uchar)'\230', (uchar)'\231', (uchar)'\232',
(uchar)'\233', (uchar)'\234', (uchar)'\235', (uchar)'\236', (uchar)'\237',
(uchar)'\240', (uchar)'\241', (uchar)'\242', (uchar)'\243', (uchar)'\244',
(uchar)'\245', (uchar)'\246', (uchar)'\247', (uchar)'\250', (uchar)'\251',
(uchar)'\252', (uchar)'\253', (uchar)'\254', (uchar)'\255', (uchar)'\256',
(uchar)'\257', (uchar)'\260', (uchar)'\261', (uchar)'\262', (uchar)'\263',
(uchar)'\264', (uchar)'\265', (uchar)'\266', (uchar)'\267', (uchar)'\270',
(uchar)'\271', (uchar)'\272', (uchar)'\273', (uchar)'\274', (uchar)'\275',
(uchar)'\276', (uchar)'\277', (uchar)'\300', (uchar)'\301', (uchar)'\302',
(uchar)'\303', (uchar)'\304', (uchar)'\305', (uchar)'\306', (uchar)'\307',
(uchar)'\310', (uchar)'\311', (uchar)'\312', (uchar)'\313', (uchar)'\314',
(uchar)'\315', (uchar)'\316', (uchar)'\317', (uchar)'\320', (uchar)'\321',
(uchar)'\322', (uchar)'\323', (uchar)'\324', (uchar)'\325', (uchar)'\326',
(uchar)'\327', (uchar)'\330', (uchar)'\331', (uchar)'\332', (uchar)'\333',
(uchar)'\334', (uchar)'\335', (uchar)'\336', (uchar)'\337', (uchar)'\340',
(uchar)'\341', (uchar)'\342', (uchar)'\343', (uchar)'\344', (uchar)'\345',
(uchar)'\346', (uchar)'\347', (uchar)'\350', (uchar)'\351', (uchar)'\352',
(uchar)'\353', (uchar)'\354', (uchar)'\355', (uchar)'\356', (uchar)'\357',
(uchar)'\360', (uchar)'\361', (uchar)'\362', (uchar)'\363', (uchar)'\364',
(uchar)'\365', (uchar)'\366', (uchar)'\367', (uchar)'\370', (uchar)'\371',
(uchar)'\372', (uchar)'\373', (uchar)'\374', (uchar)'\375', (uchar)'\376',
(uchar)'\377',
};
static unsigned char sort_order_tis620[] = {
'\000', '\001', '\002', '\003', '\004',
'\005', '\006', '\007', '\010', '\011',
'\012', '\013', '\014', '\015', '\016',
'\017', '\020', '\021', '\022', '\023',
'\024', '\025', '\026', '\027', '\030',
'\031', '\032', '\033', '\034', '\035',
'\036', '\037', ' ', '!', '"',
'#', '$', '%', '&', '\'',
'(', ')', '*', '+', ',',
'-', '.', '/', '0', '1',
'2', '3', '4', '5', '6',
'7', '8', '9', ':', ';',
'<', '=', '>', '?', '@',
'A', 'B', 'C', 'D', 'E',
'F', 'G', 'H', 'I', 'J',
'K', 'L', 'M', 'N', 'O',
'P', 'Q', 'R', 'S', 'T',
'U', 'V', 'W', 'X', 'Y',
'Z', '\\', ']', '[', '^',
'_', 'E', 'A', 'B', 'C',
'D', 'E', 'F', 'G', 'H',
'I', 'J', 'K', 'L', 'M',
'N', 'O', 'P', 'Q', 'R',
'S', 'T', 'U', 'V', 'W',
'X', 'Y', 'Z', '{', '|',
'}', 'Y', '\177', (uchar)'\200', (uchar)'\201',
(uchar)'\202', (uchar)'\203', (uchar)'\204', (uchar)'\205', (uchar)'\206',
(uchar)'\207', (uchar)'\210', (uchar)'\211', (uchar)'\212', (uchar)'\213',
(uchar)'\214', (uchar)'\215', (uchar)'\216', (uchar)'\217', (uchar)'\220',
(uchar)'\221', (uchar)'\222', (uchar)'\223', (uchar)'\224', (uchar)'\225',
(uchar)'\226', (uchar)'\227', (uchar)'\230', (uchar)'\231', (uchar)'\232',
(uchar)'\233', (uchar)'\234', (uchar)'\235', (uchar)'\236', (uchar)'\237',
(uchar)'\240', (uchar)'\241', (uchar)'\242', (uchar)'\243', (uchar)'\244',
(uchar)'\245', (uchar)'\246', (uchar)'\247', (uchar)'\250', (uchar)'\251',
(uchar)'\252', (uchar)'\253', (uchar)'\254', (uchar)'\255', (uchar)'\256',
(uchar)'\257', (uchar)'\260', (uchar)'\261', (uchar)'\262', (uchar)'\263',
(uchar)'\264', (uchar)'\265', (uchar)'\266', (uchar)'\267', (uchar)'\270',
(uchar)'\271', (uchar)'\272', (uchar)'\273', (uchar)'\274', (uchar)'\275',
(uchar)'\276', (uchar)'\277', (uchar)'\300', (uchar)'\301', (uchar)'\302',
(uchar)'\303', (uchar)'\304', (uchar)'\305', (uchar)'\306', (uchar)'\307',
(uchar)'\310', (uchar)'\311', (uchar)'\312', (uchar)'\313', (uchar)'\314',
(uchar)'\315', (uchar)'\316', (uchar)'\317', (uchar)'\320', (uchar)'\321',
(uchar)'\322', (uchar)'\323', (uchar)'\324', (uchar)'\325', (uchar)'\326',
(uchar)'\327', (uchar)'\330', (uchar)'\331', (uchar)'\332', (uchar)'\333',
(uchar)'\334', (uchar)'\335', (uchar)'\336', (uchar)'\337', (uchar)'\340',
(uchar)'\341', (uchar)'\342', (uchar)'\343', (uchar)'\344', (uchar)'\345',
(uchar)'\346', (uchar)'\347', (uchar)'\350', (uchar)'\351', (uchar)'\352',
(uchar)'\353', (uchar)'\354', (uchar)'\355', (uchar)'\356', (uchar)'\357',
(uchar)'\360', (uchar)'\361', (uchar)'\362', (uchar)'\363', (uchar)'\364',
(uchar)'\365', (uchar)'\366', (uchar)'\367', (uchar)'\370', (uchar)'\371',
(uchar)'\372', (uchar)'\373', (uchar)'\374', (uchar)'\375', (uchar)'\376',
(uchar)'\377',
};
static unsigned short cs_to_uni_tis620[256] = {
0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008,
0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F, 0x0010, 0x0011,
0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001A,
0x001B, 0x001C, 0x001D, 0x001E, 0x001F, 0x0020, 0x0021, 0x0022, 0x0023,
0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002A, 0x002B, 0x002C,
0x002D, 0x002E, 0x002F, 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035,
0x0036, 0x0037, 0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E,
0x003F, 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047,
0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F, 0x0050,
0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059,
0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F, 0x0060, 0x0061, 0x0062,
0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006A, 0x006B,
0x006C, 0x006D, 0x006E, 0x006F, 0x0070, 0x0071, 0x0072, 0x0073, 0x0074,
0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D,
0x007E, 0x007F, 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086,
0x0087, 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098,
0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F, 0xFFFD, 0x0E01,
0x0E02, 0x0E03, 0x0E04, 0x0E05, 0x0E06, 0x0E07, 0x0E08, 0x0E09, 0x0E0A,
0x0E0B, 0x0E0C, 0x0E0D, 0x0E0E, 0x0E0F, 0x0E10, 0x0E11, 0x0E12, 0x0E13,
0x0E14, 0x0E15, 0x0E16, 0x0E17, 0x0E18, 0x0E19, 0x0E1A, 0x0E1B, 0x0E1C,
0x0E1D, 0x0E1E, 0x0E1F, 0x0E20, 0x0E21, 0x0E22, 0x0E23, 0x0E24, 0x0E25,
0x0E26, 0x0E27, 0x0E28, 0x0E29, 0x0E2A, 0x0E2B, 0x0E2C, 0x0E2D, 0x0E2E,
0x0E2F, 0x0E30, 0x0E31, 0x0E32, 0x0E33, 0x0E34, 0x0E35, 0x0E36, 0x0E37,
0x0E38, 0x0E39, 0x0E3A, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0x0E3F, 0x0E40,
0x0E41, 0x0E42, 0x0E43, 0x0E44, 0x0E45, 0x0E46, 0x0E47, 0x0E48, 0x0E49,
0x0E4A, 0x0E4B, 0x0E4C, 0x0E4D, 0x0E4E, 0x0E4F, 0x0E50, 0x0E51, 0x0E52,
0x0E53, 0x0E54, 0x0E55, 0x0E56, 0x0E57, 0x0E58, 0x0E59, 0x0E5A, 0x0E5B,
0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD};
static unsigned char pl00_tis620[256] = {
0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, 0x0008,
0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F, 0x0010, 0x0011,
0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001A,
0x001B, 0x001C, 0x001D, 0x001E, 0x001F, 0x0020, 0x0021, 0x0022, 0x0023,
0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002A, 0x002B, 0x002C,
0x002D, 0x002E, 0x002F, 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035,
0x0036, 0x0037, 0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E,
0x003F, 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047,
0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F, 0x0050,
0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, 0x0058, 0x0059,
0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F, 0x0060, 0x0061, 0x0062,
0x0063, 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006A, 0x006B,
0x006C, 0x006D, 0x006E, 0x006F, 0x0070, 0x0071, 0x0072, 0x0073, 0x0074,
0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D,
0x007E, 0x007F, 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086,
0x0087, 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, 0x0098,
0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000};
static unsigned char pl0E[256] = {
0x0000, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8,
0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, 0x00B0, 0x00B1,
0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x00BA,
0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, 0x00C0, 0x00C1, 0x00C2, 0x00C3,
0x00C4, 0x00C5, 0x00C6, 0x00C7, 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC,
0x00CD, 0x00CE, 0x00CF, 0x00D0, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5,
0x00D6, 0x00D7, 0x00D8, 0x00D9, 0x00DA, 0x0000, 0x0000, 0x0000, 0x0000,
0x00DF, 0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7,
0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF, 0x00F0,
0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7, 0x00F8, 0x00F9,
0x00FA, 0x00FB, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000};
static unsigned char plFF[256] = {
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,
0x0000, 0x00FF, 0x0000, 0x0000};
static unsigned char *uni_to_cs_tis620[256] = {
pl00_tis620, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, pl0E, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, plFF};

120
deps/oblib/src/lib/charset/ob_ctype_uca.cc vendored
View File

@ -2906,8 +2906,8 @@ ObCharsetInfo ob_charset_utf8mb4_unicode_ci=
NULL,
NULL,
&ob_uca_v400,
//NULL,
//NULL,
NULL,
NULL,
&ob_unicase_default,
NULL,
NULL,
@ -2941,8 +2941,8 @@ ObCharsetInfo ob_charset_utf16_unicode_ci=
NULL,
NULL,
&ob_uca_v400,
// NULL,
// NULL,
NULL,
NULL,
&ob_unicase_default,
NULL,
NULL,
@ -2974,6 +2974,8 @@ ObCharsetInfo ob_charset_utf8mb4_0900_ai_ci = {
nullptr,
nullptr,
&ob_uca_v900,
NULL,
NULL,
&ob_unicase_unicode900,
nullptr,
nullptr,
@ -3007,6 +3009,8 @@ ObCharsetInfo ob_charset_utf8mb4_zh_0900_as_cs = {
nullptr,
nullptr,
&ob_uca_v900,
NULL,
NULL,
&ob_unicase_unicode900,
nullptr,
nullptr,
@ -3040,6 +3044,8 @@ ObCharsetInfo ob_charset_utf8mb4_zh2_0900_as_cs = {
nullptr,
nullptr,
&ob_uca_v900,
NULL,
NULL,
&ob_unicase_unicode900,
nullptr,
nullptr,
@ -3073,6 +3079,8 @@ ObCharsetInfo ob_charset_utf8mb4_zh3_0900_as_cs = {
nullptr,
nullptr,
&ob_uca_v900,
NULL,
NULL,
&ob_unicase_unicode900,
nullptr,
nullptr,
@ -3140,6 +3148,8 @@ ObCharsetInfo ob_charset_utf8mb4_0900_bin = {
nullptr, // to_upper
nullptr, // sort_order
nullptr, // uca
NULL,
NULL,
&ob_unicase_unicode900, // caseinfo
nullptr, // state_map
nullptr, // ident_map
@ -3158,3 +3168,105 @@ ObCharsetInfo ob_charset_utf8mb4_0900_bin = {
&ob_charset_utf8mb4_handler,
&ob_collation_utf8mb4_0900_bin_handler,
NO_PAD};
ObCharsetInfo ob_charset_utf8mb4_croatian_uca_ci = {
245, 0, 0, // numbers
OB_CS_UTF8MB4_UCA_FLAGS, // flags
OB_UTF8MB4, // cs name
OB_UTF8MB4 "_croatian_ci", // name
"", // comment
croatian, // tailoring
nullptr, // coll_param
ctype_utf8, // ctype
nullptr, // to_lower
nullptr, // to_upper
nullptr, // sort_order
nullptr, // uca
nullptr, // tab_to_uni
nullptr, // tab_from_uni
&ob_unicase_default, // caseinfo
nullptr, // state_map
nullptr, // ident_map
8, // strxfrm_multiply
1, // caseup_multiply
1, // casedn_multiply
1, // mbminlen
4, // mbmaxlen
1, // mbmaxlenlen
9, // min_sort_char
0xFFFF, // max_sort_char
' ', // pad char
false, // escape_with_backslash_is_dangerous
1, // levels_for_compare
1, // levels_for_order
&ob_charset_utf8mb4_handler,
&ob_collation_any_uca_handler,
PAD_SPACE};
ObCharsetInfo ob_charset_utf8mb4_unicode_520_ci = {
246, 0, 0, // numbers
OB_CS_UTF8MB4_UCA_FLAGS, // flags
OB_UTF8MB4, // cs name
OB_UTF8MB4 "_unicode_520_ci", // name
"", // comment
"", // tailoring
nullptr, // coll_param
ctype_utf8, // ctype
nullptr, // to_lower
nullptr, // to_upper
nullptr, // sort_order
&ob_uca_v520, // uca
nullptr, // tab_to_uni
nullptr, // tab_from_uni
&ob_unicase_unicode520, // caseinfo
nullptr, // state_map
nullptr, // ident_map
8, // strxfrm_multiply
1, // caseup_multiply
1, // casedn_multiply
1, // mbminlen
4, // mbmaxlen
1, // mbmaxlenlen
9, // min_sort_char
0x10FFFF, // max_sort_char
' ', // pad char
false, // escape_with_backslash_is_dangerous
1, // levels_for_compare
1, // levels_for_order
&ob_charset_utf8mb4_handler,
&ob_collation_any_uca_handler,
PAD_SPACE};
ObCharsetInfo ob_charset_utf8mb4_czech_uca_ci = {
234, 0, 0, // numbers
OB_CS_UTF8MB4_UCA_FLAGS, // flags
OB_UTF8MB4, // cs name
OB_UTF8MB4 "_czech_ci", // name
"", // comment
czech, // tailoring
nullptr, // coll_param
ctype_utf8, // ctype
nullptr, // to_lower
nullptr, // to_upper
nullptr, // sort_order
nullptr, // uca
nullptr, // tab_to_uni
nullptr, // tab_from_uni
&ob_unicase_default, // caseinfo
nullptr, // state_map
nullptr, // ident_map
8, // strxfrm_multiply
1, // caseup_multiply
1, // casedn_multiply
1, // mbminlen
4, // mbmaxlen
1, // mbmaxlenlen
9, // min_sort_char
0xFFFF, // max_sort_char
' ', // pad char
false, // escape_with_backslash_is_dangerous
1, // levels_for_compare
1, // levels_for_order
&ob_charset_utf8mb4_handler,
&ob_collation_any_uca_handler,
PAD_SPACE};

6
deps/oblib/src/lib/charset/ob_ctype_utf16.cc vendored
View File

@ -48,7 +48,6 @@ ob_tosort_utf16(ObUnicaseInfo *uni_plane, ob_wc_t *wc)
static unsigned int ob_mbcharlen_utf16(const ObCharsetInfo *cs __attribute__((unused)),
unsigned int c __attribute__((unused)))
{
ob_charset_assert(0);
return OB_UTF16_HIGH_HEAD(c) ? 4 : 2;
}
@ -1129,6 +1128,7 @@ PAD_MIN_MAX:
ObCharsetHandler ob_charset_utf16_handler=
{
NULL,
ob_ismbchar_utf16,
ob_mbcharlen_utf16,
ob_numchars_utf16,
@ -1199,6 +1199,8 @@ ObCharsetInfo ob_charset_utf16_bin=
NULL,
NULL,
NULL,
NULL,
NULL,
&ob_unicase_default,
NULL,
NULL,
@ -1233,6 +1235,8 @@ ObCharsetInfo ob_charset_utf16_general_ci=
NULL,
NULL,
NULL,
NULL,
NULL,
&ob_unicase_default,
NULL,
NULL,

10
deps/oblib/src/lib/charset/ob_ctype_utf8.cc vendored
View File

@ -52,7 +52,7 @@ static int ob_valid_mbcharlen_utf8mb3(const uchar *s, const uchar *e)
return 3;
}
static int ob_valid_mbcharlen_utf8mb4(const ObCharsetInfo *cs __attribute__((unused)), const uchar *s, const uchar *e)
static inline int ob_valid_mbcharlen_utf8mb4(const ObCharsetInfo *cs __attribute__((unused)), const uchar *s, const uchar *e)
{
uchar c;
if (s >= e)
@ -97,7 +97,7 @@ static uint ob_mbcharlen_utf8mb4(const ObCharsetInfo *cs __attribute__((unused))
return 0; /* Illegal mb head */;
}
static size_t ob_well_formed_len_utf8mb4(const ObCharsetInfo *cs,
static inline size_t ob_well_formed_len_utf8mb4(const ObCharsetInfo *cs,
const char *b, const char *e,
size_t pos, int *error)
{
@ -443,6 +443,7 @@ void ob_strnxfrm_unicode_help(uchar **dst,
if ((flags & OB_STRXFRM_PAD_TO_MAXLEN) && *dst < *de)
*dst += ob_strxfrm_pad_unicode(*dst, *de);
}
size_t ob_strnxfrm_unicode(const ObCharsetInfo *cs,
uchar *dst, size_t dstlen, uint nweights,
const uchar *src, size_t srclen, uint flags, bool *is_valid_unicode)
@ -944,6 +945,7 @@ int ob_mb_wc_utf8mb4_thunk(const ObCharsetInfo *cs __attribute__((unused)),
ObCharsetHandler ob_charset_utf8mb4_handler=
{
NULL,
ob_ismbchar_utf8mb4,
ob_mbcharlen_utf8mb4,
ob_numchars_mb,
@ -1015,6 +1017,8 @@ ObCharsetInfo ob_charset_utf8mb4_general_ci=
to_upper_utf8mb4,
to_upper_utf8mb4,
NULL,
NULL,
NULL,
&ob_unicase_default,
NULL,
NULL,
@ -1050,6 +1054,8 @@ ObCharsetInfo ob_charset_utf8mb4_bin=
to_upper_utf8mb4,
NULL,
NULL,
NULL,
NULL,
&ob_unicase_default,
NULL,
NULL,

119
deps/oblib/src/lib/geo/ob_geo_to_s2_visitor.cpp vendored
View File

@ -63,6 +63,7 @@ int ObWkbToS2Visitor::MakeS2Point(T_IBIN *geo, S2Cell *&res)
LOG_WARN("failed to alloc s2cell", K(ret));
} else {
res = p;
bounder_.AddPoint(S2Point(latlng));
}
}
return ret;
@ -129,6 +130,7 @@ int ObWkbToS2Visitor::MakeProjS2Point(T_IBIN *geo, S2Cell *&res)
LOG_WARN("failed to alloc s2cell", K(ret));
} else {
res = p;
bounder_.AddPoint(point);
}
}
}
@ -149,6 +151,8 @@ int ObWkbToS2Visitor::MakeS2Polyline(T_IBIN *geo, S2Polyline *&res)
LOG_WARN("failed to add cell from point", K(ret));
} else if (OB_FAIL(vector_push_back<S2LatLng>(vertices, latlng))) {
LOG_WARN("failed to add vertice", K(ret));
} else {
bounder_.AddPoint(S2Point(latlng));
}
}
if (OB_SUCC(ret)) {
@ -178,6 +182,8 @@ int ObWkbToS2Visitor::MakeProjS2Polyline(T_IBIN *geo, S2Polyline *&res)
LOG_WARN("failed to add cell from point", K(ret));
} else if (OB_FAIL(vector_push_back<S2Point>(vertices, p))) {
LOG_WARN("failed to add vertice", K(ret));
} else {
bounder_.AddPoint(p);
}
}
@ -212,6 +218,8 @@ int ObWkbToS2Visitor::MakeS2Polygon(T_IBIN *geo, S2Polygon *&res)
LOG_WARN("failed to add cell from point", K(ret));
} else if (OB_FAIL(vector_push_back<S2Point>(vertices, tmp))) {
LOG_WARN("failed to add vertice", K(ret));
} else {
bounder_.AddPoint(tmp);
}
}
if (OB_SUCC(ret)) {
@ -240,6 +248,8 @@ int ObWkbToS2Visitor::MakeS2Polygon(T_IBIN *geo, S2Polygon *&res)
LOG_WARN("failed to add cell from point", K(ret));
} else if (OB_FAIL(vector_push_back<S2Point>(vertices, tmp))) {
LOG_WARN("failed to add vertice", K(ret));
} else {
bounder_.AddPoint(tmp);
}
}
if (OB_SUCC(ret)) {
@ -286,6 +296,8 @@ int ObWkbToS2Visitor::MakeProjS2Polygon(T_IBIN *geo, S2Polygon *&res)
LOG_WARN("failed to add cell from point", K(ret));
} else if (OB_FAIL(vector_push_back<S2Point>(vertices, tmp))) {
LOG_WARN("failed to add vertice", K(ret));
} else {
bounder_.AddPoint(tmp);
}
}
if (OB_SUCC(ret)) {
@ -313,6 +325,8 @@ int ObWkbToS2Visitor::MakeProjS2Polygon(T_IBIN *geo, S2Polygon *&res)
LOG_WARN("failed to add cell from point", K(ret));
} else if (OB_FAIL(vector_push_back<S2Point>(vertices, tmp))) {
LOG_WARN("failed to add vertice", K(ret));
} else {
bounder_.AddPoint(tmp);
}
}
if (OB_SUCC(ret)) {
@ -448,30 +462,23 @@ int64_t ObWkbToS2Visitor::get_cellids(ObS2Cellids &cells, bool is_query, bool ne
LOG_WARN("fail to push_back cellid", K(ret));
}
} else {
S2CellUnion cellids;
S2RegionCoverer coverer(options_);
uint32_t s2v_size = s2v_.size();
for (int i = 0; i < s2v_size; i++) {
S2CellUnion tmp = coverer.GetCovering(*s2v_[i]);
cellids = cellids.Union(tmp);
}
if (need_buffer) {
const int max_level_diff = 2;
cellids.Expand(distance, max_level_diff);
cell_union_.Expand(distance, max_level_diff);
}
if (s2v_size > 1) {
cellids.Normalize();
cell_union_.Normalize();
}
S2CellId prev_id = S2CellId::None();
for (int i = 0; OB_SUCC(ret) && i < cellids.size(); i++) {
if (OB_FAIL(cells.push_back(cellids[i].id()))) {
for (int i = 0; OB_SUCC(ret) && i < cell_union_.size(); i++) {
if (OB_FAIL(cells.push_back(cell_union_[i].id()))) {
LOG_WARN("fail to push_back cellid", K(ret));
}
if (OB_SUCC(ret) && is_query) {
int level = cellids[i].level();
int level = cell_union_[i].level();
while (OB_SUCC(ret) && (level -= options_.level_mod()) >= options_.min_level()) {
S2CellId ancestor_id = cellids[i].parent(level);
S2CellId ancestor_id = cell_union_[i].parent(level);
if (prev_id != S2CellId::None() && prev_id.level() > level &&
prev_id.parent(level) == ancestor_id) {
break;
@ -481,7 +488,7 @@ int64_t ObWkbToS2Visitor::get_cellids(ObS2Cellids &cells, bool is_query, bool ne
}
}
}
prev_id = cellids[i];
prev_id = cell_union_[i];
}
if (OB_SUCC(ret) && has_reset_ && OB_FAIL(cells.push_back(exceedsBoundsCellID))) {
LOG_WARN("fail to push_back cellid", K(ret));
@ -490,6 +497,63 @@ int64_t ObWkbToS2Visitor::get_cellids(ObS2Cellids &cells, bool is_query, bool ne
return ret;
}
bool ObWkbToS2Visitor::is_full_range_cell_union(S2CellUnion &cellids)
{
bool b_ret = false;
if (is_geog_) {
const uint8_t cell_faces = 6;
if (cellids.size() != cell_faces) {
// do nothing
} else {
uint8_t curr_faces = 0;
for (uint32_t i = 0; i < cellids.size(); i++) {
if (cellids[i].level() == 0) {
curr_faces++;
LOG_INFO("cell id", K(static_cast<uint64_t>(cellids[i].id())));
}
}
if (curr_faces == cell_faces) {
b_ret = true;
}
}
} else {
for (uint32_t i = 0; i < cellids.size() && !b_ret; i++) {
if (cellids[i].face() != 0) {
b_ret = true;
LOG_INFO("cell id", K(static_cast<uint64_t>(cellids[i].id())));
}
}
}
return b_ret;
}
int ObWkbToS2Visitor::get_s2_cell_union()
{
int ret = OB_SUCCESS;
if (!invalid_) {
S2RegionCoverer coverer(options_);
uint32_t s2v_size = s2v_.size();
for (int i = 0; i < s2v_size; i++) {
S2CellUnion tmp = coverer.GetCovering(*s2v_[i]);
cell_union_ = cell_union_.Union(tmp);
}
if (is_full_range_cell_union(cell_union_)) {
S2LatLng margin = S2LatLng::FromDegrees(0.00001, 0.00001);
S2LatLngRect rect = bounder_.GetBound().Expanded(margin);
cell_union_ = coverer.GetCovering(rect);
mbr_ = rect;
LOG_INFO("generate new mbr: ", K(rect.lo().ToStringInDegrees().c_str()), K(rect.hi().ToStringInDegrees().c_str()));
S2cells_.clear();
for (uint8_t i = 0; i < 4 && OB_SUCC(ret); i++) {
if (OB_FAIL(add_cell_from_point(rect.GetVertex(i)))) {
LOG_WARN("fail to push_back cellid", K(ret));
}
}
}
}
return ret;
}
int64_t ObWkbToS2Visitor::get_cellids_and_unrepeated_ancestors(ObS2Cellids &cells,
ObS2Cellids &ancestors,
bool need_buffer,
@ -501,8 +565,6 @@ int64_t ObWkbToS2Visitor::get_cellids_and_unrepeated_ancestors(ObS2Cellids &cell
LOG_WARN("fail to push_back cellid", K(ret));
}
} else {
S2CellUnion cellids;
S2RegionCoverer coverer(options_);
uint32_t s2v_size = s2v_.size();
hash::ObHashSet<uint64_t> cellid_set;
if (OB_FAIL(cellid_set.create(128, "CellidSet", "HashNode"))) {
@ -511,30 +573,26 @@ int64_t ObWkbToS2Visitor::get_cellids_and_unrepeated_ancestors(ObS2Cellids &cell
ret = OB_NOT_INIT;
LOG_WARN("fail to init cellid set", K(ret));
} else {
for (int i = 0; i < s2v_size && OB_SUCC(ret); i++) {
S2CellUnion tmp = coverer.GetCovering(*s2v_[i]);
cellids = cellids.Union(tmp);
}
if (need_buffer) {
const int max_level_diff = 2;
cellids.Expand(distance, max_level_diff);
cell_union_.Expand(distance, max_level_diff);
}
if (s2v_size > 1) {
cellids.Normalize();
cell_union_.Normalize();
}
S2CellId prev_id = S2CellId::None();
for (int i = 0; OB_SUCC(ret) && i < cellids.size(); i++) {
int hash_ret = cellid_set.exist_refactored(cellids[i].id());
for (int i = 0; OB_SUCC(ret) && i < cell_union_.size(); i++) {
int hash_ret = cellid_set.exist_refactored(cell_union_[i].id());
if (OB_HASH_NOT_EXIST == hash_ret) {
if (OB_FAIL(cellid_set.set_refactored(cellids[i].id()))) {
if (OB_FAIL(cellid_set.set_refactored(cell_union_[i].id()))) {
LOG_WARN("failed to add cellid into set", K(ret));
} else if (OB_FAIL(cells.push_back(cellids[i].id()))) {
} else if (OB_FAIL(cells.push_back(cell_union_[i].id()))) {
LOG_WARN("fail to push_back cellid", K(ret));
}
if (OB_SUCC(ret)) {
int level = cellids[i].level();
int level = cell_union_[i].level();
while (OB_SUCC(ret) && (level -= options_.level_mod()) >= options_.min_level()) {
S2CellId ancestor_id = cellids[i].parent(level);
S2CellId ancestor_id = cell_union_[i].parent(level);
if (prev_id != S2CellId::None() && prev_id.level() > level &&
prev_id.parent(level) == ancestor_id) {
break;
@ -556,7 +614,7 @@ int64_t ObWkbToS2Visitor::get_cellids_and_unrepeated_ancestors(ObS2Cellids &cell
ret = hash_ret;
LOG_WARN("fail to check if key exist", K(ret), K(i));
}
prev_id = cellids[i];
prev_id = cell_union_[i];
}
if (OB_SUCC(ret) && has_reset_ && OB_FAIL(cells.push_back(exceedsBoundsCellID))) {
LOG_WARN("fail to push_back cellid", K(ret));
@ -607,6 +665,9 @@ void ObWkbToS2Visitor::reset()
S2cells_.clear();
invalid_ = false;
has_reset_ = true;
cell_union_.Clear();
// reset to empty rectangle
bounder_.~S2LatLngRectBounder();
}
template <typename ElementType>

8
deps/oblib/src/lib/geo/ob_geo_to_s2_visitor.h vendored
View File

@ -46,7 +46,9 @@ public:
options_(options),
is_geog_(is_geog),
invalid_(false),
has_reset_(false)
has_reset_(false),
cell_union_(),
bounder_()
{
mbr_ = S2LatLngRect::Empty();
}
@ -97,12 +99,14 @@ public:
int64_t get_mbr(S2LatLngRect &mbr, bool need_buffer, S1Angle distance);
bool is_invalid() { return invalid_; }
void reset();
int get_s2_cell_union();
private:
double stToUV(double s);
bool exceedsBounds(double x, double y);
S2Point MakeS2PointFromXy(double x, double y);
int add_cell_from_point(S2Point point);
int add_cell_from_point(S2LatLng point);
bool is_full_range_cell_union(S2CellUnion &cellids);
template <typename ElementType>
static int vector_push_back(std::vector<ElementType> &vector, ElementType &element);
static int vector_emplace_back(std::vector<std::unique_ptr<S2Loop>> &vector, S2Loop *element);
@ -117,6 +121,8 @@ private:
bool is_geog_;
bool invalid_;
bool has_reset_;
S2CellUnion cell_union_;
S2LatLngRectBounder bounder_;
DISALLOW_COPY_AND_ASSIGN(ObWkbToS2Visitor);
};

4
deps/oblib/src/lib/geo/ob_s2adapter.cpp vendored
View File

@ -350,6 +350,8 @@ int64_t ObS2Adapter::init(const ObString &swkb, const ObSrsBoundsItem *bound)
geo_ = geo;
if (OB_FAIL(geo->do_visit(*visitor_))) {
LOG_WARN("fail to do_visit by ObWkbToS2Visitor", K(ret));
} else if (OB_FAIL(visitor_->get_s2_cell_union())) {
LOG_WARN("fail to get s2 cell union", K(ret));
} else if (visitor_->is_invalid()) {
// 1. get valid geo inside bounds
ObGeometry *corrected_geo = NULL;
@ -370,6 +372,8 @@ int64_t ObS2Adapter::init(const ObString &swkb, const ObSrsBoundsItem *bound)
// 3. do_visit again
if (OB_FAIL(corrected_geo->do_visit(*visitor_))) {
LOG_WARN("fail to do_visit by ObWkbToS2Visitor", K(ret));
} else if (OB_FAIL(visitor_->get_s2_cell_union())) {
LOG_WARN("fail to get s2 cell union", K(ret));
}
}
}

18
deps/oblib/src/lib/lock/ob_qsync_lock.cpp vendored
View File

@ -86,5 +86,23 @@ int ObQSyncLock::try_rdlock()
}
return ret;
}
int ObQSyncLock::try_wrlock()
{
int ret = OB_SUCCESS;
if (!ATOMIC_BCAS(&write_flag_, 0, 1)) {
ret = OB_EAGAIN;
} else {
bool sync_success = false;
for (int64_t i = 0; !sync_success && i < TRY_SYNC_COUNT; i++) {
sync_success = qsync_.try_sync();
}
if (!sync_success) {
ATOMIC_STORE(&write_flag_, 0);
ret = OB_EAGAIN;
}
}
return ret;
}
}
}

1
deps/oblib/src/lib/lock/ob_qsync_lock.h vendored
View File

@ -32,6 +32,7 @@ public:
void rdunlock();
int wrlock();
void wrunlock();
int try_wrlock();
int try_rdlock();
private:
static const int64_t TRY_SYNC_COUNT = 16;

17
deps/oblib/src/lib/mysqlclient/ob_isql_connection.h vendored
View File

@ -135,8 +135,9 @@ public:
const share::schema::ObRoutineInfo &routine_info,
const common::ObIArray<const pl::ObUserDefinedType *> &udts,
const ObTimeZoneInfo *tz_info,
ObObj *result) = 0;
virtual int prepare(const char *sql) {
ObObj *result,
bool is_sql) = 0;
virtual int prepare(const char *sql, int64_t param_count, ObIAllocator *allocator = NULL) {
UNUSED(sql);
return OB_NOT_SUPPORTED;
}
@ -144,9 +145,10 @@ public:
void *param,
int64_t param_size,
int32_t datatype,
int32_t &indicator)
int32_t &indicator,
bool is_out_param)
{
UNUSEDx(position, param, param_size, datatype);
UNUSEDx(position, param, param_size, datatype, indicator, is_out_param);
return OB_NOT_SUPPORTED;
}
virtual int bind_array_type_by_pos(uint64_t position,
@ -206,6 +208,7 @@ public:
virtual void set_is_load_data_exec(bool v) { UNUSED(v); }
virtual void set_force_remote_exec(bool v) { UNUSED(v); }
virtual void set_use_external_session(bool v) { UNUSED(v); }
virtual void set_ob_enable_pl_cache(bool v) { UNUSED(v); }
virtual int64_t get_cluster_id() const { return common::OB_INVALID_ID; }
void set_session_init_status(bool status) { is_inited_ = status;}
virtual void set_user_timeout(int64_t user_timeout) { UNUSED(user_timeout); }
@ -253,8 +256,8 @@ public:
}
return ret;
}
void set_group_id(const int64_t v) {consumer_group_id_ = v; }
int64_t get_group_id() const {return consumer_group_id_; }
void set_group_id(const uint64_t v) {consumer_group_id_ = v; }
uint64_t get_group_id() const {return consumer_group_id_; }
void set_reverse_link_creadentials(bool flag) { has_reverse_link_credentials_ = flag; }
bool get_reverse_link_creadentials() { return has_reverse_link_credentials_; }
void set_usable(bool flag) { usable_ = flag; }
@ -274,7 +277,7 @@ protected:
uint64_t dblink_id_; // for dblink, record dblink_id of a connection used by dblink
DblinkDriverProto dblink_driver_proto_; //for dblink, record DblinkDriverProto of a connection used by dblink
uint32_t sessid_;
int64_t consumer_group_id_; //for resource isolation
uint64_t consumer_group_id_; //for resource isolation
bool has_reverse_link_credentials_; // for dblink, mark if this link has credentials set
bool usable_; // usable_ = false: connection is unusable, should not execute query again.
char *last_set_sql_mode_cstr_; // for mysql dblink to set sql mode

83
deps/oblib/src/lib/mysqlclient/ob_mysql_connection.cpp vendored
View File

@ -21,6 +21,7 @@
#include "lib/string/ob_sql_string.h"
#include "lib/mysqlclient/ob_mysql_read_context.h"
#include "lib/mysqlclient/ob_dblink_error_trans.h"
#include "share/schema/ob_routine_info.h"
namespace oceanbase
{
@ -95,7 +96,7 @@ void ObMySQLConnection::reset()
int ObMySQLConnection::prepare_statement(ObMySQLPreparedStatement &stmt, const char *sql)
{
int ret = OB_SUCCESS;
if (OB_FAIL(stmt.init(*this, sql))) {
if (OB_FAIL(stmt.init(*this, sql, 0))) {
LOG_WARN("fail to init prepared statement", K(ret));
}
return ret;
@ -518,19 +519,25 @@ int ObMySQLConnection::execute_proc(const uint64_t tenant_id,
const share::schema::ObRoutineInfo &routine_info,
const common::ObIArray<const pl::ObUserDefinedType *> &udts,
const ObTimeZoneInfo *tz_info,
ObObj *result)
ObObj *result,
bool is_sql)
{
int ret = OB_SUCCESS;
if (OB_UNLIKELY(closed_)) {
ret = OB_NOT_INIT;
LOG_WARN("connection not established. call connect first", K(ret));
} else {
ObMySQLProcStatement stmt;
if (OB_FAIL(create_statement(stmt, tenant_id, sql.ptr()))) {
int64_t real_param_cnt = routine_info.is_function() ? 1 : 0;
for (int64_t i = 0; i < params.count(); i++) {
if (!params.at(i).is_pl_mock_default_param()) {
real_param_cnt++;
}
}
if (OB_FAIL(prepare_proc_stmt(sql.ptr(), real_param_cnt, &allocator))) {
LOG_WARN("create statement failed", K(sql), K(ret));
} else if (OB_FAIL(stmt.execute_proc(allocator, params, routine_info, tz_info))) {
} else if (OB_FAIL(proc_stmt_.execute_proc(allocator, params, routine_info, tz_info, result, is_sql))) {
LOG_WARN("statement execute update failed", K(sql), K(ret));
} else if (OB_FAIL(stmt.close())) {
} else if (OB_FAIL(proc_stmt_.close())) {
LOG_WARN("fail to close stmt", K(ret));
}
}
@ -692,6 +699,70 @@ int ObMySQLConnection::connect_dblink(const bool use_ssl, int64_t sql_request_le
return ret;
}
int ObMySQLConnection::prepare(const char *sql, int64_t param_count, ObIAllocator *allocator)
{
int ret = OB_SUCCESS;
if (OB_FAIL(prepare_proc_stmt(sql, param_count, allocator))) {
LOG_WARN("prepare proc stmt failed", K(ret), KCSTRING(sql));
}
return ret;
}
int ObMySQLConnection::prepare_proc_stmt(const char *sql, int64_t param_count, ObIAllocator *allocator)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(allocator)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("allocator is NULL", K(ret));
} else if (FALSE_IT(proc_stmt_.set_allocator(allocator))) {
} else if (OB_UNLIKELY(closed_)) {
ret = OB_NOT_INIT;
LOG_WARN("connection not established. call connect first", K(ret));
} else if (OB_FAIL(create_statement(proc_stmt_, OB_INVALID_TENANT_ID, sql, param_count))) {
LOG_WARN("create statement failed", K(ret), KCSTRING(sql));
}
return ret;
}
int ObMySQLConnection::bind_basic_type_by_pos(uint64_t position,
void *param,
int64_t param_size,
int32_t datatype,
int32_t &indicator,
bool is_out_param)
{
int ret = OB_SUCCESS;
if (OB_FAIL(proc_stmt_.bind_basic_type_by_pos(position - 1, param, param_size, datatype, indicator, is_out_param))) {
LOG_WARN("bind basic type failed", K(ret), K(position), K(param_size), K(datatype));
}
return ret;
}
int ObMySQLConnection::bind_array_type_by_pos(uint64_t position,
void *array,
int32_t *indicators,
int64_t ele_size,
int32_t ele_datatype,
uint64_t array_size,
uint32_t *out_valid_array_size)
{
int ret = OB_SUCCESS;
if (OB_FAIL(proc_stmt_.bind_array_type_by_pos(position - 1, array, indicators, ele_size,
ele_datatype, array_size, out_valid_array_size))) {
LOG_WARN("bind array type failed", K(ret), K(position), K(ele_size), K(ele_datatype), K(array_size));
}
return ret;
}
int ObMySQLConnection::execute_proc()
{
int ret = OB_SUCCESS;
if (OB_FAIL(proc_stmt_.execute_proc())) {
LOG_WARN("failed to execute proc", K(ret));
}
return ret;
}
} // end namespace sqlclient
} // end namespace common
} // end namespace oceanbase

28
deps/oblib/src/lib/mysqlclient/ob_mysql_connection.h vendored
View File

@ -19,6 +19,7 @@
#include "lib/container/ob_se_array.h"
#include "lib/allocator/ob_malloc.h"
#include "lib/net/ob_addr.h"
#include "lib/mysqlclient/ob_mysql_prepared_statement.h"
namespace oceanbase
{
namespace common
@ -56,7 +57,7 @@ public:
virtual bool is_closed() const;
// use user provided the statement
template<typename T>
int create_statement(T &stmt, const uint64_t tenant_id, const char *sql);
int create_statement(T &stmt, const uint64_t tenant_id, const char *sql, int64_t param_count = 0);
int prepare_statement(ObMySQLPreparedStatement &stmt, const char *sql);
int escape(const char *from, const int64_t from_size, char *to,
const int64_t to_size, int64_t &out_size);
@ -91,7 +92,8 @@ public:
const share::schema::ObRoutineInfo &routine_info,
const common::ObIArray<const pl::ObUserDefinedType *> &udts,
const ObTimeZoneInfo *tz_info,
ObObj *result) override;
ObObj *result,
bool is_sql) override;
virtual int start_transaction(const uint64_t &tenant_id, bool with_snap_shot = false) override;
virtual int rollback() override;
virtual int commit() override;
@ -121,7 +123,22 @@ public:
// dblink.
virtual int connect_dblink(const bool use_ssl, int64_t sql_request_level);
int prepare(const char *sql, int64_t param_count, ObIAllocator *allocator);
int prepare_proc_stmt(const char *sql, int64_t param_count, ObIAllocator *allocator);
int bind_basic_type_by_pos(uint64_t position,
void *param,
int64_t param_size,
int32_t datatype,
int32_t &indicator,
bool is_out_param);
int bind_array_type_by_pos(uint64_t position,
void *array,
int32_t *indicators,
int64_t ele_size,
int32_t ele_datatype,
uint64_t array_size,
uint32_t *out_valid_array_size);
int execute_proc();
private:
int switch_tenant(const uint64_t tenant_id);
int reset_read_consistency();
@ -145,6 +162,7 @@ private:
const char *db_name_;
uint64_t tenant_id_;
int64_t read_consistency_;
ObMySQLProcStatement proc_stmt_;
DISALLOW_COPY_AND_ASSIGN(ObMySQLConnection);
};
inline bool ObMySQLConnection::is_busy() const
@ -181,14 +199,14 @@ inline int64_t ObMySQLConnection::connection_version() const
}
template<typename T>
int ObMySQLConnection::create_statement(T &stmt, const uint64_t tenant_id, const char *sql)
int ObMySQLConnection::create_statement(T &stmt, const uint64_t tenant_id, const char *sql, int64_t param_count)
{
int ret = OB_SUCCESS;
if (OB_FAIL(switch_tenant(tenant_id))) {
_OB_LOG(WARN, "switch tenant failed, tenant_id=%ld, ret=%d", tenant_id, ret);
} else if (OB_FAIL(reset_read_consistency())) {
_OB_LOG(WARN, "fail to set read consistency, ret=%d", ret);
} else if (OB_FAIL(stmt.init(*this, sql))) {
} else if (OB_FAIL(stmt.init(*this, sql, param_count))) {
_OB_LOG(WARN, "fail to init statement, ret=%d", ret);
}
return ret;

3
deps/oblib/src/lib/mysqlclient/ob_mysql_connection_pool.cpp vendored
View File

@ -464,8 +464,7 @@ int ObMySQLConnectionPool::acquire(const uint64_t tenant_id, ObMySQLConnection *
}
if (OB_ISNULL(connection)) {
//overwrite ret
ret = OB_ERR_UNEXPECTED;
ret = OB_SUCC(ret) ? OB_ERR_UNEXPECTED: ret;
LOG_WARN("failed to acquire connection",
K(this), K(tenant_id), K(server_count), K(busy_conn_count_), K(ret));
obsys::ObRLockGuard lock(get_lock_);

11
deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_param.cpp vendored
View File

@ -49,10 +49,11 @@ int ObMySQLPreparedParam::init()
ret = OB_ERR_SQL_CLIENT;
} else if (0 == param_count_) {
// insert or replace that do not produce result sets
} else if (OB_ISNULL(bind_ = reinterpret_cast<MYSQL_BIND *>(alloc_.alloc(sizeof(MYSQL_BIND) * param_count_)))) {
} else if (OB_ISNULL(bind_ = reinterpret_cast<MYSQL_BIND *>(alloc_->alloc(sizeof(MYSQL_BIND) * param_count_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("out of memory, alloc mem for mysql_bind error", K(ret));
} else {
MEMSET(bind_, 0, sizeof(MYSQL_BIND) * param_count_);
LOG_DEBUG("statement field", K(param_count_));
}
return ret;
@ -75,8 +76,10 @@ int ObMySQLPreparedParam::bind_param()
void ObMySQLPreparedParam::close()
{
if (OB_LIKELY(NULL != bind_)) {
alloc_.free(bind_);
alloc_->free(bind_);
bind_ = NULL;
param_count_ = 0;
alloc_ = NULL;
}
}
@ -93,6 +96,10 @@ int ObMySQLPreparedParam::bind_param(ObBindParam &param)
bind_[param.col_idx_].length = &param.length_;
bind_[param.col_idx_].is_null = &param.is_null_;
bind_[param.col_idx_].is_unsigned = param.is_unsigned_;
bind_[param.col_idx_].long_data_used = 0;
#ifdef OB_BUILD_ORACLE_PL
bind_[param.col_idx_].mysql = stmt_.get_conn_handler();
#endif
} else {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid index", K(param), K(param_count_));

3
deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_param.h vendored
View File

@ -29,6 +29,7 @@ class ObBindParam;
class ObMySQLPreparedStatement;
class ObMySQLPreparedParam
{
friend ObMySQLPreparedStatement;
public:
explicit ObMySQLPreparedParam(ObMySQLPreparedStatement &stmt);
~ObMySQLPreparedParam();
@ -40,7 +41,7 @@ public:
private:
ObMySQLPreparedStatement &stmt_;
common::ObIAllocator &alloc_;
common::ObIAllocator *alloc_;
int64_t param_count_;
MYSQL_BIND *bind_;
};

7
deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_result.cpp vendored
View File

@ -50,11 +50,12 @@ int ObMySQLPreparedResult::init()
ret = OB_ERR_SQL_CLIENT;
} else if (result_column_count_ == 0) {
// insert or replace that do not produce result sets
} else if (OB_ISNULL(bind_ = reinterpret_cast<MYSQL_BIND *>(alloc_.alloc(sizeof(MYSQL_BIND) *
} else if (OB_ISNULL(bind_ = reinterpret_cast<MYSQL_BIND *>(alloc_->alloc(sizeof(MYSQL_BIND) *
result_column_count_)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
LOG_ERROR("out of memory, alloc mem for mysql bind error", K(ret));
} else {
MEMSET(bind_, 0, sizeof(MYSQL_BIND) * result_column_count_);
LOG_TRACE("statemen field count = ", K(result_column_count_));
}
return ret;
@ -77,8 +78,9 @@ int ObMySQLPreparedResult::bind_result_param()
void ObMySQLPreparedResult::close()
{
result_column_count_ = 0;
alloc_.free(bind_);
alloc_->free(bind_);
bind_ = NULL;
alloc_ = NULL;
}
int ObMySQLPreparedResult::next()
@ -120,6 +122,7 @@ int ObMySQLPreparedResult::bind_result(ObBindParam &param)
bind_[param.col_idx_].length = &param.length_;
bind_[param.col_idx_].is_null = &param.is_null_;
bind_[param.col_idx_].is_unsigned = param.is_unsigned_;
bind_[param.col_idx_].error = &bind_[param.col_idx_].error_value;
} else {
ret = OB_INVALID_ARGUMENT;
LOG_WARN("invalid index", K(param), K(result_column_count_));

4
deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_result.h vendored
View File

@ -29,6 +29,7 @@ class ObBindParam;
class ObMySQLPreparedStatement;
class ObMySQLPreparedResult
{
friend ObMySQLPreparedStatement;
public:
explicit ObMySQLPreparedResult(ObMySQLPreparedStatement &stmt);
~ObMySQLPreparedResult();
@ -50,9 +51,10 @@ public:
int64_t get_result_column_count() const { return result_column_count_; }
int bind_result(ObBindParam &param);
MYSQL_BIND *&get_bind() { return bind_; }
private:
ObMySQLPreparedStatement &stmt_;
common::ObIAllocator &alloc_;
common::ObIAllocator *alloc_;
int64_t result_column_count_;
MYSQL_BIND *bind_;
};

971
deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_statement.cpp vendored
View File

File diff suppressed because it is too large Load Diff

102
deps/oblib/src/lib/mysqlclient/ob_mysql_prepared_statement.h vendored
View File

@ -29,30 +29,37 @@ namespace sqlclient
struct ObBindParam
{
ObBindParam() : col_idx_(-1), buffer_type_(enum_field_types::MAX_NO_FIELD_TYPES), buffer_(nullptr),
buffer_len_(0), length_(0), is_unsigned_(0), is_null_(0)
buffer_len_(0), length_(0), is_unsigned_(0), is_null_(0), array_buffer_(nullptr),
ele_size_(0), max_array_size_(0), out_valid_array_size_(nullptr)
{
}
ObBindParam(int64_t col_idx, enum_field_types buffer_type,
void *buffer, int64_t buffer_len, unsigned long length)
: col_idx_(col_idx), buffer_type_(buffer_type), buffer_(buffer), buffer_len_(buffer_len),
length_(length), is_unsigned_(0), is_null_(0)
length_(length), is_unsigned_(0), is_null_(0), array_buffer_(nullptr),
ele_size_(0), max_array_size_(0), out_valid_array_size_(nullptr)
{
}
ObBindParam(int64_t col_idx, enum_field_types buffer_type,
void *buffer, int64_t buffer_len, unsigned long length,
my_bool is_unsigned, my_bool is_null)
: col_idx_(col_idx), buffer_type_(buffer_type), buffer_(buffer), buffer_len_(buffer_len),
length_(length), is_unsigned_(is_unsigned), is_null_(is_null)
length_(length), is_unsigned_(is_unsigned), is_null_(is_null), array_buffer_(nullptr),
ele_size_(0), max_array_size_(0), out_valid_array_size_(nullptr)
{
}
int assign(const ObBindParam &other);
void assign(const ObBindParam &other);
TO_STRING_KV(K_(col_idx),
K_(buffer_type),
K_(buffer),
K_(buffer_len),
K_(length),
K_(is_unsigned),
K_(is_null));
K_(is_null),
K_(array_buffer),
K_(ele_size),
K_(max_array_size),
K_(out_valid_array_size));
public:
int64_t col_idx_;
@ -62,6 +69,10 @@ public:
unsigned long length_;
my_bool is_unsigned_;
my_bool is_null_;
void *array_buffer_;
int64_t ele_size_;
int64_t max_array_size_;
uint32_t *out_valid_array_size_;
};
class ObBindParamEncode
@ -70,9 +81,10 @@ public:
#define ENCODE_FUNC_ARG_DECL const int64_t col_idx, \
const bool is_output_param, \
const ObTimeZoneInfo &tz_info, \
ObObjParam &param, \
ObObj &param, \
ObBindParam &bind_param, \
ObIAllocator &allocator
ObIAllocator &allocator, \
enum_field_types buffer_type
#define DEF_ENCODE_FUNC(name) \
static int encode_##name(ENCODE_FUNC_ARG_DECL);
using EncodeFunc = int (*)(ENCODE_FUNC_ARG_DECL);
@ -87,10 +99,12 @@ public:
DEF_ENCODE_FUNC(number);
DEF_ENCODE_FUNC(unumber);
DEF_ENCODE_FUNC(datetime);
DEF_ENCODE_FUNC(timestamp);
DEF_ENCODE_FUNC(date);
DEF_ENCODE_FUNC(time);
DEF_ENCODE_FUNC(year);
DEF_ENCODE_FUNC(string);
DEF_ENCODE_FUNC(number_float);
static int encode_not_supported(ENCODE_FUNC_ARG_DECL);
public:
@ -103,7 +117,7 @@ public:
#define DECODE_FUNC_ARG_DECL const enum_field_types field_type, \
const ObTimeZoneInfo &tz_info, \
const ObBindParam &bind_param, \
ObObjParam &param, \
ObObj &param, \
ObIAllocator &allocator
#define DEF_DECODE_FUNC(name) \
static int decode_##name(DECODE_FUNC_ARG_DECL);
@ -119,9 +133,11 @@ public:
DEF_DECODE_FUNC(number);
DEF_DECODE_FUNC(unumber);
DEF_DECODE_FUNC(datetime);
DEF_DECODE_FUNC(timestamp);
DEF_DECODE_FUNC(time);
DEF_DECODE_FUNC(year);
DEF_DECODE_FUNC(string);
DEF_DECODE_FUNC(number_float);
static int decode_not_supported(DECODE_FUNC_ARG_DECL);
public:
@ -133,14 +149,15 @@ class ObMySQLPreparedStatement
public:
ObMySQLPreparedStatement();
virtual ~ObMySQLPreparedStatement();
ObIAllocator &get_allocator();
ObIAllocator *get_allocator();
void set_allocator(ObIAllocator *alloc);
ObMySQLConnection *get_connection();
MYSQL_STMT *get_stmt_handler();
MYSQL *get_conn_handler();
int close();
int init(ObMySQLConnection &conn, const char *sql);
int bind_param(const ObBindParam &param);
int bind_result(const ObBindParam &param);
virtual int close();
virtual int init(ObMySQLConnection &conn, const char *sql, int64_t param_count);
int bind_param(ObBindParam &param);
int bind_result(ObBindParam &param);
int bind_param_int(const int64_t col_idx, int64_t *out_buf);
int bind_param_varchar(const int64_t col_idx, char *out_buf, unsigned long res_len);
@ -188,39 +205,78 @@ protected:
class ObMySQLProcStatement : public ObMySQLPreparedStatement
{
public:
ObMySQLProcStatement()
ObMySQLProcStatement() : ObMySQLPreparedStatement()
{
in_out_map_.reset();
proc_ = NULL;
}
~ObMySQLProcStatement()
{
in_out_map_.reset();
proc_ = NULL;
}
virtual int init(ObMySQLConnection &conn, const char *sql, int64_t param_count);
virtual int close();
virtual void free_resouce();
virtual int close_mysql_stmt();
int execute_proc(ObIAllocator &allocator,
ParamStore &params,
const share::schema::ObRoutineInfo &routine_info,
const ObTimeZoneInfo *tz_info);
const ObTimeZoneInfo *tz_info,
ObObj *result,
bool is_sql);
int execute_proc();
int bind_basic_type_by_pos(uint64_t position,
void *param_buffer,
int64_t param_size,
int32_t datatype,
int32_t &indicator,
bool is_out_param);
int bind_array_type_by_pos(uint64_t position,
void *array,
int32_t *indicators,
int64_t ele_size,
int32_t ele_datatype,
uint64_t array_size,
uint32_t *out_valid_array_size);
inline void set_proc(const char *sql) { proc_ = sql; }
private:
int bind_proc_param(ObIAllocator &allocator,
ParamStore &params,
const share::schema::ObRoutineInfo &routine_info,
common::ObIArray<int64_t> &basic_out_param,
const ObTimeZoneInfo *tz_info);
common::ObIArray<std::pair<int64_t, int64_t>> &basic_out_param,
const ObTimeZoneInfo *tz_info,
ObObj *result,
bool is_sql);
int bind_param(const int64_t col_idx,
const int64_t param_idx,
const bool is_output_param,
const ObTimeZoneInfo *tz_info,
ObObjParam &obj,
ObObj &obj,
const share::schema::ObRoutineInfo &routine_info,
ObIAllocator &allocator);
int process_proc_output_params(ObIAllocator &allocator,
ParamStore &params,
const share::schema::ObRoutineInfo &routine_info,
common::ObIArray<int64_t> &basic_out_param,
const ObTimeZoneInfo *tz_info);
int convert_proc_output_param_result(const ObTimeZoneInfo &tz_info,
common::ObIArray<std::pair<int64_t, int64_t>> &basic_out_param,
const ObTimeZoneInfo *tz_info,
ObObj *result,
bool is_sql);
int convert_proc_output_param_result(int64_t out_param_idx,
const ObTimeZoneInfo &tz_info,
const ObBindParam &bind_param,
ObObjParam &param,
ObObj *param,
const share::schema::ObRoutineInfo &routine_info,
ObIAllocator &allocator);
ObIAllocator &allocator,
bool is_return_value);
int execute_stmt_v2_interface();
int handle_data_truncated(ObIAllocator &allocator);
private:
common::ObSEArray<bool, 8> in_out_map_;
const char * proc_;
};
} //namespace sqlclient
}

18
deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp vendored
View File

@ -205,6 +205,7 @@ int ObCommonSqlProxy::write(const uint64_t tenant_id, const ObString sql,
conn->set_is_load_data_exec(param->is_load_data_exec_);
conn->set_use_external_session(param->use_external_session_);
conn->set_group_id(param->consumer_group_id_);
conn->set_ob_enable_pl_cache(param->enable_pl_cache_);
if (param->is_load_data_exec_) {
is_user_sql = true;
}
@ -643,13 +644,16 @@ int ObDbLinkProxy::dblink_execute_proc(ObISQLConnection *dblink_conn)
}
int ObDbLinkProxy::dblink_prepare(sqlclient::ObISQLConnection *dblink_conn, const char *sql)
int ObDbLinkProxy::dblink_prepare(sqlclient::ObISQLConnection *dblink_conn,
const char *sql,
int64_t param_count,
ObIAllocator *allocator)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(dblink_conn) || OB_ISNULL(sql)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("null ptr", K(ret), KP(dblink_conn), KP(sql));
} else if (OB_FAIL(dblink_conn->prepare(sql))) {
} else if (OB_FAIL(dblink_conn->prepare(sql, param_count, allocator))) {
LOG_WARN("prepare to dblink failed", K(ret), K(ObString(sql)));
}
return ret;
@ -660,13 +664,14 @@ int ObDbLinkProxy::dblink_bind_basic_type_by_pos(sqlclient::ObISQLConnection *db
void *param,
int64_t param_size,
int32_t datatype,
int32_t &indicator)
int32_t &indicator,
bool is_out_param)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(dblink_conn)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("null ptr", K(ret), KP(dblink_conn));
} else if (OB_FAIL(dblink_conn->bind_basic_type_by_pos(position, param, param_size, datatype, indicator))) {
} else if (OB_FAIL(dblink_conn->bind_basic_type_by_pos(position, param, param_size, datatype, indicator, is_out_param))) {
LOG_WARN("bind_basic_type_by_pos to dblink failed", K(ret));
} else {
LOG_DEBUG("succ to bind_basic_type_by_pos dblink", K(ret));
@ -740,14 +745,15 @@ int ObDbLinkProxy::dblink_execute_proc(const uint64_t tenant_id,
const share::schema::ObRoutineInfo &routine_info,
const common::ObIArray<const pl::ObUserDefinedType *> &udts,
const ObTimeZoneInfo *tz_info,
ObObj *result)
ObObj *result,
bool is_sql)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(dblink_conn) || sql.empty()) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("null ptr", K(ret), KP(dblink_conn), K(sql));
} else if (OB_FAIL(dblink_conn->execute_proc(tenant_id, allocator, params, sql,
routine_info, udts, tz_info, result))) {
routine_info, udts, tz_info, result, is_sql))) {
LOG_WARN("call procedure to dblink failed", K(ret), K(dblink_conn), K(sql));
} else {
LOG_DEBUG("succ to call procedure by dblink", K(sql));

16
deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h vendored
View File

@ -91,8 +91,8 @@ struct ObSessionParam final
{
public:
ObSessionParam()
: sql_mode_(nullptr), tz_info_wrap_(nullptr), ddl_info_(), is_load_data_exec_(false), use_external_session_(false), consumer_group_id_(0), nls_formats_{}
{}
: sql_mode_(nullptr), tz_info_wrap_(nullptr), ddl_info_(), is_load_data_exec_(false),
use_external_session_(false), consumer_group_id_(0), nls_formats_{}, enable_pl_cache_(true) {}
~ObSessionParam() = default;
public:
int64_t *sql_mode_;
@ -102,6 +102,7 @@ public:
bool use_external_session_; // need init remote inner sql conn with sess getting from sess mgr
int64_t consumer_group_id_;
common::ObString nls_formats_[common::ObNLSFormatEnum::NLS_MAX];
bool enable_pl_cache_;
};
// thread safe sql proxy
@ -212,14 +213,19 @@ public:
const share::schema::ObRoutineInfo &routine_info,
const common::ObIArray<const pl::ObUserDefinedType *> &udts,
const ObTimeZoneInfo *tz_info,
ObObj *result);
int dblink_prepare(sqlclient::ObISQLConnection *dblink_conn, const char *sql);
ObObj *result,
bool is_sql);
int dblink_prepare(sqlclient::ObISQLConnection *dblink_conn,
const char *sql,
int64_t param_count,
ObIAllocator *allocator = NULL);
int dblink_bind_basic_type_by_pos(sqlclient::ObISQLConnection *dblink_conn,
uint64_t position,
void *param,
int64_t param_size,
int32_t datatype,
int32_t &indicator);
int32_t &indicator,
bool is_out_param);
int dblink_bind_array_type_by_pos(sqlclient::ObISQLConnection *dblink_conn,
uint64_t position,
void *array,

141
deps/oblib/src/lib/mysqlclient/ob_mysql_result.h vendored
View File

@ -1072,6 +1072,9 @@
ObCastCtx cast_ctx(&allocator, no_dtc_params ? NULL : &dtc_params, CM_NONE, column.get_collation_type());\
if (is_cur_default_value && column.is_default_expr_v2_column())\
{ \
if (lib::is_mysql_mode()) { \
res_obj.set_collation_type(CS_TYPE_UTF8MB4_BIN); \
} \
res_obj.set_varchar(str_value); \
ret = (class_obj).set_##column_name(res_obj); \
} \
@ -1179,6 +1182,144 @@
} \
} ()
#define EXTRACT_DEFAULT_VALUE_FIELD_MYSQL_V2(result, data_type, class_obj,is_cur_default_value, default_value_v2_version, tenant_id) \
{ \
if (OB_SUCC(ret)) \
{ \
ObString str_value; \
ObObj res_obj; \
ret = (result).get_varchar("cur_default_value", str_value); \
\
if (OB_ERR_NULL_VALUE == ret) \
{ /* without default value */ \
if (!default_value_v2_version) { \
res_obj.set_null(); \
ret = (class_obj).set_cur_default_value(res_obj, column.is_default_expr_v2_column()); \
} \
else { \
ret = OB_SUCCESS; \
} \
} \
else if (OB_ERR_COLUMN_NOT_FOUND == ret) \
{ \
SQL_LOG(WARN, "column not found, ignore", "column_name", "cur_default_value"); \
ret = OB_SUCCESS; \
} \
else if (OB_SUCC(ret)) \
{ /*big stack check ObSchemaRetrieveUtils::fill_column_schema*/ \
ObObj def_obj; \
ObArenaAllocator allocator(ObModIds::OB_SCHEMA); \
ObTimeZoneInfo tz_info; \
const ObDataTypeCastParams dtc_params(&tz_info); \
bool no_dtc_params = (ob_is_bit_tc(data_type) || ob_is_enum_or_set_type(data_type)); \
ObCastCtx cast_ctx(&allocator, no_dtc_params ? NULL : &dtc_params, CM_NONE, column.get_collation_type());\
if (is_cur_default_value && column.is_default_expr_v2_column()) \
{ \
if (lib::is_mysql_mode()) { \
res_obj.set_collation_type(CS_TYPE_UTF8MB4_BIN); \
} \
res_obj.set_varchar(str_value); \
ret = (class_obj).set_cur_default_value(res_obj, column.is_default_expr_v2_column()); \
} \
else if (IS_DEFAULT_NOW_STR(data_type, str_value)) \
{ \
res_obj.set_ext(ObActionFlag::OP_DEFAULT_NOW_FLAG); \
res_obj.set_scale(column.get_data_scale()); \
ret = (class_obj).set_cur_default_value(res_obj, column.is_default_expr_v2_column()); \
} \
else if (column.is_generated_column()) { \
res_obj.set_string(data_type, str_value); \
res_obj.meta_.set_collation_type(CS_TYPE_UTF8MB4_BIN); \
res_obj.meta_.set_collation_level(CS_LEVEL_IMPLICIT); \
ret = (class_obj).set_cur_default_value(res_obj, column.is_default_expr_v2_column()); \
} \
else if (column.is_identity_column() || ob_is_string_type(data_type) || ob_is_geometry(data_type) || ob_is_collection_sql_type(data_type)) \
{ \
res_obj.set_string(data_type, str_value); \
res_obj.meta_.set_collation_type(column.get_collation_type()); \
/* will override the collaction level set in set_varchar */ \
res_obj.meta_.set_collation_level(CS_LEVEL_IMPLICIT); \
/* only support full inrow data, all lobs from systable should be made inrow */ \
if (res_obj.is_outrow_lob()) { \
ret = OB_INVALID_ARGUMENT; \
SQL_LOG(WARN, "outrow lob unsupported", "column_name", "cur_default_value"); \
} \
else { \
if (ob_is_text_tc(data_type) || ob_is_geometry(data_type) || ob_is_collection_sql_type(data_type)) { res_obj.set_inrow(); } \
ret = (class_obj).set_cur_default_value(res_obj, column.is_default_expr_v2_column()); \
} \
} \
else { \
if (ob_is_bit_tc(data_type) || ob_is_enum_or_set_type(data_type)) \
{ \
def_obj.set_varchar(str_value); \
ObObj tmp_obj; \
if(OB_FAIL(ObObjCaster::to_type(ObUInt64Type, cast_ctx, def_obj, tmp_obj))) \
{ \
SQL_LOG(WARN, "cast obj failed, ", "src type", def_obj.get_type()); \
} \
else \
{ \
if (ObBitType == data_type) { \
res_obj.set_bit(tmp_obj.get_uint64()); \
res_obj.set_scale(column.get_data_precision()); \
} else if (ObEnumType == data_type) { \
res_obj.set_enum(tmp_obj.get_uint64()); \
} else {/*set type*/ \
res_obj.set_set(tmp_obj.get_uint64()); \
} \
ret = (class_obj).set_cur_default_value(res_obj, column.is_default_expr_v2_column());\
} \
} \
else if (ob_is_json(data_type)) \
{ \
def_obj.set_type(data_type); \
if (is_mysql_mode()) { \
if (OB_FAIL(def_obj.build_not_strict_default_value(column.get_data_precision()))) { \
SQL_LOG(WARN, "failed to build not strict default json value", K(ret)); \
} else { \
res_obj.set_json_value(data_type, def_obj.get_string().ptr(), \
def_obj.get_string().length()); \
} \
} else { \
def_obj.set_json_value(data_type, str_value.ptr(), str_value.length()); \
if (OB_FAIL(ObObjCaster::to_type(data_type, cast_ctx, def_obj, res_obj))) { \
SQL_LOG(WARN, "cast obj failed, ", "src type", def_obj.get_type(), "dest type", data_type); \
} else { \
res_obj.set_inrow(); \
} \
} \
if (OB_SUCC(ret)) { \
res_obj.meta_.set_collation_level(CS_LEVEL_IMPLICIT); \
ret = (class_obj).set_cur_default_value(res_obj, column.is_default_expr_v2_column());\
} \
} \
else \
{ \
def_obj.set_varchar(str_value); \
if (OB_FAIL(OTTZ_MGR.get_tenant_tz(tenant_id, tz_info.get_tz_map_wrap()))) \
{ \
SQL_LOG(WARN, "get tenant timezone map failed", K(ret)); \
} \
else if (OB_FAIL(ObObjCaster::to_type(data_type, cast_ctx, def_obj, res_obj))) \
{ \
SQL_LOG(WARN, "cast obj failed, ", "src type", def_obj.get_type(), "dest type", data_type); \
} \
else \
{ \
res_obj.set_scale(column.get_data_scale()); \
ret = (class_obj).set_cur_default_value(res_obj, column.is_default_expr_v2_column());\
} \
} \
} \
} \
else \
{ \
SQL_LOG(WARN, "fail to default value field mysql. ", K(ret)); \
} \
} \
}
#define EXTRACT_CREATE_TIME_FIELD_MYSQL(result, column_name, field, type) \
EXTRACT_INT_FIELD_MYSQL(result, column_name, field, type)

2
deps/oblib/src/lib/mysqlclient/ob_mysql_statement.cpp vendored
View File

@ -54,7 +54,7 @@ ObMySQLConnection *ObMySQLStatement::get_connection()
return conn_;
}
int ObMySQLStatement::init(ObMySQLConnection &conn, const char *sql)
int ObMySQLStatement::init(ObMySQLConnection &conn, const char *sql, int64_t param_count)
{
int ret = OB_SUCCESS;
conn_ = &conn;

2
deps/oblib/src/lib/mysqlclient/ob_mysql_statement.h vendored
View File

@ -31,7 +31,7 @@ public:
ObMySQLConnection *get_connection();
MYSQL *get_stmt_handler();
MYSQL *get_conn_handler();
int init(ObMySQLConnection &conn, const char *sql);
int init(ObMySQLConnection &conn, const char *sql, int64_t param_count = 0);
/*
* close statement

2
deps/oblib/src/lib/number/ob_number_v2.cpp vendored
View File

@ -1592,7 +1592,7 @@ int ObNumber::check_range(bool *is_valid_uint64, bool *is_valid_int64,
} else {
*is_valid = false; //no break
}
} else {
} else if (digit != 0) {
decimal_parts = digit;
break;
}

119
deps/oblib/src/lib/ob_define.h vendored
View File

@ -372,6 +372,8 @@ const int64_t OB_MAX_PASSWORD_LENGTH = 128;
const int64_t OB_MAX_PASSWORD_BUF_LENGTH = OB_MAX_PASSWORD_LENGTH + 1;
// After each sha1 is 41 characters, the incremental backup is up to 64 times, and the maximum password required for recovery is 64*(41+1)=2,688
const int64_t OB_MAX_ENCRYPTED_PASSWORD_LENGTH = OB_MAX_PASSWORD_LENGTH * 4;
const int64_t OB_MAX_EXTERNAL_TABLE_PROPERTIES_ITEM_LENGTH = 128;
const int64_t OB_MAX_ENCRYPTED_EXTERNAL_TABLE_PROPERTIES_ITEM_LENGTH = OB_MAX_EXTERNAL_TABLE_PROPERTIES_ITEM_LENGTH * 4;
const int64_t OB_MAX_PASSWORD_ARRAY_LENGTH = 4096;
const int64_t OB_MAX_ERROR_MSG_LEN = 512;
const int64_t OB_MAX_RESULT_MESSAGE_LENGTH = 1024;
@ -409,6 +411,7 @@ const int64_t OB_MAX_DIRECTORY_NAME_LENGTH = 128; // Compatible with Oracle
const int64_t OB_MAX_DIRECTORY_PATH_LENGTH = 4000; // Compatible with Oracle
const uint64_t OB_MAX_INTERVAL_PARTITIONS = 1048575; // interval parted table max partitions
const int64_t OB_MAX_BALANCE_GROUP_NAME_LENGTH = 512;
const int64_t OB_SERVICE_NAME_LENGTH = 64;
//plan cache
const int64_t OB_PC_NOT_PARAM_COUNT = 8;
@ -444,7 +447,7 @@ const int64_t OB_SCHEMA_START_VERSION = 100;
const int64_t OB_SYS_PARAM_ROW_KEY_LENGTH = 192;
const int64_t OB_MAX_SYS_PARAM_NAME_LENGTH = 128;
const int64_t OB_MAX_SYS_PARAM_VALUE_LENGTH = 1024;
const int64_t OB_MAX_SYS_PARAM_NUM = 600;
const int64_t OB_MAX_SYS_PARAM_NUM = 700;
const int64_t OB_MAX_PREPARE_STMT_NUM_PER_SESSION = 512;
const uint32_t INVALID_SESSID = UINT32_MAX;
const int64_t OB_MAX_VAR_NUM_PER_SESSION = 1024;
@ -855,6 +858,10 @@ const uint64_t DEFAULT_CUSTOMIZED_CG_NUM = 2;
const int64_t OB_CG_NAME_PREFIX_LENGTH = 5; // length of cg prefix like "__cg_"
const int64_t OB_MAX_COLUMN_GROUP_NAME_LENGTH = OB_MAX_COLUMN_NAME_LENGTH * OB_MAX_CHAR_LEN + OB_CG_NAME_PREFIX_LENGTH; //(max_column_name_length(128) * ob_max_char_len(3)) + prefix
const int64_t MAX_NAME_CHAR_LEN = 64;
const int64_t MAX_AUDIT_FILTER_NAME_LENGTH = 64;
const int64_t MAX_AUDIT_FILTER_NAME_LENGTH_BYTE = 4 * MAX_AUDIT_FILTER_NAME_LENGTH;
const int64_t MAX_AUDIT_USER_NAME_LENGTH_BYTE = 4 * OB_MAX_USER_NAME_LENGTH_STORE;
const int64_t MAX_AUDIT_HOST_NAME_LENGTH_BYTE = 4 * OB_MAX_HOST_NAME_LENGTH;
//Oracle
const int64_t MAX_ORACLE_COMMENT_LENGTH = 4000;
@ -1525,6 +1532,11 @@ const char *const OB_MYSQL_PROXY_VEERSION = "__proxy_version";
const char *const OB_MYSQL_CLIENT_VERSION = "__ob_client_version";
const char *const OB_MYSQL_CLIENT_NAME = "__ob_client_name";
const char *const OB_MYSQL_FAILOVER_MODE = "__proxy_failover_mode";
const char *const OB_MYSQL_FAILOVER_MODE_OFF = "off";
const char *const OB_MYSQL_FAILOVER_MODE_ON = "on";
const char *const OB_MYSQL_SERVICE_NAME = "__proxy_service_name";
const char *const OB_MYSQL_JDBC_CLIENT_NAME = "OceanBase Connector/J";
const char *const OB_MYSQL_OCI_CLIENT_NAME = "OceanBase Connector/C";
// for java client
@ -2084,17 +2096,19 @@ enum ObFreezeStatus
};
/*
* |---- 2 bits ---|--- 4 bits ---|--- 2 bits ---|--- 2 bits ---| LSB
* |-- encryption--|--- clog ---|-- SSStore ---|--- MemStore--| LSB
* |---- 2 bits ---|---- 2 bits ---|--- 4 bits ---|--- 2 bits ---|--- 2 bits ---| LSB
* |--column-store-|-- encryption--|--- clog ---|-- SSStore ---|--- MemStore--| LSB
*/
const int64_t MEMSTORE_BITS_SHIFT = 0;
const int64_t SSSTORE_BITS_SHIFT = 2;
const int64_t CLOG_BITS_SHIFT = 4;
const int64_t ENCRYPTION_BITS_SHIFT = 8;
const int64_t COLUMNSTORE_BITS_SHIFT = 10;
const int64_t REPLICA_TYPE_MEMSTORE_MASK = (0x3UL << MEMSTORE_BITS_SHIFT);
const int64_t REPLICA_TYPE_SSSTORE_MASK = (0x3UL << SSSTORE_BITS_SHIFT);
const int64_t REPLICA_TYPE_CLOG_MASK = (0xFUL << CLOG_BITS_SHIFT);
const int64_t REPLICA_TYPE_ENCRYPTION_MASK = (0x3UL << ENCRYPTION_BITS_SHIFT);
const int64_t REPLICA_TYPE_COLUMNSTORE_MASK = (0x3UL << COLUMNSTORE_BITS_SHIFT);
// replica type associated with memstore
const int64_t WITH_MEMSTORE = 0;
const int64_t WITHOUT_MEMSTORE = 1;
@ -2107,16 +2121,22 @@ const int64_t ASYNC_CLOG = 1 << CLOG_BITS_SHIFT;
// replica type associated with encryption
const int64_t WITHOUT_ENCRYPTION = 0 << ENCRYPTION_BITS_SHIFT;
const int64_t WITH_ENCRYPTION = 1 << ENCRYPTION_BITS_SHIFT;
// replica type associated with columnstore
const int64_t NOT_COLUMNSTORE = 0 << COLUMNSTORE_BITS_SHIFT;
const int64_t COLUMNSTORE = 1 << COLUMNSTORE_BITS_SHIFT;
// tracepoint, refer to OB_MAX_CONFIG_xxx
const int64_t OB_MAX_TRACEPOINT_NAME_LEN = 128;
const int64_t OB_MAX_TRACEPOINT_DESCRIBE_LEN = 4096;
// Need to manually maintain the replica_type_to_str function in utility.cpp,
// Currently there are only three types: REPLICA_TYPE_FULL, REPLICA_TYPE_READONLY, and REPLICA_TYPE_LOGONLY
// Please modify the replica_type_to_string and string_to_replica_type function
// in ob_share_util.cpp when adding new replica_type.
enum ObReplicaType
{
// Almighty copy: is a member of paxos; has ssstore; has memstore
// Invalid replica_type, value of which is -1.
// Attention: Please DO use REPLICA_TYPE_INVALID as initial value. DO NOT use REPLICA_TYPE_MAX.
REPLICA_TYPE_INVALID = -1,
// Fully functional copy: is a member of paxos; has ssstore; has memstore
REPLICA_TYPE_FULL = (SYNC_CLOG | WITH_SSSTORE | WITH_MEMSTORE), // 0
// Backup copy: Paxos member; ssstore; no memstore
REPLICA_TYPE_BACKUP = (SYNC_CLOG | WITH_SSSTORE | WITHOUT_MEMSTORE), // 1
@ -2133,54 +2153,44 @@ enum ObReplicaType
REPLICA_TYPE_ARBITRATION = (ASYNC_CLOG | WITHOUT_SSSTORE | WITHOUT_MEMSTORE), // 21
// Encrypted log copy: encrypted; paxos member; no sstore; no memstore
REPLICA_TYPE_ENCRYPTION_LOGONLY = (WITH_ENCRYPTION | SYNC_CLOG | WITHOUT_SSSTORE | WITHOUT_MEMSTORE), // 261
// invalid value
// Column-store copy: column-store, not a member of paxos; ssstore; memstore
REPLICA_TYPE_COLUMNSTORE = (COLUMNSTORE | ASYNC_CLOG | WITH_SSSTORE | WITH_MEMSTORE), // 1040
// max value
REPLICA_TYPE_MAX,
};
static inline int replica_type_to_string(const ObReplicaType replica_type, char *name_str, const int64_t str_len)
{
int ret = OB_SUCCESS;
switch(replica_type) {
case REPLICA_TYPE_FULL: {
strncpy(name_str ,"FULL", str_len);
break;
}
case REPLICA_TYPE_BACKUP: {
strncpy(name_str ,"BACKUP", str_len);
break;
}
case REPLICA_TYPE_LOGONLY: {
strncpy(name_str ,"LOGONLY", str_len);
break;
}
case REPLICA_TYPE_READONLY: {
strncpy(name_str ,"READONLY", str_len);
break;
}
case REPLICA_TYPE_MEMONLY: {
strncpy(name_str ,"MEMONLY", str_len);
break;
}
case REPLICA_TYPE_ENCRYPTION_LOGONLY: {
strncpy(name_str ,"ENCRYPTION_LOGONLY", str_len);
break;
}
default: {
ret = OB_INVALID_ARGUMENT;
strncpy(name_str ,"INVALID", str_len);
break;
} // default
} // switch
return ret;
}
// full replica
const char *const FULL_REPLICA_STR = "FULL";
const char *const F_REPLICA_STR = "F";
// logonly replica
const char *const LOGONLY_REPLICA_STR = "LOGONLY";
const char *const L_REPLICA_STR = "L";
// backup replica
const char *const BACKUP_REPLICA_STR = "BACKUP";
const char *const B_REPLICA_STR = "B";
// readonly replica
const char *const READONLY_REPLICA_STR = "READONLY";
const char *const R_REPLICA_STR = "R";
// memonly replica
const char *const MEMONLY_REPLICA_STR = "MEMONLY";
const char *const M_REPLICA_STR = "M";
// encryption logonly replica
const char *const ENCRYPTION_LOGONLY_REPLICA_STR = "ENCRYPTION_LOGONLY";
const char *const E_REPLICA_STR = "E";
// columnstore replica
const char *const COLUMNSTORE_REPLICA_STR = "COLUMNSTORE";
const char *const C_REPLICA_STR = "C";
class ObReplicaTypeCheck
{
public:
// Currently only three types are valid,
// including REPLICA_TYPE_FULL, REPLICA_TYPE_READONLY, and REPLICA_TYPE_COLUMNSTORE
static bool is_replica_type_valid(const int32_t replica_type)
{
return REPLICA_TYPE_FULL == replica_type
|| REPLICA_TYPE_READONLY == replica_type;
|| REPLICA_TYPE_READONLY == replica_type
|| REPLICA_TYPE_COLUMNSTORE == replica_type;
}
static bool is_can_elected_replica(const int32_t replica_type)
{
@ -2194,6 +2204,10 @@ public:
{
return (REPLICA_TYPE_READONLY == replica_type);
}
static bool is_columnstore_replica(const int32_t replica_type)
{
return (REPLICA_TYPE_COLUMNSTORE == replica_type);
}
static bool is_log_replica(const int32_t replica_type)
{
return (REPLICA_TYPE_LOGONLY == replica_type || REPLICA_TYPE_ENCRYPTION_LOGONLY == replica_type);
@ -2208,13 +2222,18 @@ public:
return (replica_type >= REPLICA_TYPE_FULL && replica_type <= REPLICA_TYPE_LOGONLY)
|| (REPLICA_TYPE_ENCRYPTION_LOGONLY == replica_type);
}
static bool is_non_paxos_replica(const int32_t replica_type)
{
return (REPLICA_TYPE_READONLY == replica_type || REPLICA_TYPE_COLUMNSTORE == replica_type);
}
static bool is_writable_replica(const int32_t replica_type)
{
return (REPLICA_TYPE_FULL == replica_type);
}
static bool is_readable_replica(const int32_t replica_type)
{
return (REPLICA_TYPE_FULL == replica_type || REPLICA_TYPE_READONLY == replica_type);
return (REPLICA_TYPE_FULL == replica_type || REPLICA_TYPE_READONLY == replica_type
|| REPLICA_TYPE_COLUMNSTORE == replica_type);
}
static bool is_replica_with_memstore(const ObReplicaType replica_type)
{
@ -2228,15 +2247,11 @@ public:
{
return (REPLICA_TYPE_FULL == replica_type || REPLICA_TYPE_READONLY == replica_type);
}
static bool can_as_data_source(const int32_t dest_replica_type, const int32_t src_replica_type)
{
return (dest_replica_type == src_replica_type
|| REPLICA_TYPE_FULL == src_replica_type); // TODO temporarily only supports the same type or F as the data source
}
//Currently only copies of F and R can be used for machine reading, not L
static bool can_slave_read_replica(const int32_t replica_type)
{
return (REPLICA_TYPE_FULL == replica_type || REPLICA_TYPE_READONLY == replica_type);
return (REPLICA_TYPE_FULL == replica_type || REPLICA_TYPE_READONLY == replica_type
|| REPLICA_TYPE_COLUMNSTORE == replica_type);
}
static bool change_replica_op_allow(const ObReplicaType source, const ObReplicaType target)
@ -2245,6 +2260,8 @@ public:
if (REPLICA_TYPE_LOGONLY == source || REPLICA_TYPE_LOGONLY == target) {
bool_ret = false;
} else if (REPLICA_TYPE_COLUMNSTORE == source || REPLICA_TYPE_COLUMNSTORE == target) {
bool_ret = false;
} else if (REPLICA_TYPE_FULL == source) {
bool_ret = true;
} else if (REPLICA_TYPE_READONLY == source && REPLICA_TYPE_FULL == target) {

1
deps/oblib/src/lib/ob_errno.h vendored
View File

@ -182,6 +182,7 @@ constexpr int OB_ERR_UNEXPECTED_TZ_TRANSITION = -5297;
constexpr int OB_ERR_INVALID_TIMEZONE_REGION_ID = -5341;
constexpr int OB_ERR_INVALID_HEX_NUMBER = -5342;
constexpr int OB_ERR_FIELD_NOT_FOUND_IN_DATETIME_OR_INTERVAL = -5352;
constexpr int OB_CANT_AGGREGATE_3COLLATIONS = -5356;
constexpr int OB_ERR_INVALID_JSON_TEXT = -5411;
constexpr int OB_ERR_INVALID_JSON_TEXT_IN_PARAM = -5412;
constexpr int OB_ERR_INVALID_JSON_BINARY_DATA = -5413;

7
deps/oblib/src/lib/ob_name_def.h vendored
View File

@ -758,6 +758,7 @@
#define N_CURRENT_USER "current_user"
#define N_CURRENT_USER_PRIV "current_user_priv"
#define N_CURRENT_ROLE "current_role"
#define N_IS_ENABLED_ROLE "is_enabled_role"
#define N_USER "user"
#define N_HOST_IP "host_ip"
#define N_RPC_PORT "rpc_port"
@ -1066,6 +1067,7 @@
#define N_ST_WITHIN "st_within"
#define N_SQL_MODE_CONVERT "sql_mode_convert"
#define N_EXTERNAL_FILE_COLUMN_PREFIX "metadata$filecol"
#define N_EXTERNAL_TABLE_COLUMN_PREFIX "external$tablecol"
#define N_PARTITION_LIST_COL "metadata$partition_list_col"
#define N_EXTERNAL_FILE_URL "metadata$fileurl"
#define N_EXTERNAL_FILE_ROW "external$filerow"
@ -1154,4 +1156,9 @@
#define N_RB_TO_STRING "rb_to_string"
#define N_RB_FROM_STRING "rb_from_string"
#define N_GET_PATH "get_path"
#define N_AUDIT_LOG_FILTER_SET_FILTER "audit_log_filter_set_filter"
#define N_AUDIT_LOG_FILTER_REMOVE_FILTER "audit_log_filter_remove_filter"
#define N_AUDIT_LOG_FILTER_SET_USER "audit_log_filter_set_user"
#define N_AUDIT_LOG_FILTER_REMOVE_USER "audit_log_filter_remove_user"
#define N_CAN_ACCESS_TRIGGER "can_access_trigger"
#endif //OCEANBASE_LIB_OB_NAME_DEF_H_

5
deps/oblib/src/lib/restore/ob_storage_cos_base.cpp vendored
View File

@ -551,6 +551,11 @@ int ObStorageCosBase::open(
} else if (OB_FAIL(handle_.build_bucket_and_object_name(uri))) {
OB_LOG(WARN, "failed to build bucket and object name", K(ret), K(uri));
}
#ifdef ERRSIM
if (OB_NOT_NULL(storage_info) && (OB_SUCCESS != EventTable::EN_ENABLE_LOG_OBJECT_STORAGE_CHECKSUM_TYPE)) {
OB_LOG(ERROR, "errsim backup io with checksum type", "checksum_type", storage_info->get_checksum_type_str());
}
#endif
return ret;
}

19
deps/oblib/src/lib/restore/ob_storage_info.cpp vendored
View File

@ -35,7 +35,8 @@ const char *get_storage_checksum_type_str(const ObStorageChecksumType &type)
//***********************ObObjectStorageInfo***************************
ObObjectStorageInfo::ObObjectStorageInfo()
: device_type_(ObStorageType::OB_STORAGE_MAX_TYPE),
: delete_mode_(ObIStorageUtil::DELETE),
device_type_(ObStorageType::OB_STORAGE_MAX_TYPE),
checksum_type_(ObStorageChecksumType::OB_MD5_ALGO)
{
endpoint_[0] = '\0';
@ -51,6 +52,7 @@ ObObjectStorageInfo::~ObObjectStorageInfo()
void ObObjectStorageInfo::reset()
{
delete_mode_ = ObIStorageUtil::DELETE;
device_type_ = ObStorageType::OB_STORAGE_MAX_TYPE;
checksum_type_ = ObStorageChecksumType::OB_MD5_ALGO;
endpoint_[0] = '\0';
@ -221,6 +223,8 @@ int ObObjectStorageInfo::parse_storage_info_(const char *storage_info, bool &has
const char *checksum_type_str = token + strlen(CHECKSUM_TYPE);
if (OB_FAIL(set_checksum_type_(checksum_type_str))) {
OB_LOG(WARN, "fail to set checksum type", K(ret), K(checksum_type_str));
} else if (OB_FAIL(set_storage_info_field_(token, extension_, sizeof(extension_)))) {
LOG_WARN("fail to set checksum type into extension", K(ret), K(token));
}
} else {
}
@ -228,8 +232,8 @@ int ObObjectStorageInfo::parse_storage_info_(const char *storage_info, bool &has
}
return ret;
}
int ObObjectStorageInfo::check_delete_mode_(const char *delete_mode) const
//TODO(shifagndan): define delete mode as enum
int ObObjectStorageInfo::check_delete_mode_(const char *delete_mode)
{
int ret = OB_SUCCESS;
if (OB_ISNULL(delete_mode)) {
@ -238,6 +242,10 @@ int ObObjectStorageInfo::check_delete_mode_(const char *delete_mode) const
} else if (0 != strcmp(delete_mode, "delete") && 0 != strcmp(delete_mode, "tagging")) {
ret = OB_INVALID_ARGUMENT;
OB_LOG(WARN, "delete mode is invalid", K(ret), K(delete_mode));
} else if (0 == strcmp(delete_mode, "delete")) {
delete_mode_ = ObIStorageUtil::DELETE;
} else {
delete_mode_ = ObIStorageUtil::TAGGING;
}
return ret;
}
@ -322,6 +330,7 @@ int ObObjectStorageInfo::set_storage_info_field_(const char *info, char *field,
int ObObjectStorageInfo::assign(const ObObjectStorageInfo &storage_info)
{
int ret = OB_SUCCESS;
delete_mode_ = storage_info.delete_mode_;
device_type_ = storage_info.device_type_;
checksum_type_ = storage_info.checksum_type_;
MEMCPY(endpoint_, storage_info.endpoint_, sizeof(endpoint_));
@ -346,9 +355,7 @@ int ObObjectStorageInfo::get_storage_info_str(char *storage_info, const int64_t
} else if (OB_STORAGE_FILE != device_type_) {
if (OB_FAIL(get_access_key_(key, sizeof(key)))) {
LOG_WARN("failed to get access key", K(ret));
} else if (OB_FAIL(databuff_printf(storage_info, info_len, "%s&%s&%s&%s%s",
endpoint_, access_id_, key,
CHECKSUM_TYPE, get_checksum_type_str()))) {
} else if (OB_FAIL(databuff_printf(storage_info, info_len, "%s&%s&%s", endpoint_, access_id_, key))) {
LOG_WARN("failed to set storage info", K(ret), K(info_len));
}
}

4
deps/oblib/src/lib/restore/ob_storage_info.h vendored
View File

@ -76,6 +76,7 @@ public:
ObStorageChecksumType get_checksum_type() const;
const char *get_checksum_type_str() const;
virtual int get_storage_info_str(char *storage_info, const int64_t info_len) const;
int get_delete_mode() const { return delete_mode_; }
virtual bool is_valid() const;
virtual void reset();
@ -88,11 +89,12 @@ public:
protected:
virtual int get_access_key_(char *key_buf, const int64_t key_buf_len) const;
virtual int parse_storage_info_(const char *storage_info, bool &has_appid);
int check_delete_mode_(const char *delete_mode) const;
int check_delete_mode_(const char *delete_mode);
int set_checksum_type_(const char *checksum_type_str);
int set_storage_info_field_(const char *info, char *field, const int64_t length);
public:
int delete_mode_;
// TODO: Rename device_type_ to storage_protocol_type_ for better clarity
// Prefix in the storage_info string, such as 's3://', indicates the protocol used to access the target
// Currently, both OBS and GCS are accessed via the s3 protocol,

5
deps/oblib/src/lib/restore/ob_storage_oss_base.cpp vendored
View File

@ -452,6 +452,11 @@ int ObStorageOssBase::init_with_storage_info(common::ObObjectStorageInfo *storag
OB_LOG(WARN, "aos pool or oss option is NULL", K(aos_pool_), K(oss_option_));
} else {
checksum_type_ = storage_info->get_checksum_type();
#ifdef ERRSIM
if (OB_NOT_NULL(storage_info) && (OB_SUCCESS != EventTable::EN_ENABLE_LOG_OBJECT_STORAGE_CHECKSUM_TYPE)) {
OB_LOG(ERROR, "errsim backup io with checksum type", "checksum_type", storage_info->get_checksum_type_str());
}
#endif
if (OB_UNLIKELY(!is_oss_supported_checksum(checksum_type_))) {
ret = OB_CHECKSUM_TYPE_NOT_SUPPORTED;
OB_LOG(WARN, "that checksum algorithm is not supported for oss", K(ret), K_(checksum_type));

5
deps/oblib/src/lib/restore/ob_storage_s3_base.cpp vendored
View File

@ -992,6 +992,11 @@ int ObStorageS3Base::open(const ObString &uri, ObObjectStorageInfo *storage_info
OB_LOG(WARN, "faied to get s3 client", K(ret));
} else {
checksum_type_ = storage_info->get_checksum_type();
#ifdef ERRSIM
if (OB_NOT_NULL(storage_info) && (OB_SUCCESS != EventTable::EN_ENABLE_LOG_OBJECT_STORAGE_CHECKSUM_TYPE)) {
OB_LOG(ERROR, "errsim backup io with checksum type", "checksum_type", storage_info->get_checksum_type_str());
}
#endif
if (OB_UNLIKELY(!is_s3_supported_checksum(checksum_type_))) {
ret = OB_CHECKSUM_TYPE_NOT_SUPPORTED;
OB_LOG(WARN, "that checksum algorithm is not supported for s3", K(ret), K_(checksum_type));

15
deps/oblib/src/lib/stat/ob_latch_define.h vendored
View File

@ -80,7 +80,7 @@ LATCH_DEF(SERVER_STATUS_LOCK, 53, "server status lock", LATCH_READ_PREFER, 2000,
LATCH_DEF(SERVER_MAINTAINCE_LOCK, 54, "server maintaince lock", LATCH_READ_PREFER, 2000, 0, true)
LATCH_DEF(UNIT_MANAGER_LOCK, 55, "unit manager lock", LATCH_READ_PREFER, 2000, 0, true)
LATCH_DEF(ZONE_MANAGER_LOCK, 56, "zone manager maintaince lock", LATCH_READ_PREFER, 2000, 0, true)
LATCH_DEF(ALLOC_OBJECT_LOCK, 57, "object set lock", LATCH_READ_PREFER, 2000, 0, true)
LATCH_DEF(ALLOC_OBJECT_LOCK, 57, "object set lock", LATCH_READ_PREFER, 1, 0, true)
LATCH_DEF(ALLOC_BLOCK_LOCK, 58, "block set lock", LATCH_READ_PREFER, 2000, 0, true)
LATCH_DEF(TRACE_RECORDER_LOCK, 59, "normal trace recorder lock", LATCH_FIFO, 2000, 0, true)
LATCH_DEF(SESSION_TRACE_RECORDER_LOCK, 60, "session trace recorder lock", LATCH_FIFO, 2000, 0, true)
@ -290,9 +290,9 @@ LATCH_DEF(ARCHIVE_ROUND_MGR_LOCK, 263, "archive round mgr lock", LATCH_FIFO, 200
LATCH_DEF(ARCHIVE_PERSIST_MGR_LOCK, 264, "archive persist mgr lock", LATCH_FIFO, 2000, 0, true)
LATCH_DEF(ARCHIVE_TASK_QUEUE_LOCK, 265, "archive task queue lock", LATCH_FIFO, 2000, 0, true)
LATCH_DEF(EXT_SVR_BLACKLIST_LOCK, 266, "external server blacklist lock", LATCH_FIFO, 2000, 0, true)
LATCH_DEF(SSTABLE_INSERT_TABLE_CONTEXT_LOCK, 267, "direct insert table context lock", LATCH_FIFO, 2000, 0, true)
LATCH_DEF(SSTABLE_INSERT_TABLET_CONTEXT_LOCK, 268, "direct insert tablet context lock", LATCH_FIFO, 2000, 0, true)
LATCH_DEF(SSTABLE_INSERT_TABLE_MANAGER_LOCK, 269, "direct insert table manager lock", LATCH_FIFO, 2000, 0, true)
LATCH_DEF(SSTABLE_INSERT_TABLE_CONTEXT_LOCK, 267, "direct insert table context lock", LATCH_FIFO, 2000, 0, false)
LATCH_DEF(SSTABLE_INSERT_TABLET_CONTEXT_LOCK, 268, "direct insert tablet context lock", LATCH_FIFO, 2000, 0, false)
LATCH_DEF(SSTABLE_INSERT_TABLE_MANAGER_LOCK, 269, "direct insert table manager lock", LATCH_FIFO, 2000, 0, false)
LATCH_DEF(COMPLEMENT_DATA_CONTEXT_LOCK, 270, "complement data context lock", LATCH_FIFO, 2000, 0, true)
LATCH_DEF(TABLET_DDL_KV_MGR_LOCK, 271, "tablet ddl kv mgr lock", LATCH_FIFO, 2000, 0, true)
LATCH_DEF(TABLET_AUTO_INCREMENT_MGR_LOCK, 272, "tablet auto increment mgr lock", LATCH_FIFO, 2000, 0, true)
@ -374,7 +374,12 @@ LATCH_DEF(DI_SUMMARY_LOCK, 340, "diagnostic info summary lock", LATCH_FIFO, 2000
LATCH_DEF(DI_ALLOCATE_LOCK, 341, "diagnostic info allocator lock", LATCH_FIFO, 2000, 0, false)
LATCH_DEF(DI_COLLECTOR_LOCK, 342, "diagnostic info collector lock", LATCH_FIFO, 2000, 0, false)
LATCH_DEF(LATCH_END, 343, "latch end", LATCH_FIFO, 2000, 0, true)
LATCH_DEF(DAG_NET_SCHEDULER, 343, "dag net scheduler lock", LATCH_FIFO, 2000, 0, true)
LATCH_DEF(DAG_PRIO_SCHEDULER, 344, "dag prio scheduler lock", LATCH_FIFO, 2000, 0, true)
LATCH_DEF(LS_RESERVED_SNAPSHOT_LOCK, 345, "ls reserved snapshot lock", LATCH_FIFO, 2000, 0, true)
LATCH_DEF(LATCH_END, 346, "latch end", LATCH_FIFO, 2000, 0, true)
#endif

7
deps/oblib/src/lib/thread/thread_define.h vendored
View File

@ -19,12 +19,11 @@ TG_DEF(TEST3, test3, DEDUP_QUEUE, 1, 8, 8, 16L << 20, 16L << 20,8192, "test")
TG_DEF(TEST4, test4, THREAD_POOL, 1)
TG_DEF(TEST5, test5, ASYNC_TASK_QUEUE, 1, 16)
TG_DEF(TEST6, test6, MAP_QUEUE_THREAD, 2)
TG_DEF(TEST7, test7, QUEUE_THREAD, 10, 10)
// TG_DEF(TEST7, test7, QUEUE_THREAD, 10, 10)
TG_DEF(TEST8, test8, REENTRANT_THREAD_POOL, 1)
// other
TG_DEF(MEMORY_DUMP, memDump, THREAD_POOL, 1)
TG_DEF(SchemaRefTask, SchemaRefTask, DEDUP_QUEUE, 1, 1024, 1024, 1L << 30, 512L << 20, common::OB_MALLOC_BIG_BLOCK_SIZE, "SchemaDedupQueu")
TG_DEF(ReqMemEvict, ReqMemEvict, TIMER)
TG_DEF(replica_control, replica_control, THREAD_POOL, 1)
// TG_DEF(SchemaRefTask, SchemaRefTask, DEDUP_QUEUE, 1, 1024, 1024, 1L << 30, 512L << 20, common::OB_MALLOC_BIG_BLOCK_SIZE, "SchemaDedupQueu")
// TG_DEF(replica_control, replica_control, THREAD_POOL, 1)
TG_DEF(SYSLOG_COMPRESS, SyslogCompress, THREAD_POOL, 1)
#endif

27
deps/oblib/src/lib/utility/ob_macro_utils.h vendored
View File

@ -685,6 +685,26 @@ for (__typeof__((c).at(0)) *it = ((extra_condition) && (c).count() > 0 ? &(c).at
} while(false)
#endif
#ifndef ENABLE_DEBUG_LOG
#define OB_SAFE_ASSERT(x) \
do{ \
bool v=(x); \
if(OB_UNLIKELY(!(v))) { \
_OB_LOG_RET(ERROR, oceanbase::common::OB_ERROR, "assert fail, exp=%s", #x); \
BACKTRACE_RET(ERROR, oceanbase::common::OB_ERROR, 1, "assert fail"); \
} \
} while(false)
#else
#define OB_SAFE_ASSERT(x) \
do{ \
bool v=(x); \
if(OB_UNLIKELY(!(v))) { \
_OB_LOG_RET(ERROR, oceanbase::common::OB_ERROR, "assert fail, exp=%s", #x); \
BACKTRACE_RET(ERROR, oceanbase::common::OB_ERROR, 1, "assert fail"); \
assert(v); \
} \
} while(false)
#endif
#define OB_ASSERT_MSG(x, msg...) \
do{ \
@ -709,6 +729,13 @@ for (__typeof__((c).at(0)) *it = ((extra_condition) && (c).count() > 0 ? &(c).at
//#define ob_assert(x) OB_ASSERT(x)
#define ob_assert(x) ob_release_assert(x)
#ifndef ENABLE_DEBUG_LOG
#define OB_SAFE_ABORT()
#else
#define OB_SAFE_ABORT() ob_abort()
#endif
////////////////////////////////////////////////////////////////
// interval

2
deps/oblib/src/lib/utility/ob_tracepoint_def.h vendored
View File

@ -439,6 +439,7 @@ GLOBAL_ERRSIM_POINT_DEF(748, EN_COMPACTION_ITER_SET_BATCH_CNT, "");
// please add new trace point after 750
GLOBAL_ERRSIM_POINT_DEF(751, EN_SESSION_LEAK_COUNT_THRESHOLD, "used to control the threshold of report session leak ERROR");
GLOBAL_ERRSIM_POINT_DEF(760, EN_DISABLE_TABLET_MINOR_MERGE, "used to stop scheduling minor merge");
GLOBAL_ERRSIM_POINT_DEF(761, EN_FAST_RECLAIM_THREAD, "used to speed up reclaiming thread");
GLOBAL_ERRSIM_POINT_DEF(800, EN_END_PARTICIPANT, "");
// compaction 801 - 899
@ -495,6 +496,7 @@ GLOBAL_ERRSIM_POINT_DEF(1113, EN_RESTORE_TABLET_TASK_FAILED, "");
GLOBAL_ERRSIM_POINT_DEF(1114, EN_INSERT_USER_RECOVER_JOB_FAILED, "");
GLOBAL_ERRSIM_POINT_DEF(1115, EN_INSERT_AUX_TENANT_RESTORE_JOB_FAILED, "");
GLOBAL_ERRSIM_POINT_DEF(1116, EN_RESTORE_CREATE_LS_FAILED, "");
GLOBAL_ERRSIM_POINT_DEF(1117, EN_ENABLE_LOG_OBJECT_STORAGE_CHECKSUM_TYPE, "");
// END OF STORAGE HA - 1101 - 2000
// sql parameterization 1170-1180

27
deps/oblib/src/lib/utility/utility.cpp vendored
View File

@ -1658,33 +1658,6 @@ int long_to_str10(int64_t val,char *dst, const int64_t buf_len, const bool is_si
////////////////////////////////////////////////////////////////////////////////////////////////////
const char *replica_type_to_str(const ObReplicaType &type)
{
const char *str = "";
switch (type) {
case REPLICA_TYPE_FULL:
str = "REPLICA_TYPE_FULL";
break;
case REPLICA_TYPE_BACKUP:
str = "REPLICA_TYPE_BACKUP";
break;
case REPLICA_TYPE_LOGONLY:
str = "REPLICA_TYPE_LOGONLY";
break;
case REPLICA_TYPE_READONLY:
str = "REPLICA_TYPE_READONLY";
break;
case REPLICA_TYPE_MEMONLY:
str = "REPLICA_TYPE_MEMONLY";
break;
default:
str = "REPLICA_TYPE_UNKNOWN";
}
return str;
}
bool ez2ob_addr(ObAddr &addr, easy_addr_t& ez)
{
bool ret = false;

2
deps/oblib/src/lib/utility/utility.h vendored
View File

@ -1273,8 +1273,6 @@ private:
void get_addr_by_proxy_sessid(const uint64_t session_id, ObAddr &addr);
const char *replica_type_to_str(const ObReplicaType &type);
int ob_atoll(const char *str, int64_t &res);
int ob_strtoll(const char *str, char *&endptr, int64_t &res);
int ob_strtoull(const char *str, char *&endptr, uint64_t &res);

29
deps/oblib/src/lib/worker.cpp vendored
View File

@ -44,18 +44,34 @@ int __attribute__((weak)) common_yield()
return OB_SUCCESS;
}
}
int __attribute__((weak)) SET_GROUP_ID(uint64_t group_id)
{
int ret = OB_SUCCESS;
THIS_WORKER.set_group_id_(group_id);
return ret;
}
int __attribute__((weak)) CONVERT_FUNCTION_TYPE_TO_GROUP_ID(const uint8_t function_type, uint64_t &group_id)
{
int ret = OB_SUCCESS;
UNUSED(function_type);
group_id = GET_GROUP_ID();
return ret;
}
} // namespace lib
} // namespace oceanbase
__thread Worker *Worker::self_;
Worker::Worker()
: allocator_(nullptr),
: group_(nullptr),
allocator_(nullptr),
st_current_priority_(0),
session_(nullptr),
cur_request_(nullptr),
worker_level_(INT32_MAX),
curr_request_level_(0),
is_th_worker_(false),
group_id_(0),
rpc_stat_srv_(nullptr),
timeout_ts_(INT64_MAX),
@ -85,15 +101,6 @@ Worker::Status Worker::check_wait()
return ret_status;
}
void Worker::set_group_id(int32_t group_id)
{
if (OBCG_DEFAULT_GROUP_ID == group_id_ || (is_user_group(group_id_) && is_valid_resource_group(group_id))) {
group_id_ = group_id;
} else {
LOG_ERROR_RET(OB_INNER_STAT_ERROR, "group_id is unexpected", K(group_id_), K(group_id));
}
}
bool Worker::sched_wait()
{
return true;

89
deps/oblib/src/lib/worker.h vendored
View File

@ -82,10 +82,15 @@ public:
OB_INLINE void set_curr_request_level(const int32_t level) { curr_request_level_ = level; }
OB_INLINE int32_t get_curr_request_level() const { return curr_request_level_; }
OB_INLINE bool is_th_worker() const { return is_th_worker_; }
OB_INLINE void set_group_id_(const uint64_t group_id) { group_id_ = group_id;}
void set_group_id(int32_t group_id);
OB_INLINE int32_t get_group_id() const { return group_id_; }
OB_INLINE uint64_t get_group_id() const { return group_id_; }
OB_INLINE void set_group(void *group) { group_ = group; };
OB_INLINE void *get_group() { return group_;};
OB_INLINE bool is_group_worker() const { return OB_NOT_NULL(group_); }
OB_INLINE void set_func_type_(uint8_t func_type) { func_type_ = func_type; }
OB_INLINE uint8_t get_func_type() const { return func_type_; }
OB_INLINE void set_rpc_stat_srv(void *rpc_stat_srv) { rpc_stat_srv_ = rpc_stat_srv; }
OB_INLINE void *get_rpc_stat_srv() const { return rpc_stat_srv_; }
@ -125,12 +130,14 @@ public:
static void set_module_type(const ObErrsimModuleType &module_type);
static ObErrsimModuleType get_module_type();
#endif
protected:
OB_INLINE void set_is_th_worker(bool is_th_worker) { is_th_worker_ = is_th_worker; }
public:
static __thread Worker *self_;
public:
common::ObDLinkNode<Worker*> worker_node_;
void *group_;
protected:
// 线程运行时内存从此分配器分配
// 初始tenant_id=500, 在处理request时，tenant_id被更新成request的租户id
@ -143,7 +150,9 @@ private:
// whether worker is in blocking
int32_t worker_level_;
int32_t curr_request_level_;
int32_t group_id_;
bool is_th_worker_;
uint64_t group_id_;
uint8_t func_type_;
void *rpc_stat_srv_;
int64_t timeout_ts_;
@ -190,6 +199,76 @@ inline Worker &this_worker()
#define THIS_WORKER oceanbase::lib::Worker::self()
#define GET_FUNC_TYPE() (THIS_WORKER.get_func_type())
#define GET_GROUP_ID() (THIS_WORKER.get_group_id())
int SET_GROUP_ID(uint64_t group_id);
int CONVERT_FUNCTION_TYPE_TO_GROUP_ID(const uint8_t function_type, uint64_t &group_id);
class ConsumerGroupIdGuard
{
public:
ConsumerGroupIdGuard(uint64_t group_id)
: thread_group_id_(GET_GROUP_ID()), group_changed_(false), ret_(OB_SUCCESS)
{
group_changed_ = group_id != thread_group_id_;
if (group_changed_) {
ret_ = SET_GROUP_ID(group_id);
}
}
~ConsumerGroupIdGuard()
{
if (group_changed_) {
SET_GROUP_ID(thread_group_id_);
}
}
int get_ret()
{
return ret_;
}
private:
uint64_t thread_group_id_;
bool group_changed_;
int ret_;
};
#define CONSUMER_GROUP_ID_GUARD(group_id) oceanbase::lib::ConsumerGroupIdGuard consumer_group_id_guard_(group_id)
class ConsumerGroupFuncGuard
{
public:
ConsumerGroupFuncGuard(uint8_t func_type)
: thread_group_id_(GET_GROUP_ID()), thread_func_type_(GET_FUNC_TYPE()), group_changed_(false), ret_(OB_SUCCESS)
{
THIS_WORKER.set_func_type_(func_type);
uint64_t group_id = 0;
ret_ = CONVERT_FUNCTION_TYPE_TO_GROUP_ID(func_type, group_id);
if (OB_SUCCESS == ret_ && is_user_group(group_id) && group_id != thread_group_id_) {
group_changed_ = true;
ret_ = SET_GROUP_ID(group_id);
}
}
~ConsumerGroupFuncGuard()
{
if (group_changed_) {
SET_GROUP_ID(thread_group_id_);
THIS_WORKER.set_func_type_(thread_func_type_);
}
}
int get_ret()
{
return ret_;
}
private:
uint64_t thread_group_id_;
uint8_t thread_func_type_;
bool group_changed_;
int ret_;
};
#define CONSUMER_GROUP_FUNC_GUARD(func_type) oceanbase::lib::ConsumerGroupFuncGuard consumer_group_func_guard_(func_type)
class DisableSchedInterGuard
{
public:

1
deps/oblib/src/rpc/frame/ob_req_transport.cpp vendored
View File

@ -97,6 +97,7 @@ int async_cb(easy_request_t *r)
ret = OB_LIBEASY_ERROR;
} else if (OB_FAIL(cb->decode(r->ipacket))) {
cb->set_error(ret);
cb->on_invalid();
LOG_WARN("decode failed", K(ret), K(pcode));
} else if (OB_PACKET_CLUSTER_ID_NOT_MATCH == cb->get_rcode()) {

10
deps/oblib/src/rpc/frame/ob_req_transport.h vendored
View File

@ -94,7 +94,8 @@ public:
{
public:
AsyncCB(int pcode)
: low_level_cb_(NULL), dst_(), timeout_(0), tenant_id_(0),
: low_level_cb_(NULL), gtid_(0), pkt_id_(0),
dst_(), timeout_(0), tenant_id_(0),
err_(0), pcode_(pcode), send_ts_(0), payload_(0)
{}
virtual ~AsyncCB() {}
@ -110,7 +111,10 @@ public:
virtual bool get_cloned() = 0;
// invoke when get a valid packet on protocol level, but can't decode it.
virtual void on_invalid() { RPC_FRAME_LOG_RET(ERROR, common::OB_INVALID_ARGUMENT, "invalid packet"); }
virtual void on_invalid() {
int ret = err_;
RPC_FRAME_LOG(ERROR, "rpc response decode failed, tenant oom or deserialization failed", K_(pcode), K_(tenant_id), K_(dst));
}
// invoke when can't get a valid or completed packet.
virtual void on_timeout() { RPC_FRAME_LOG(DEBUG, "packet timeout"); }
virtual int on_error(int err);
@ -127,6 +131,8 @@ public:
obrpc::ObRpcPacketCode get_pcode() const { return static_cast<obrpc::ObRpcPacketCode>(pcode_); }
void* low_level_cb_;
uint64_t gtid_;
uint32_t pkt_id_;
private:
static const int64_t REQUEST_ITEM_COST_RT = 100 * 1000; // 100ms
protected:

2
deps/oblib/src/rpc/obmysql/ob_mysql_global.h vendored
View File

@ -261,6 +261,8 @@ enum EMySQLFieldType
MYSQL_TYPE_ORA_BLOB = 210,
MYSQL_TYPE_ORA_CLOB = 211,
MYSQL_TYPE_ROARINGBITMAP = 215,
MYSQL_TYPE_OB_VECTOR = 216, // used in cdc/oms not used in client
MYSQL_TYPE_OB_ARRAY = 217, // used in cdc/oms not used in client
MYSQL_TYPE_JSON = 245,
MYSQL_TYPE_NEWDECIMAL = 246,
MYSQL_TYPE_ENUM = 247,

8
deps/oblib/src/rpc/obmysql/ob_sql_nio.cpp vendored
View File

@ -1233,17 +1233,11 @@ void ObSqlNio::destroy()
}
}
int __attribute__((weak)) sql_nio_add_cgroup(const uint64_t tenant_id)
{
return 0;
}
void ObSqlNio::run(int64_t idx)
{
if (NULL != impl_) {
lib::set_thread_name("sql_nio", idx);
// if (tenant_id_ != common::OB_INVALID_ID) {
// obmysql::sql_nio_add_cgroup(tenant_id_);
// }
// SET_GROUP_ID(OBCG_SQL_NIO);
while(!has_set_stop() && !(OB_NOT_NULL(&lib::Thread::current()) ? lib::Thread::current().has_set_stop() : false)) {
impl_[idx].do_work();
}

1
deps/oblib/src/rpc/obmysql/ob_sql_nio.h vendored
View File

@ -71,7 +71,6 @@ private:
uint64_t dispatch_idx_;
uint64_t tenant_id_;
};
extern int sql_nio_add_cgroup(const uint64_t tenant_id);
}; // end namespace obmysql
}; // end namespace oceanbase

2
deps/oblib/src/rpc/obmysql/obsm_struct.h vendored
View File

@ -80,6 +80,7 @@ public:
client_sessid_ = INVALID_SESSID;
client_addr_port_ = 0;
client_create_time_ = 0;
has_service_name_ = false;
}
obmysql::ObCompressType get_compress_type() {
@ -215,6 +216,7 @@ public:
uint32_t client_sessid_;
int32_t client_addr_port_;
int64_t client_create_time_;
bool has_service_name_;
};
} // end of namespace observer
} // end of namespace oceanbase

17
deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.cpp vendored
View File

@ -67,7 +67,8 @@ int ObSyncRespCallback::wait(const int64_t wait_timeout_us, const int64_t pcode,
const struct timespec ts = {1, 0};
bool has_terminated = false;
while(ATOMIC_LOAD(&cond_) == 0) {
if (OB_UNLIKELY((obrpc::OB_REMOTE_SYNC_EXECUTE == pcode || obrpc::OB_REMOTE_EXECUTE == pcode)
if (OB_UNLIKELY((obrpc::OB_REMOTE_SYNC_EXECUTE == pcode || obrpc::OB_REMOTE_EXECUTE == pcode
|| proxy_.is_detect_session_killed())
&& !has_terminated
&& OB_ERR_SESSION_INTERRUPTED == THIS_WORKER.check_status())) {
RPC_LOG(INFO, "check session killed, will execute pn_terminate_pkt", K(gtid_), K(pkt_id_));
@ -128,7 +129,7 @@ int ObAsyncRespCallback::handle_resp(int io_err, const char* buf, int64_t sz)
int64_t after_decode_time = 0;
int64_t after_process_time = 0;
ObRpcPacketCode pcode = OB_INVALID_RPC_CODE;
ObRpcPacket* ret_pkt = NULL;
ObRpcPacket ret_pkt;
if (buf != NULL && sz > easy_head_size) {
EVENT_INC(RPC_PACKET_IN);
EVENT_ADD(RPC_PACKET_IN_BYTES, sz);
@ -151,22 +152,24 @@ int ObAsyncRespCallback::handle_resp(int io_err, const char* buf, int64_t sz)
}
} else if (NULL == buf) {
ucb_->on_timeout();
} else if (OB_FAIL(rpc_decode_ob_packet(pool_, buf, sz, ret_pkt))) {
} else if (OB_FAIL(rpc_decode_ob_packet(buf, sz, ret_pkt))) {
ucb_->set_error(ret);
ucb_->on_invalid();
RPC_LOG(WARN, "rpc_decode_ob_packet fail", K(ret));
} else if (OB_FALSE_IT(ObCurTraceId::set(ret_pkt->get_trace_id()))) {
} else if (OB_FALSE_IT(ObCurTraceId::set(ret_pkt.get_trace_id()))) {
}
#ifdef ERRSIM
else if (OB_FALSE_IT(THIS_WORKER.set_module_type(ret_pkt->get_module_type()))) {
else if (OB_FALSE_IT(THIS_WORKER.set_module_type(ret_pkt.get_module_type()))) {
}
#endif
else if (OB_FAIL(ucb_->decode(ret_pkt))) {
else if (OB_FAIL(ucb_->decode(&ret_pkt))) {
ucb_->set_error(ret);
ucb_->on_invalid();
RPC_LOG(WARN, "ucb.decode fail", K(ret));
} else {
after_decode_time = ObTimeUtility::current_time();
int tmp_ret = OB_SUCCESS;
pcode = ret_pkt->get_pcode();
pcode = ret_pkt.get_pcode();
if (OB_SUCCESS != (tmp_ret = ucb_->process())) {
RPC_LOG(WARN, "ucb.process fail", K(tmp_ret));
}

14
deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.h vendored
View File

@ -28,8 +28,8 @@ namespace obrpc
class ObSyncRespCallback
{
public:
ObSyncRespCallback(ObRpcMemPool& pool)
: pkt_nio_cb_(NULL), pool_(pool), resp_(NULL), sz_(0), cond_(0), send_ret_(common::OB_SUCCESS), gtid_(0), pkt_id_(0){}
ObSyncRespCallback(ObRpcMemPool& pool, ObRpcProxy& proxy)
: pkt_nio_cb_(NULL), pool_(pool), proxy_(proxy), resp_(NULL), sz_(0), cond_(0), send_ret_(common::OB_SUCCESS){}
~ObSyncRespCallback() {}
void* alloc(int64_t sz) { return pool_.alloc(sz); }
int handle_resp(int io_err, const char* buf, int64_t sz);
@ -45,6 +45,7 @@ public:
private:
void* pkt_nio_cb_;
ObRpcMemPool& pool_;
const ObRpcProxy& proxy_;
char* resp_;
int64_t sz_;
int cond_;
@ -77,9 +78,6 @@ private:
void* pkt_nio_cb_;
ObRpcMemPool& pool_;
UAsyncCB* ucb_;
public:
uint64_t gtid_;
uint32_t pkt_id_;
};
void init_ucb(ObRpcProxy& proxy, UAsyncCB* ucb, const common::ObAddr& addr, int64_t send_ts, int64_t payload_sz);
@ -125,7 +123,7 @@ public:
auto &set = obrpc::ObRpcPacketSet::instance();
const char* pcode_label = set.name_of_idx(set.idx_of_pcode(pcode));
ObRpcMemPool pool(src_tenant_id, pcode_label);
ObSyncRespCallback cb(pool);
ObSyncRespCallback cb(pool, proxy);
char* req = NULL;
int64_t req_sz = 0;
const char* resp = NULL;
@ -244,8 +242,8 @@ public:
set_ucb_args(newcb, args);
init_ucb(proxy, cb->get_ucb(), addr, start_ts, req_sz);
}
cb->gtid_ = (pnio_group_id<<32) + thread_id;
pkt_id_ptr = &cb->pkt_id_;
ucb->gtid_ = (pnio_group_id<<32) + thread_id;
pkt_id_ptr = &ucb->pkt_id_;
}
IGNORE_RETURN snprintf(rpc_timeguard_str, sizeof(rpc_timeguard_str), "sz=%ld,pcode=%x,id=%ld", req_sz, pcode, src_tenant_id);
timeguard.click(rpc_timeguard_str);

14
deps/oblib/src/rpc/obrpc/ob_rpc_endec.cpp vendored
View File

@ -71,16 +71,14 @@ int init_packet(ObRpcProxy& proxy, ObRpcPacket& pkt, ObRpcPacketCode pcode, cons
return proxy.init_pkt(&pkt, pcode, opts, unneed_response);
}
int rpc_decode_ob_packet(ObRpcMemPool& pool, const char* buf, int64_t sz, ObRpcPacket*& ret_pkt)
int rpc_decode_ob_packet(const char* buf, int64_t sz, ObRpcPacket& pkt)
{
int ret = common::OB_SUCCESS;
ObRpcPacket* pkt = (ObRpcPacket*)pool.alloc(sizeof(ObRpcPacket));
if (NULL == pkt) {
ret = common::OB_ALLOCATE_MEMORY_FAILED;
} else {
new(pkt)ObRpcPacket();
if (OB_SUCC(pkt->decode(buf, sz))) {
ret_pkt = pkt;
if (OB_SUCC(pkt.decode(buf, sz))) {
const int64_t fly_ts = ObTimeUtility::current_time() - pkt.get_timestamp();
if (pkt.get_timestamp() > 0 && fly_ts > oceanbase::common::OB_MAX_PACKET_FLY_TS && TC_REACH_TIME_INTERVAL(100 * 1000)) {
RPC_LOG_RET(WARN, common::OB_ERR_TOO_MUCH_TIME, "PNIO packet wait too much time between response and client_cb", "pcode", pkt.get_pcode(),
"fly_ts", fly_ts, "send_timestamp", pkt.get_timestamp(), K(sz));
}
}
return ret;

2
deps/oblib/src/rpc/obrpc/ob_rpc_endec.h vendored
View File

@ -175,7 +175,7 @@ int rpc_decode_resp(const char* resp_buf, int64_t resp_sz, T& result, ObRpcPacke
return ret;
}
int rpc_decode_ob_packet(ObRpcMemPool& pool, const char* buf, int64_t sz, ObRpcPacket*& ret_pkt);
int rpc_decode_ob_packet(const char* buf, int64_t sz, ObRpcPacket& ret_pkt);
int rpc_encode_ob_packet(ObRpcMemPool& pool, ObRpcPacket* pkt, char*& buf, int64_t& sz, int64_t reserve_buf_size);
}; // end namespace obrpc

12
deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h vendored
View File

@ -238,7 +238,7 @@ PCODE_DEF(OB_PARALLEL_CREATE_TABLE, 0x276)
PCODE_DEF(OB_PARALLEL_SET_COMMENT, 0x277)
PCODE_DEF(OB_PARALLEL_CREATE_INDEX, 0x278)
PCODE_DEF(OB_PARALLEL_UPDATE_INDEX_STATUS, 0x279)
PCODE_DEF(OB_PARALLEL_DROP_TABLE, 0x27A)
// system admin command
PCODE_DEF(OB_ADMIN_SWITCH_REPLICA_ROLE, 0x280)
@ -285,7 +285,7 @@ PCODE_DEF(OB_REMOVE_CLUSTER_INFO_FROM_ARB_SERVER, 0x2A8)
#endif
PCODE_DEF(OB_DROP_LOB, 0x2A9)
PCODE_DEF(OB_EXCHANGE_PARTITION, 0x2AA)
// PCODE_DEF(OB_REFRESH_SERVICE_NAME, 0x2AB)
PCODE_DEF(OB_REFRESH_SERVICE_NAME, 0x2AB)
PCODE_DEF(OB_CREATE_OUTLINE, 0x350)
PCODE_DEF(OB_DROP_OUTLINE, 0x351)
@ -876,12 +876,16 @@ PCODE_DEF(OB_REMOTE_WRITE_DDL_INC_COMMIT_LOG, 0x967)
//PCODE_DEF(OB_PREPARE_TABLET_SPLIT_TASK_RANGES, 0x96C)
PCODE_DEF(OB_BATCH_GET_TABLET_BINDING, 0x96D)
//PCODE_DEF(OB_BATCH_UPGRADE_TABLE_SCHEMA, 0x96E)
//PCODE_DEF(OB_SPLIT_TABLET_DATA_START_REQUEST, 0x96F)
//PCODE_DEF(OB_BATCH_GET_TABLET_SPLIT, 0x970)
// Depedency Detector
PCODE_DEF(OB_DETECTOR_LCL_MESSAGE, 0x9F0)
PCODE_DEF(OB_DETECTOR_COLLECT_INFO_MESSAGE, 0x9F1)
PCODE_DEF(OB_DETECTOR_NOTIFY_PARENT_MESSAGE, 0x9F2)
PCODE_DEF(OB_CHECK_AND_CANCEL_DELETE_LOB_META_ROW_DAG, 0x9F3)
PCODE_DEF(OB_RPC_ASSEMBLE, 0x1000)
// Table API (by zhuweng.yzf)
@ -1039,6 +1043,10 @@ PCODE_DEF(OB_LOG_ACQUIRE_REBUILD_INFO, 0x1525)
// for ob_admin
PCODE_DEF(OB_LOG_FORCE_SET_TENANT_LOG_DISK, 0x1526)
PCODE_DEF(OB_LOG_SYNC_BASE_LSN_REQ, 0x1527)
#ifdef OB_BUILD_SHARED_STORAGE
PCODE_DEF(OB_LOG_GET_UPLOADED_PROGRESS, 0x1528)
PCODE_DEF(OB_LOG_FLUSH_UNTIL, 0x1529)
#endif
// 1531-1550 for obesi
// PCODE_DEF(OB_ESI_IS_EXIST, 0x1531)

3
deps/oblib/src/rpc/obrpc/ob_rpc_proxy.cpp vendored
View File

@ -184,8 +184,7 @@ int ObRpcProxy::init_pkt(
if (0 != get_group_id()) {
pkt->set_group_id(get_group_id());
} else if (this_worker().get_group_id() == OBCG_LQ ||
(is_user_group(this_worker().get_group_id()) &&
(is_virtual_tenant_id(ob_get_tenant_id()) || ob_get_tenant_id() != tenant_id_))) {
(is_user_group(this_worker().get_group_id()) && ob_get_tenant_id() != tenant_id_)) {
pkt->set_group_id(0);
} else {
pkt->set_group_id(this_worker().get_group_id());

11
deps/oblib/src/rpc/obrpc/ob_rpc_proxy.h vendored
View File

@ -119,7 +119,8 @@ public:
max_process_handler_time_(0), compressor_type_(common::INVALID_COMPRESSOR),
src_cluster_id_(common::OB_INVALID_CLUSTER_ID),
dst_cluster_id_(common::OB_INVALID_CLUSTER_ID), init_(false),
active_(true), is_trace_time_(false), do_ratelimit_(false), is_bg_flow_(0), rcode_() {}
active_(true), is_trace_time_(false), do_ratelimit_(false),
do_detect_session_killed_(false), is_bg_flow_(0), rcode_() {}
virtual ~ObRpcProxy() = default;
int init(const rpc::frame::ObReqTransport *transport,
@ -133,6 +134,8 @@ public:
int64_t get_timeout() const { return timeout_; }
void set_trace_time(const bool is_trace_time) { is_trace_time_ = is_trace_time; }
void set_ratelimit(const bool do_ratelimit) { do_ratelimit_ = do_ratelimit; }
void set_detect_session_killed(const bool do_detect) { do_detect_session_killed_ = do_detect; }
bool is_detect_session_killed() const { return do_detect_session_killed_; }
void set_bg_flow(const int8_t is_bg_flow) { is_bg_flow_ = is_bg_flow;}
void set_max_process_handler_time(const uint32_t max_process_handler_time)
{ max_process_handler_time_ = max_process_handler_time; }
@ -261,6 +264,7 @@ protected:
bool active_;
bool is_trace_time_;
bool do_ratelimit_;
bool do_detect_session_killed_;
int8_t is_bg_flow_;
ObRpcResultCode rcode_;
};
@ -391,6 +395,11 @@ extern ObRpcProxy::NoneT None;
set_ratelimit(do_ratelimit); \
return *this; \
} \
inline CLS& detect_session_killed(const bool do_detect) \
{ \
set_detect_session_killed(do_detect); \
return *this; \
} \
inline CLS& bg_flow(const uint32_t is_bg_flow) \
{ \
set_bg_flow(is_bg_flow); \

4
deps/oblib/src/rpc/obrpc/ob_rpc_proxy.ipp vendored
View File

@ -55,7 +55,7 @@ int SSHandle<pcodeStruct>::get_more(typename pcodeStruct::Response &result)
auto &set = obrpc::ObRpcPacketSet::instance();
const char* pcode_label = set.name_of_idx(set.idx_of_pcode(pcode_));
ObRpcMemPool pool(src_tenant_id, pcode_label);
ObSyncRespCallback cb(pool);
ObSyncRespCallback cb(pool, proxy_);
char* pnio_req = NULL;
int64_t pnio_req_sz = 0, resp_sz = 0;
const char* resp = NULL;
@ -205,7 +205,7 @@ int SSHandle<pcodeStruct>::abort()
auto &set = obrpc::ObRpcPacketSet::instance();
const char* pcode_label = set.name_of_idx(set.idx_of_pcode(pcode_));
ObRpcMemPool pool(src_tenant_id, pcode_label);
ObSyncRespCallback cb(pool);
ObSyncRespCallback cb(pool, proxy_);
char* pnio_req = NULL;
int64_t pnio_req_sz = 0, resp_sz = 0;
const char* resp = NULL;

6
deps/oblib/src/rpc/obrpc/ob_rpc_session_handler.cpp vendored
View File

@ -241,9 +241,9 @@ int ObRpcSessionHandler::wait_for_next_request(int64_t sessid,
// when waiting for OB_REMOTE_EXECUTE/OB_REMOTE_SYNC_EXECUTE/OB_INNER_SQL_SYNC_TRANSMIT request more than 30s,
// try to send reverse keepalive request.
if (current_time_us >= keepalive_timeout_us && reverse_keepalive_arg.is_valid()) {
get_next_cond_(wait_object.thid_).unlock();
get_next_cond_(thid).unlock();
ret = stream_rpc_reverse_probe(reverse_keepalive_arg);
get_next_cond_(wait_object.thid_).lock();
get_next_cond_(thid).lock();
if (OB_FAIL(ret)) {
LOG_WARN("stream rpc sender has been aborted, unneed to wait", K(sessid), K(timeout), K(reverse_keepalive_arg));
break;
@ -276,7 +276,7 @@ int ObRpcSessionHandler::wait_for_next_request(int64_t sessid,
K(hash_ret), K(sessid), K(req));
}
get_next_cond_(wait_object.thid_).unlock();
get_next_cond_(thid).unlock();
ATOMIC_DEC(&waiting_thread_count_);
} else {
//do nothing

1
deps/oblib/unittest/lib/CMakeLists.txt vendored
View File

@ -6,6 +6,7 @@
# oblib_addtest(time/test_ob_time_utility.cpp)
# oblib_addtest(timezone/test_ob_timezone_utils.cpp)
oblib_addtest(parquet/test_parquet.cpp)
# oblib_addtest(orc/test_orc.cpp)
oblib_addtest(alloc/test_alloc_struct.cpp)
oblib_addtest(alloc/test_block_set.cpp)
oblib_addtest(alloc/test_chunk_mgr.cpp)

175
deps/oblib/unittest/lib/charset/test_charset.cpp vendored
View File

@ -22,6 +22,7 @@
#include "lib/allocator/page_arena.h"
#include "lib/charset/ob_charset.h"
#include "lib/charset/ob_template_helper.h"
#include "lib/string/ob_string.h"
#include "lib/utility/ob_print_utils.h"
#include "unicode_map.h"
@ -854,6 +855,89 @@ TEST_F(TestCharset, check_mbmaxlenlen)
}
}
std::vector<const char *> test_strings = {"1", "abcdef", "ab1dc4", "你好", "b今a天", "1abad "};
TEST_F(TestCharset, basic_collation_handler_test)
{
ObArenaAllocator alloc;
for (int i = CS_TYPE_INVALID; i < CS_TYPE_EXTENDED_MARK; i++) {
ObCollationType coll = static_cast<ObCollationType>(i);
const ObCharsetInfo * cs = ObCharset::get_charset(coll);
const char *coll_name = ObCharset::collation_name(coll);
if (OB_NOT_NULL(cs)) {
std::cout << "#TEST Coll = " << coll_name << std::endl;
for (const char* utf8_str:test_strings) {
ObString dst;
if (cs->mbmaxlen <= 1) {
dst = ObString(utf8_str);
} else {
ASSERT_EQ(0, ObCharset::charset_convert(alloc, ObString(utf8_str), CS_TYPE_UTF8MB4_BIN, coll, dst));
}
char*str = dst.ptr();
char*end = dst.ptr() + dst.length();
if (OB_NOT_NULL(cs->coll->strnncoll)) {
fprintf(stdout, ">> strnncoll = %d for text = \"%s\"\n",
cs->coll->strnncoll(cs, pointer_cast<const uchar*>(str), end-str, pointer_cast<const uchar*>(str), end-str, true), utf8_str);
}
if (OB_NOT_NULL(cs->coll->strnncollsp)) {
fprintf(stdout, ">> strnncollsp = %d for text = \"%s\"\n",
cs->coll->strnncollsp(cs, pointer_cast<const uchar*>(str), end-str, pointer_cast<const uchar*>(str), end-str, true), utf8_str);
}
if (OB_NOT_NULL(cs->coll->strnxfrm)) {
char temp[100];
bool is_valid_unicode = false;
fprintf(stdout, ">> strnxfrm = %ld for text = \"%s\", is_valid_unicode = %d\n",
cs->coll->strnxfrm(cs, reinterpret_cast<unsigned char *>(temp), 100, UINT32_MAX,
reinterpret_cast<unsigned char *>(str), end-str, 0, &is_valid_unicode), utf8_str, is_valid_unicode);
}
if (OB_NOT_NULL(cs->coll->strnxfrmlen)) {
fprintf(stdout, ">> strnxfrmlen = %ld for text = \"%s\"\n",
cs->coll->strnxfrmlen(cs, end-str), utf8_str);
}
if (OB_NOT_NULL(cs->coll->strnxfrm_varlen)) {
fprintf(stdout, ">> strnxfrmlen = %ld for text = \"%s\"\n",
cs->coll->strnxfrmlen(cs, end-str), utf8_str);
}
if (OB_NOT_NULL(cs->coll->like_range)) {
char temp1[100];
char temp2[100];
size_t len1, len2;
fprintf(stdout, ">> like_range = %d for text = \"%s\", min = %.*s, max = %.*s\n",
cs->coll->like_range(cs, str, end-str, '\\', '_', '%', 100, temp1, temp2, &len1, &len2), utf8_str,
(int)len1, temp1, (int)len2, temp2);
}
if (OB_NOT_NULL(cs->coll->wildcmp)) {
const char *wild_str = "%";
fprintf(stdout, ">> wildcmp = %d for text = \"%s\"\n",
cs->coll->wildcmp(cs, str, end, wild_str, wild_str + strlen(wild_str), '\\', '_', '%'), utf8_str);
}
if (OB_NOT_NULL(cs->coll->strcasecmp)) {
fprintf(stdout, ">> strcasecmp = %d for text = \"%s\"\n",
cs->coll->strcasecmp(cs, str, end), utf8_str);
}
if (OB_NOT_NULL(cs->coll->instr)) {
ob_match_t m_match_t[2];
unsigned int nmatch = 1;
const char *temp = "1";
fprintf(stdout, ">> instr = %d for text = \"%s\" nmatch = %u match_mb_len = %u\n",
cs->coll->instr(cs, temp, strlen(temp), str, end-str, m_match_t, nmatch), utf8_str,
nmatch, m_match_t[0].mb_len);
}
if (OB_NOT_NULL(cs->coll->hash_sort)) {
ulong nr1, nr2;
cs->coll->hash_sort(cs, pointer_cast<const uchar*>(str), end-str, &nr1, &nr2, true, NULL);
fprintf(stdout, ">> hash_sort for text = \"%s\" nr1 = %lu nr1 = %lu\n", utf8_str, nr1, nr2);
}
if (OB_NOT_NULL(cs->coll->propagate)) {
fprintf(stdout, ">> propagate = %d for text = \"%s\"\n",
cs->coll->propagate(cs, pointer_cast<const uchar*>(str), end-str), utf8_str);
}
}
}
}
}
TEST_F(TestCharset, foreach_char) {
const char *data = "豫章故郡，洪都新府。星分翼轸，地接衡庐。襟三江而带五湖，控蛮荆而引瓯越。物华天宝，龙光射牛斗之墟"
"人杰地灵，徐孺下陈蕃之榻。雄州雾列，俊采星驰。台隍枕夷夏之交，宾主尽东南之美。都督阎公之雅望，棨戟遥临"
@ -971,7 +1055,7 @@ TEST_F(TestCharset, foreach_char) {
fprintf(stdout, "Raw Impl\n");
start_timer();
ASSERT_EQ(OB_SUCCESS, ObCharsetUtils::foreach_char(input, test_collation_type, do_nothing));
ASSERT_EQ(OB_SUCCESS, ObCharsetUtils::foreach_char(input, test_collation_type, do_nothing, true));
end_timer();
@ -985,11 +1069,98 @@ TEST_F(TestCharset, foreach_char) {
ASSERT_EQ(OB_SUCCESS, ObFastStringScanner::foreach_char(input, test_cs_type, do_nothing, false));
end_timer();
}
}
TEST_F(TestCharset, basic_charset_handler_test)
{
ObArenaAllocator alloc;
for (int i = CHARSET_INVALID; i < CHARSET_MAX; i++) {
if (ObCharset::is_valid_charset(static_cast<ObCharsetType>(i))) {
ObCollationType coll = static_cast<ObCollationType>(ObCharset::get_default_collation(static_cast<ObCharsetType>(i)));
const char *coll_name = ObCharset::collation_name(coll);
const ObCharsetInfo * cs = ObCharset::get_charset(coll);
std::cout << "#TEST Coll = " << coll_name << std::endl;
for (const char* utf8_str:test_strings) {
ObString dst;
if (cs->mbmaxlen <= 1) {
dst = ObString(utf8_str);
} else {
ASSERT_EQ(0, ObCharset::charset_convert(alloc, ObString(utf8_str), CS_TYPE_UTF8MB4_BIN, coll, dst));
}
const char*str = dst.ptr();
const char*end = dst.ptr() + dst.length();
if (OB_NOT_NULL(cs->cset->ismbchar)) {
fprintf(stdout, ">> ismbchar = %d for text = \"%s\"\n",
cs->cset->ismbchar(cs, str, end), utf8_str);
}
if (OB_NOT_NULL(cs->cset->mbcharlen)) {
fprintf(stdout, ">> mbcharlen = %d for text = \"%s\"\n",
cs->cset->mbcharlen(cs, str[0]), utf8_str);
}
if (OB_NOT_NULL(cs->cset->numchars)) {
fprintf(stdout, ">> numchars = %ld for text = \"%s\"\n",
cs->cset->numchars(cs, str, end), utf8_str);
}
if (OB_NOT_NULL(cs->cset->charpos)) {
size_t pos = 3;
fprintf(stdout, ">> charpos = %ld pos = %ld for text = \"%s\"\n",
cs->cset->charpos(cs, str, end, pos), pos, utf8_str);
}
if (OB_NOT_NULL(cs->cset->max_bytes_charpos)) {
size_t max_bytes = 5;
size_t char_len = 0;
fprintf(stdout, ">> max_bytes_charpos = %ld max_bytes = %ld char_len = %ld for text = \"%s\"\n",
cs->cset->max_bytes_charpos(cs, str, end, max_bytes, &char_len), max_bytes, char_len, utf8_str);
}
if (OB_NOT_NULL(cs->cset->well_formed_len)) {
int error = 0;
fprintf(stdout, ">> well_formed_len = %ld error = %d text = \"%s\"\n",
cs->cset->well_formed_len(cs, str, end, INT64_MAX, &error), error, utf8_str);
}
if (OB_NOT_NULL(cs->cset->lengthsp)) {
fprintf(stdout, ">> lengthsp = %ld text = \"%s\"\n",
cs->cset->lengthsp(cs, str, end-str), utf8_str);
}
if (OB_NOT_NULL(cs->cset->mb_wc)) {
ob_wc_t wchar = 0;
fprintf(stdout, ">> mb_wc = %d wchar = %ld text = \"%s\"\n",
cs->cset->mb_wc(cs, &wchar, pointer_cast<const uchar*>(str), pointer_cast<const uchar*>(end)), wchar, utf8_str);
}
if (OB_NOT_NULL(cs->cset->wc_mb)) {
ob_wc_t wchar = 41;
unsigned char temp[10];
MEMSET(temp, 0, 10);
fprintf(stdout, ">> wc_mb = %d A = %.*s text = \"%s\"\n",
cs->cset->wc_mb(cs, wchar, temp, temp + 10), 10, temp, utf8_str);
}
if (OB_NOT_NULL(cs->cset->ctype)) {
int ctype = 0;
fprintf(stdout, ">> ctype = %d ctype = %d text = \"%s\"\n",
cs->cset->ctype(cs, &ctype, pointer_cast<const uchar*>(str), pointer_cast<const uchar*>(end)), ctype, utf8_str);
}
if (cs->casedn_multiply <= 1 && OB_NOT_NULL(cs->cset->casedn)) {
ObString temp;
ASSERT_EQ(0, ob_write_string(alloc, dst, temp));
fprintf(stdout, ">> casedn = %ld res = %.*s text = \"%s\"\n",
cs->cset->casedn(cs, temp.ptr(), temp.length(), temp.ptr(), temp.length()), temp.length(), temp.ptr(), utf8_str);
}
if (cs->caseup_multiply <= 1 && OB_NOT_NULL(cs->cset->caseup)) {
ObString temp;
ASSERT_EQ(0, ob_write_string(alloc, dst, temp));
fprintf(stdout, ">> caseup = %ld res = %.*s text = \"%s\"\n",
cs->cset->caseup(cs, temp.ptr(), temp.length(), temp.ptr(), temp.length()), temp.length(), temp.ptr(), utf8_str);
}
if (OB_NOT_NULL(cs->cset->fill)) {
char temp[10];
cs->cset->fill(cs, temp, 10, 0x42);
fprintf(stdout, ">> fill res = %.*s text = \"%s\"\n", 10, temp, utf8_str);
}
}
}
}
}
int main(int argc, char **argv)

603
deps/oblib/unittest/lib/orc/test_orc.cpp vendored Normal file
View File

@ -0,0 +1,603 @@
/**
* Copyright (c) 2023 OceanBase
* OceanBase CE is licensed under Mulan PubL v2.
* You can use this software according to the terms and conditions of the Mulan PubL v2.
* You may obtain a copy of Mulan PubL v2 at:
* http://license.coscl.org.cn/MulanPubL-2.0
* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
* EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
* MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
* See the Mulan PubL v2 for more details.
*/
#define USING_LOG_PREFIX SQL
#include "gtest/gtest.h"
#include "lib/oblog/ob_log.h"
#include "lib/oblog/ob_log_module.h"
#include <orc/OrcFile.hh>
#include <orc/MemoryPool.hh>
#include <orc/Writer.hh>
#include <orc/Reader.hh>
#include <orc/Vector.hh>
#include <iostream>
#include <cstdlib>
#include "lib/allocator/page_arena.h"
#include "lib/file/ob_file.h"
#include "lib/file/file_directory_utils.h"
#include "lib/charset/ob_template_helper.h"
#include "lib/net/ob_net_util.h"
#define USING_LOG_PREFIX SQL
using namespace oceanbase::common;
class TestOrc: public ::testing::Test
{
public:
TestOrc() {};
virtual ~TestOrc() {};
virtual void SetUp();
virtual void TearDown();
};
void TestOrc::SetUp()
{
}
void TestOrc::TearDown()
{
}
class OrcMemoryPool : public ::orc::MemoryPool
{
public:
virtual char* malloc(uint64_t size) override {
return (char* )alloc_.alloc(size);
}
virtual void free(char* p) override {
//do nothing
}
private:
oceanbase::common::ObArenaAllocator alloc_;
};
// Result<std::unique_ptr<liborc::Type>> GetOrcType(const DataType& type) {
// Type::type kind = type.id();
// switch (kind) {
// case Type::type::BOOL:
// return liborc::createPrimitiveType(liborc::TypeKind::BOOLEAN);
// case Type::type::INT8:
// return liborc::createPrimitiveType(liborc::TypeKind::BYTE);
// case Type::type::INT16:
// return liborc::createPrimitiveType(liborc::TypeKind::SHORT);
// case Type::type::INT32:
// return liborc::createPrimitiveType(liborc::TypeKind::INT);
// case Type::type::INT64:
// return liborc::createPrimitiveType(liborc::TypeKind::LONG);
// case Type::type::FLOAT:
// return liborc::createPrimitiveType(liborc::TypeKind::FLOAT);
// case Type::type::DOUBLE:
// return liborc::createPrimitiveType(liborc::TypeKind::DOUBLE);
// // Use STRING instead of VARCHAR for now, both use UTF-8
// case Type::type::STRING:
// case Type::type::LARGE_STRING:
// return liborc::createPrimitiveType(liborc::TypeKind::STRING);
// case Type::type::BINARY:
// case Type::type::LARGE_BINARY:
// case Type::type::FIXED_SIZE_BINARY:
// return liborc::createPrimitiveType(liborc::TypeKind::BINARY);
// case Type::type::DATE32:
// return liborc::createPrimitiveType(liborc::TypeKind::DATE);
// case Type::type::DATE64:
// return liborc::createPrimitiveType(liborc::TypeKind::TIMESTAMP);
// case Type::type::TIMESTAMP: {
// const auto& timestamp_type = checked_cast<const TimestampType&>(type);
// if (!timestamp_type.timezone().empty()) {
// // The timestamp values stored in the arrow array are normalized to UTC.
// // TIMESTAMP_INSTANT type is always preferred over TIMESTAMP type.
// return liborc::createPrimitiveType(liborc::TypeKind::TIMESTAMP_INSTANT);
// }
// // The timestamp values stored in the arrow array can be in any timezone.
// return liborc::createPrimitiveType(liborc::TypeKind::TIMESTAMP);
// }
// case Type::type::DECIMAL128: {
// const uint64_t precision =
// static_cast<uint64_t>(checked_cast<const Decimal128Type&>(type).precision());
// const uint64_t scale =
// static_cast<uint64_t>(checked_cast<const Decimal128Type&>(type).scale());
// return liborc::createDecimalType(precision, scale);
// }
// case Type::type::LIST:
// case Type::type::FIXED_SIZE_LIST:
// case Type::type::LARGE_LIST: {
// const auto& value_field = checked_cast<const BaseListType&>(type).value_field();
// ARROW_ASSIGN_OR_RAISE(auto orc_subtype, GetOrcType(*value_field->type()));
// SetAttributes(value_field, orc_subtype.get());
// return liborc::createListType(std::move(orc_subtype));
// }
// case Type::type::STRUCT: {
// std::unique_ptr<liborc::Type> out_type = liborc::createStructType();
// std::vector<std::shared_ptr<Field>> arrow_fields =
// checked_cast<const StructType&>(type).fields();
// for (auto it = arrow_fields.begin(); it != arrow_fields.end(); ++it) {
// std::string field_name = (*it)->name();
// ARROW_ASSIGN_OR_RAISE(auto orc_subtype, GetOrcType(*(*it)->type()));
// SetAttributes(*it, orc_subtype.get());
// out_type->addStructField(field_name, std::move(orc_subtype));
// }
// return out_type;
// }
// case Type::type::MAP: {
// const auto& key_field = checked_cast<const MapType&>(type).key_field();
// const auto& item_field = checked_cast<const MapType&>(type).item_field();
// ARROW_ASSIGN_OR_RAISE(auto key_orc_type, GetOrcType(*key_field->type()));
// ARROW_ASSIGN_OR_RAISE(auto item_orc_type, GetOrcType(*item_field->type()));
// SetAttributes(key_field, key_orc_type.get());
// SetAttributes(item_field, item_orc_type.get());
// return liborc::createMapType(std::move(key_orc_type), std::move(item_orc_type));
// }
// case Type::type::DENSE_UNION:
// case Type::type::SPARSE_UNION: {
// std::unique_ptr<liborc::Type> out_type = liborc::createUnionType();
// std::vector<std::shared_ptr<Field>> arrow_fields =
// checked_cast<const UnionType&>(type).fields();
// for (const auto& arrow_field : arrow_fields) {
// std::shared_ptr<DataType> arrow_child_type = arrow_field->type();
// ARROW_ASSIGN_OR_RAISE(auto orc_subtype, GetOrcType(*arrow_child_type));
// SetAttributes(arrow_field, orc_subtype.get());
// out_type->addUnionChild(std::move(orc_subtype));
// }
// return out_type;
// }
// default: {
// return Status::NotImplemented("Unknown or unsupported Arrow type: ",
// type.ToString());
// }
// }
// }
// class ObOrcRandomAccess : public :orc::InputStream {
// public:
// ObOrcRandomAccess(oceanbase::sql::ObExternalDataAccessDriver &file_reader, const char* file_name, orc::MemoryPool *pool)
// : file_reader_(file_reader), file_name_(file_name), pool_(pool) {
// }
// uint64_t getLength() const override {
// return totalLength;
// }
// uint64_t getNaturalReadSize() const override {
// return 128 * 1024;
// }
// void read(void* buf,
// uint64_t length,
// uint64_t offset) override {
// int64_t bytesRead = 0;
// int ret = file_reader_.pread(buf, length, offset, bytesRead);
// totalLength += bytesRead;
// if (ret != OB_SUCCESS) {
// throw orc::ParseError("Bad read of " + std::string(file_name_));
// }
// }
// const std::string& getName() const override {
// return file_name_;
// }
// private:
// oceanbase::sql::ObExternalDataAccessDriver &file_reader_;
// const std::string file_name_;
// orc::MemoryPool *pool_;
// uint64_t totalLength;
// };
void wirte_orc_file() {
std::unique_ptr<orc::OutputStream> outStream = orc::writeLocalFile("my-file.orc");
//std::unique_ptr<orc::Type> schema(orc::Type::buildTypeFromString("struct<x:int,y:int>"));
std::unique_ptr<orc::Type> schema = orc::createStructType();
schema->addStructField("x", orc::createPrimitiveType(orc::TypeKind::INT));
schema->addStructField("y", orc::createPrimitiveType(orc::TypeKind::BOOLEAN));
std::unique_ptr<orc::Type> sub_schema = orc::createStructType();
sub_schema->addStructField("z", orc::createPrimitiveType(orc::TypeKind::FLOAT));
sub_schema->addStructField("d", orc::createPrimitiveType(orc::TypeKind::DATE));
schema->addStructField("S2", std::move(sub_schema));
orc::WriterOptions options;
OrcMemoryPool pool;
options.setMemoryPool(&pool);
options.setCompression(orc::CompressionKind::CompressionKind_ZLIB);
std::unique_ptr<orc::Writer> writer = orc::createWriter(*schema, outStream.get(), options);
uint64_t batchSize = 8, rowCount = 100;
std::unique_ptr<orc::ColumnVectorBatch> batch =
writer->createRowBatch(batchSize);
orc::StructVectorBatch *root =
dynamic_cast<orc::StructVectorBatch *>(batch.get());
orc::LongVectorBatch *x =
dynamic_cast<orc::LongVectorBatch *>(root->fields[0]);
orc::LongVectorBatch *y =
dynamic_cast<orc::LongVectorBatch *>(root->fields[1]);
orc::DoubleVectorBatch *z =
dynamic_cast<orc::DoubleVectorBatch *>(dynamic_cast<orc::StructVectorBatch *>(root->fields[2])->fields[0]);
uint64_t rows = 0;
for (uint64_t i = 0; i < rowCount; ++i) {
if (i % 5 == 0) {
x->notNull[rows] = 0;
y->notNull[rows] = 0;
z->notNull[rows] = 0;
x->hasNulls = true;
y->hasNulls = true;
z->hasNulls = true;
x->data[rows] = 0;
y->data[rows] = 0;
z->data[rows] = i * 1.1 + 0.01;
rows++;
} else {
x->notNull[rows] = true;
y->notNull[rows] = true;
z->notNull[rows] = true;
x->data[rows] = i + 1;
y->data[rows] = i * 3 + 1;
z->data[rows] = i * 1.1 + 0.01;
rows++;
}
if (rows == batchSize) {
root->numElements = rows;
x->numElements = rows;
y->numElements = rows;
z->numElements = rows;
writer->add(*batch);
rows = 0;
}
}
if (rows != 0) {
root->numElements = rows;
x->numElements = rows;
y->numElements = rows;
//z->numElements = rows;
writer->add(*batch);
rows = 0;
}
writer->close();
}
void read_orc_file() {
std::unique_ptr<orc::InputStream> inStream = orc::readLocalFile("my-file.orc");
orc::ReaderOptions options;
OrcMemoryPool pool;
options.setMemoryPool(pool);
std::unique_ptr<orc::Reader> reader = orc::createReader(std::move(inStream), options);
orc::RowReaderOptions rowReaderOptions;
std::unique_ptr<orc::RowReader> rowReader = reader->createRowReader(rowReaderOptions);
std::unique_ptr<orc::ColumnVectorBatch> batch = rowReader->createRowBatch(2);
//std::cout <<"root field size: " << root->fields.size() << std::endl;
while (rowReader->next(*batch)) {
std::cout<<"column batch:" << batch->toString() <<"\n";
for (uint64_t r = 0; r < batch->numElements; ++r) {
orc::StructVectorBatch *root =
dynamic_cast<orc::StructVectorBatch *>(batch.get());
orc::ColumnVectorBatch *col[10] = {NULL};
int k = 0;
std::cout<< "row:" << r ;
for (int i = 0; i < reader->getType().getSubtypeCount(); i++) {
const uint8_t* valid_bytes = NULL;
switch (reader->getType().getSubtype(i)->getKind()) {
case orc::TypeKind::BOOLEAN:
case orc::TypeKind::BYTE:
case orc::TypeKind::SHORT:
case orc::TypeKind::INT:
case orc::TypeKind::LONG:
case orc::TypeKind::DATE:
//valid_bytes = reinterpret_cast<const uint8_t*>( dynamic_cast<orc::LongVectorBatch *>(root->fields[i])->notNull.data()) + r;
std::cout<<" col" << i <<":" << dynamic_cast<orc::LongVectorBatch *>(root->fields[i])->data[r] << " is not null:"<< bool(root->fields[i]->notNull[r]);
break;
case orc::TypeKind::FLOAT:
case orc::TypeKind::DOUBLE:
std::cout<< " col" << i <<":" << dynamic_cast<orc::DoubleVectorBatch *>(root->fields[i])->data[r] ;
std::cout<<" has NULL:"<<(root->fields[i])->hasNulls<<" is not null:"<< bool(root->fields[i]->notNull[r]);
break;
case orc::TypeKind::STRING:
case orc::TypeKind::VARCHAR:
case orc::TypeKind::CHAR:
case orc::TypeKind::BINARY:
std::cout<< " col" << i <<":" << dynamic_cast<orc::StringVectorBatch *>(root->fields[i])->data[r] <<" is not null:"<< bool(root->fields[i]->notNull[r]);
break;
case orc::TypeKind::LIST:
case orc::TypeKind::MAP:
case orc::TypeKind::UNION:
//not supported
break;
case orc::TypeKind::DECIMAL:
//std::cout<< "row:" << r << " col" << i <<":" << dynamic_cast<orc::Decimal128VectorBatch *>(root->fields[i])->data[r] <<" is not null:"<< (bool)dynamic_cast<orc::LongVectorBatch *>(root->fields[i])->notNull[r];
break;
case orc::TypeKind::TIMESTAMP:
case orc::TypeKind::TIMESTAMP_INSTANT:
//std::cout<< "row:" << r << " col" << i <<":" << dynamic_cast<orc::TimestampVectorBatch *>(root->fields[i])->data[r] <<" is not null:"<< (bool)dynamic_cast<orc::LongVectorBatch *>(root->fields[i])->notNull[r];
break;
case orc::TypeKind::STRUCT:
std::cout<< " col" << i <<":" << dynamic_cast<orc::DoubleVectorBatch *>(dynamic_cast<orc::StructVectorBatch *>(root->fields[i])->fields[0])->data[r];
break;
default:
//error
break;
}
// if (reader->getType().getColumnId() == reader->getType().getMaximumColumnId()) {//is primitive
// col[k++] = x;
// }
}
std::cout<<"\n";
}
}
}
void printType(const orc::Type &type) {
std::cout << " type:" << type.toString() <<" type subTypeCount:" << type.getSubtypeCount()<< " typeKind: " << type.getKind() << " ColumnId:" << type.getColumnId() << " maxColumnId:" << type.getMaximumColumnId() << "\n";
for (int i = 0; i < type.getSubtypeCount(); i++) {
printType(*type.getSubtype(i));
}
}
bool is_primitive_Type(const orc::Type &type) {
if (type.getColumnId() == type.getMaximumColumnId()) {
return true;
}
return false;
}
void read_file_footer() {
std::unique_ptr<orc::InputStream> inStream = orc::readLocalFile("my-file.orc");
orc::ReaderOptions options;
OrcMemoryPool pool;
//options.setMemoryPool(pool);
std::unique_ptr<orc::Reader> reader = orc::createReader(std::move(inStream), options);
const orc::Type& type = reader->getType();
printType(type);
for (int i = 0; i < reader->getType().getSubtypeCount(); i++) {
std::cout << "Subfield" << i << ": " << type.getFieldName(i);
printType(*type.getSubtype(i));
}
for (int i = 0; i <= reader->getType().getMaximumColumnId(); i++) {
std::cout<<"column" <<i<<": " << reader->getColumnStatistics(i)->toString() <<"\n";
}
}
// Result<std::shared_ptr<Schema>> GetArrowSchema(const liborc::Type& type) {
// if (type.getKind() != liborc::STRUCT) {
// return Status::NotImplemented(
// "Only ORC files with a top-level struct "
// "can be handled");
// }
// int size = static_cast<int>(type.getSubtypeCount());
// std::vector<std::shared_ptr<Field>> fields;
// fields.reserve(size);
// for (int child = 0; child < size; ++child) {
// const std::string& name = type.getFieldName(child);
// ARROW_ASSIGN_OR_RAISE(auto elem_field, GetArrowField(name, type.getSubtype(child)));
// fields.push_back(std::move(elem_field));
// }
// ARROW_ASSIGN_OR_RAISE(auto metadata, ReadMetadata());
// return std::make_shared<Schema>(std::move(fields), std::move(metadata));
// }
// Result<std::shared_ptr<const KeyValueMetadata>> ReadMetadata() {
// const std::list<std::string> keys = reader_->getMetadataKeys();
// auto metadata = std::make_shared<KeyValueMetadata>();
// for (const auto& key : keys) {
// metadata->Append(key, reader_->getMetadataValue(key));
// }
// return std::const_pointer_cast<const KeyValueMetadata>(metadata);
// }
void read_column() {
std::cout<<"=================== test read column ===================\n";
std::unique_ptr<orc::InputStream> inStream = orc::readLocalFile("my-file.orc");
orc::ReaderOptions options;
// OrcMemoryPool pool;
// options.setMemoryPool(pool);
std::list<std::string> include_names_list;
include_names_list.push_front(std::string("y"));
include_names_list.push_front(std::string("S2.z"));
include_names_list.push_front(std::string("x"));
std::unique_ptr<orc::Reader> reader = orc::createReader(std::move(inStream), options);
orc::RowReaderOptions rowReaderOptions;
rowReaderOptions.include(include_names_list);
std::unique_ptr<orc::RowReader> rowReader = reader->createRowReader(rowReaderOptions);
std::unique_ptr<orc::ColumnVectorBatch> batch = rowReader->createRowBatch(1024);
std::cout<<"column batch:" << batch->toString() <<"\n";
printType(rowReader->getSelectedType());
const orc::Type &type = rowReader->getSelectedType();
int size = static_cast<int>(type.getSubtypeCount());
for (int child = 0; child < size; ++child) {
const std::string& name = type.getFieldName(child);
std::cout<<"field "<< child << " name:" << name <<std::endl;
printType(*type.getSubtype(child));
}
// const std::list<std::string> keys = reader->getMetadataKeys();
// auto metadata = std::make_shared<KeyValueMetadata>();
// for (const auto& key : keys) {
// metadata->Append(key, reader_->getMetadataValue(key));
// }
// const orc::Type& type = reader->getType();
// printType(type);
// for (int i = 0; i < reader->getType().getSubtypeCount(); i++) {
// std::cout << "Subfield" << i << ": " << type.getFieldName(i);
// printType(*type.getSubtype(i));
// }
// for (int i = 0; i <= reader->getType().getMaximumColumnId(); i++) {
// std::cout<<"column" <<i<<": " << reader->getColumnStatistics(i)->toString() <<"\n";
// }
}
void read_schema(std::unique_ptr<orc::RowReader> &rowReader) {
const orc::Type &type = rowReader->getSelectedType();
if (type.getKind() != orc::STRUCT) {
throw std::runtime_error("Only ORC files with a top-level struct can be handled");
} else {
}
}
void read_stripe() {
// int64_t nstripes = reader_->getNumberOfStripes();
// stripes_.resize(static_cast<size_t>(nstripes));
// std::unique_ptr<liborc::StripeInformation> stripe;
// uint64_t first_row_of_stripe = 0;
// for (int i = 0; i < nstripes; ++i) {
// stripe = reader_->getStripe(i);
// stripes_[i] = StripeInformation({static_cast<int64_t>(stripe->getOffset()),
// static_cast<int64_t>(stripe->getLength()),
// static_cast<int64_t>(stripe->getNumberOfRows()),
// static_cast<int64_t>(first_row_of_stripe)});
// first_row_of_stripe += stripe->getNumberOfRows();
// }
}
void generate_orc_file(const bool output_null) {
std::unique_ptr<orc::OutputStream> outStream = orc::writeLocalFile("test1.orc");
//std::unique_ptr<orc::Type> schema(orc::Type::buildTypeFromString("struct<x:int,y:int>"));
//std::unique_ptr<orc::Type> schema = orc::createStructType();
// schema->addStructField("age", orc::createPrimitiveType(orc::TypeKind::INT))
// ->addStructField("is_male", orc::createPrimitiveType(orc::TypeKind::BOOLEAN))
// ->addStructField("name", orc::createPrimitiveType(orc::TypeKind::Varchar))
// ->addStructField("comment", orc::createPrimitiveType(orc::TypeKind::String));
// std::unique_ptr<Type> schema(Type::buildTypeFromString("
// struct<name:varchar(100), age:int,
// sex:char(1), comment:string, in_date:date, job_time:timestamp,
// language:binary, has_house:boolean, money:bigint,
// height:decimal(5, 2), weight:double, width:float>"));
// std::unique_ptr<orc::Type> sub_schema = orc::createStructType();
// sub_schema->addStructField("z", orc::createPrimitiveType(orc::TypeKind::FLOAT));
// schema->addStructField("S2", std::move(sub_schema));
// orc::WriterOptions options;
// OrcMemoryPool pool;
// options.setCompression(orc::CompressionKind::CompressionKind_SNAPPY);
// //options.setStripeSize(1024);
// options.setCompressionBlockSize(1024);
// options.setMemoryPool(&pool);
// std::unique_ptr<orc::Writer> writer = orc::createWriter(*schema, outStream.get(), options);
// uint64_t batchSize = 128, rowCount = 12800;
// std::unique_ptr<orc::ColumnVectorBatch> batch = writer->createRowBatch(batchSize);
// orc::StructVectorBatch *root = dynamic_cast<orc::StructVectorBatch *>(batch.get());
// uint64_t rows = 0;
// for (uint64_t i = 0; i < rowCount; ++i) {
// if (output_null) {
// int put_null = std::rand() % 10;
// if (put_null == 1) {
// }
// }
// if (i % 5 == 0) {
// x->notNull[rows] = 0;
// y->notNull[rows] = 0;
// z->notNull[rows] = 0;
// x->hasNulls = true;
// y->hasNulls = true;
// z->hasNulls = true;
// x->data[rows] = 0;
// y->data[rows] = 0;
// z->data[rows] = i * 1.1 + 0.01;
// rows++;
// } else {
// x->notNull[rows] = true;
// y->notNull[rows] = true;
// z->notNull[rows] = true;
// x->data[rows] = i + 1;
// y->data[rows] = i * 3 + 1;
// z->data[rows] = i * 1.1 + 0.01;
// rows++;
// }
// if (rows == batchSize) {
// root->numElements = rows;
// x->numElements = rows;
// y->numElements = rows;
// z->numElements = rows;
// writer->add(*batch);
// rows = 0;
// }
// }
// if (rows != 0) {
// root->numElements = rows;
// x->numElements = rows;
// y->numElements = rows;
// //z->numElements = rows;
// writer->add(*batch);
// rows = 0;
// }
// writer->close();
}
void read_test_orc_file()
{
std::cout<<"=================== test read orc file column ===================\n";
std::unique_ptr<orc::InputStream> inStream = orc::readLocalFile("/data/1/mingye.swj/work/support_master_orc/data/t.orc");
orc::ReaderOptions options;
// OrcMemoryPool pool;
// options.setMemoryPool(pool);
// std::list<std::string> include_names_list;
// include_names_list.push_front(std::string("S2.z"));
// include_names_list.push_front(std::string("x"));
std::unique_ptr<orc::Reader> reader = orc::createReader(std::move(inStream), options);
orc::RowReaderOptions rowReaderOptions;
//rowReaderOptions.include(include_names_list);
std::unique_ptr<orc::RowReader> rowReader = reader->createRowReader(rowReaderOptions);
std::unique_ptr<orc::ColumnVectorBatch> batch = rowReader->createRowBatch(256);
while (rowReader->next(*batch)) {
std::cout<<"column batch:" << batch->toString() <<"\n";
for (uint64_t r = 0; r < batch->numElements; ++r) {
orc::StructVectorBatch *root =
dynamic_cast<orc::StructVectorBatch *>(batch.get());
std::cout<<" row" << r <<":" << dynamic_cast<orc::LongVectorBatch *>(root->fields[0])->data[r] << " is not null:"<< bool(root->fields[0]->notNull[r]);
}
}
printType(rowReader->getSelectedType());
const orc::Type &type = rowReader->getSelectedType();
int size = static_cast<int>(type.getSubtypeCount());
for (int child = 0; child < size; ++child) {
const std::string& name = type.getFieldName(child);
printType(*type.getSubtype(child));
}
}
TEST_F(TestOrc, read_write_orc_file_test)
{
wirte_orc_file();
read_orc_file();
read_file_footer();
read_column();
//read_test_orc_file();
}
int main(int argc, char **argv)
{
OB_LOGGER.set_log_level("INFO");
testing::InitGoogleTest(&argc,argv);
return RUN_ALL_TESTS();
}

8
deps/oblib/unittest/lib/restore/test_storage_info.cpp vendored
View File

@ -111,7 +111,7 @@ TEST(ObObjectStorageInfo, cos)
storage_info = "host=xxx.com&access_id=111&access_key=222";
ASSERT_EQ(OB_INVALID_BACKUP_DEST, info1.set(uri, storage_info));
storage_info = "host=xxx.com&access_id=111&access_key=222&checksum_type=md5&appid=333";
storage_info = "host=xxx.com&access_id=111&access_key=222&appid=333&checksum_type=md5";
ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info));
char buf[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 };
@ -153,15 +153,15 @@ TEST(ObObjectStorageInfo, s3)
ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info));
info1.reset();
storage_info = "host=xxx.com&access_id=111&access_key=222&checksum_type=md5&s3_region=333";
storage_info = "host=xxx.com&access_id=111&access_key=222&s3_region=333&checksum_type=md5";
ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info));
ASSERT_EQ(0, ::strcmp("s3_region=333", info1.extension_));
ASSERT_EQ(0, ::strcmp("s3_region=333&checksum_type=md5", info1.extension_));
char buf[OB_MAX_BACKUP_STORAGE_INFO_LENGTH] = { 0 };
ASSERT_EQ(OB_SUCCESS, info1.get_storage_info_str(buf, sizeof(buf)));
ASSERT_STREQ(storage_info, buf);
storage_info = "host=xxx.com&access_id=111&access_key=222&checksum_type=md5&s3_region=333&delete_mode=delete";
storage_info = "host=xxx.com&access_id=111&access_key=222&s3_region=333&delete_mode=delete&checksum_type=md5";
info1.reset();
ASSERT_EQ(OB_SUCCESS, info1.set(uri, storage_info));
ASSERT_EQ(OB_SUCCESS, info1.get_storage_info_str(buf, sizeof(buf)));

2571
docs/coding_standard.md Normal file
View File

File diff suppressed because it is too large Load Diff

364
docs/logging.md Normal file
View File

@ -0,0 +1,364 @@
# OceanBase System Log Introduction
## Introduction
This document mainly introduces the system logs of Oceanbase, including the classification and level of the log, how to output logs in the program, and the details of some log implementation.
## System Log Introduction
Similar to common application systems, system logs are one of the important means for Oceanbase developers to investigate problems.
Oceanbase's system log is stored under the log directory under the observer installation path. The system log is mainly divided into two categories:
1. Ordinary logs: with ".log" suffix, printed all logs (including warning logs) of a certain module.
2. Warning log: with ".log.wf" suffix, only printed the warn level of a module and above.
| log file name | record information |
| ------------------ | ----------------------------- |
| observer.log[.wf] | General logs (warning logs, general query logs, other logs) |
| rootservice.log[.wf] | rootservice module log (including global DDL log) |
| election.log[.wf] | Election related logs |
| trace.log | Full link tracking log |
Especially, trace.log does not have the corresponding ".wf" log.
In addition to output ordinary logs, `wf` logs also have a special `info` log, that is, every time the log file is created, some current systems and processes information will be recorded.
### Log Parameters
There are 7 parameters related to syslog, which are dynamically effective, that is, it can be adjusted dynamically during runtime.
| Configuration Item | Type | Range | Default Value | Describtion |
| --------------------------- | ---- | ----------------------------------------- | ------ | ------------------------------------- |
| enable_syslog_recycle | Boolean | | False | Whether to recycle the old log files |
| enable_syslog_wf | Boolean | | True | Whether to print the WARN log level and above to a separate WF file |
| enable_async_syslog | Boolean | | True | Whether to print the log asynchronous |
| max_syslog_file_count | Integer | \[0, +∞) | 0 | The maximum number of each log file |
| syslog_io_bandwidth_limit | String | 0, Other legal size | "30MB" | Log IO bandwidth limit |
| syslog_level | String | DEBUG, TRACE, WDIAG, EDIAG, INFO, WARN, ERROR | WDIAG | Log level|
| diag_syslog_per_error_limit | Integer | \[0, +∞) | 200 | The maximum number of each error code of DIAG system log per second. |
> All the parameters here are cluster-level and dynamic effect.
> Refer to ob_parameter_seed.ipp file for more details.
## Log File Recycle
OceanBase's log can be configured with the upper limit of the number of files to prevent the log file from occupying too much disk space.
If `enable_syslog_recycle = true` and `max_syslog_file_count > 0`, the number of each type log files can not exceed `max_syslog_file_count`. OceanBase will detect and delete old log files periodically.
The new log files will print a special log at the beginning. The information contains the IP and ports of the current node, version number, and some system information. Refer to `ObLogger::log_new_file_info` for more details.
```
[2023-12-26 13:15:58.612579] INFO New syslog file info: [address: "127.0.0.1:2882", observer version: OceanBase_CE 4.2.1.1, revision: 101010012023111012-2f6924cd5a576f09d6e7f212fac83f1a15ff531a, sysname: Linux, os release: 3.10.0-327.ali2019.alios7.x86_64, machine: x86_64, tz GMT offset: 08:00]
```
## Log Level
Similar to the common system, Oceanbase also provides log macro to print different levels of logs:
| Level | Macro | Describtion |
| ----- | --------- | ---- |
| DEBUG | LOG_DEBUG | Developers debug logs |
| TRACE | LOG_TRACE | Incident tracking logs are usually viewed by developers |
| INFO | LOG_INFO | System state change log |
| WARN | LOG_DBA_WARN | For DBA. observer can provide services, but the behavior not meet expectations |
| ERROR | LOG_DBA_ERROR | For DBA. observer cannot provide services, such as the disk full of monitoring ports occupied. Need DBA intervention to restore service |
| WDIAG | LOG_WARN | Warning Diagnosis. Assisting the diagnostic information of fault investigation, and the errors in the expected expectations, if the function returns failure. The level is the same as WARN |
| EDIAG | LOG_ERROR | Error Diagnosis. Assisting the diagnostic information of faulty investigation, unexpected logical errors, such as the function parameters do not meet the expected, are usually Oceanbase program bugs. The level is the same as ERROR |
> Only the most commonly used log levels are introduced here. For more detailed information, please refer to the configuration of syslog_level in `ob_parameter_seed.ipp`, and macro definitions such as `LOG_ERROR` in the `ob_log_module.h` file.
**How to set up log level?**
There are three ways to adjust the log level:
- When the OceanBase process starts, it reads the log level config from configuration file or command line parameters. The configuration item name is `syslog_level`;
- After startup, you can also connect through the MySQL client and execute the SQL command `alter system set syslog_level='DEBUG'`;
- Modify the log level when the request is executed through the SQL Hint. For example `select /*+ log_level("ERROR") */ * from foo;`. This method is only effective for the current SQL request related logs.
You can refer to the code of dynamic modification log settings `ObReloadConfig::reload_ob_logger_set`。
```cpp
if (OB_FAIL(OB_LOGGER.parse_set(conf_->syslog_level,
static_cast<int32_t>(STRLEN(conf_->syslog_level)),
(conf_->syslog_level).version()))) {
OB_LOG(ERROR, "fail to parse_set syslog_level",
K(conf_->syslog_level.str()), K((conf_->syslog_level).version()), K(ret));
```
## How to Print Logs
Common systems use C ++ Stream mode or C fprintf style printing log, but Oceanbase is slightly different. Let's start with the example to see how to print logs.
### An Example of Printing Log
Unlike `fprintf`, Oceanbase's system log does not have a format string, but only "info" parameter, and each parameter information. For example:
```cpp
LOG_INFO("start stmt", K(ret),
K(auto_commit),
K(session_id),
K(snapshot),
K(savepoint),
KPC(tx_desc),
K(plan_type),
K(stmt_type),
K(has_for_update),
K(query_start_time),
K(use_das),
K(nested_level),
KPC(session),
K(plan),
"consistency_level_in_plan_ctx", plan_ctx->get_consistency_level(),
K(trans_result));
```
Among the example, "start stmt" is the `INFO` information, and we uses the `K` macro to print objects.
### Log Field Introduction
A output of the example code above:
```text
[2023-12-11 18:00:55.711877] INFO [SQL.EXE] start_stmt (ob_sql_trans_control.cpp:619)
[99178][T1004_TeRec][T1003][YD9F97F000001-00060C36119D4757-0-0] [lt=15]
start stmt(ret=0, auto_commit=true, session_id=1,
snapshot={this:0x7f3184fca0e8, valid:true, source:2,
core:{version:{val:1702288855549635029, v:0}, tx_id:{txid:167035},
scn:1702288855704049}, uncertain_bound:0, snapshot_lsid:{id:1},
snapshot_ls_role:0, parts:[{left:{id:1}, right:491146514786417}]},
savepoint=1702288855704049, tx_desc={this:0x7f31df697420,
tx_id:{txid:167035}, state:2, addr:"127.0.0.1:55801", tenant_id:1003,
session_id:1, assoc_session_id:1, xid:NULL, xa_mode:"",
xa_start_addr:"0.0.0.0:0", access_mode:0, tx_consistency_type:0,
isolation:1, snapshot_version:{val:18446744073709551615, v:3},
snapshot_scn:0, active_scn:1702288855704040, op_sn:6, alloc_ts:1702288855706134,
active_ts:1702288855706134, commit_ts:-1, finish_ts:-1, timeout_us:29999942,
lock_timeout_us:-1, expire_ts:1702288885706076, coord_id:{id:-1},
parts:[{id:{id:1}, addr:"127.0.0.1:55801", epoch:491146514786417,
first_scn:1702288855704043, last_scn:1702288855704048, last_touch_ts:1702288855704044}],
exec_info_reap_ts:1702288855704043, commit_version:{val:18446744073709551615, v:3},
commit_times:0, commit_cb:null, cluster_id:1, cluster_version:17180065792,
flags_.SHADOW:false, flags_.INTERRUPTED:false, flags_.BLOCK:false,
flags_.REPLICA:false, can_elr:true, cflict_txs:[], abort_cause:0,
commit_expire_ts:0, commit_task_.is_registered():false, ref:2},
plan_type=1, stmt_type=5, has_for_update=false, query_start_time=1702288855711692,
use_das=false, nested_level=0, session={this:0x7f31de2521a0, id:1,
deser:false, tenant:"sys", tenant_id:1, effective_tenant:"sys",
effective_tenant_id:1003, database:"oceanbase", user:"root@%",
consistency_level:3, session_state:0, autocommit:true, tx:0x7f31df697420},
plan=0x7f31565ba050, consistency_level_in_plan_ctx=3,
trans_result={incomplete:false, parts:[], touched_ls_list:[],
cflict_txs:[]})
```
> NOTE: The log output is wrapped for readability.
A log mainly contains the following parts:
| field | example | description |
| -------- | ------------------------------------ | ----------------------------------- |
| time | [2023-12-11 18:00:55.711877] | The time of printing this log |
| level | INFO | The log level |
| module | [SQL.EXE] | The module printing the log |
| function name | start_stmt | The function printing the log |
| code location | (ob_sql_trans_control.cpp:619) | The location of code, including file name and line |
| thread identifier | [99178][T1004_TeRec] | The thread ID and name |
| tenant id | [T1003] | The tenant ID |
| Trace ID | [YD9F97F000001-00060C36119D4757-0-0] | The global ID of a specific request. You can usually get all logs related one request according the trace ID |
| The cost of printing log | [lt=15] | The cost in microsecond of printing last log |
| information | start stmt(...) | The log information |
### Commonly Used Log Parameters Macro Introduction
For developers, we only need to care about how to output our object information. Usually we write `K(obj)` to output the information we want in the log. Below are some details。
In order to avoid some errors in format string, OceanBase uses automatic recognition of types and then serialization to solve this problem. Any parameter in the log will be identified as multiple Key Value pairs, where Key is the name of the field to be printed and Value is the value of the field. For example, `"consistency_level_in_plan_ctx", plan_ctx->get_consistency_level()` in the above example prints the name and value of a field. OceanBase automatically recognizes the type of Value and converts it to a string. The final output in the log may be "consistency_level_in_plan_ctx=3".
Because most logs print the original name and value of the specified object, OceanBase provides some macros to simplify the operation of printing logs. The most commonly used one is `K`. Taking the above example `K(ret)`, its expansion in the code is:
```cpp
"ret", ret
```
The final information in the log is:
```cpp
ret=-5595
```
OceanBase also provides some other macros, which are used in different scenarios.
> Log parameter macro definitions can be found in the `ob_log_module.h` file.
| macro | example | description |
| ------ | --------------- | ----------- |
| K | K(ret) | After expansion, it is `"ret", ret`. The parameter can be a simple value or an ordinary object |
| K_ | K_(consistency_level) | After expansion, it is `"consistency_level", consistency_level_`. Different from K, the `_` suffix will be automatically added after the expanded Value, which is used for printing class member variables. |
| KR | KR(ret) | After expansion, it is `"ret", ret, "ret", common::ob_error_name(ret)`. This macro is for the convenience of printing error code and error code name. In OceanBase, `ret` is usually used as the return value of a function, and each return value has a corresponding string description. `ob_error_name` can get the string description corresponding to the error code. Note that this macro can only be used in non-lib code |
| KCSTRING/<br/>KCSTRING_ | KCSTRING(consistency_level_name) | After expansion, it is `"consistency_level_name", consistency_level_name`. This macro is used to print C-formatted strings. Since a variable of type `const char *` does not necessarily represent a string in C++, such as a binary buffer, when printing the value of this variable, if it is printed as a C string, an illegal memory access error will occur, so this macro has been added to explicitly print C strings |
| KP/KP_ | KP(plan) | After expansion, it is `"plan", plan`, where `plan` is a pointer. This macro will print out the hexadecimal value of a pointer |
| KPC/KPC_ | KPC(session) | The input parameters are object pointers. If it is NULL, "NULL" will be output. Otherwise, the `to_string` method of the pointer will be called to output the string. |
| KTIME | KTIME(cur_time) | Convert timestamp converted to string. Timestamp unit microseconds |
| KTIMERANGE/<br/>KTIMERANGE_ | KTIMERANGE(cur_time, HOUR, SECOND) | Convert the timestamp to a string and only obtain the specified range, such as the hour to second period in the example |
| KPHEX/KPHEX_ | KPHEX(buf, 20) | Print buf content in hexadecimal |
| KERRMSG | KERRMSG | Output system error code information |
| KERRNOMSG | KERRNOMSG(2) | Specify error code to output system error information |
## Some Implementation Details in the Log
### How to Convert Value to String
OceanBase automatically identifies the type of value you want to print in the log and converts it to a string. For example, in the above example, `ret` is an `int` type variable, and `plan_ctx->get_consistency_level()` returns an `enum` type variable. Both variables will be converted to strings.
However, since OceanBase does not know how to convert an ordinary object into a string, the user needs to implement a `TO_STRING_KV` function to convert the object into a string. For example, in the above example, `snapshot` is an object of type `ObTxReadSnapshot`. This object implements the `TO_STRING_KV` function, so it can be printed directly.
**Convert normal value to string**
OceanBase can automatically identify simple type values, such as `int`, `int64_t`, `double`, `bool`, `const char *`, etc., and convert them into strings. For enumeration types, they will be treated as numbers. For pointers, the pointer value will be output in hexadecimal format.
**Convert class object to string**
Since C++ does not have a reflection mechanism, it cannot automatically identify the member variables of a class object and convert them into strings. Therefore, the user needs to implement a `TO_STRING_KV` function to convert the object into a string. For example, in the above example, `snapshot` is an object of type `ObTxReadSnapshot`. This object implements the `TO_STRING_KV` function. You can refer to the implementation code as follows:
```cpp
class ObTxReadSnapshot {
...
TO_STRING_KV(KP(this),
K_(valid),
K_(source),
K_(core),
K_(uncertain_bound),
K_(snapshot_lsid),
K_(parts));
};
```
As you can see, in `TO_STRING_KV`, you can directly use a macro similar to printing logs to "list" the member variable names you want to output.
> NOTE: TO_STRING_KV is actually a macro definition. For specific implementation, please refer to `ob_print_utils.h`. TO_STRING_KV converts input parameters into strings and outputs them to a buffer.
### Log Module
OceanBase's logs are module-specific and can support sub-modules. For example, in the above example, `[SQL.EXE]` is a module, `SQL` is a main module, and `EXE` is a submodule. For the definition of the log module, please refer to the `LOG_MOD_BEGIN` and `DEFINE_LOG_SUB_MOD` related codes in the `ob_log_module.h` file.
**How does the log module output to the log?**
Normally, we just use macros like `LOG_WARN` to print logs, and different modules will be output, which is also achieved through macro definitions. Still taking the above log as an example, you can see a macro definition `#define USING_LOG_PREFIX SQL_EXE` at the beginning of the `ob_sql_trans_control.cpp` file. This macro defines the log module of the current file, that is, all logs in the current file the module `[SQL.EXE]` will be printed.
> There is also an issue here, that is, the header file introduced in the current implementation file will also use this module to print logs by default.
**How to specify module name explicitly?**
The above method is indeed a bit inflexible. OceanBase has another way to specify the module name, which is to use the macro `OB_MOD_LOG` or `OB_SUB_MOD_LOG`. The usage of these two macros is similar to `LOG_WARN`, except that there are additional module parameters and log levels:
```cpp
OB_MOD_LOG(parMod, level, info_string, args...)
OB_SUB_MOD_LOG(parMod, subMod, level, info_string, args...)
```
**Set the module's log level**
In addition to setting the global and current thread log levels, OceanBase can also adjust the log level of a certain module. Currently, you can use `SQL HINT` to modify the log level of a module when executing a request, for example:
```sql
select /*+ log_level("SHARE.SCHEMA:ERROR") */ * from foo;
```
Where `SHARE` is the main module, `SCHEMA` is the submodule, and `ERROR` is the log level. The function of this SQL HINT is to set the log level of the `SHARE.SCHEMA` module to `ERROR`, and is only valid for the current request.
### Log Time
OceanBase's log time is the number of microseconds in the current local time.
Since converting a timestamp into a string is a time-consuming task, OceanBase caches the timestamp conversion to speed up the process. For details, please refer to the `ob_fast_localtime` function.
### Thread Identifier
Currently, two information related to thread will be recorded:
- Thread ID: the information returned by the system call `__NR_gettid` (the system call is relatively inefficient, and this value will be cached);
- Thread name: The thread name field may contain the tenant ID, thread pool type, and thread pool index. The thread name of OceanBase is set through the `set_thread_name` function and will also be displayed in the `top` command.
> NOTE：The thread name is determined by the created thread. Since the tenant of the created thread may be different from the tenant of subsequent runs of this thread, the tenant in the thread name may be incorrect.
### Log Rate Limit
OceanBase supports two log rate limits: a common system log disk IO bandwidth limit and a WDIAG system log limit.
**System log bandwidth rate limit**
OceanBase will limit log output according to disk bandwidth. The log bandwidth rate limit does not limit the rate for different log levels. If the log rate is limited, the rate limit log may be printed with the keyword `REACH SYSLOG RATE LIMIT`.
Rate limit log example:
```txt
[2023-12-26 09:46:04.621435] INFO [SHARE.LOCATION] fetch_vtable_location_ (ob_vtable_location_service.cpp:281) [35675][VTblLocAsyncUp0][T0][YB427F000001-00060D52A9614571-0-0] [lt=0] REACH SYSLOG RATE LIMIT [bandwidth]
```
The rate limit can be adjusted through the configuration item `syslog_io_bandwidth_limit`.
Please refer to the `check_tl_log_limiter` function for rate limiting code details.
**WDIAG log rate limit**
OceanBase has implemented a current limit for WARN level logs. Each error code is limited to 200 logs per second by default. If the limit is exceeded, the current limiting log will be output, keyword `Throttled WDIAG logs in last second`. The current limiting threshold can be adjusted through the configuration item `diag_syslog_per_error_limit`.
Limiting log example:
```txt
[2023-12-25 18:01:15.527519] WDIAG [SHARE] refresh (ob_task_define.cpp:402) [35585][LogLimiterRefre][T0][Y0-0000000000000000-0-0] [lt=8][errcode=0] Throttled WDIAG logs in last second(details {error code, dropped logs, earliest tid}=[{errcode:-4006, dropped:31438, tid:35585}])
```
Limiting code reference `ObSyslogPerErrLimiter::do_acquire`。
## Some Other Details
### Logs for DBA
There are also two types of special logs in OceanBase, LOG_DBA_WARN and LOG_DBA_ERROR, which correspond to WARN and ERROR logs respectively. Since the volume of OceanBase logs is extremely large, and most of them can only be understood by R&D personnel, it brings a certain burden to DBA operation and maintenance troubleshooting problems. Therefore, these two types of logs are added, hoping that the DBA can only focus on a small amount of these two types of logs to troubleshoot system problems. The logs output using LOG_WARN and LOG_ERROR are converted into WDIAG and EDIAG logs to help developers troubleshoot problems.
### Output Prompt Information to the User Terminal
Sometimes we want to output the error message directly to the user's terminal, so that it can be more convenient for users to understand what error is currently occurring. At this time we can use `LOG_USER_ERROR`, `LOG_USER_WARN`, `LOG_USER_INFO` and other macros to print logs. Each error code has a corresponding `USER_ERROR_MSG`. If this `USER_ERROR_MSG` requires input parameters, then we also need to provide the corresponding parameters when printing the log. For example, the error code `OB_NOT_SINGLE_RESOURCE_POOL` has the corresponding `OB_NOT_SINGLE_RESOURCE_POOL__USER_ERROR_MSG`, and it's message is "create tenant only support single resource pool now, but pool list is %s", we just need to provide a string.
The LOG_USER_ERROR macro is defined as follows：
```cpp
#define LOG_USER_ERROR(errcode, args...)
```
The usage of other macros is similar.
> Error code definitions can be found in `src/share/ob_errno.h`.
Since `LOG_USER_XXX` provides fixed error information, if we want to output some customized information, we can use `FORWARD_USER_XXX`, such as `FORWARD_USER_ERROR`, `FORWARD_USER_WARN`, etc. Taking `FORWARD_USER_ERROR` as an example, its definition is as follows:
```cpp
#define FORWARD_USER_ERROR(errcode, args...)
```
### Health Log
OceanBase will periodically output some internal status information, such as the memory information of each module and tenant, to the log to facilitate problem finding. This kind of log usually outputs multiple lines of data in one log, such as:
```txt
[2023-12-26 13:15:58.608131] INFO [LIB] print_usage (ob_tenant_ctx_allocator.cpp:176) [35582][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=116]
[MEMORY] tenant_id= 500 ctx_id= GLIBC hold= 4,194,304 used= 1,209,328 limit= 9,223,372,036,854,775,807
[MEMORY] idle_size= 0 free_size= 0
[MEMORY] wash_related_chunks= 0 washed_blocks= 0 washed_size= 0
[MEMORY] hold= 858,240 used= 575,033 count= 3,043 avg_used= 188 block_cnt= 93 chunk_cnt= 2 mod=glibc_malloc
[MEMORY] hold= 351,088 used= 104,389 count= 3,290 avg_used= 31 block_cnt= 51 chunk_cnt= 1 mod=Buffer
[MEMORY] hold= 1,209,328 used= 679,422 count= 6,333 avg_used= 107 mod=SUMMARY
```
This kind of data can be helpful for finding historical issues.
### ERROR Log
For general errors that occur in the system, such as an exception when processing a certain request, logs will be output at WARN level. Only when the normal operation of the OceanBase process is affected, or if there is a serious problem, the log will be output at the ERROR level. Therefore, if a process exits abnormally or cannot be started, searching the ERROR log will more effectively find the cause of the problem.

151
docs/memory.md Normal file
View File

@ -0,0 +1,151 @@
# Introduction
Memory management is one of the most important modules in any large C++ project. Since OceanBase also needs to deal with the issue of multi-tenant memory resource isolation, OceanBase's memory management is more complicated than ordinary C++ projects. Generally, a good memory management module needs to consider the following issues:
- Easy to use. The designed interface must be understood and used by the container, otherwise the code will be difficult to read and maintain, and memory errors will be more likely to occur;
- Efficient. An efficient memory allocator has a crucial impact on performance, especially in high-concurrency scenarios;
- Diagnosis. As the amount of code increases, bugs are inevitable. Common memory errors, such as memory leaks, memory out-of-bounds, wild pointers and other problems cause headaches for development and operation and maintenance. How to write a function that can help us avoid or troubleshoot these problems is also an important indicator to measure the quality of the memory management module.
For the multi-tenant model, the impact of memory management design mainly includes the following aspects:
- Transparent interface design. How to make developers have no awareness or little need to care about the memory management of different tenants;
- Efficient and accurate. Sufficient memory must be applied successfully, and tenant memory exhaustion must be detected in time, which is the most basic condition for multi-tenant memory management.
This article will introduce the commonly used memory allocation interfaces and memory management related idioms in OceanBase. For technical details of memory management, please refer to [Memory Management](https://open.oceanbase.com/blog/8501613072)( In Chinese).
# Common Interfaces and Methods of OceanBase Memory Management
OceanBase provides different memory allocators for different scenarios. In addition, in order to improve program execution efficiency, there are some conventional implementations, such as reset/reuse, etc.
## ob_malloc
OceanBase has developed a set of libc-style interface functions ob_malloc/ob_free/ob_realloc. This set of interfaces will dynamically apply for memory blocks of size based on tenant_id, ctx_id, label and other attributes, and mark the memory blocks to determine ownership. This not only facilitates multi-tenant resource management, but is also very helpful in diagnosing memory problems.
ob_malloc will index to the corresponding ObTenantCtxAllocator based on tenant_id and ctx_id, and ObTenantCtxAllocator will allocate memory according to the current tenant context.
ob_free uses offset operation to find the object allocator corresponding to the memory to be released, and then returns the memory to the memory pool.
ob_realloc is different from libc's realloc. It does not expand the original address, but first copies the data to another memory through ob_malloc+memcpy, and then calls ob_free to release the original memory.
```cpp
inline void *ob_malloc(const int64_t nbyte, const ObMemAttr &attr = default_memattr);
inline void ob_free(void *ptr);
inline void *ob_realloc(void *ptr, const int64_t nbyte, const ObMemAttr &attr);
```
## OB_NEW/OB_NEWx
Similar to ob_malloc, OB_NEW provides a set of "C++" interfaces that call the object's constructor and destructor when allocating and releasing memory.
```cpp
/// T is the type, label is the memory label and it can be a const string
#define OB_NEW(T, label, ...)
#define OB_NEW_ALIGN32(T, label, ...)
#define OB_DELETE(T, label, ptr)
#define OB_DELETE_ALIGN32(T, label, ptr)
/// T is the type, pool is the memory pool allocator
#define OB_NEWx(T, pool, ...)
```
> There is no OB_DELETEx, but you can release the memory by yourself.
## ObArenaAllocator
The design feature is to allocate release multiple times and only release once. Only reset or destruction can truly release the memory. The memory allocated before will not have any effect even if `free` is actively called.
ObArenaAllocator is suitable for scenarios where many small memory allocates are released in a short period of time. For example, in a SQL request, many small block memories will be frequently allocated, and the life cycle of these small memories will last for the entire request period. Usually, the processing time of an SQL request is also very short. This memory allocation method is very effective for small memory and avoiding memory leaks. In OceanBase's code, don't be surprised if you see there is only apply for memory but cannot find a place to release it.
> Code reference `page_arena.h`
## ObMemAttr Introduction
OceanBase uses `ObMemAttr` to mark a section of memory.
```cpp
struct ObMemAttr
{
uint64_t tenant_id_; // tenant
ObLabel label_; // label or module
uint64_t ctx_id_; // refer to ob_mod_define.h, each ctx id is corresponding to a ObTenantCtxAllocator
uint64_t sub_ctx_id_; // please ignore it
ObAllocPrio prio_; // priority
};
```
> reference file alloc_struct.h
**tenant_id**
Memory allocation management perform resource statistics and restrictions based on tenant maintenance.
**label**
At the beginning, OceanBase uses a predefined method to create memory labels for each module. However, as the amount of code increases, the method of predefined labels is not suitable. Currently, ObLabel is constructed directly using constant strings. When using ob_malloc, you can also directly pass in a constant string as the ObLabel parameter, such as `buf = ob_malloc(disk_addr.size_, "ReadBuf");`.
**ctx_id**
ctx id is predefined, please refer to `alloc_struct.h`. Each ctx_id of each tenant will create an `ObTenantCtxAllocator` object, which can separately count the related memory allocation usage. Normally use `DEFAULT_CTX_ID` as ctx id. For some special modules, for example, if we want to more conveniently observe memory usage or troubleshoot problems, we define special ctx ids for them, such as libeasy communication library (LIBEASY) and Plan Cache cache usage (PLAN_CACHE_CTX_ID). We can see periodic memory statistics in log files, such as:
```txt
[2024-01-02 20:05:50.375549] INFO [LIB] operator() (ob_malloc_allocator.cpp:537) [47814][MemDumpTimer][T0][Y0-0000000000000000-0-0] [lt=10] [MEMORY] tenant: 500, limit: 9,223,372,036,854,775,807 hold: 800,768,000 rpc_hold: 0 cache_hold: 0 cache_used: 0 cache_item_count: 0
[MEMORY] ctx_id= DEFAULT_CTX_ID hold_bytes= 270,385,152 limit= 2,147,483,648
[MEMORY] ctx_id= GLIBC hold_bytes= 8,388,608 limit= 9,223,372,036,854,775,807
[MEMORY] ctx_id= CO_STACK hold_bytes= 106,954,752 limit= 9,223,372,036,854,775,807
[MEMORY] ctx_id= LIBEASY hold_bytes= 4,194,304 limit= 9,223,372,036,854,775,807
[MEMORY] ctx_id= LOGGER_CTX_ID hold_bytes= 12,582,912 limit= 9,223,372,036,854,775,807
[MEMORY] ctx_id= PKT_NIO hold_bytes= 17,969,152 limit= 9,223,372,036,854,775,807
[MEMORY] ctx_id= SCHEMA_SERVICE hold_bytes= 135,024,640 limit= 9,223,372,036,854,775,807
[MEMORY] ctx_id= UNEXPECTED_IN_500 hold_bytes= 245,268,480 limit= 9,223,372,036,854,775,807
```
**prio**
Currently, two memory allocation priorities are supported, Normal and High. The default is Normal. For definition, please refer to the `enum ObAllocPrio` in file `alloc_struct.h`. High priority memory can allocate memory from urgent (memory_reserved) memory, otherwise it cannot. Refer to `AChunkMgr::update_hold` implementation.
> You can use the configuration item `memory_reserved` to view the reserved memory size.
## init/destroy/reset/reuse
Caching is one of the important methods to improve program performance. Object reuse is also a way of caching. On the one hand, it reduces the frequency of memory allocate and release, and on the other hand, it can reduce some construction and destruction overhead. There is a lot of object reuse in OceanBase, and some conventions have been formed, such as the reset and reuse functions.
**reset**
Used to reset objects. Restore the object's state to the state after the constructor or init function was executed. For example `ObNewRow::reset`.
**reuse**
Compared with reset, it is more lightweight. Try not to release some expensive resources, such as `PageArena::reuse`.
**init/destroy**
There are two other common interfaces in OceanBase: `init` and `destroy`. `init` is used to initizalize object and `destory` to release resources. Only do some very lightweight initialization work in the constructor, such as initializing the pointer to `nullptr`.
## SMART_VAR/HEAP_VAR
SMART_VAR is an auxiliary interface for defining local variables. Variables using this interface are always allocated from the stack first. When the stack memory is insufficient, they will be allocated from the heap. For those large local variables (>8K) that are not easy to optimize, this interface not only ensures the performance of regular scenarios, but also safely reduces the stack capacity. The interface is defined as follows:
```cpp
SMART_VAR(Type, Name, Args...) {
// do...
}
```
It allocate from the stack when the following conditions are met, otherwise allocate from the heap
```cpp
sizeof(T) < 8K || (stack_used < 256K && stack_free > sizeof(T) + 64K)
```
> SMART_VAR was created to solve historical problems. It try to reduce the amount of stack memory occupied by large memory objects.
HEAP_VAR is similar to SMART_VAR, except that it must allocate memory on the heap.
## SMART_CALL
SMART_CALL is used to "quasi-transparently" resolve recursive function calls that may explode the stack on threads with very small stacks. This interface accepts a function call as a parameter. It will automatically check the current stack usage before calling the function. Once it is found that the available stack space is insufficient, a new stack execution function will be created on this thread immediately. After the function ends, it will continue to return to the original stack. This ensures performance when the stack is sufficient, and can also avoid stack explosion scenarios.
```cpp
SMART_CALL(func(args...))
```
Notice:
1. The return value of func must be an int type representing the error code.
2. SMART_CALL will return an error code. This may be an internal mechanism or a func call.
3. It supports stack cascade expansion, each time a 2M stack is expanded (there is a hard-coded total upper limit of 10M)
Compared with direct calling, SMART_CALL only call `check_stack_overflow` to check stack.

2
mittest/logservice/env/ob_simple_log_server.cpp vendored
View File

@ -644,7 +644,7 @@ int ObLogDeliver::init(const common::ObAddr &self, const bool is_bootstrap)
// init_all_propocessor_();
if (is_bootstrap && OB_FAIL(ObMittestBlacklist::init(self))) {
SERVER_LOG(WARN, "ObMittestBlacklist init failed", K(ret));
} else if (OB_FAIL(TG_CREATE_TENANT(lib::TGDefIDs::TEST7, tg_id_))) {
} else if (OB_FAIL(TG_CREATE_TENANT(lib::TGDefIDs::COMMON_QUEUE_THREAD, tg_id_))) {
SERVER_LOG(WARN, "ObSimpleThreadPool::init failed", K(ret));
} else {
is_inited_ = true;

69
mittest/logservice/test_ob_simple_arb_server_single_replica.cpp
View File

@ -13,7 +13,7 @@
#define private public
#include "env/ob_simple_log_cluster_env.h"
#undef private
#include <thread>
const std::string TEST_NAME = "single_arb_server";
using namespace oceanbase::common;
@ -327,6 +327,73 @@ TEST_F(TestObSimpleMutilArbServer, restart_arb)
EXPECT_EQ(OB_SUCCESS, restart_server(0));
}
TEST_F(TestObSimpleMutilArbServer, multi_thread)
{
SET_CASE_LOG_FILE(TEST_NAME, "restart_arb");
OB_LOGGER.set_log_level("TRACE");
ObISimpleLogServer *iserver = get_cluster()[0];
EXPECT_EQ(true, iserver->is_arb_server());
ObSimpleArbServer *arb_server = dynamic_cast<ObSimpleArbServer*>(iserver);
palflite::PalfEnvLiteMgr *palf_env_mgr = &arb_server->palf_env_mgr_;
int64_t cluster_id = 100;
arbserver::GCMsgEpoch epoch = arbserver::GCMsgEpoch(1, 1);
// test add tenant without cluster, generate placeholder
EXPECT_EQ(OB_SUCCESS, palf_env_mgr->create_palf_env_lite(palflite::PalfEnvKey(cluster_id, 1)));
EXPECT_TRUE(palf_env_mgr->is_cluster_placeholder_exists(cluster_id));
std::vector<int64_t> ls_ids = {1001, 1002, 1003, 1004, 1005, 1006, 1007};
int64_t create_success_count = 0;
auto create_func = [&]() {
for (auto ls_id : ls_ids) {
int ret = palf_env_mgr->create_arbitration_instance(
palflite::PalfEnvKey(cluster_id, 1), arb_server->self_, 1001,
ObTenantRole(ObTenantRole::PRIMARY_TENANT));
if (OB_SUCCESS == ret) {
ATOMIC_INC(&create_success_count);
}
if (OB_SUCCESS != ret) {
ASSERT_EQ(false, true);
} else {
}
}
};
int64_t remove_success_count = 0;
auto remove_func = [&] () {
for (auto ls_id : ls_ids) {
int ret = arb_server->palf_env_mgr_.delete_arbitration_instance(
palflite::PalfEnvKey(cluster_id, 1), arb_server->self_, ls_id);
if (OB_SUCCESS == ret) {
ATOMIC_INC(&remove_success_count);
}
if (OB_SUCCESS != ret) {
ASSERT_EQ(false, true);
} else {
}
}
};
int64_t thread_count = 8;
std::vector<std::thread> create_threads;
create_threads.reserve(thread_count);
for (int i = 0; i < thread_count; i++) {
create_threads.emplace_back(std::thread(create_func));
}
for (int i = 0; i < thread_count; i++) {
create_threads[i].join();
}
ASSERT_EQ(thread_count*ls_ids.size(), create_success_count);
std::vector<std::thread> remove_threads;
remove_threads.reserve(thread_count);
for (int i = 0; i < thread_count; i++) {
remove_threads.emplace_back(std::thread(remove_func));
}
for (int i = 0; i < thread_count; i++) {
remove_threads[i].join();
}
ASSERT_EQ(thread_count*ls_ids.size(), remove_success_count);
}
} // end unittest
} // end oceanbase

12
mittest/mtlenv/mock_tenant_module_env.h
View File

@ -95,6 +95,7 @@
#include "observer/table/ob_table_session_pool.h"
#include "share/index_usage/ob_index_usage_info_mgr.h"
#include "storage/tenant_snapshot/ob_tenant_snapshot_service.h"
#include "storage/tmp_file/ob_tmp_file_manager.h" // ObTenantTmpFileManager
#include "storage/memtable/ob_lock_wait_mgr.h"
namespace oceanbase
@ -658,6 +659,10 @@ int MockTenantModuleEnv::init_before_start_mtl()
STORAGE_LOG(WARN, "fail to init env", K(ret));
} else if (OB_FAIL(oceanbase::palf::election::GLOBAL_INIT_ELECTION_MODULE())) {
STORAGE_LOG(WARN, "fail to init env", K(ret));
} else if (OB_FAIL(tmp_file::ObTmpBlockCache::get_instance().init("tmp_block_cache", 1))) {
STORAGE_LOG(WARN, "init tmp block cache failed", KR(ret));
} else if (OB_FAIL(tmp_file::ObTmpPageCache::get_instance().init("tmp_page_cache", 1))) {
STORAGE_LOG(WARN, "init tmp page cache failed", KR(ret));
} else if (OB_SUCCESS != (ret = bandwidth_throttle_.init(1024 * 1024 * 60))) {
STORAGE_LOG(ERROR, "failed to init bandwidth_throttle_", K(ret));
} else if (OB_FAIL(TG_START(lib::TGDefIDs::ServerGTimer))) {
@ -691,6 +696,7 @@ int MockTenantModuleEnv::init()
} else {
oceanbase::ObClusterVersion::get_instance().update_data_version(DATA_CURRENT_VERSION);
MTL_BIND2(ObTenantIOManager::mtl_new, ObTenantIOManager::mtl_init, mtl_start_default, mtl_stop_default, nullptr, ObTenantIOManager::mtl_destroy);
MTL_BIND2(mtl_new_default, tmp_file::ObTenantTmpFileManager::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default);
MTL_BIND2(mtl_new_default, omt::ObSharedTimer::mtl_init, omt::ObSharedTimer::mtl_start, omt::ObSharedTimer::mtl_stop, omt::ObSharedTimer::mtl_wait, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObTenantSchemaService::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default);
MTL_BIND2(mtl_new_default, ObStorageLogger::mtl_init, ObStorageLogger::mtl_start, ObStorageLogger::mtl_stop, ObStorageLogger::mtl_wait, mtl_destroy_default);
@ -753,8 +759,6 @@ int MockTenantModuleEnv::init()
STORAGE_LOG(ERROR, "reload memory config failed", K(ret));
} else if (OB_FAIL(start_())) {
STORAGE_LOG(ERROR, "mock env start failed", K(ret));
} else if (OB_FAIL(ObTmpFileManager::get_instance().init())) {
STORAGE_LOG(WARN, "init_tmp_file_manager failed", K(ret));
} else {
inited_ = true;
}
@ -853,7 +857,6 @@ void MockTenantModuleEnv::destroy()
ObKVGlobalCache::get_instance().destroy();
ObServerCheckpointSlogHandler::get_instance().destroy();
SLOGGERMGR.destroy();
ObTmpFileManager::get_instance().destroy();
OB_SERVER_BLOCK_MGR.stop();
OB_SERVER_BLOCK_MGR.wait();
@ -867,7 +870,8 @@ void MockTenantModuleEnv::destroy()
net_frame_.stop();
net_frame_.wait();
net_frame_.destroy();
tmp_file::ObTmpBlockCache::get_instance().destroy();
tmp_file::ObTmpPageCache::get_instance().destroy();
TG_STOP(lib::TGDefIDs::ServerGTimer);
TG_WAIT(lib::TGDefIDs::ServerGTimer);
TG_DESTROY(lib::TGDefIDs::ServerGTimer);

1
mittest/mtlenv/storage/CMakeLists.txt
View File

@ -42,5 +42,6 @@ add_subdirectory(checkpoint)
add_subdirectory(blocksstable)
add_subdirectory(tenant_snapshot)
add_subdirectory(tablet_memtable)
add_subdirectory(tmp_file)
target_link_libraries(test_memtable PUBLIC mock_tx_ctx mock_tx_log_adapter)

4
mittest/mtlenv/storage/blocksstable/test_index_dumper.cpp
View File

@ -383,8 +383,8 @@ TEST_F(TestIndexDumper, get_from_disk)
}
}
// close
ObTableFlag table_flag;
table_flag.clear();
ObTableBackupFlag table_backup_flag;
table_backup_flag.clear();
ASSERT_EQ(OB_SUCCESS, macro_meta_dumper.close(meta_block_info));
STORAGE_LOG(INFO, "test print meta block info", K(meta_block_info));
ASSERT_FALSE(meta_block_info.in_mem_);

76
mittest/mtlenv/storage/blocksstable/test_index_tree.cpp
View File

@ -668,8 +668,8 @@ void TestIndexTree::mock_compaction(const int64_t test_row_num,
ASSERT_EQ(sec_blk.original_size_, sec_blk.data_zsize_);
ASSERT_EQ(OB_SUCCESS, data_writer.close());
ASSERT_EQ(OB_SUCCESS, sstable_builder->close(res));
ASSERT_EQ(false, res.table_flag_.has_backup());
ASSERT_EQ(true, res.table_flag_.has_local());
ASSERT_EQ(false, res.table_backup_flag_.has_backup());
ASSERT_EQ(true, res.table_backup_flag_.has_local());
ObSSTableMergeRes tmp_res;
ASSERT_EQ(OB_ERR_UNEXPECTED, data_writer.close()); // not re-entrant
@ -677,8 +677,8 @@ void TestIndexTree::mock_compaction(const int64_t test_row_num,
ASSERT_EQ(tmp_res.root_desc_.buf_, res.root_desc_.buf_);
ASSERT_EQ(tmp_res.data_root_desc_.buf_, res.data_root_desc_.buf_);
ASSERT_EQ(tmp_res.data_blocks_cnt_, res.data_blocks_cnt_);
ASSERT_EQ(false, tmp_res.table_flag_.has_backup());
ASSERT_EQ(true, tmp_res.table_flag_.has_local());
ASSERT_EQ(false, tmp_res.table_backup_flag_.has_backup());
ASSERT_EQ(true, tmp_res.table_backup_flag_.has_local());
OK(data_write_ctxs.push_back(sstable_builder->roots_[0]->data_write_ctx_));
roots = &(sstable_builder->roots_);
@ -802,8 +802,8 @@ void TestIndexTree::mock_cg_compaction(const int64_t test_row_num,
ASSERT_EQ(sec_blk.original_size_, sec_blk.data_zsize_);
ASSERT_EQ(OB_SUCCESS, data_writer.close());
ASSERT_EQ(OB_SUCCESS, sstable_builder->close(res));
ASSERT_EQ(false, res.table_flag_.has_backup());
ASSERT_EQ(true, res.table_flag_.has_local());
ASSERT_EQ(false, res.table_backup_flag_.has_backup());
ASSERT_EQ(true, res.table_backup_flag_.has_local());
ObSSTableMergeRes tmp_res;
ASSERT_EQ(OB_ERR_UNEXPECTED, data_writer.close()); // not re-entrant
@ -811,8 +811,8 @@ void TestIndexTree::mock_cg_compaction(const int64_t test_row_num,
ASSERT_EQ(tmp_res.root_desc_.buf_, res.root_desc_.buf_);
ASSERT_EQ(tmp_res.data_root_desc_.buf_, res.data_root_desc_.buf_);
ASSERT_EQ(tmp_res.data_blocks_cnt_, res.data_blocks_cnt_);
ASSERT_EQ(false, tmp_res.table_flag_.has_backup());
ASSERT_EQ(true, tmp_res.table_flag_.has_local());
ASSERT_EQ(false, tmp_res.table_backup_flag_.has_backup());
ASSERT_EQ(true, tmp_res.table_backup_flag_.has_local());
OK(data_write_ctxs.push_back(sstable_builder->roots_[0]->data_write_ctx_));
roots = &(sstable_builder->roots_);
@ -991,8 +991,8 @@ TEST_F(TestIndexTree, test_empty_index_tree)
ASSERT_EQ(0, sstable_builder.roots_.count());
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(res.root_desc_.is_empty());
ASSERT_EQ(false, res.table_flag_.has_backup());
ASSERT_EQ(false, res.table_flag_.has_local());
ASSERT_EQ(false, res.table_backup_flag_.has_backup());
ASSERT_EQ(false, res.table_backup_flag_.has_local());
// test rebuild macro blocks
ASSERT_EQ(OB_SUCCESS, data_writer.open(data_desc.get_desc(), data_seq));
@ -1006,8 +1006,8 @@ TEST_F(TestIndexTree, test_empty_index_tree)
ret = sstable_builder.close(res);
ASSERT_EQ(OB_SUCCESS, ret);
ASSERT_TRUE(res.root_desc_.is_empty());
ASSERT_EQ(false, res.table_flag_.has_backup());
ASSERT_EQ(false, res.table_flag_.has_local());
ASSERT_EQ(false, res.table_backup_flag_.has_backup());
ASSERT_EQ(false, res.table_backup_flag_.has_local());
// test easy index tree
ObDatumRow row;
@ -1130,8 +1130,8 @@ TEST_F(TestIndexTree, test_multi_writers_with_close)
res.reset();
sstable_builder.index_block_loader_.reset();
OK(sstable_builder.close(res));
ASSERT_EQ(false, res.table_flag_.has_backup());
ASSERT_EQ(true, res.table_flag_.has_local());
ASSERT_EQ(false, res.table_backup_flag_.has_backup());
ASSERT_EQ(true, res.table_backup_flag_.has_local());
ObIndexTreeRootBlockDesc &root_desc = res.root_desc_;
ASSERT_TRUE(root_desc.is_valid());
@ -1356,8 +1356,8 @@ TEST_F(TestIndexTree, test_meta_builder_mem_and_disk)
ASSERT_EQ(10, sst_builder->roots_[1]->meta_block_info_.get_row_count());
ASSERT_EQ(OB_SUCCESS, sst_builder->close(res));
ASSERT_EQ(false, res.table_flag_.has_backup());
ASSERT_EQ(true, res.table_flag_.has_local());
ASSERT_EQ(false, res.table_backup_flag_.has_backup());
ASSERT_EQ(true, res.table_backup_flag_.has_local());
ObSSTableMergeRes tmp_res;
@ -1763,8 +1763,8 @@ TEST_F(TestIndexTree, test_reuse_macro_block)
for (int64_t i = 0; i < res.data_blocks_cnt_; ++i) {
ASSERT_EQ(res.data_block_ids_.at(i), reused_res.data_block_ids_.at(i));
}
ASSERT_EQ(false, res.table_flag_.has_backup());
ASSERT_EQ(true, res.table_flag_.has_local());
ASSERT_EQ(false, res.table_backup_flag_.has_backup());
ASSERT_EQ(true, res.table_backup_flag_.has_local());
sst_builder->~ObSSTableIndexBuilder();
}
@ -1902,8 +1902,8 @@ TEST_F(TestIndexTree, test_rebuilder)
OK(data_writer.close());
ObSSTableMergeRes res1;
OK(sstable_builder1.close(res1));
ASSERT_EQ(false, res1.table_flag_.has_backup());
ASSERT_EQ(true, res1.table_flag_.has_local());
ASSERT_EQ(false, res1.table_backup_flag_.has_backup());
ASSERT_EQ(true, res1.table_backup_flag_.has_local());
ObIndexBlockRebuilder rebuilder;
OK(rebuilder.init(sstable_builder2, nullptr, 0));
@ -1934,8 +1934,8 @@ TEST_F(TestIndexTree, test_rebuilder)
OK(rebuilder.close());
ObSSTableMergeRes res2;
OK(sstable_builder2.close(res2));
ASSERT_EQ(false, res2.table_flag_.has_backup());
ASSERT_EQ(true, res2.table_flag_.has_local());
ASSERT_EQ(false, res2.table_backup_flag_.has_backup());
ASSERT_EQ(true, res2.table_backup_flag_.has_local());
// compare merge res
ASSERT_EQ(res1.root_desc_.height_, res2.root_desc_.height_);
ASSERT_EQ(res1.root_desc_.height_, 2);
@ -2128,8 +2128,8 @@ TEST_F(TestIndexTree, test_absolute_offset)
ObSSTableMergeRes res2;
OK(sstable_builder.close(res2));
ASSERT_GT(res2.root_desc_.height_, 2);
ASSERT_EQ(false, res2.table_flag_.has_backup());
ASSERT_EQ(true, res2.table_flag_.has_local());
ASSERT_EQ(false, res2.table_backup_flag_.has_backup());
ASSERT_EQ(true, res2.table_backup_flag_.has_local());
ObMicroBlockReaderHelper reader_helper;
ObIMicroBlockReader *micro_reader;
@ -2227,8 +2227,8 @@ TEST_F(TestIndexTree, test_rebuilder_backup_all_mem)
OK(data_writer.close());
ObSSTableMergeRes res1;
OK(sstable_builder1.close(res1));
ASSERT_EQ(false, res1.table_flag_.has_backup());
ASSERT_EQ(true, res1.table_flag_.has_local());
ASSERT_EQ(false, res1.table_backup_flag_.has_backup());
ASSERT_EQ(true, res1.table_backup_flag_.has_local());
ObIndexBlockRebuilder rebuilder;
OK(mock_init_backup_rebuilder(rebuilder, sstable_builder2));
@ -2265,8 +2265,8 @@ TEST_F(TestIndexTree, test_rebuilder_backup_all_mem)
ASSERT_EQ(true, rebuilder.index_tree_root_ctx_->index_tree_info_.in_mem());
rebuilder.~ObIndexBlockRebuilder();
OK(sstable_builder2.close(res2));
ASSERT_EQ(false, res2.table_flag_.has_backup());
ASSERT_EQ(true, res2.table_flag_.has_local());
ASSERT_EQ(false, res2.table_backup_flag_.has_backup());
ASSERT_EQ(true, res2.table_backup_flag_.has_local());
ASSERT_EQ(true, res2.data_root_desc_.is_mem_type());
ASSERT_EQ(true, res2.root_desc_.is_mem_type());
// compare merge res
@ -2336,8 +2336,8 @@ TEST_F(TestIndexTree, test_rebuilder_backup_all_disk)
OK(data_writer.close());
ObSSTableMergeRes res1;
OK(sstable_builder1.close(res1));
ASSERT_EQ(false, res1.table_flag_.has_backup());
ASSERT_EQ(true, res1.table_flag_.has_local());
ASSERT_EQ(false, res1.table_backup_flag_.has_backup());
ASSERT_EQ(true, res1.table_backup_flag_.has_local());
ObIndexBlockRebuilder rebuilder;
OK(mock_init_backup_rebuilder(rebuilder, sstable_builder2));
@ -2374,8 +2374,8 @@ TEST_F(TestIndexTree, test_rebuilder_backup_all_disk)
ObSSTableMergeRes res2;
OK(sstable_builder2.close(res2));
ASSERT_EQ(false, res2.table_flag_.has_backup());
ASSERT_EQ(true, res2.table_flag_.has_local());
ASSERT_EQ(false, res2.table_backup_flag_.has_backup());
ASSERT_EQ(true, res2.table_backup_flag_.has_local());
// compare merge res
ASSERT_EQ(res1.root_desc_.height_, res2.root_desc_.height_);
// ASSERT_EQ(res1.root_desc_.height_, 2);
@ -2443,8 +2443,8 @@ TEST_F(TestIndexTree, test_rebuilder_backup_mem_and_disk)
OK(data_writer.close());
ObSSTableMergeRes res1;
OK(sstable_builder1.close(res1));
ASSERT_EQ(false, res1.table_flag_.has_backup());
ASSERT_EQ(true, res1.table_flag_.has_local());
ASSERT_EQ(false, res1.table_backup_flag_.has_backup());
ASSERT_EQ(true, res1.table_backup_flag_.has_local());
ObIndexBlockRebuilder rebuilder_mem;
OK(mock_init_backup_rebuilder(rebuilder_mem, sstable_builder2));
ObIndexBlockRebuilder rebuilder;
@ -2496,8 +2496,8 @@ TEST_F(TestIndexTree, test_rebuilder_backup_mem_and_disk)
rebuilder.~ObIndexBlockRebuilder();
ObSSTableMergeRes res2;
OK(sstable_builder2.close(res2));
ASSERT_EQ(false, res2.table_flag_.has_backup());
ASSERT_EQ(true, res2.table_flag_.has_local());
ASSERT_EQ(false, res2.table_backup_flag_.has_backup());
ASSERT_EQ(true, res2.table_backup_flag_.has_local());
}
TEST_F(TestIndexTree, test_cg_compaction_all_mem)
@ -2554,8 +2554,8 @@ TEST_F(TestIndexTree, test_cg_compaction_all_mem)
OK(sstable_builder2.close(res2));
ASSERT_EQ(false, res2.table_flag_.has_backup());
ASSERT_EQ(true, res2.table_flag_.has_local());
ASSERT_EQ(false, res2.table_backup_flag_.has_backup());
ASSERT_EQ(true, res2.table_backup_flag_.has_local());
ASSERT_EQ(true, res2.data_root_desc_.is_mem_type());
ASSERT_EQ(true, res2.root_desc_.is_mem_type());
// compare merge res

Some files were not shown because too many files have changed in this diff Show More