diff --git a/.gitignore b/.gitignore index 4895fcbebc..b720dfdd05 100644 --- a/.gitignore +++ b/.gitignore @@ -231,6 +231,7 @@ tools/deploy/single-with-proxy.yaml tools/deploy/single.yaml tools/deploy/tpchobp_test.yaml tools/deploy/bmsqlobp_test.yaml +tools/deploy/oci_test.yaml tools/log_parser/slog_parser tools/log_tool/log_statistics tools/log_tool/log_tool @@ -286,6 +287,8 @@ tools/deploy/distributed.yaml tools/deploy/obd_profile.sh tools/deploy/single-with-proxy.yaml tools/deploy/single.yaml +tools/docker/standalone/boot +tools/docker/standalone/init_store_for_fast_start.py ############# unittest ############# unittest/**/test_* diff --git a/CMakeLists.txt b/CMakeLists.txt index 93da78a3b0..d18d5217c6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,14 +55,17 @@ if(ENABLE_SMART_VAR_CHECK) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DENABLE_SMART_VAR_CHECK") endif() -if(ENABLE_SERIALIZATION_CHECK) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DENABLE_SERIALIZATION_CHECK") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DENABLE_SERIALIZATION_CHECK") -endif() - if(ENABLE_DEBUG_LOG) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DENABLE_DEBUG_LOG") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DENABLE_DEBUG_LOG") + if(NOT DEFINED ENABLE_SERIALIZATION_CHECK) + set(ENABLE_SERIALIZATION_CHECK ${ENABLE_DEBUG_LOG}) + endif() +endif() + +if(ENABLE_SERIALIZATION_CHECK) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DENABLE_SERIALIZATION_CHECK") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DENABLE_SERIALIZATION_CHECK") endif() if (OB_GPERF_MODE) diff --git a/cmake/Env.cmake b/cmake/Env.cmake index abe25ef9a4..5396ca20c8 100644 --- a/cmake/Env.cmake +++ b/cmake/Env.cmake @@ -18,7 +18,6 @@ ob_define(ENABLE_MEMORY_DIAGNOSIS OFF) ob_define(ENABLE_OBJ_LEAK_CHECK OFF) ob_define(ENABLE_FATAL_ERROR_HANG ON) ob_define(ENABLE_SMART_VAR_CHECK OFF) -ob_define(ENABLE_SERIALIZATION_CHECK OFF) ob_define(ENABLE_COMPILE_DLL_MODE OFF) ob_define(OB_CMAKE_RULES_CHECK ON) ob_define(OB_STATIC_LINK_LGPL_DEPS ON) @@ -273,7 +272,7 @@ if( ${ARCHITECTURE} STREQUAL "x86_64" ) set(ARCH_LDFLAGS "") set(OCI_DEVEL_INC "${DEP_3RD_DIR}/usr/include/oracle/12.2/client64") else() - set(MARCH_CFLAGS "-march=armv8-a+crc" ) + set(MARCH_CFLAGS "-march=armv8-a+crc+lse" ) set(MTUNE_CFLAGS "-mtune=generic" ) set(ARCH_LDFLAGS "-l:libatomic.a") set(OCI_DEVEL_INC "${DEP_3RD_DIR}/usr/include/oracle/19.10/client64") diff --git a/deps/init/oceanbase.el7.aarch64.deps b/deps/init/oceanbase.el7.aarch64.deps index a4dba81c62..3e23698f83 100644 --- a/deps/init/oceanbase.el7.aarch64.deps +++ b/deps/init/oceanbase.el7.aarch64.deps @@ -28,7 +28,7 @@ devdeps-boost-1.74.0-22022110914.el7.aarch64.rpm devdeps-s2geometry-0.9.0-12023092021.el7.aarch64.rpm devdeps-icu-69.1-72022112416.el7.aarch64.rpm devdeps-cos-c-sdk-5.0.16-52023070517.el7.aarch64.rpm -devdeps-s3-cpp-sdk-1.11.156-62023101011.el7.aarch64.rpm +devdeps-s3-cpp-sdk-1.11.156-102023122011.el7.aarch64.rpm [tools] obdevtools-binutils-2.30-12022100413.el7.aarch64.rpm diff --git a/deps/init/oceanbase.el7.x86_64.deps b/deps/init/oceanbase.el7.x86_64.deps index 6954028cca..1999d30bc5 100644 --- a/deps/init/oceanbase.el7.x86_64.deps +++ b/deps/init/oceanbase.el7.x86_64.deps @@ -31,7 +31,7 @@ devdeps-s2geometry-0.9.0-12023092021.el7.x86_64.rpm devdeps-icu-69.1-72022112416.el7.x86_64.rpm devdeps-cos-c-sdk-5.0.16-52023070517.el7.x86_64.rpm devdeps-cloud-qpl-1.1.0-272023061419.el7.x86_64.rpm -devdeps-s3-cpp-sdk-1.11.156-62023101011.el7.x86_64.rpm +devdeps-s3-cpp-sdk-1.11.156-102023122011.el7.x86_64.rpm [tools] obdevtools-binutils-2.30-12022100413.el7.x86_64.rpm diff --git a/deps/init/oceanbase.el8.aarch64.deps b/deps/init/oceanbase.el8.aarch64.deps index ad9b9f8d06..ff58370486 100644 --- a/deps/init/oceanbase.el8.aarch64.deps +++ b/deps/init/oceanbase.el8.aarch64.deps @@ -28,7 +28,7 @@ devdeps-boost-1.74.0-22022110914.el8.aarch64.rpm devdeps-s2geometry-0.9.0-12023092021.el8.aarch64.rpm devdeps-icu-69.1-72022112416.el8.aarch64.rpm devdeps-cos-c-sdk-5.0.16-52023070517.el8.aarch64.rpm -devdeps-s3-cpp-sdk-1.11.156-62023101011.el8.aarch64.rpm +devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.aarch64.rpm [tools] obdevtools-binutils-2.30-12022100413.el8.aarch64.rpm diff --git a/deps/init/oceanbase.el8.x86_64.deps b/deps/init/oceanbase.el8.x86_64.deps index 3a1f1028f5..3c84ae1bd3 100644 --- a/deps/init/oceanbase.el8.x86_64.deps +++ b/deps/init/oceanbase.el8.x86_64.deps @@ -30,7 +30,7 @@ devdeps-s2geometry-0.9.0-12023092021.el8.x86_64.rpm devdeps-icu-69.1-72022112416.el8.x86_64.rpm devdeps-cos-c-sdk-5.0.16-52023070517.el8.x86_64.rpm devdeps-cloud-qpl-1.1.0-272023061419.el8.x86_64.rpm -devdeps-s3-cpp-sdk-1.11.156-62023101011.el8.x86_64.rpm +devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.x86_64.rpm [tools] obdevtools-binutils-2.30-12022100413.el8.x86_64.rpm diff --git a/deps/init/oceanbase.el9.aarch64.deps b/deps/init/oceanbase.el9.aarch64.deps index 88c2632b0b..17b1d66939 100644 --- a/deps/init/oceanbase.el9.aarch64.deps +++ b/deps/init/oceanbase.el9.aarch64.deps @@ -32,7 +32,7 @@ devdeps-boost-1.74.0-22022110914.el8.aarch64.rpm devdeps-s2geometry-0.9.0-12023092021.el8.aarch64.rpm devdeps-icu-69.1-72022112416.el8.aarch64.rpm devdeps-cos-c-sdk-5.0.16-52023070517.el8.aarch64.rpm -devdeps-s3-cpp-sdk-1.11.156-62023101011.el8.aarch64.rpm +devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.aarch64.rpm [deps-el9] devdeps-apr-1.6.5-232023090616.el9.aarch64.rpm target=el9 diff --git a/deps/init/oceanbase.el9.x86_64.deps b/deps/init/oceanbase.el9.x86_64.deps index 50a5b30cc0..77d166d71b 100644 --- a/deps/init/oceanbase.el9.x86_64.deps +++ b/deps/init/oceanbase.el9.x86_64.deps @@ -34,7 +34,7 @@ devdeps-s2geometry-0.9.0-12023092021.el8.x86_64.rpm devdeps-icu-69.1-72022112416.el8.x86_64.rpm devdeps-cos-c-sdk-5.0.16-52023070517.el8.x86_64.rpm devdeps-cloud-qpl-1.1.0-272023061419.el8.x86_64.rpm -devdeps-s3-cpp-sdk-1.11.156-62023101011.el8.x86_64.rpm +devdeps-s3-cpp-sdk-1.11.156-102023122011.el8.x86_64.rpm [deps-el9] devdeps-apr-1.6.5-232023090616.el9.x86_64.rpm target=el9 diff --git a/deps/oblib/src/common/object/ob_obj_funcs.h b/deps/oblib/src/common/object/ob_obj_funcs.h index 0d610a118c..b285f584c2 100644 --- a/deps/oblib/src/common/object/ob_obj_funcs.h +++ b/deps/oblib/src/common/object/ob_obj_funcs.h @@ -2320,8 +2320,8 @@ template <> inline int obj_val_serialize(const ObObj &obj, char* buf, const int64_t buf_len, int64_t& pos) { int ret = OB_SUCCESS; - OB_UNIS_ENCODE(obj.get_ext()); if (obj.is_pl_extend()) { + OB_UNIS_ENCODE(obj.get_ext()); COMMON_LOG(ERROR, "Unexpected serialize", K(OB_NOT_SUPPORTED), K(obj), K(obj.get_meta().get_extend_type())); return OB_NOT_SUPPORTED; //TODO:@ryan.ly: close this feature before composite refactor if (NULL == serialize_composite_callback) { @@ -2329,6 +2329,18 @@ inline int obj_val_serialize(const ObObj &obj, char* buf, const in } else { ret = serialize_composite_callback(obj, buf, buf_len, pos); } + } else if (obj.is_ext_sql_array()) { + int64_t v = 0; + OB_UNIS_ENCODE(v); + const ObSqlArrayObj *array_obj = reinterpret_cast(obj.get_ext()); + if (OB_SUCC(ret) && NULL != array_obj) { + int64_t len = array_obj->get_serialize_size(); + int64_t tmp_pos = pos; + OB_UNIS_ENCODE(len); + OB_UNIS_ENCODE(*array_obj); + } + } else { + OB_UNIS_ENCODE(obj.get_ext()); } return ret; } @@ -2347,6 +2359,20 @@ inline int obj_val_deserialize(ObObj &obj, const char* buf, const } else { ret = deserialize_composite_callback(obj, buf, data_len, pos); } + } else if (obj.is_ext_sql_array()) { + if (OB_UNLIKELY(v != 0)) { + ret = OB_NOT_SUPPORTED; + COMMON_LOG(WARN, "using such type in upgrade period", K(ret)); + } else { + int64_t len = 0; + int64_t tmp_pos = pos; + OB_UNIS_DECODE(len); + /* record the buffer and delay it's deserialize. + * should call ObSqlArrayObj::do_real_deserialize which need an allocator + */ + obj.set_extend(reinterpret_cast(buf + pos), T_EXT_SQL_ARRAY, int32_t(len)); + pos += len; + } } else { obj.set_obj_value(v); } @@ -2357,8 +2383,8 @@ template <> inline int64_t obj_val_get_serialize_size(const ObObj &obj) { int64_t len = 0; - OB_UNIS_ADD_LEN(obj.get_ext()); if (obj.is_pl_extend()) { + OB_UNIS_ADD_LEN(obj.get_ext()); COMMON_LOG_RET(ERROR, OB_NOT_SUPPORTED, "Unexpected serialize", K(OB_NOT_SUPPORTED), K(obj), K(obj.get_meta().get_extend_type())); return len; //TODO:@ryan.ly: close this feature before composite refactor if (NULL == composite_serialize_size_callback) { @@ -2366,6 +2392,17 @@ inline int64_t obj_val_get_serialize_size(const ObObj &obj) } else { len += composite_serialize_size_callback(obj); } + } else if (obj.is_ext_sql_array()) { + int64_t v = 0; + OB_UNIS_ADD_LEN(v); + const ObSqlArrayObj *array_obj = reinterpret_cast(obj.get_ext()); + if (NULL != array_obj) { + int64_t array_obj_len = array_obj->get_serialize_size(); + OB_UNIS_ADD_LEN(array_obj_len); + OB_UNIS_ADD_LEN(*array_obj); + } + } else { + OB_UNIS_ADD_LEN(obj.get_ext()); } return len; } diff --git a/deps/oblib/src/common/object/ob_obj_type.cpp b/deps/oblib/src/common/object/ob_obj_type.cpp index 299f6a10d2..45f773537a 100644 --- a/deps/oblib/src/common/object/ob_obj_type.cpp +++ b/deps/oblib/src/common/object/ob_obj_type.cpp @@ -650,24 +650,6 @@ bool is_match_alter_integer_column_online_ddl_rules(const common::ObObjMeta& src || (src_meta.is_unsigned_integer() && dst_meta.is_unsigned_integer())) // both are singed or unsigned integer && src_meta.get_type() <= dst_meta.get_type())) { // (unsigned) integer can be changed into larger by online ddl is_online_ddl = true; - } else if (src_meta.is_unsigned_integer() && dst_meta.is_signed_integer()) { - if (src_meta.is_utinyint()) { - if (dst_meta.get_type() >= ObSmallIntType && dst_meta.get_type() <= ObIntType) { // unsigned tinyint -> smallint mediumint int bigint - is_online_ddl = true; - } - } else if (src_meta.is_usmallint()) { - if (dst_meta.get_type() >= ObMediumIntType && dst_meta.get_type() <= ObIntType) { // unsigned smallint -> mediumint int bigint - is_online_ddl = true; - } - } else if (src_meta.is_umediumint()) { - if (dst_meta.get_type() >= ObInt32Type && dst_meta.get_type() <= ObIntType) { // unsigned mediumint -> int bigint - is_online_ddl = true; - } - } else if (src_meta.is_uint32()) { - if (dst_meta.is_int()) { // unsigned int -> bigint - is_online_ddl = true; - } - } } return is_online_ddl; } diff --git a/deps/oblib/src/common/object/ob_object.cpp b/deps/oblib/src/common/object/ob_object.cpp index 9397054447..b875052b7f 100644 --- a/deps/oblib/src/common/object/ob_object.cpp +++ b/deps/oblib/src/common/object/ob_object.cpp @@ -2227,35 +2227,6 @@ DEFINE_SERIALIZE(ObObjParam) if (OB_SUCC(ret)) { OB_UNIS_ENCODE(accuracy_); OB_UNIS_ENCODE(res_flags_); - if (OB_SUCC(ret) && is_ext_sql_array()) { - const ObSqlArrayObj *array_obj = reinterpret_cast(get_ext()); - int64_t n = sizeof(ObSqlArrayObj); - if (OB_ISNULL(array_obj)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected NULL ptr", K(ret), KP(array_obj)); - } else if (buf_len - pos < n) { - ret = OB_BUF_NOT_ENOUGH; - LOG_WARN("serialize buf not enough", K(ret), "remain", buf_len - pos, "needed", n); - } else { - MEMCPY(buf + pos, array_obj, n); - pos += n; - if (array_obj->count_ == 0) { - /* do nothing */ - } else if (OB_ISNULL(array_obj->data_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("data is NULL ptr", K(ret), KP(array_obj->data_)); - } else { - n = sizeof(array_obj->data_[0]) * array_obj->count_; - if (buf_len - pos < n) { - ret = OB_BUF_NOT_ENOUGH; - LOG_WARN("serialize buf not enough", K(ret), "remain", buf_len - pos, "needed", n); - } else { - MEMCPY(buf + pos, static_cast(array_obj->data_), n); - pos += n; - } - } - } - } } return ret; } @@ -2266,30 +2237,6 @@ DEFINE_DESERIALIZE(ObObjParam) if (OB_SUCC(ret)) { OB_UNIS_DECODE(accuracy_); OB_UNIS_DECODE(res_flags_); - if (OB_SUCC(ret) && is_ext_sql_array()) { - ObSqlArrayObj *array_obj = NULL; - int64_t n = sizeof(ObSqlArrayObj); - if (data_len - pos < n) { - ret = OB_BUF_NOT_ENOUGH; - LOG_WARN("deserialize buf not enough", K(ret), "remain", data_len - pos, "needed", n); - } else { - array_obj = reinterpret_cast(const_cast(buf + pos)); - pos += n; - } - if (OB_SUCC(ret) && array_obj->count_ > 0) { - n = sizeof(ObObjParam) * array_obj->count_; - if (data_len - pos < n) { - ret = OB_BUF_NOT_ENOUGH; - LOG_WARN("deserialize buf not enough", K(ret), "remain", data_len - pos, "needed", n); - } else { - array_obj->data_ = reinterpret_cast(const_cast(buf + pos)); - pos += n; - } - } - if (OB_SUCC(ret)) { - set_extend(reinterpret_cast(array_obj), T_EXT_SQL_ARRAY); - } - } } return ret; } @@ -2299,18 +2246,6 @@ DEFINE_GET_SERIALIZE_SIZE(ObObjParam) int64_t len = ObObj::get_serialize_size(); OB_UNIS_ADD_LEN(accuracy_); OB_UNIS_ADD_LEN(res_flags_); - if (is_ext_sql_array()) { - len += sizeof(ObSqlArrayObj); - const ObSqlArrayObj *array_obj = reinterpret_cast(get_ext()); - if (NULL != array_obj) { - len += sizeof(ObSqlArrayObj); - if (array_obj->count_ == 0) { - /* do nothing */ - } else if (NULL != array_obj->data_) { - len += sizeof(array_obj->data_[0]) * array_obj->count_; - } - } - } return len; } @@ -2451,3 +2386,58 @@ int64_t ObHexEscapeSqlStr::get_extra_length() const } return ret_length; } + +int ObSqlArrayObj::do_real_deserialize(common::ObIAllocator &allocator, char *buf, int64_t data_len, + ObSqlArrayObj *&array_obj) +{ + int ret = OB_SUCCESS; + int64_t n = sizeof(ObSqlArrayObj); + void *array_buf = allocator.alloc(n); + int64_t pos = 0; + if (OB_ISNULL(array_buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate memory failed", K(ret)); + } else { + array_obj = new (array_buf) ObSqlArrayObj(); + if (OB_FAIL(array_obj->deserialize(allocator, buf, data_len, pos))) { + LOG_WARN("failed to deserialize ObSqlArrayObj", K(ret)); + } + } + return ret; +} + +DEFINE_SERIALIZE(ObSqlArrayObj) +{ + int ret = OB_SUCCESS; + int64_t len = 0; + OB_UNIS_ENCODE(element_); + OB_UNIS_ENCODE_ARRAY(data_, count_); + return ret; +} + +int ObSqlArrayObj::deserialize(ObIAllocator &allocator, const char* buf, const int64_t data_len, + int64_t& pos) +{ + int ret = OB_SUCCESS; + OB_UNIS_DECODE(element_); + OB_UNIS_DECODE(count_); + if (OB_SUCC(ret) && count_ > 0) { + void *data_buf = allocator.alloc(sizeof(ObObjParam) * count_); + if (OB_ISNULL(data_buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate memory failed", K(ret)); + } else { + data_ = new (data_buf) common::ObObjParam[count_]; + OB_UNIS_DECODE_ARRAY(data_, count_); + } + } + return ret; +} + +DEFINE_GET_SERIALIZE_SIZE(ObSqlArrayObj) +{ + int64_t len = 0; + OB_UNIS_ADD_LEN(element_); + OB_UNIS_ADD_LEN_ARRAY(data_, count_); + return len; +} diff --git a/deps/oblib/src/common/object/ob_object.h b/deps/oblib/src/common/object/ob_object.h index 13446a8d37..203fd535d3 100644 --- a/deps/oblib/src/common/object/ob_object.h +++ b/deps/oblib/src/common/object/ob_object.h @@ -4150,6 +4150,10 @@ struct ObSqlArrayObj } typedef common::ObArrayWrap DataArray; static ObSqlArrayObj *alloc(common::ObIAllocator &allocator, int64_t count); + static int do_real_deserialize(common::ObIAllocator &allocator, char *buf, int64_t data_len, ObSqlArrayObj *&array_obj); + int serialize(char* buf, const int64_t buf_len, int64_t& pos) const; + int deserialize(common::ObIAllocator &allocator, const char* buf, const int64_t data_len, int64_t& pos); + int64_t get_serialize_size(void) const; TO_STRING_KV("data", DataArray(data_, count_), K_(count), K_(element)); common::ObObjParam *data_; int64_t count_; diff --git a/deps/oblib/src/lib/compress/zlib_lite/zlib_lite_adaptor.cpp b/deps/oblib/src/lib/compress/zlib_lite/zlib_lite_adaptor.cpp index 114ffd6939..3635bc4b2f 100644 --- a/deps/oblib/src/lib/compress/zlib_lite/zlib_lite_adaptor.cpp +++ b/deps/oblib/src/lib/compress/zlib_lite/zlib_lite_adaptor.cpp @@ -11,6 +11,9 @@ */ #include "zlib_lite_adaptor.h" + +#include + #include "codec_deflate_qpl.h" #include "zlib_lite_src/deflate.h" #include "zlib_lite_src/zlib.h" @@ -21,6 +24,47 @@ namespace common { namespace ZLIB_LITE { + +static bool check_support_qpl(void) +{ + bool bret = false; + unsigned int eax, ebx, ecx, edx; + //LEVEL=0: Highest Function Parameter and Manufacturer ID + __cpuid(0, eax, ebx, ecx, edx); + //Determine whether it is an Intel manufacturer. + if (ebx == 0x756e6547 && edx == 0x49656e69 && ecx == 0x6c65746e) { + //LEVEL=1: Processor Info and Feature Bit + __cpuid(1, eax, ebx, ecx, edx); + unsigned int family, model; + /* + If the Family ID field is equal to 15, the family is equal to the sum of + the Extended Family ID and the Family ID fields. + Otherwise, the family is equal to the value of the Family ID field. + */ + family = (eax >> 8) & 0xF; + if (family == 0xF) { + family += (eax >> 20) & 0xF; + } + /* + If the Family ID field is either 6 or 15, the model is equal to the sum of + the Extended Model ID field shifted left by 4 bits and the Model field, + Otherwise, the model is equal to the value of the Model field. + */ + model = (eax >> 4) & 0xF; + if (family >= 0x6) { + model += ((eax >> 16) & 0xF) << 4; + } + + // if Model less then 0X8F or family not equal 6, QPL compress and + // decompress is not supported, will use zlib instead. + if (family == 6 && model >= 0X8F) { + bret = true; + } + + } + return bret; +} + /*zlib_lite supports two algorithms. On the platform that supports qpl, the qpl compression algorithm will be used, otherwise the zlib algorithm will be used.*/ @@ -37,12 +81,15 @@ int ObZlibLiteAdaptor::init(allocator alloc, deallocator dealloc, int32_t io_thr int ret = 0; #ifdef ENABLE_QPL_COMPRESSION - QplAllocator allocator; - allocator.allocate = alloc; - allocator.deallocate = dealloc; - int qpl_ret = qpl_init(allocator, io_thread_count); - if (0 != qpl_ret) { - ret = -1; + qpl_support_ = check_support_qpl(); + if (qpl_support_) { + QplAllocator allocator; + allocator.allocate = alloc; + allocator.deallocate = dealloc; + int qpl_ret = qpl_init(allocator, io_thread_count); + if (0 != qpl_ret) { + ret = -1; + } } #endif @@ -52,7 +99,9 @@ int ObZlibLiteAdaptor::init(allocator alloc, deallocator dealloc, int32_t io_thr void ObZlibLiteAdaptor::deinit() { #ifdef ENABLE_QPL_COMPRESSION - qpl_deinit(); + if (qpl_support_) { + qpl_deinit(); + } #endif } @@ -129,8 +178,10 @@ int64_t ObZlibLiteAdaptor::compress(const char* src_buffer, { #ifdef ENABLE_QPL_COMPRESSION - return qpl_compress(src_buffer, dst_buffer, static_cast(src_data_size), - static_cast(dst_buffer_size)); + if (qpl_support_) { + return qpl_compress(src_buffer, dst_buffer, static_cast(src_data_size), + static_cast(dst_buffer_size)); + } #endif // ENABLE_QPL_COMPRESSION @@ -152,9 +203,11 @@ int64_t ObZlibLiteAdaptor::decompress(const char* src_buffer, { #ifdef ENABLE_QPL_COMPRESSION - return qpl_decompress(src_buffer, dst_buffer, - static_cast(src_data_size), - static_cast(dst_buffer_size)); + if (qpl_support_) { + return qpl_decompress(src_buffer, dst_buffer, + static_cast(src_data_size), + static_cast(dst_buffer_size)); + } #endif int64_t decompress_ret_size = dst_buffer_size; @@ -170,7 +223,9 @@ int64_t ObZlibLiteAdaptor::decompress(const char* src_buffer, const char *ObZlibLiteAdaptor::compression_method() const { #ifdef ENABLE_QPL_COMPRESSION - return qpl_hardware_enabled() ? "qpl_hardware" : "qpl_software"; + if (qpl_support_) { + return qpl_hardware_enabled() ? "qpl_hardware" : "qpl_software"; + } #endif return "zlib_native"; diff --git a/deps/oblib/src/lib/compress/zlib_lite/zlib_lite_adaptor.h b/deps/oblib/src/lib/compress/zlib_lite/zlib_lite_adaptor.h index 2539ca01dd..81112e7ce0 100644 --- a/deps/oblib/src/lib/compress/zlib_lite/zlib_lite_adaptor.h +++ b/deps/oblib/src/lib/compress/zlib_lite/zlib_lite_adaptor.h @@ -54,6 +54,8 @@ private: int zlib_decompress(char *dest, int64_t *dest_len, const char *source, int64_t source_len); private: + bool qpl_support_ = false; + //zlib compress level,default is 1. static constexpr int compress_level = 1; diff --git a/deps/oblib/src/lib/json_type/ob_json_base.cpp b/deps/oblib/src/lib/json_type/ob_json_base.cpp index 25f9b8e563..db4b8c6846 100644 --- a/deps/oblib/src/lib/json_type/ob_json_base.cpp +++ b/deps/oblib/src/lib/json_type/ob_json_base.cpp @@ -2147,53 +2147,63 @@ bool ObIJsonBase::is_real_json_null(const ObIJsonBase* ptr) const return ret_bool; } -int ObIJsonBase::trans_json_node(ObIAllocator* allocator, ObIJsonBase* &left, ObIJsonBase* &right) const +// left is scalar, right is ans of subpath +int ObIJsonBase::trans_json_node(ObIAllocator* allocator, ObIJsonBase* &scalar, ObIJsonBase* &path_res) const { INIT_SUCC(ret); - ObJsonNodeType left_type = left->json_type(); - ObJsonNodeType right_type = right->json_type(); - // 左边的需要根据右边的类型转换 + ObJsonNodeType left_type = scalar->json_type(); + ObJsonNodeType right_type = path_res->json_type(); if (left_type == ObJsonNodeType::J_STRING) { - ObString str(left->get_data_length(), left->get_data()); + ObString str(scalar->get_data_length(), scalar->get_data()); if (is_json_number(right_type)) { // fail is normal - ret = trans_to_json_number(allocator, str, left); + ret = trans_to_json_number(allocator, str, scalar); } else if (right_type == ObJsonNodeType::J_DATE || right_type == ObJsonNodeType::J_DATETIME || right_type == ObJsonNodeType::J_TIME || right_type == ObJsonNodeType::J_ORACLEDATE) { - ret = trans_to_date_timestamp(allocator, str, left, true); + ret = trans_to_date_timestamp(allocator, str, scalar, true); } else if (right_type == ObJsonNodeType::J_TIMESTAMP || right_type == ObJsonNodeType::J_OTIMESTAMP || right_type == ObJsonNodeType::J_OTIMESTAMPTZ) { - ret = trans_to_date_timestamp(allocator, str, left, false); + ret = trans_to_date_timestamp(allocator, str, scalar, false); } else if (right_type == ObJsonNodeType::J_BOOLEAN) { - ret = trans_to_boolean(allocator, str, left); + // when scalar is string, path_res is boolean, case compare + if (str.case_compare("true") == 0 || str.case_compare("false") == 0) { + ret = trans_to_boolean(allocator, str, scalar); + } else { + ret = OB_NOT_SUPPORTED; + } } else if (right_type != ObJsonNodeType::J_ARRAY && right_type != ObJsonNodeType::J_OBJECT) { ret = ret = OB_INVALID_ARGUMENT; LOG_WARN("CAN'T TRANS", K(ret)); } - // 右边需要根据左边的转换 + } else if (left_type == ObJsonNodeType::J_NULL) { + // return error code, mean can't cast, return false ans directly + ret = OB_NOT_SUPPORTED; } else if (right_type == ObJsonNodeType::J_STRING) { - ObString str(right->get_data_length(), right->get_data()); + ObString str(path_res->get_data_length(), path_res->get_data()); if (is_json_number(left_type)) { // fail is normal - ret = trans_to_json_number(allocator, str, right); + ret = trans_to_json_number(allocator, str, path_res); } else if (left_type == ObJsonNodeType::J_DATE || left_type == ObJsonNodeType::J_DATETIME || left_type == ObJsonNodeType::J_TIME || left_type == ObJsonNodeType::J_ORACLEDATE) { - ret = trans_to_date_timestamp(allocator, str, right, true); + ret = trans_to_date_timestamp(allocator, str, path_res, true); } else if (left_type == ObJsonNodeType::J_TIMESTAMP || left_type == ObJsonNodeType::J_OTIMESTAMP || left_type == ObJsonNodeType::J_OTIMESTAMPTZ) { - ret = trans_to_date_timestamp(allocator, str, right, false); + ret = trans_to_date_timestamp(allocator, str, path_res, false); } else if (left_type == ObJsonNodeType::J_BOOLEAN) { - ret = trans_to_boolean(allocator, str, right); + ret = trans_to_boolean(allocator, str, path_res); } else if (left_type != ObJsonNodeType::J_ARRAY && left_type != ObJsonNodeType::J_OBJECT) { - ret = ret = OB_INVALID_ARGUMENT; + ret = OB_INVALID_ARGUMENT; LOG_WARN("CAN'T TRANS", K(ret)); } + } else if (left_type == ObJsonNodeType::J_BOOLEAN || is_json_number(left_type)) { + // scalar is boolean or number, and path_res is not string, return false + ret = OB_NOT_SUPPORTED; } else { // do nothing LOG_WARN("CAN'T TRANS", K(ret)); @@ -2247,7 +2257,7 @@ int ObIJsonBase::cmp_to_right_recursively(ObIAllocator* allocator, const ObJsonB // 但只要有一个找到，且为123或"123"则为true ObIJsonBase* left = jb_ptr; ObIJsonBase* right = right_arg; - if (OB_FAIL(trans_json_node(allocator, left, right))) { + if (OB_FAIL(trans_json_node(allocator, right, left))) { // fail is normal, it is not an error. ret = OB_SUCCESS; cmp_result = false; @@ -2270,7 +2280,7 @@ int ObIJsonBase::cmp_to_right_recursively(ObIAllocator* allocator, const ObJsonB // 不相同的类型，同上 ObIJsonBase* left = hit[i]; ObIJsonBase* right = right_arg; - if (OB_FAIL(trans_json_node(allocator, left, right))) { + if (OB_FAIL(trans_json_node(allocator, right, left))) { // fail is normal, it is not an error. ret = OB_SUCCESS; cmp_result = false; @@ -2291,7 +2301,6 @@ int ObIJsonBase::cmp_to_right_recursively(ObIAllocator* allocator, const ObJsonB } // for compare ——> ( scalar/sql_var, subpath) -// 左边调用compare，右边是数组时自动解包 // 只要有一个结果为true则返回true，找不到或结果为false均返回false int ObIJsonBase::cmp_to_left_recursively(ObIAllocator* allocator, const ObJsonBaseVector& hit, const ObJsonPathNodeType node_type, @@ -2310,44 +2319,7 @@ int ObIJsonBase::cmp_to_left_recursively(ObIAllocator* allocator, const ObJsonBa } else if (hit[i]->json_type() == ObJsonNodeType::J_NULL && !is_real_json_null(hit[i])) { cmp_result = false; } else { - // error is ok - // if is array, compare with every node - // but only autowrap once - if (hit[i]->json_type() == ObJsonNodeType::J_ARRAY) { - uint64_t size = hit[i]->element_count(); - ObIJsonBase *jb_ptr = NULL; - for (uint32_t array_i = 0; array_i < size && !cmp_result && OB_SUCC(ret); ++array_i) { - jb_ptr = NULL; // reset jb_ptr to NULL - ret = hit[i]->get_array_element(array_i, jb_ptr); - int cmp_res = -3; - // 类型相同可以直接用compare函数比较 - if(OB_FAIL(ret) || OB_ISNULL(jb_ptr)) { - ret = OB_ERR_NULL_VALUE; - LOG_WARN("compare value is null.", K(ret)); - } else if (is_same_type(left_arg, jb_ptr)) { - if (OB_SUCC(left_arg->compare((*jb_ptr), cmp_res, true))) { - cmp_based_on_node_type(node_type, cmp_res, cmp_result); - } - } else { - // 不相同的类型，oracle会将string类型转换为对应类型再进行比较 - // 转换或比较失败也正常，并不报错 - // 例如: [*].a == 123 - // 里面可能有多个元素无法转换成数字或无法和数字比较甚至找不到.a - // 但只要有一个找到，且为123或"123"则为true - ObIJsonBase* left = left_arg; - ObIJsonBase* right = jb_ptr; - if (OB_FAIL(trans_json_node(allocator, left, right))) { - // fail is normal, it is not an error. - ret = OB_SUCCESS; - cmp_result = false; - } else if (OB_SUCC(left->compare((*right), cmp_res, true))) { - cmp_based_on_node_type(node_type, cmp_res, cmp_result); - } else { - cmp_result = false; - } - } - } - } else if (hit[i]->json_type() == ObJsonNodeType::J_OBJECT) { + if (hit[i]->json_type() == ObJsonNodeType::J_OBJECT || hit[i]->json_type() == ObJsonNodeType::J_ARRAY) { cmp_result = false; } else { int cmp_res = -3; diff --git a/deps/oblib/src/lib/json_type/ob_json_path.cpp b/deps/oblib/src/lib/json_type/ob_json_path.cpp index a2823c8eb4..ea48b2d61c 100644 --- a/deps/oblib/src/lib/json_type/ob_json_path.cpp +++ b/deps/oblib/src/lib/json_type/ob_json_path.cpp @@ -786,13 +786,16 @@ int ObJsonPath::change_json_expr_res_type_if_need(common::ObIAllocator &allocato switch (func_node->get_node_type()) { case JPN_BOOLEAN : case JPN_BOOL_ONLY : { - ret_node.type_ = T_CAST_ARGUMENT; - ret_node.value_ = 0; - ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] = T_VARCHAR; - ret_node.int16_values_[OB_NODE_CAST_COLL_IDX] = 0; - ret_node.int32_values_[OB_NODE_CAST_C_LEN_IDX] = 20; - ret_node.length_semantics_ = 0; - ret_node.is_hidden_const_ = 1; + if (json_expr_flag == OPT_JSON_QUERY && ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] == T_JSON) { // do nothing + } else { + ret_node.type_ = T_CAST_ARGUMENT; + ret_node.value_ = 0; + ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] = T_VARCHAR; + ret_node.int16_values_[OB_NODE_CAST_COLL_IDX] = 0; + ret_node.int32_values_[OB_NODE_CAST_C_LEN_IDX] = 20; + ret_node.length_semantics_ = 0; + ret_node.is_hidden_const_ = 1; + } break; } case JPN_DATE : { @@ -824,14 +827,16 @@ int ObJsonPath::change_json_expr_res_type_if_need(common::ObIAllocator &allocato case JPN_NUMBER : case JPN_FLOOR : case JPN_CEILING : { - ret_node.value_ = 0; - if (ret_node.type_ == T_NULL) { + if (ret_node.type_ == T_NULL + || (json_expr_flag == OPT_JSON_QUERY && ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] == T_JSON)) { + ret_node.value_ = 0; ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] = T_VARCHAR; ret_node.int16_values_[OB_NODE_CAST_COLL_IDX] = 0; ret_node.int32_values_[OB_NODE_CAST_C_LEN_IDX] = 4000; ret_node.length_semantics_ = 0; ret_node.is_hidden_const_ = 1; } else { + ret_node.value_ = 0; ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] = T_NUMBER; ret_node.int16_values_[OB_NODE_CAST_N_PREC_IDX] = -1; /* precision */ ret_node.int16_values_[OB_NODE_CAST_N_SCALE_IDX] = -85; /* scale */ @@ -855,24 +860,29 @@ int ObJsonPath::change_json_expr_res_type_if_need(common::ObIAllocator &allocato case JPN_TYPE: case JPN_STR_ONLY : case JPN_STRING : { - ret_node.type_ = T_CAST_ARGUMENT; - ret_node.value_ = 0; - ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] = T_VARCHAR; - ret_node.int16_values_[OB_NODE_CAST_COLL_IDX] = 0; - ret_node.int32_values_[OB_NODE_CAST_C_LEN_IDX] = 4000; - ret_node.length_semantics_ = 0; - ret_node.is_hidden_const_ = 1; + if (json_expr_flag == OPT_JSON_QUERY && ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] == T_JSON) { + } else { + ret_node.type_ = T_CAST_ARGUMENT; + ret_node.value_ = 0; + ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] = T_VARCHAR; + ret_node.int16_values_[OB_NODE_CAST_COLL_IDX] = 0; + ret_node.int32_values_[OB_NODE_CAST_C_LEN_IDX] = 4000; + ret_node.length_semantics_ = 0; + ret_node.is_hidden_const_ = 1; + } } case JPN_UPPER: case JPN_LOWER: { - ret_node.type_ = T_CAST_ARGUMENT; - ret_node.value_ = 0; - ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] = T_VARCHAR; - ret_node.int16_values_[OB_NODE_CAST_COLL_IDX] = 0; - ret_node.int32_values_[OB_NODE_CAST_C_LEN_IDX] = 75; - ret_node.length_semantics_ = 0; - ret_node.is_hidden_const_ = 1; - break; + if (json_expr_flag == OPT_JSON_QUERY && ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] == T_JSON) { + } else { + ret_node.type_ = T_CAST_ARGUMENT; + ret_node.value_ = 0; + ret_node.int16_values_[OB_NODE_CAST_TYPE_IDX] = T_VARCHAR; + ret_node.int16_values_[OB_NODE_CAST_COLL_IDX] = 0; + ret_node.int32_values_[OB_NODE_CAST_C_LEN_IDX] = 75; + ret_node.length_semantics_ = 0; + ret_node.is_hidden_const_ = 1; + } break; } default : { @@ -2123,7 +2133,7 @@ int ObJsonPath::parse_name_with_rapidjson(char*& str, uint64_t& len) } // if keyname without double quote, end with ' ', '.', '[', '*' -bool ObJsonPathUtil::is_mysql_terminator(char ch) +bool ObJsonPathUtil::is_key_name_terminator(char ch) { bool ret_bool = false; switch (ch) { @@ -2143,6 +2153,18 @@ bool ObJsonPathUtil::is_mysql_terminator(char ch) ret_bool = true; break; } + case '(': { + ret_bool = true; + break; + } + case '?': { + ret_bool = true; + break; + } + case '-': { + ret_bool = true; + break; + } default: { break; } @@ -2150,98 +2172,6 @@ bool ObJsonPathUtil::is_mysql_terminator(char ch) return ret_bool; } -// process JPN_MEMBER get keyname -// @param[in,out] name Keyname -// @param[in] is_quoted -// @return the error code. -int ObJsonPath::get_mysql_origin_key_name(char*& str, uint64_t& length, bool is_quoted) -{ - INIT_SUCC(ret); - uint64_t start = 0; - uint64_t end = 0; - - int len = expression_.length(); - - if (index_ < len) { - if (is_quoted) { - // with quote, check quote - if (expression_[index_] == ObJsonPathItem::DOUBLE_QUOTE) { - start = index_; - ++index_; - - while (index_ < len && end == 0) { - if (expression_[index_] == '\\') { - index_ += 2; - } else if (expression_[index_] == ObJsonPathItem::DOUBLE_QUOTE) { - end = index_; - ++index_; - } else { - ++index_; - } - } - - if (end == 0 && index_ == len) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("should end with DOUBLE_QUOTE!", K(ret), K(index_), K(expression_)); - } - } else { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("should start with DOUBLE_QUOTE!", K(ret), K(index_), K(expression_)); - } - } else { - start = index_; - while (index_ < len && end == 0) { - if (ObJsonPathUtil::is_mysql_terminator(expression_[index_])) { - end = index_ - 1; - } else { - ++index_; - } - } - if (index_ == len) { - end = index_ - 1; - } - } - } else { - ret = OB_ARRAY_OUT_OF_RANGE; - LOG_WARN("index out of range!", K(ret), K(index_), K(expression_)); - } - - if (OB_SUCC(ret)) { - if (end < start) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("get keyname: end (allocator_->alloc(length)); - if (OB_ISNULL(str)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to allocate memory for member_name.", - K(ret), K(len),KCSTRING(start_ptr)); - } else { - str[0] = ObJsonPathItem::DOUBLE_QUOTE; - MEMCPY(str + 1, start_ptr, len); - str[len + 1] = ObJsonPathItem::DOUBLE_QUOTE; - } - } else { - length = len; - str = static_cast (allocator_->alloc(length)); - if (OB_ISNULL(str)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to allocate memory for member_name.", - K(ret), K(len),KCSTRING(start_ptr)); - } else { - MEMCPY(str, start_ptr, len); - } - } - } - } - - return ret; -} - // parse JPN_MEMBER_WILDCARD // @return the error code. int ObJsonPath::parse_member_wildcard_node() @@ -2331,9 +2261,11 @@ int ObJsonPath::parse_mysql_member_node() char* name = nullptr; uint64_t name_len = 0; + bool is_func = false; + bool with_escape = false; // get name // add double quote for rapidjson requires - if (OB_FAIL(get_mysql_origin_key_name(name, name_len, is_quoted))) { + if (OB_FAIL(get_origin_key_name(name, name_len, is_quoted, is_func, with_escape))) { LOG_WARN("fail to get keyname!", K(ret), K(index_), K(expression_)); } else { if (OB_FAIL(parse_name_with_rapidjson(name, name_len))) { @@ -2431,6 +2363,31 @@ bool ObJsonPathUtil::is_scalar(const ObJsonPathNodeType node_type) return ret_bool; } +bool ObJsonPathUtil::is_escape(char ch) +{ + return (('\n' == ch) || (ch == '\t') || (ch == '\r') || (ch == '\f') || (ch == '\e')); +} +int ObJsonPathUtil::append_character_of_escape(ObJsonBuffer& buf, char ch) +{ + INIT_SUCC(ret); + if ('\n' == ch) { + ret = buf.append("n"); + } else if ('\t' == ch) { + ret = buf.append("t"); + } else if ('\r' == ch) { + ret = buf.append("r"); + } else if ('\f' == ch) { + ret = buf.append("f"); + } else if ('\e' == ch) { + ret = buf.append("e"); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("should be escape", K(ch), K(ret)); + } + return ret; +} + + void ObJsonPathUtil::skip_whitespace(const ObString &path, uint64_t& idx) { while (idx < path.length() && ObJsonPathUtil::is_whitespace(path[idx])) { @@ -3039,7 +2996,7 @@ bool ObJsonPathUtil::is_oracle_keyname(const char* name, uint64_t length) // @param[in,out] name Keyname // @param[in] is_quoted // @return the error code. -int ObJsonPath::get_oracle_origin_key_name(char*& str, uint64_t& length, bool is_quoted, bool& is_func) +int ObJsonPath::get_origin_key_name(char*& str, uint64_t& length, bool is_quoted, bool& is_func, bool& with_escape) { INIT_SUCC(ret); uint64_t start = 0; @@ -3060,6 +3017,9 @@ int ObJsonPath::get_oracle_origin_key_name(char*& str, uint64_t& length, bool is } else if (expression_[index_] == ObJsonPathItem::DOUBLE_QUOTE) { end = index_; ++index_; + } else if (ObJsonPathUtil::is_escape(expression_[index_])) { + with_escape = true; + ++index_; } else { ++index_; } @@ -3074,35 +3034,28 @@ int ObJsonPath::get_oracle_origin_key_name(char*& str, uint64_t& length, bool is LOG_WARN("should start with DOUBLE_QUOTE!", K(ret), K(index_), K(expression_)); } } else { - if (ObJsonPathUtil::letter_or_not(expression_[index_]) || (expression_[index_] == '_')) { - // without '""' - start = index_; - ++index_; - while (index_ < len) { - if (!(ObJsonPathUtil::letter_or_not(expression_[index_]) - || ObJsonPathUtil::is_digit(expression_[index_]) - || (expression_[index_] == '_'))) { + start = index_; + if (ObJsonPathUtil::is_digit(expression_[index_])) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("shouldn't start with number!", K(ret), K(index_), K(expression_)); + } else { + while (index_ < len && end == 0) { + if (ObJsonPathUtil::is_key_name_terminator(expression_[index_])) { end = index_ - 1; break; } else { ++index_; } } - if (index_ == len) { end = index_ - 1; } else { ObJsonPathUtil::skip_whitespace(expression_, index_); // fun_name + () - if (index_ < expression_.length()) { - if (expression_[index_] == '(') { + if (index_ < expression_.length() && expression_[index_] == '(') { is_func = true; - } - }//now, index could equal to len + } } - } else { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("wrong keyname!", K(ret), K(start), K(end), K(expression_)); } } } else { @@ -3115,11 +3068,11 @@ int ObJsonPath::get_oracle_origin_key_name(char*& str, uint64_t& length, bool is ret = OB_INVALID_ARGUMENT; LOG_WARN("get keyname: end (allocator_->alloc(length)); if (OB_ISNULL(str)) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -3182,6 +3135,31 @@ int ObJsonPath::parse_func_node(char*& name, uint64_t& len) return ret; } +int ObJsonPath::deal_with_escape(char* &str, uint64_t& len) +{ + INIT_SUCC(ret); + ObJsonBuffer buf(allocator_); + for (int i = 0; i < len && OB_SUCC(ret); ++i) { + char* tmp = str + i; + if (OB_ISNULL(tmp)) { + } else if (ObJsonPathUtil::is_escape(*tmp)) { + if (OB_FAIL(buf.append("\\"))) { + LOG_WARN("fail to append \\.", K(i), K(ret)); + } else if (OB_FAIL(ObJsonPathUtil::append_character_of_escape(buf, *tmp))) { + LOG_WARN("fail to append_character_of_escape.", K(*tmp), K(i), K(ret)); + } + } else { + ret = buf.append(tmp, 1); + } + } + + if (OB_SUCC(ret)) { + str = buf.ptr(); + len = buf.length(); + } + return ret; +} + // parse JPN_ORACLE_MEMBER // @return the error code. int ObJsonPath::parse_oracle_member_node() @@ -3191,6 +3169,7 @@ int ObJsonPath::parse_oracle_member_node() if (index_ < expression_.length()) { bool is_quoted = false; bool is_func = false; + bool with_escape = false; // check double quote if (expression_[index_] == ObJsonPathItem::DOUBLE_QUOTE) is_quoted = true; @@ -3199,12 +3178,19 @@ int ObJsonPath::parse_oracle_member_node() uint64_t name_len = 0; // get name // add double quote for rapidjson requires - if (OB_FAIL(get_oracle_origin_key_name(name, name_len, is_quoted, is_func))) { + if (OB_FAIL(get_origin_key_name(name, name_len, is_quoted, is_func, with_escape))) { LOG_WARN("fail to get keyname!", K(ret), K(index_), K(expression_)); } else if (is_quoted || (!is_func)) { - // with "", must be member node - if (OB_FAIL(parse_name_with_rapidjson(name, name_len))) { - LOG_WARN("fail to parse name with rapidjson",K(ret), K(index_), K(expression_),K(name)); + if (lib::is_oracle_mode() && with_escape && OB_FAIL(ObJsonPath::deal_with_escape(name, name_len))) { + LOG_WARN("fail to deal escape!", K(ret), K(index_), K(expression_)); + } else if (OB_FAIL(parse_name_with_rapidjson(name, name_len))) { + LOG_WARN("fail to parse name with rapidjson", + K(ret), K(index_), K(expression_),KCSTRING(name)); + } else if (!is_quoted) { + if (!ObJsonPathUtil::is_ecmascript_identifier(name, name_len)) { + LOG_WARN("the key name isn't ECMAScript identifier!", + K(ret), KCSTRING(name)); + } } if (OB_SUCC(ret)) { @@ -3932,7 +3918,8 @@ int ObJsonPath::parse_comp_string_num(ObJsonPathFilterNode* filter_comp_node, bo char* str = nullptr; uint64_t name_len = 0; bool is_func = false; - if (OB_FAIL(get_oracle_origin_key_name(str, name_len, true, is_func))) { + bool with_escape = false; + if (OB_FAIL(get_origin_key_name(str, name_len, true, is_func, with_escape))) { LOG_WARN("fail to get string scalar",K(ret), K(index_), K(expression_),K(str)); } else { if (OB_ISNULL(str)) { diff --git a/deps/oblib/src/lib/json_type/ob_json_path.h b/deps/oblib/src/lib/json_type/ob_json_path.h index 8c487e1285..20d5e93671 100644 --- a/deps/oblib/src/lib/json_type/ob_json_path.h +++ b/deps/oblib/src/lib/json_type/ob_json_path.h @@ -359,8 +359,8 @@ private: bool& from_end1, bool& from_end2); int add_single_array_node(bool is_cell_type, uint64_t& index1, uint64_t& index2, bool& from_end1, bool& from_end2); - int get_mysql_origin_key_name(char* &str, uint64_t& length, bool is_quoted); - int get_oracle_origin_key_name(char* &str, uint64_t& length, bool is_quoted, bool& is_func); + int get_origin_key_name(char* &str, uint64_t& length, bool is_quoted, bool& is_func, bool& with_escape); + int deal_with_escape(char* &str, uint64_t& length); int parse_name_with_rapidjson(char*& str, uint64_t& len); int parse_func_node(char*& name, uint64_t& len); int get_char_comparison_type(ObJsonPathFilterNode* filter_comp_node); @@ -455,7 +455,7 @@ public: static bool is_oracle_keyname(const char* name, uint64_t length); // add quote and static int double_quote(ObString &name, ObJsonBuffer* tmp_name); - static bool is_mysql_terminator(char ch); + static bool is_key_name_terminator(char ch); static bool is_begin_field_name(char ch); static bool is_end_of_comparission(char ch); static bool letter_or_not(char ch); @@ -480,6 +480,8 @@ public: bool auto_wrap); static bool is_letter(unsigned codepoint, const char* ori, uint64_t start, uint64_t end); + static bool is_escape(char ch); + static int append_character_of_escape(ObJsonBuffer& buf, char ch); static bool is_connector_punctuation(unsigned codepoint); static bool unicode_combining_mark(unsigned codepoint); static bool is_utf8_unicode_charator(const char* ori, uint64_t& start, int64_t len); diff --git a/deps/oblib/src/lib/mysqlclient/ob_tenant_oci_envs.h b/deps/oblib/src/lib/mysqlclient/ob_tenant_oci_envs.h index f33004e754..e1c201f520 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_tenant_oci_envs.h +++ b/deps/oblib/src/lib/mysqlclient/ob_tenant_oci_envs.h @@ -22,6 +22,7 @@ namespace sqlclient class ObTenantOciEnvs { public: + static int mtl_new(ObTenantOciEnvs *&tenant_oci_envs) { return OB_SUCCESS; } static int mtl_init(ObTenantOciEnvs *&tenant_oci_envs) { return OB_SUCCESS; } static void mtl_destroy(ObTenantOciEnvs *&tenant_oci_envs) { } }; @@ -30,4 +31,4 @@ public: } //sqlclient } // namespace common } // namespace oceanbase -#endif //OBTENANOCIENVS_H \ No newline at end of file +#endif //OBTENANOCIENVS_H diff --git a/deps/oblib/src/lib/ob_define.h b/deps/oblib/src/lib/ob_define.h index 6492b709c2..f2cc1425b2 100644 --- a/deps/oblib/src/lib/ob_define.h +++ b/deps/oblib/src/lib/ob_define.h @@ -1670,7 +1670,7 @@ OB_INLINE bool is_bootstrap_resource_pool(const uint64_t resource_pool_id) const int64_t OB_MALLOC_NORMAL_BLOCK_SIZE = (1LL << 13) - 256; // 8KB const int64_t OB_MALLOC_MIDDLE_BLOCK_SIZE = (1LL << 16) - 128; // 64KB const int64_t OB_MALLOC_BIG_BLOCK_SIZE = (1LL << 21) - ACHUNK_PRESERVE_SIZE;// 2MB (-17KB) -const int64_t OB_MALLOC_REQ_NORMAL_BLOCK_SIZE = (256LL << 10); // 256KB +const int64_t OB_MALLOC_REQ_NORMAL_BLOCK_SIZE = (240LL << 10); // 240KB const int64_t OB_MAX_MYSQL_RESPONSE_PACKET_SIZE = OB_MALLOC_BIG_BLOCK_SIZE; diff --git a/deps/oblib/src/lib/oblog/ob_async_log_struct.h b/deps/oblib/src/lib/oblog/ob_async_log_struct.h index ddc5bd3022..98ea697f21 100644 --- a/deps/oblib/src/lib/oblog/ob_async_log_struct.h +++ b/deps/oblib/src/lib/oblog/ob_async_log_struct.h @@ -118,8 +118,6 @@ public: int64_t file_size_; struct stat stat_; struct stat wf_stat_; - std::deque file_list_;//to store the names of log-files - std::deque wf_file_list_;//to store the names of warning log-files }; } // common diff --git a/deps/oblib/src/lib/oblog/ob_log.cpp b/deps/oblib/src/lib/oblog/ob_log.cpp index 87ce50061b..e008822668 100644 --- a/deps/oblib/src/lib/oblog/ob_log.cpp +++ b/deps/oblib/src/lib/oblog/ob_log.cpp @@ -743,7 +743,7 @@ void ObLogger::rotate_log(const int64_t size, const bool redirect_flag, if (OB_LIKELY(size > 0) && max_file_size_ > 0 && log_struct.file_size_ >= max_file_size_) { if (OB_LIKELY(0 == pthread_mutex_trylock(&file_size_mutex_))) { rotate_log(log_struct.filename_, fd_type, redirect_flag, log_struct.fd_, - log_struct.wf_fd_, log_struct.file_list_, log_struct.wf_file_list_); + log_struct.wf_fd_, file_list_, wf_file_list_); (void)ATOMIC_SET(&log_struct.file_size_, 0); if (fd_type <= FD_ELEC_FILE) { (void)log_new_file_info(log_struct); @@ -1255,17 +1255,17 @@ int ObLogger::record_old_log_file() int ret = OB_SUCCESS; if (max_file_index_ <= 0 || !rec_old_file_flag_) { } else { + int tmp_ret = OB_SUCCESS; + ObSEArray files; + ObSEArray wf_files; for (int type = FD_SVR_FILE; type < FD_AUDIT_FILE; ++type) { - ObSEArray files; - ObSEArray wf_files; - if (OB_FAIL(get_log_files_in_dir(log_file_[type].filename_, &files, &wf_files))) { + if (OB_TMP_FAIL(get_log_files_in_dir(log_file_[type].filename_, &files, &wf_files))) { OB_LOG(WARN, "Get log files in log dir error", K(ret)); - } else if (OB_FAIL(add_files_to_list(&files, &wf_files, log_file_[type].file_list_, log_file_[type].wf_file_list_))) { - OB_LOG(WARN, "Add files to list error", K(ret)); - } else { - // do nothing } } + if (OB_FAIL(add_files_to_list(&files, &wf_files, file_list_, wf_file_list_))) { + OB_LOG(WARN, "Add files to list error", K(ret)); + } } return ret; } @@ -1348,6 +1348,28 @@ int ObLogger::get_log_files_in_dir(const char *filename, void *files, void *wf_f return ret; } +int compare_log_filename_by_date_suffix(const void *v1, const void *v2) +{ + int ret = 0; + if (NULL == v1) { + ret = -1; + } else if (NULL == v2) { + ret = 1; + } else { + const int DATE_LENGTH = 17; + const char *str1 = static_cast(v1); + const char *str2 = static_cast(v2); + if (strlen(str1) < DATE_LENGTH) { + ret = -1; + } else if (strlen(str2) < DATE_LENGTH) { + ret = 1; + } else { + ret = str_cmp(str1 + strlen(str1) - DATE_LENGTH, str2 + strlen(str2) - DATE_LENGTH); + } + } + return ret; +} + int ObLogger::add_files_to_list(void *files, void *wf_files, std::deque &file_list, @@ -1362,10 +1384,10 @@ int ObLogger::add_files_to_list(void *files, ObIArray *wf_files_arr = static_cast *>(wf_files); //sort files if (files_arr->count() > 0) { - qsort(&files_arr->at(0), files_arr->count(), sizeof(FileName), str_cmp); + qsort(&files_arr->at(0), files_arr->count(), sizeof(FileName), compare_log_filename_by_date_suffix); } if (wf_files_arr->count() > 0) { - qsort(&wf_files_arr->at(0), wf_files_arr->count(), sizeof(FileName), str_cmp); + qsort(&wf_files_arr->at(0), wf_files_arr->count(), sizeof(FileName), compare_log_filename_by_date_suffix); } //Add to file_list diff --git a/deps/oblib/src/lib/oblog/ob_log.h b/deps/oblib/src/lib/oblog/ob_log.h index b4ec2dc24c..424a3f0a7c 100644 --- a/deps/oblib/src/lib/oblog/ob_log.h +++ b/deps/oblib/src/lib/oblog/ob_log.h @@ -865,6 +865,8 @@ private: // This info will be logged when log file created. const char *new_file_info_; bool info_as_wdiag_; + std::deque file_list_;//to store the names of log-files + std::deque wf_file_list_;//to store the names of warning log-files }; inline ObLogger& ObLogger::get_logger() diff --git a/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.cpp b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.cpp index 5b86ad2fe4..967cfcb92f 100644 --- a/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.cpp +++ b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.cpp @@ -23,6 +23,7 @@ #include "cos_auth.h" #include "cos_sys_util.h" #include "apr_errno.h" +#include "cos_crc64.h" #include "ob_cos_wrapper.h" @@ -38,6 +39,7 @@ constexpr int OB_INVALID_ARGUMENT = -4002; constexpr int OB_INIT_TWICE = -4005; constexpr int OB_ALLOCATE_MEMORY_FAILED = -4013; constexpr int OB_SIZE_OVERFLOW = -4019; +constexpr int OB_CHECKSUM_ERROR = -4103; constexpr int OB_BACKUP_FILE_NOT_EXIST = -9011; constexpr int OB_COS_ERROR = -9060; constexpr int OB_IO_LIMIT = -9061; @@ -814,6 +816,7 @@ int ObCosWrapper::pread( int64_t offset, char *buf, int64_t buf_size, + const bool is_range_read, int64_t &read_size) { int ret = OB_SUCCESS; @@ -849,42 +852,57 @@ int ObCosWrapper::pread( const char* const COS_RANGE_KEY = "Range"; char range_size[COS_RANGE_SIZE]; - // Cos read range of [10, 100] include the start offset 10 and the end offset 100. - // But what we except is [10, 100) which does not include the end. - // So we subtract 1 from the end. - int n = snprintf(range_size, COS_RANGE_SIZE, "bytes=%ld-%ld", offset, offset + buf_size - 1); - if (0 >= n || COS_RANGE_SIZE <= n) { - ret = OB_SIZE_OVERFLOW; - cos_warn_log("[COS]fail to format range size,n=%d, ret=%d\n", n, ret); - } else if (NULL == (headers = cos_table_make(ctx->mem_pool, 1))) { + if (NULL == (headers = cos_table_make(ctx->mem_pool, 1))) { ret = OB_ALLOCATE_MEMORY_FAILED; cos_warn_log("[COS]fail to make cos headers, ret=%d\n", ret); } else { - apr_table_set(headers, COS_RANGE_KEY, range_size); + if (is_range_read) { + // Cos read range of [10, 100] include the start offset 10 and the end offset 100. + // But what we except is [10, 100) which does not include the end. + // So we subtract 1 from the end. + int n = snprintf(range_size, COS_RANGE_SIZE, "bytes=%ld-%ld", offset, offset + buf_size - 1); + if (0 >= n || COS_RANGE_SIZE <= n) { + ret = OB_SIZE_OVERFLOW; + cos_warn_log("[COS]fail to format range size,n=%d, ret=%d\n", n, ret); + } else { + apr_table_set(headers, COS_RANGE_KEY, range_size); + } + } - if (NULL == (cos_ret = cos_get_object_to_buffer(ctx->options, &bucket, &object, headers, NULL, &buffer, &resp_headers)) || - !cos_status_is_ok(cos_ret)) { + if (OB_SUCCESS != ret) { + } else if (NULL == (cos_ret = cos_get_object_to_buffer(ctx->options, &bucket, &object, headers, NULL, &buffer, &resp_headers)) || + !cos_status_is_ok(cos_ret)) { convert_io_error(cos_ret, ret); cos_warn_log("[COS]fail to get object to buffer, ret=%d\n", ret); log_status(cos_ret); } else { + read_size = 0; int64_t size = 0; int64_t buf_pos = 0; cos_buf_t *content = NULL; + int64_t needed_size = -1; cos_list_for_each_entry(cos_buf_t, content, &buffer, node) { size = cos_buf_size(content); - if (buf_pos + size > buf_size) { + needed_size = size; + if (buf_size - buf_pos < size) { + needed_size = buf_size - buf_pos; + } + if (is_range_read && (buf_pos + size > buf_size)) { ret = OB_COS_ERROR; cos_warn_log("[COS]unexpected error, too much data returned, ret=%d, range_size=%s, buf_pos=%ld, size=%ld, req_id=%s.\n", ret, range_size, buf_pos, size, cos_ret->req_id); log_status(cos_ret); break; } else { // copy to buf - memcpy(buf + buf_pos, content->pos, (size_t)size); - buf_pos += size; - read_size += size; + memcpy(buf + buf_pos, content->pos, (size_t)needed_size); + buf_pos += needed_size; + read_size += needed_size; + + if (buf_pos >= buf_size) { + break; + } } - } + } // end cos_list_for_each_entry } } } @@ -900,62 +918,8 @@ int ObCosWrapper::get_object( int64_t buf_size, int64_t &read_size) { - int ret = OB_SUCCESS; - - CosContext *ctx = reinterpret_cast(h); - read_size = 0; - - if (NULL == h) { - ret = OB_INVALID_ARGUMENT; - cos_warn_log("[COS]cos handle is null, ret=%d\n", ret); - } else if (bucket_name.empty()) { - ret = OB_INVALID_ARGUMENT; - cos_warn_log("[COS]bucket name is null, ret=%d\n", ret); - } else if (object_name.empty()) { - ret = OB_INVALID_ARGUMENT; - cos_warn_log("[COS]object name is null, ret=%d\n", ret); - } else if (NULL == buf || 0 >= buf_size) { - ret = OB_INVALID_ARGUMENT; - cos_warn_log("[COS]buffer is null, ret=%d\n", ret); - } else { - cos_string_t bucket; - cos_string_t object; - cos_str_set(&bucket, bucket_name.data_); - cos_str_set(&object, object_name.data_); - cos_table_t *headers = NULL; - cos_table_t *resp_headers = NULL; - cos_status_t *cos_ret = NULL; - - cos_list_t buffer; - cos_list_init(&buffer); - - if (NULL == (cos_ret = cos_get_object_to_buffer(ctx->options, &bucket, &object, headers, NULL, &buffer, &resp_headers)) || - !cos_status_is_ok(cos_ret)) { - convert_io_error(cos_ret, ret); - cos_warn_log("[COS]fail to get object to buffer, ret=%d\n", ret); - log_status(cos_ret); - } else { - int64_t size = 0; - int64_t buf_pos = 0; - cos_buf_t *content = NULL; - cos_list_for_each_entry(cos_buf_t, content, &buffer, node) { - size = cos_buf_size(content); - if (buf_pos + size > buf_size) { - ret = OB_COS_ERROR; - cos_warn_log("[COS]unexpected error, too much data returned, ret=%d, buf_pos=%ld, size=%ld, buf_size=%ld, req_id=%s.\n", ret, buf_pos, size, buf_size, cos_ret->req_id); - log_status(cos_ret); - break; - } else { - // copy to buf - memcpy(buf + buf_pos, content->pos, (size_t)size); - buf_pos += size; - read_size += size; - } - } - } - } - - return ret; + return ObCosWrapper::pread(h, bucket_name, object_name, + 0, buf, buf_size, false/*is_range_read*/, read_size); } int ObCosWrapper::is_object_tagging( @@ -1444,7 +1408,8 @@ int ObCosWrapper::upload_part_from_buffer( const CosStringBuffer &upload_id_str, const int part_num, const char *buf, - const int64_t buf_size) + const int64_t buf_size, + uint64_t &total_crc) { int ret = OB_SUCCESS; CosContext *ctx = reinterpret_cast(h); @@ -1488,6 +1453,9 @@ int ObCosWrapper::upload_part_from_buffer( convert_io_error(cos_ret, ret); cos_warn_log("[COS]fail to upload part to cos, ret=%d, part_num=%d\n", ret, part_num); log_status(cos_ret); + } else { + // TODO @fangdan: supports parallel uploads + total_crc = cos_crc64(total_crc, (void *)buf, buf_size); } } } @@ -1498,7 +1466,8 @@ int ObCosWrapper::complete_multipart_upload( Handle *h, const CosStringBuffer &bucket_name, const CosStringBuffer &object_name, - const CosStringBuffer &upload_id_str) + const CosStringBuffer &upload_id_str, + const uint64_t total_crc) { int ret = OB_SUCCESS; CosContext *ctx = reinterpret_cast(h); @@ -1575,6 +1544,16 @@ int ObCosWrapper::complete_multipart_upload( convert_io_error(cos_ret, ret); cos_warn_log("[COS]fail to complete multipart upload, ret=%d\n", ret); log_status(cos_ret); + } else { + const char *expected_crc_str = (const char *)(apr_table_get(resp_headers, COS_HASH_CRC64_ECMA)); + if (NULL != expected_crc_str) { + uint64_t expected_crc = cos_atoui64(expected_crc_str); + if (expected_crc != total_crc) { + ret = OB_CHECKSUM_ERROR; + cos_warn_log("[COS]multipart object crc is not equal to returned crc, ret=%d, upload_id=%s, total_crc=%llu, expected_crc=%llu\n", + ret, upload_id_str.data_, total_crc, expected_crc); + } + } } } diff --git a/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.h b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.h index b035bbe176..0472defcd2 100644 --- a/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.h +++ b/deps/oblib/src/lib/restore/cos/ob_cos_wrapper.h @@ -269,6 +269,7 @@ public: int64_t offset, char *buf, int64_t buf_size, + const bool is_range_read, int64_t &read_size); // Get whole object @@ -372,13 +373,15 @@ public: const CosStringBuffer &upload_id_str, const int part_num, /*the sequence number of this part, [1, 10000]*/ const char *buf, - const int64_t buf_size); + const int64_t buf_size, + uint64_t &total_crc); static int complete_multipart_upload( Handle *h, const CosStringBuffer &bucket_name, const CosStringBuffer &object_name, - const CosStringBuffer &upload_id_str); + const CosStringBuffer &upload_id_str, + const uint64_t total_crc); static int abort_multipart_upload( Handle *h, diff --git a/deps/oblib/src/lib/restore/ob_storage_cos_base.cpp b/deps/oblib/src/lib/restore/ob_storage_cos_base.cpp index 2685d3fa28..58d0419fbe 100644 --- a/deps/oblib/src/lib/restore/ob_storage_cos_base.cpp +++ b/deps/oblib/src/lib/restore/ob_storage_cos_base.cpp @@ -786,13 +786,38 @@ int ObStorageCosReader::pread( ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "invalid argument", K(ret), KP(buf), K(buf_size), K(offset)); } else { - qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer( - handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length()); - qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer( - handle_.get_object_name().ptr(), handle_.get_object_name().length()); - if (OB_FAIL(qcloud_cos::ObCosWrapper::pread(handle_.get_ptr(), bucket_name, - object_name, offset, buf, buf_size, read_size))) { - OB_LOG(WARN, "fail to read object from cos", K(ret), KP(buf), K(buf_size), K(offset)); + // When is_range_read is true, it indicates that only a part of the data is read. + // When false, it indicates that the entire object is read + bool is_range_read = true; + int64_t get_data_size = buf_size; + if (has_meta_) { + if (file_length_ < offset) { + ret = OB_FILE_LENGTH_INVALID; + OB_LOG(WARN, "File lenth is invilid", K_(file_length), K(offset), + K(handle_.get_bucket_name()), K(handle_.get_object_name()), K(ret)); + } else { + get_data_size = MIN(buf_size, file_length_ - offset); + if (get_data_size == file_length_) { + // read entire object + is_range_read = false; + } + } + } + + if (OB_FAIL(ret)) { + } else if (get_data_size == 0) { + read_size = 0; + } else { + qcloud_cos::CosStringBuffer bucket_name = qcloud_cos::CosStringBuffer( + handle_.get_bucket_name().ptr(), handle_.get_bucket_name().length()); + qcloud_cos::CosStringBuffer object_name = qcloud_cos::CosStringBuffer( + handle_.get_object_name().ptr(), handle_.get_object_name().length()); + + if (OB_FAIL(qcloud_cos::ObCosWrapper::pread(handle_.get_ptr(), bucket_name, + object_name, offset, buf, get_data_size, is_range_read, read_size))) { + OB_LOG(WARN, "fail to read object from cos", K(ret), K(is_range_read), + KP(buf), K(buf_size), K(offset), K(get_data_size), K_(has_meta)); + } } } return ret; @@ -1026,7 +1051,8 @@ ObStorageCosMultiPartWriter::ObStorageCosMultiPartWriter() base_buf_pos_(0), upload_id_(NULL), partnum_(0), - file_length_(-1) + file_length_(-1), + total_crc_(0) {} ObStorageCosMultiPartWriter::~ObStorageCosMultiPartWriter() @@ -1084,6 +1110,7 @@ int ObStorageCosMultiPartWriter::open(const ObString &uri, common::ObObjectStora is_opened_ = true; base_buf_pos_ = 0; file_length_ = 0; + total_crc_ = 0; } } return ret; @@ -1152,7 +1179,7 @@ int ObStorageCosMultiPartWriter::write_single_part() qcloud_cos::CosStringBuffer upload_id_str = qcloud_cos::CosStringBuffer( upload_id_, strlen(upload_id_)); if (OB_FAIL(qcloud_cos::ObCosWrapper::upload_part_from_buffer(handle_.get_ptr(), bucket_name, - object_name, upload_id_str, partnum_, base_buf_, base_buf_pos_))) { + object_name, upload_id_str, partnum_, base_buf_, base_buf_pos_, total_crc_))) { OB_LOG(WARN, "fail to upload part to cos", K(ret), KP_(upload_id)); if (OB_TMP_FAIL(cleanup())) { OB_LOG(WARN, "fail to abort multiupload", K(ret), K(tmp_ret), KP_(upload_id)); @@ -1192,7 +1219,7 @@ int ObStorageCosMultiPartWriter::close() upload_id_, strlen(upload_id_)); if (OB_FAIL(qcloud_cos::ObCosWrapper::complete_multipart_upload(handle_.get_ptr(), bucket_name, - object_name, upload_id_str))) { + object_name, upload_id_str, total_crc_))) { OB_LOG(WARN, "fail to complete multipart upload", K(ret), KP_(upload_id)); if (OB_TMP_FAIL(cleanup())) { OB_LOG(WARN, "fail to abort multiupload", K(ret), K(tmp_ret), KP_(upload_id)); diff --git a/deps/oblib/src/lib/restore/ob_storage_cos_base.h b/deps/oblib/src/lib/restore/ob_storage_cos_base.h index 5d2caff698..95753fdfea 100644 --- a/deps/oblib/src/lib/restore/ob_storage_cos_base.h +++ b/deps/oblib/src/lib/restore/ob_storage_cos_base.h @@ -191,6 +191,7 @@ private: char *upload_id_; int partnum_; int64_t file_length_; + uint64_t total_crc_; DISALLOW_COPY_AND_ASSIGN(ObStorageCosMultiPartWriter); }; diff --git a/deps/oblib/src/lib/restore/ob_storage_oss_base.cpp b/deps/oblib/src/lib/restore/ob_storage_oss_base.cpp index e3a5778d76..f2b1f88ea6 100644 --- a/deps/oblib/src/lib/restore/ob_storage_oss_base.cpp +++ b/deps/oblib/src/lib/restore/ob_storage_oss_base.cpp @@ -17,6 +17,7 @@ #include "common/ob_string_buf.h" #include "apr_errno.h" #include "ob_storage.h" +#include "aos_crc64.h" #include "lib/hash/ob_hashset.h" #include "lib/utility/ob_tracepoint.h" @@ -643,7 +644,8 @@ ObStorageOssMultiPartWriter::ObStorageOssMultiPartWriter() object_(), partnum_(0), is_opened_(false), - file_length_(-1) + file_length_(-1), + total_crc_(0) { upload_id_.len = -1; upload_id_.data = NULL; @@ -697,6 +699,7 @@ int ObStorageOssMultiPartWriter::open(const ObString &uri, common::ObObjectStora is_opened_ = true; base_buf_pos_ = 0; file_length_ = 0; + total_crc_ = 0; } } } @@ -790,6 +793,9 @@ int ObStorageOssMultiPartWriter::write_single_part() K_(base_buf_pos), K_(bucket), K_(object), K(ret)); print_oss_info(resp_headers, aos_ret); cleanup(); + } else { + // TODO @fangdan: supports parallel uploads + total_crc_ = aos_crc64(total_crc_, base_buf_, base_buf_pos_); } bool is_slow = false; print_access_storage_log("oss upload one part ", object_, start_time, base_buf_pos_, &is_slow); @@ -894,6 +900,16 @@ int ObStorageOssMultiPartWriter::close() OB_LOG(WARN, "fail to complete multipart upload", K_(bucket), K_(object), K(ret)); print_oss_info(resp_headers, aos_ret); cleanup(); + } else { + const char *expected_crc_str = (const char *)(apr_table_get(resp_headers, OSS_HASH_CRC64_ECMA)); + if (OB_NOT_NULL(expected_crc_str)) { + uint64_t expected_crc = aos_atoui64(expected_crc_str); + if (OB_UNLIKELY(expected_crc != total_crc_)) { + ret = OB_CHECKSUM_ERROR; + OB_LOG(WARN, "multipart object crc is not equal to returned crc", + K(ret), K_(total_crc), K(expected_crc)); + } + } } } } @@ -1011,15 +1027,29 @@ int ObStorageOssReader::pread( ret = OB_INVALID_ARGUMENT; OB_LOG(WARN, "aos pool or oss option is NULL", K(aos_pool), K(oss_option), K(ret)); } else { - if (has_meta_ && file_length_ == offset) { + // When is_range_read is true, it indicates that only a part of the data is read. + // When false, it indicates that the entire object is read + bool is_range_read = true; + int64_t get_data_size = buf_size; + if (has_meta_) { + if (file_length_ < offset) { + ret = OB_FILE_LENGTH_INVALID; + OB_LOG(WARN, "File lenth is invilid", K_(file_length), + K(offset), K_(bucket), K_(object), K(ret)); + } else { + get_data_size = MIN(buf_size, file_length_ - offset); + if (get_data_size == file_length_) { + // read entire object + is_range_read = false; + } + } + } + + if (OB_FAIL(ret)) { + } else if (get_data_size == 0) { read_size = 0; - } else if (has_meta_ && file_length_ < offset) { - ret = OB_FILE_LENGTH_INVALID; - OB_LOG(WARN, "File lenth is invilid", K(file_length_), K(offset), - K(bucket_), K(object_), K(ret)); } else { const int64_t start_time = ObTimeUtility::current_time(); - int64_t get_data_size = buf_size; aos_string_t bucket; aos_string_t object; aos_str_set(&bucket, bucket_.ptr()); @@ -1037,49 +1067,61 @@ int ObStorageOssReader::pread( const char *const OSS_RANGE_KEY = "Range"; char range_size[OSS_RANGE_SIZE]; - //oss read size is [10, 100] include the 10 and 100 bytes - //we except is [10, 100) not include the end, so we subtraction 1 to the end - int n = snprintf(range_size, OSS_RANGE_SIZE, "bytes=%ld-%ld", offset, offset + get_data_size - 1); - if(n <=0 || n >= OSS_RANGE_SIZE) { - ret = OB_SIZE_OVERFLOW; - OB_LOG(WARN, "fail to get range size", K(n), K(OSS_RANGE_SIZE), K(ret)); - } else if (NULL == (headers = aos_table_make(aos_pool, AOS_TABLE_INIT_SIZE))) { + if (NULL == (headers = aos_table_make(aos_pool, AOS_TABLE_INIT_SIZE))) { ret = OB_OSS_ERROR; OB_LOG(WARN, "fail to make oss headers", K(ret)); } else { - apr_table_set(headers, OSS_RANGE_KEY, range_size); + if (is_range_read) { + // oss read size is [10, 100] include the 10 and 100 bytes + // we except is [10, 100) not include the end, so we subtraction 1 to the end + if (OB_FAIL(databuff_printf(range_size, OSS_RANGE_SIZE, "bytes=%ld-%ld", + offset, offset + get_data_size - 1))) { + OB_LOG(WARN, "fail to get range size", K(ret), + K(offset), K(get_data_size), K(OSS_RANGE_SIZE)); + } else { + apr_table_set(headers, OSS_RANGE_KEY, range_size); + } + } - if (NULL == (aos_ret = oss_get_object_to_buffer(oss_option, &bucket, &object, headers, params, + if (OB_FAIL(ret)) { + } else if (NULL == (aos_ret = oss_get_object_to_buffer(oss_option, &bucket, &object, headers, params, &buffer, &resp_headers)) || !aos_status_is_ok(aos_ret)) { - convert_io_error(aos_ret, ret); - OB_LOG(WARN, "fail to get object to buffer", K(bucket_), K(object_), K(ret)); + convert_io_error(aos_ret, ret); + OB_LOG(WARN, "fail to get object to buffer", K_(bucket), K_(object), K(ret)); print_oss_info(resp_headers, aos_ret); } else { //check date len aos_list_for_each_entry(aos_buf_t, content, &buffer, node) { len += aos_buf_size(content); } - if(len > get_data_size) { + // For appendable file, there may be data written in between the time when the reader is opened and pread is called. + // At this point, the actual size of the file could be larger than the determined file length at the time of opening. + // Therefore, if it's not a range read, the data read could exceed the expected amount to be read + if (is_range_read && len > get_data_size) { ret = OB_OSS_ERROR; - OB_LOG(WARN, "get data size error", K(get_data_size), K(len), K(bucket_), - K(object_), K(ret)); + OB_LOG(WARN, "get data size error", K(get_data_size), K(len), K_(bucket), + K_(object), K(ret), K_(has_meta), K(offset)); } else { //copy to buf + read_size = 0; + int64_t needed_size = -1; aos_list_for_each_entry(aos_buf_t, content, &buffer, node) { size = aos_buf_size(content); - if (buf_pos + size > get_data_size) { + needed_size = MIN(size, get_data_size - buf_pos); + if (is_range_read && (buf_pos + size > get_data_size)) { ret = OB_SIZE_OVERFLOW; OB_LOG(WARN, "the size is too long", K(buf_pos), K(size), K(get_data_size), K(ret)); break; } else { - memcpy(buf + buf_pos, content->pos, (size_t)size); - buf_pos += size; - } - } + memcpy(buf + buf_pos, content->pos, (size_t)needed_size); + buf_pos += needed_size; + read_size += needed_size; - if (OB_SUCC(ret)) { - read_size = len; - } + if (buf_pos >= get_data_size) { + break; + } + } + } // end aos_list_for_each_entry } } } diff --git a/deps/oblib/src/lib/restore/ob_storage_oss_base.h b/deps/oblib/src/lib/restore/ob_storage_oss_base.h index 53c8a74bc5..d5951514ad 100644 --- a/deps/oblib/src/lib/restore/ob_storage_oss_base.h +++ b/deps/oblib/src/lib/restore/ob_storage_oss_base.h @@ -186,6 +186,7 @@ private: int partnum_; bool is_opened_; int64_t file_length_; + uint64_t total_crc_; DISALLOW_COPY_AND_ASSIGN(ObStorageOssMultiPartWriter); }; diff --git a/deps/oblib/src/lib/restore/ob_storage_s3_base.cpp b/deps/oblib/src/lib/restore/ob_storage_s3_base.cpp index 6a617d85d5..586ad16ad9 100644 --- a/deps/oblib/src/lib/restore/ob_storage_s3_base.cpp +++ b/deps/oblib/src/lib/restore/ob_storage_s3_base.cpp @@ -1161,14 +1161,19 @@ int ObStorageS3Reader::pread_(char *buf, } else { Aws::S3::Model::GetObjectRequest request; Aws::S3::Model::GetObjectOutcome outcome; + const bool check_crc = (default_s3_crc_algo == ObStorageCRCAlgorithm::OB_CRC32_ALGO); + if (check_crc) { + request.SetChecksumMode(Aws::S3::Model::ChecksumMode::ENABLED); + } char range_read[64] = { 0 }; request.WithBucket(bucket_.ptr()).WithKey(object_.ptr()); request.SetAdditionalCustomHeaderValue("Connection", "keep-alive"); + const int64_t get_data_size = has_meta_ ? MIN(buf_size, file_length_ - offset) : buf_size; if (OB_FAIL(databuff_printf(range_read, sizeof(range_read), - "bytes=%ld-%ld", offset, offset + buf_size - 1))) { - OB_LOG(WARN, "fail to set range to read", - K(ret), K_(bucket), K_(object), K(offset), K(buf_size)); + "bytes=%ld-%ld", offset, offset + get_data_size - 1))) { + OB_LOG(WARN, "fail to set range to read", K(ret), + K_(bucket), K_(object), K(offset), K(buf_size), K_(has_meta), K_(file_length)); } else if (FALSE_IT(request.SetRange(range_read))) { } else if (OB_FAIL(s3_client_->get_object(request, outcome))) { OB_LOG(WARN, "failed to get s3 object", K(ret), K(range_read)); @@ -1952,6 +1957,7 @@ int ObStorageS3MultiPartWriter::close_() } // list parts + const bool check_crc = (default_s3_crc_algo == ObStorageCRCAlgorithm::OB_CRC32_ALGO); Aws::S3::Model::CompletedMultipartUpload completedMultipartUpload; int64_t part_num = 0; if (OB_SUCC(ret)) { @@ -1972,6 +1978,9 @@ int ObStorageS3MultiPartWriter::close_() for (int64_t i = 0; OB_SUCC(ret) && i < parts.size(); i++) { Aws::S3::Model::CompletedPart tmp_part; tmp_part.WithPartNumber(parts[i].GetPartNumber()).WithETag(parts[i].GetETag()); + if (check_crc) { + tmp_part.SetChecksumCRC32(parts[i].GetChecksumCRC32()); + } completedMultipartUpload.AddParts(std::move(tmp_part)); } } @@ -1984,7 +1993,7 @@ int ObStorageS3MultiPartWriter::close_() Aws::S3::Model::CompleteMultipartUploadRequest request; request.WithBucket(bucket_.ptr()).WithKey(object_.ptr()); request.WithUploadId(upload_id_).WithMultipartUpload(completedMultipartUpload); - const bool check_crc = (default_s3_crc_algo == ObStorageCRCAlgorithm::OB_CRC32_ALGO); + Aws::String complete_crc; if (check_crc) { if (OB_ISNULL(sum_hash_)) { @@ -2070,6 +2079,7 @@ int ObStorageS3MultiPartWriter::write_single_part_() ret = OB_ERR_UNEXPECTED; OB_LOG(WARN, "sum_hash should not be null", K(ret)); } else { + // TODO @fangdan: supports parallel uploads Aws::Utils::Crypto::CRC32 part_hash; request.SetChecksumAlgorithm(Aws::S3::Model::ChecksumAlgorithm::CRC32); Aws::String part_str(base_buf_, base_buf_pos_); diff --git a/deps/oblib/src/lib/restore/ob_storage_s3_base.h b/deps/oblib/src/lib/restore/ob_storage_s3_base.h index c79c86db21..c0b13382d3 100644 --- a/deps/oblib/src/lib/restore/ob_storage_s3_base.h +++ b/deps/oblib/src/lib/restore/ob_storage_s3_base.h @@ -63,7 +63,7 @@ int init_s3_env(); void fin_s3_env(); // default s3 checksum algorithm -static ObStorageCRCAlgorithm default_s3_crc_algo = ObStorageCRCAlgorithm::OB_INVALID_CRC_ALGO; +static ObStorageCRCAlgorithm default_s3_crc_algo = ObStorageCRCAlgorithm::OB_CRC32_ALGO; // set checksum algorithm for writing object into s3 void set_s3_checksum_algorithm(const ObStorageCRCAlgorithm crc_algo); // get current checksum algorithm diff --git a/deps/oblib/src/lib/thread/thread.cpp b/deps/oblib/src/lib/thread/thread.cpp index 859ddecd47..917b148a05 100644 --- a/deps/oblib/src/lib/thread/thread.cpp +++ b/deps/oblib/src/lib/thread/thread.cpp @@ -61,7 +61,8 @@ Thread::Thread(Threads *threads, int64_t idx, int64_t stack_size) tid_before_stop_(0), tid_(0), thread_list_node_(this), - cpu_time_(0) + cpu_time_(0), + create_ret_(OB_NOT_RUNNING) {} Thread::~Thread() @@ -98,9 +99,14 @@ int Thread::start() if (pret == 0) { stop_ = false; pret = pthread_create(&pth_, &attr, __th_start, this); + while (ATOMIC_LOAD(&create_ret_) == OB_NOT_RUNNING) { + sched_yield(); + } if (pret != 0) { LOG_ERROR("pthread create failed", K(pret), K(errno)); pth_ = 0; + } else if (OB_FAIL(create_ret_)) { + LOG_ERROR("thread create failed", K(create_ret_)); } } if (0 != pret) { @@ -320,6 +326,7 @@ void* Thread::__th_start(void *arg) WITH_CONTEXT(*mem_context) { try { in_try_stmt = true; + ATOMIC_STORE(&th->create_ret_, OB_SUCCESS); th->run(); in_try_stmt = false; } catch (OB_BASE_EXCEPTION &except) { @@ -327,6 +334,10 @@ void* Thread::__th_start(void *arg) _LOG_ERROR("Exception caught!!! errno = %d, exception info = %s", except.get_errno(), except.what()); ret = OB_ERR_UNEXPECTED; in_try_stmt = false; + if (1 == th->threads_->get_thread_count() && !th->has_set_stop()) { + LOG_WARN("thread exit by itself without set_stop", K(ret)); + th->threads_->stop(); + } } } } @@ -335,7 +346,6 @@ void* Thread::__th_start(void *arg) } } } - ATOMIC_FAA(&total_thread_count_, -1); return nullptr; } diff --git a/deps/oblib/src/lib/thread/thread.h b/deps/oblib/src/lib/thread/thread.h index d7c4329148..e8f0fa0256 100644 --- a/deps/oblib/src/lib/thread/thread.h +++ b/deps/oblib/src/lib/thread/thread.h @@ -186,6 +186,7 @@ private: int64_t tid_; ThreadListNode thread_list_node_; int64_t cpu_time_; + int create_ret_; }; OB_INLINE bool Thread::has_set_stop() const diff --git a/deps/oblib/src/lib/thread/threads.h b/deps/oblib/src/lib/thread/threads.h index eef1fb39d7..b45c97352f 100644 --- a/deps/oblib/src/lib/thread/threads.h +++ b/deps/oblib/src/lib/thread/threads.h @@ -103,8 +103,8 @@ public: } return pth; } -protected: int64_t get_thread_count() const { return n_threads_; } +protected: uint64_t get_thread_idx() const { return thread_idx_; } void set_thread_idx(int64_t idx) { thread_idx_ = idx; } diff --git a/deps/oblib/src/lib/trace/ob_trace_event.h b/deps/oblib/src/lib/trace/ob_trace_event.h index 2786229a0f..791b832d89 100644 --- a/deps/oblib/src/lib/trace/ob_trace_event.h +++ b/deps/oblib/src/lib/trace/ob_trace_event.h @@ -62,7 +62,7 @@ public: int64_t total_time = 0; const char* event_name = NULL; const ObString nls_format; - for (int64_t i = 0; i < this->next_idx_; ++i) { + for (int64_t i = 0; i < MIN(EVENT_COUNT, this->next_idx_); ++i) { const ObTraceEvent &ev = this->events_[i]; event_name = NAME(ev.id_); if (prev_ts == 0) { diff --git a/deps/oblib/src/lib/utility/ob_print_utils.cpp b/deps/oblib/src/lib/utility/ob_print_utils.cpp index 3f4ebfdadb..7f79ae9296 100644 --- a/deps/oblib/src/lib/utility/ob_print_utils.cpp +++ b/deps/oblib/src/lib/utility/ob_print_utils.cpp @@ -291,11 +291,6 @@ const char *to_cstring(const int64_t &v) return to_cstring(v, BoolType()); } -const char *to_cstring(const int64_t v) -{ - return to_cstring(v, BoolType()); -} - //////////////////////////////////////////////////////////////// int databuff_printf(char *buf, const int64_t buf_len, const char *fmt, ...) diff --git a/deps/oblib/src/lib/utility/ob_print_utils.h b/deps/oblib/src/lib/utility/ob_print_utils.h index 10ac1a089b..5a5ef9f98e 100644 --- a/deps/oblib/src/lib/utility/ob_print_utils.h +++ b/deps/oblib/src/lib/utility/ob_print_utils.h @@ -272,8 +272,6 @@ const char *to_cstring(const char *const &str); template <> const char *to_cstring(const int64_t &v); -const char *to_cstring(const int64_t v); - template const char *to_cstring(T *obj) { diff --git a/deps/oblib/src/lib/utility/ob_tracepoint.h b/deps/oblib/src/lib/utility/ob_tracepoint.h index 651fad0305..7d90ee278d 100644 --- a/deps/oblib/src/lib/utility/ob_tracepoint.h +++ b/deps/oblib/src/lib/utility/ob_tracepoint.h @@ -638,6 +638,8 @@ class EventTable EN_ENABLE_AUTO_DOP_FORCE_PARALLEL_PLAN = 552, EN_GENERATE_PLAN_WITH_RECONSTRUCT_SQL = 553, EN_GENERATE_PLAN_WITH_NLJ = 554, + //EN_CHECK_OPERATOR_OUTPUT_ROWS = 555, + //EN_GENERATE_RANDOM_PLAN = 556, // 600-700 For PX use EN_PX_SQC_EXECUTE_FAILED = 600, @@ -804,6 +806,7 @@ class EventTable EN_ENABLE_VECTOR_CAST = 2207, EN_DISABLE_SORTKEY_SEPARATELY = 2208, EN_ENABLE_VECTOR_IN = 2209, + EN_SQL_MEMORY_MRG_OPTION = 2210, // WR && ASH EN_CLOSE_ASH = 2301, EN_DISABLE_HASH_BASE_DISTINCT = 2302, diff --git a/deps/oblib/src/lib/utility/ob_unify_serialize.cpp b/deps/oblib/src/lib/utility/ob_unify_serialize.cpp index 65c143fa9b..b497bdd7de 100644 --- a/deps/oblib/src/lib/utility/ob_unify_serialize.cpp +++ b/deps/oblib/src/lib/utility/ob_unify_serialize.cpp @@ -22,19 +22,19 @@ void begin_record_serialization() { ser_diag_record.count = 0; ser_diag_record.check_index = 0; - ser_diag_record.flag = 1; + ser_diag_record.flag = CHECK_STATUS_RECORDING; } void finish_record_serialization() { - ser_diag_record.flag = 0; + ser_diag_record.flag = CHECK_STATUS_WATING; } void begin_check_serialization() { ser_diag_record.check_index = 0; if (ser_diag_record.count > 0) { - ser_diag_record.flag = 2; + ser_diag_record.flag = CHECK_STATUS_COMPARING; } } @@ -42,8 +42,9 @@ void finish_check_serialization() { ser_diag_record.count = -1; ser_diag_record.check_index = -1; - ser_diag_record.flag = 0; + ser_diag_record.flag = CHECK_STATUS_WATING; } + } // namespace lib } // namespace oceanbase #endif \ No newline at end of file diff --git a/deps/oblib/src/lib/utility/ob_unify_serialize.h b/deps/oblib/src/lib/utility/ob_unify_serialize.h index f00aa20fcb..537f01cf5d 100644 --- a/deps/oblib/src/lib/utility/ob_unify_serialize.h +++ b/deps/oblib/src/lib/utility/ob_unify_serialize.h @@ -21,13 +21,19 @@ namespace lib { #ifdef ENABLE_SERIALIZATION_CHECK +enum ObSerializationCheckStatus +{ + CHECK_STATUS_WATING = 0, + CHECK_STATUS_RECORDING = 1, + CHECK_STATUS_COMPARING = 2 +}; static constexpr int MAX_SERIALIZE_RECORD_LENGTH = 256; struct SerializeDiagnoseRecord { uint8_t encoded_lens[MAX_SERIALIZE_RECORD_LENGTH]; int count = -1; int check_index = -1; - int flag = 0; + int flag = CHECK_STATUS_WATING; }; RLOCAL_EXTERN(SerializeDiagnoseRecord, ser_diag_record); void begin_record_serialization(); @@ -78,21 +84,31 @@ void finish_check_serialization(); } #ifdef ENABLE_SERIALIZATION_CHECK + +#define IF_TYPE_MATCH(obj, type) \ + std::is_same::value || std::is_same::value + +#define IF_NEED_TO_CHECK_SERIALIZATION(obj) \ + !(std::is_const::value || \ + IF_TYPE_MATCH(obj, uint8_t) || \ + IF_TYPE_MATCH(obj, int8_t) || \ + IF_TYPE_MATCH(obj, bool) || \ + IF_TYPE_MATCH(obj, char)) + #define OB_UNIS_ADD_LEN(obj) \ { \ int64_t this_len = NS_::encoded_length(obj); \ - if (!(std::is_same::value || std::is_same::value || \ - std::is_same::value || std::is_same::value)) { \ - if (1 == oceanbase::lib::ser_diag_record.flag && \ + if (IF_NEED_TO_CHECK_SERIALIZATION(obj)) { \ + if (oceanbase::lib::CHECK_STATUS_RECORDING == oceanbase::lib::ser_diag_record.flag && \ oceanbase::lib::ser_diag_record.count < oceanbase::lib::MAX_SERIALIZE_RECORD_LENGTH) { \ oceanbase::lib::ser_diag_record.encoded_lens[oceanbase::lib::ser_diag_record.count++] = \ static_cast(this_len); \ - } else if (2 == oceanbase::lib::ser_diag_record.flag && \ + } else if (oceanbase::lib::CHECK_STATUS_COMPARING == oceanbase::lib::ser_diag_record.flag && \ oceanbase::lib::ser_diag_record.check_index < oceanbase::lib::ser_diag_record.count) { \ int ret = OB_ERR_UNEXPECTED; \ int record_len = oceanbase::lib::ser_diag_record.encoded_lens[oceanbase::lib::ser_diag_record.check_index]; \ if (this_len != record_len) { \ - OB_LOG(ERROR, "encoded length not match", "name", MSTR(obj), K(this_len), K(record_len), K(obj)); \ + OB_LOG(ERROR, "encoded length not match", "name", MSTR(obj), K(this_len), K(record_len), "value", obj); \ } \ oceanbase::lib::ser_diag_record.check_index++; \ } \ diff --git a/deps/oblib/src/rpc/frame/ob_req_transport.cpp b/deps/oblib/src/rpc/frame/ob_req_transport.cpp index 718c7838f6..9f0cdbc215 100644 --- a/deps/oblib/src/rpc/frame/ob_req_transport.cpp +++ b/deps/oblib/src/rpc/frame/ob_req_transport.cpp @@ -144,6 +144,7 @@ int async_cb(easy_request_t *r) if (!OB_SUCC(ret)) { LOG_WARN("process async request fail", K(r), K(ret), K(pcode)); } + THIS_WORKER.get_sql_arena_allocator().reset(); const int64_t cur_time = ObTimeUtility::current_time(); const int64_t total_time = cur_time - start_time; diff --git a/deps/oblib/src/rpc/obmysql/ob_sql_sock_session.h b/deps/oblib/src/rpc/obmysql/ob_sql_sock_session.h index ddfef9100f..fccaf3a041 100644 --- a/deps/oblib/src/rpc/obmysql/ob_sql_sock_session.h +++ b/deps/oblib/src/rpc/obmysql/ob_sql_sock_session.h @@ -30,6 +30,7 @@ public: ObSqlSessionMemPool(): pool_() {} virtual ~ObSqlSessionMemPool() {} void* alloc(int64_t sz) { return pool_.alloc(sz); } + void set_tenant_id(int64_t tenant_id) { pool_.set_tenant_id(tenant_id); } void reset() { pool_.destroy(); } void reuse() { pool_.reuse(); } private: diff --git a/deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.cpp b/deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.cpp index 8a2d66d6d1..115a19b82b 100644 --- a/deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.cpp +++ b/deps/oblib/src/rpc/obrpc/ob_poc_rpc_proxy.cpp @@ -157,6 +157,7 @@ int ObAsyncRespCallback::handle_resp(int io_err, const char* buf, int64_t sz) } pool_.destroy(); ObCurTraceId::reset(); + THIS_WORKER.get_sql_arena_allocator().reset(); const int64_t cur_time = ObTimeUtility::current_time(); const int64_t total_time = cur_time - start_time; const int64_t decode_time = after_decode_time - start_time; diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_mem_pool.h b/deps/oblib/src/rpc/obrpc/ob_rpc_mem_pool.h index e75d4d18f9..d26e1a12f1 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_mem_pool.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_mem_pool.h @@ -28,6 +28,7 @@ public: ~ObRpcMemPool() { destroy(); } static ObRpcMemPool* create(int64_t tenant_id, const char* label, int64_t req_sz); void* alloc(int64_t sz); + void set_tenant_id(int64_t tenant_id) { tenant_id_ = tenant_id; } void reuse(); void destroy(); private: diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h index 2d07c80671..a308217eb2 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h @@ -985,6 +985,7 @@ PCODE_DEF(OB_LOG_GET_ARB_MEMBER_INFO, 0x1522) PCODE_DEF(OB_LOG_BATCH_FETCH_RESP, 0X1523) PCODE_DEF(OB_LOG_GET_LS_CKPT, 0x1524) PCODE_DEF(OB_LOG_ACQUIRE_REBUILD_INFO, 0x1525) +PCODE_DEF(OB_LOG_FORCE_SET_TENANT_LOG_DISK, 0x1526) // 1531-1550 for obesi // PCODE_DEF(OB_ESI_IS_EXIST, 0x1531) diff --git a/deps/oblib/unittest/lib/restore/test_object_storage.cpp b/deps/oblib/unittest/lib/restore/test_object_storage.cpp index 00562baeb0..57b3d0dc99 100644 --- a/deps/oblib/unittest/lib/restore/test_object_storage.cpp +++ b/deps/oblib/unittest/lib/restore/test_object_storage.cpp @@ -1240,6 +1240,66 @@ TEST_F(TestObjectStorage, test_cross_testing) } } +TEST_F(TestObjectStorage, test_read_single_file) +{ + int ret = OB_SUCCESS; + if (enable_test) { + ObStorageUtil util; + ASSERT_EQ(OB_SUCCESS, util.open(&info_base)); + const char *tmp_dir = "test_read_single_file"; + const int64_t ts = ObTimeUtility::current_time(); + ASSERT_EQ(OB_SUCCESS, databuff_printf(dir_uri, sizeof(dir_uri), "%s/%s/%s_%ld", + bucket, dir_name, tmp_dir, ts)); + + { + // normal + ObStorageWriter writer; + ObStorageReader reader; + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_normal", dir_uri)); + ASSERT_EQ(OB_SUCCESS, writer.open(uri, &info_base)); + ASSERT_EQ(OB_SUCCESS, writer.write("123456", 6)); + ASSERT_EQ(OB_SUCCESS, writer.close()); + + int64_t read_size = -1; + char read_buf[100]; + memset(read_buf, 0, sizeof(read_buf)); + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &info_base)); + ASSERT_EQ(OB_SUCCESS, reader.pread(read_buf, 100, 0, read_size)); + ASSERT_EQ(6, read_size); + ASSERT_EQ(0, strncmp(read_buf, "123456", 6)); + ASSERT_EQ(OB_SUCCESS, reader.close()); + + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + } + + { + // appendable + ObStorageAppender appender; + ObStorageAdaptiveReader reader; + + const char *write_content = "0123456789"; + const int64_t first_content_len = 6; + ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_appendable", dir_uri)); + ASSERT_EQ(OB_SUCCESS, appender.open(uri, &info_base)); + ASSERT_EQ(OB_SUCCESS, appender.pwrite(write_content, first_content_len, 0)); + + int64_t read_size = -1; + char read_buf[100]; + memset(read_buf, 0, sizeof(read_buf)); + ASSERT_EQ(OB_SUCCESS, reader.open(uri, &info_base)); + + ASSERT_EQ(OB_SUCCESS, appender.pwrite(write_content, 1, first_content_len)); + + ASSERT_EQ(OB_SUCCESS, reader.pread(read_buf, 100, 0, read_size)); + ASSERT_EQ(first_content_len, read_size); + ASSERT_EQ(0, strncmp(read_buf, write_content, first_content_len)); + ASSERT_EQ(OB_SUCCESS, reader.close()); + + ASSERT_EQ(OB_SUCCESS, util.del_file(uri)); + } + } +} + TEST_F(TestObjectStorage, test_multipart_write) { int ret = OB_SUCCESS; @@ -1265,6 +1325,7 @@ TEST_F(TestObjectStorage, test_multipart_write) ASSERT_EQ(OB_SUCCESS, databuff_printf(uri, sizeof(uri), "%s/test_multipart", dir_uri)); ObStorageMultiPartWriter writer; + // ObStorageWriter writer; ASSERT_EQ(OB_SUCCESS, writer.open(uri, &info_base)); ASSERT_EQ(OB_SUCCESS, writer.write(write_buf, content_size)); ASSERT_EQ(content_size, writer.get_length()); diff --git a/deps/oblib/unittest/lib/utility/test_ob_unify_serialize.cpp b/deps/oblib/unittest/lib/utility/test_ob_unify_serialize.cpp index 5dc726a081..5dd0c4fc1a 100644 --- a/deps/oblib/unittest/lib/utility/test_ob_unify_serialize.cpp +++ b/deps/oblib/unittest/lib/utility/test_ob_unify_serialize.cpp @@ -16,6 +16,7 @@ #include #include #include +#include "lib/utility/ob_print_utils.h" // TO_STRING_KV #include "lib/utility/ob_unify_serialize.h" #include "lib/random/ob_random.h" @@ -73,6 +74,7 @@ struct CVirtualTest { }; struct CEmptyTest { + TO_STRING_KV(KP(this)); OB_UNIS_VERSION(1); }; @@ -165,6 +167,7 @@ public: uint32_t vu32_; int64_t vi64_; uint64_t vu64_; + TO_STRING_KV(K_(b), K_(vi8), K_(vu8), K_(vi16), K_(vu16), K_(vi32), K_(vu32), K_(vi64), K_(vu64)); }; // end of class TestObUnifySerialize OB_SERIALIZE_MEMBER(CIntTest, b_, vi8_, vu8_, vi16_, vu16_, vi32_, vu32_, vi64_, vu64_); @@ -220,7 +223,7 @@ public: n.buf_[rlen] = 0; return n; } - + TO_STRING_KV(K_(buf)); private: char buf_[32]; }; @@ -268,6 +271,7 @@ public: } return right; } + TO_STRING_KV(K_(ia), K_(uia), K_(i64a), K_(ui64a), K_(ita)); static const CArrayTest RAND() { @@ -318,7 +322,7 @@ public: return et; } - + TO_STRING_KV(K(eval)); private: enum Eval { E0, E1, E2, E3, EMAX } eval; }; diff --git a/mittest/logservice/env/ob_simple_log_cluster_env.cpp b/mittest/logservice/env/ob_simple_log_cluster_env.cpp index 0a75ce22df..d49cc25c9b 100644 --- a/mittest/logservice/env/ob_simple_log_cluster_env.cpp +++ b/mittest/logservice/env/ob_simple_log_cluster_env.cpp @@ -349,13 +349,30 @@ int ObSimpleLogClusterTestEnv::create_paxos_group_with_arb( int64_t &leader_idx, const bool with_mock_election, PalfHandleImplGuard &leader) +{ + // if member_cnt_ is 3, arb_replica_idx should be 0,1,2 + const ObMemberList member_list = get_member_list(); + const int64_t member_cnt = get_member_cnt(); + ObMember arb_member; + return create_paxos_group_with_arb(id, loc_cb, member_list, member_cnt, arb_member, arb_replica_idx, leader_idx, with_mock_election, leader); +} + +// member_list and member_cnt contain arb member +int ObSimpleLogClusterTestEnv::create_paxos_group_with_arb( + const int64_t id, + palf::PalfLocationCacheCb *loc_cb, + ObMemberList member_list, + int64_t member_cnt, + ObMember arb_member, + int64_t &arb_replica_idx, + int64_t &leader_idx, + const bool with_mock_election, + PalfHandleImplGuard &leader) { int ret = OB_SUCCESS; PalfBaseInfo palf_base_info; palf_base_info.generate_by_default(); - // if member_cnt_ is 3, arb_replica_idx should be 0,1,2 - ObMemberList member_list = get_member_list(); - ObMember arb_replica; + ObMember arb_replica = arb_member; GlobalLearnerList learner_list; arb_replica_idx = 2; for (int i = 0; i < get_cluster().size(); i++) { @@ -367,7 +384,8 @@ int ObSimpleLogClusterTestEnv::create_paxos_group_with_arb( if (-1 == arb_replica_idx) { ret = OB_NOT_SUPPORTED; PALF_LOG(ERROR, "there is not any arb server"); - } else if (OB_FAIL(member_list.get_member_by_index(arb_replica_idx, arb_replica))) { + } else if (false == arb_replica.is_valid() && + OB_FAIL(member_list.get_member_by_index(arb_replica_idx, arb_replica))) { PALF_LOG(ERROR, "get_member_by_index failed", K(ret), K(arb_replica_idx)); } else { member_list.remove_member(arb_replica); @@ -382,7 +400,7 @@ int ObSimpleLogClusterTestEnv::create_paxos_group_with_arb( break; } else if (OB_FAIL(svr->get_palf_env()->create_palf_handle_impl(id, palf::AccessMode::APPEND, palf_base_info, handle))) { CLOG_LOG(WARN, "create_palf_handle_impl failed", K(ret), K(id), KPC(svr)); - } else if (!svr->is_arb_server() && OB_FAIL(handle->set_initial_member_list(member_list, arb_replica, get_member_cnt()-1, learner_list))) { + } else if (!svr->is_arb_server() && OB_FAIL(handle->set_initial_member_list(member_list, arb_replica, member_cnt-1, learner_list))) { CLOG_LOG(ERROR, "set_initial_member_list failed", K(ret), K(id), KPC(svr)); } else { common::ObAddr leader_addr; diff --git a/mittest/logservice/env/ob_simple_log_cluster_env.h b/mittest/logservice/env/ob_simple_log_cluster_env.h index f4eb64afb1..790c9017d0 100644 --- a/mittest/logservice/env/ob_simple_log_cluster_env.h +++ b/mittest/logservice/env/ob_simple_log_cluster_env.h @@ -166,6 +166,15 @@ public: int64_t &leader_idx, const bool with_mock_election, PalfHandleImplGuard &leader); + int create_paxos_group_with_arb(const int64_t id, + palf::PalfLocationCacheCb *loc_cb, + ObMemberList member_list, + int64_t member_cnt, + ObMember arb_member, + int64_t &arb_replica_idx, + int64_t &leader_idx, + const bool with_mock_election, + PalfHandleImplGuard &leader); int create_paxos_group_with_mock_election(const int64_t id, int64_t &leader_idx, PalfHandleImplGuard &leader); diff --git a/mittest/logservice/test_ob_simple_log_arb.cpp b/mittest/logservice/test_ob_simple_log_arb.cpp index be58ace146..a306ac5b9e 100755 --- a/mittest/logservice/test_ob_simple_log_arb.cpp +++ b/mittest/logservice/test_ob_simple_log_arb.cpp @@ -767,6 +767,38 @@ TEST_F(TestObSimpleLogClusterArbService, test_2f1a_upgrade_when_no_leader) PALF_LOG(INFO, "end test_2f1a_upgrade_when_no_leader", K(id)); } +TEST_F(TestObSimpleLogClusterArbService, test_1f1a_create_palf_group) +{ + oceanbase::common::ObClusterVersion::get_instance().cluster_version_ = CLUSTER_VERSION_4_1_0_0; + SET_CASE_LOG_FILE(TEST_NAME, "test_1f1a_create_palf_group"); + PALF_LOG(INFO, "begin test_1f1a_create_palf_group"); + OB_LOGGER.set_log_level("TRACE"); + const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s + MockLocCB loc_cb; + int ret = OB_SUCCESS; + int64_t leader_idx = 0; + int64_t arb_replica_idx = -1; + PalfHandleImplGuard leader; + const int64_t id = ATOMIC_AAF(&palf_id_, 1); + common::ObMemberList member_list = get_member_list(); + member_list.remove_server(get_cluster()[1]->get_addr()); + const int64_t member_cnt = 2; + const common::ObMember &arb_member = get_arb_member(); + EXPECT_EQ(OB_SUCCESS, create_paxos_group_with_arb(id, &loc_cb, member_list, member_cnt, arb_member, arb_replica_idx, leader_idx, false, leader)); + EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id)); + + LogConfigVersion config_version; + EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version)); + EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(ObMember(get_cluster()[1]->get_addr(), 1), 2, config_version, CONFIG_CHANGE_TIMEOUT)); + + EXPECT_EQ(2, leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_replica_num_); + EXPECT_EQ(2, leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.get_member_number()); + + leader.reset(); + delete_paxos_group(id); + PALF_LOG(INFO, "end test_2f1a_degrade_upgrade", K(id)); +} + } // end unittest } // end oceanbase diff --git a/mittest/mtlenv/mock_tenant_module_env.h b/mittest/mtlenv/mock_tenant_module_env.h index 3a0c6afe13..a9e838c40f 100644 --- a/mittest/mtlenv/mock_tenant_module_env.h +++ b/mittest/mtlenv/mock_tenant_module_env.h @@ -171,6 +171,18 @@ public: UNUSED(tenant_id); return source_.get_gts(stc, task, gts, receive_gts_ts); } + + virtual int get_gts_sync(const uint64_t tenant_id, + const MonotonicTs stc, + int64_t timeout_us, + share::SCN >s, + MonotonicTs &receive_gts_ts) + { + UNUSED(tenant_id); + UNUSED(timeout_us); + return source_.get_gts(stc, NULL, gts, receive_gts_ts); + } + virtual int get_gts(const uint64_t tenant_id, ObTsCbTask *task, share::SCN >s) { UNUSED(tenant_id); @@ -666,7 +678,7 @@ int MockTenantModuleEnv::init() STORAGE_LOG(ERROR, "init_before_start_mtl failed", K(ret)); } else { oceanbase::ObClusterVersion::get_instance().update_data_version(DATA_CURRENT_VERSION); - MTL_BIND(ObTenantIOManager::mtl_init, ObTenantIOManager::mtl_destroy); + MTL_BIND2(nullptr, ObTenantIOManager::mtl_init, nullptr, nullptr, nullptr, ObTenantIOManager::mtl_destroy); MTL_BIND2(mtl_new_default, omt::ObSharedTimer::mtl_init, omt::ObSharedTimer::mtl_start, omt::ObSharedTimer::mtl_stop, omt::ObSharedTimer::mtl_wait, mtl_destroy_default); MTL_BIND2(mtl_new_default, ObTenantSchemaService::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); MTL_BIND2(mtl_new_default, ObStorageLogger::mtl_init, ObStorageLogger::mtl_start, ObStorageLogger::mtl_stop, ObStorageLogger::mtl_wait, mtl_destroy_default); @@ -705,7 +717,7 @@ int MockTenantModuleEnv::init() MTL_BIND2(mtl_new_default, ObTableLockService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); MTL_BIND2(server_obj_pool_mtl_new, nullptr, nullptr, nullptr, nullptr, server_obj_pool_mtl_destroy); MTL_BIND2(server_obj_pool_mtl_new, nullptr, nullptr, nullptr, nullptr, server_obj_pool_mtl_destroy); - MTL_BIND(ObTenantSQLSessionMgr::mtl_init, ObTenantSQLSessionMgr::mtl_destroy); + MTL_BIND2(ObTenantSQLSessionMgr::mtl_new, ObTenantSQLSessionMgr::mtl_init, nullptr, nullptr, nullptr, ObTenantSQLSessionMgr::mtl_destroy); MTL_BIND2(mtl_new_default, ObTenantCGReadInfoMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); MTL_BIND2(mtl_new_default, ObDecodeResourcePool::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); MTL_BIND2(mtl_new_default, ObTenantDirectLoadMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); diff --git a/mittest/mtlenv/storage/blocksstable/ob_index_block_data_prepare.h b/mittest/mtlenv/storage/blocksstable/ob_index_block_data_prepare.h index adf303c2f6..34a57e47b7 100644 --- a/mittest/mtlenv/storage/blocksstable/ob_index_block_data_prepare.h +++ b/mittest/mtlenv/storage/blocksstable/ob_index_block_data_prepare.h @@ -1,5 +1,5 @@ /** - * Copyright (c) 2021 OceanBase + * Copyright (c) 2023 OceanBase * OceanBase CE is licensed under Mulan PubL v2. * You can use this software according to the terms and conditions of the Mulan PubL v2. * You may obtain a copy of Mulan PubL v2 at: @@ -65,6 +65,7 @@ public: virtual void prepare_schema(); virtual void prepare_data(const int64_t micro_block_size = 0); virtual void prepare_partial_ddl_data(); + virtual void prepare_partial_cg_data(); virtual void prepare_cg_data(); virtual void insert_data(ObMacroBlockWriter &data_writer); // override to define data in sstable virtual void insert_cg_data(ObMacroBlockWriter &data_writer); // override to define data in sstable @@ -136,6 +137,8 @@ protected: int64_t mirco_blocks_per_macro_block_; bool is_cg_data_; bool is_ddl_merge_data_; + ObGetTableParam get_table_param_; + }; ObArenaAllocator TestIndexBlockDataPrepare::allocator_; @@ -296,13 +299,18 @@ void TestIndexBlockDataPrepare::SetUp() ASSERT_EQ(OB_SUCCESS, TestTabletHelper::create_tablet(ls_handle, tablet_id, table_schema_, allocator_)); ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle_)); + get_table_param_.tablet_iter_.set_tablet_handle(tablet_handle_); + iter_param_.set_table_param(&get_table_param_); sstable_.key_.table_type_ = ObITable::TableType::COLUMN_ORIENTED_SSTABLE; - partial_sstable_.key_.table_type_ = ObITable::TableType::DDL_MERGE_CO_SSTABLE; - if (is_cg_data_) { + if (is_cg_data_ && is_ddl_merge_data_) { + prepare_partial_cg_data(); + partial_sstable_.key_.table_type_ = ObITable::TableType::DDL_MERGE_CG_SSTABLE; + } else if (is_cg_data_) { prepare_cg_data(); } else if (is_ddl_merge_data_) { prepare_partial_ddl_data(); + partial_sstable_.key_.table_type_ = ObITable::TableType::DDL_MERGE_CO_SSTABLE; } else { prepare_data(); } @@ -575,8 +583,8 @@ void TestIndexBlockDataPrepare::close_builder_and_prepare_sstable(const int64_t param.nested_offset_ = res.nested_offset_; param.nested_size_ = res.nested_size_; param.compressor_type_ = ObCompressorType::NONE_COMPRESSOR; - param.data_block_ids_ = res.data_block_ids_; - param.other_block_ids_ = res.other_block_ids_; + ASSERT_EQ(OB_SUCCESS, param.data_block_ids_.assign(res.data_block_ids_)); + ASSERT_EQ(OB_SUCCESS, param.other_block_ids_.assign(res.other_block_ids_)); param.ddl_scn_.set_min(); param.filled_tx_scn_.set_min(); param.contain_uncommitted_row_ = false; @@ -888,6 +896,51 @@ void TestIndexBlockDataPrepare::prepare_partial_ddl_data() ObTabletObjLoadHelper::free(allocator_, storage_schema); } +void TestIndexBlockDataPrepare::prepare_partial_cg_data() +{ + prepare_contrastive_sstable(); + ObMacroBlockWriter writer; + ObMacroDataSeq start_seq(0); + start_seq.set_data_block(); + row_generate_.reset(); + ObWholeDataStoreDesc desc(true/*is ddl*/); + share::SCN end_scn; + end_scn.convert_from_ts(ObTimeUtility::current_time()); + ASSERT_EQ(OB_SUCCESS, desc.init(table_schema_, ObLSID(ls_id_), ObTabletID(tablet_id_), merge_type_, SNAPSHOT_VERSION, CLUSTER_CURRENT_VERSION, end_scn)); + void *builder_buf = allocator_.alloc(sizeof(ObSSTableIndexBuilder)); + merge_root_index_builder_ = new (builder_buf) ObSSTableIndexBuilder(); + ASSERT_NE(nullptr, merge_root_index_builder_); + desc.get_desc().sstable_index_builder_ = merge_root_index_builder_; + ASSERT_TRUE(desc.is_valid()); + if (need_agg_data_) { + ASSERT_EQ(OB_SUCCESS, desc.get_desc().col_desc_->agg_meta_array_.assign(agg_col_metas_)); + } + ASSERT_EQ(OB_SUCCESS, merge_root_index_builder_->init(desc.get_desc())); + ASSERT_EQ(OB_SUCCESS, writer.open(desc.get_desc(), start_seq)); + ASSERT_EQ(OB_SUCCESS, row_generate_.init(table_schema_, &allocator_)); + const int64_t partial_row_cnt = max_partial_row_cnt_; + insert_partial_data(writer, partial_row_cnt); + ASSERT_EQ(OB_SUCCESS, writer.close()); + // data write ctx has been moved to merge_root_index_builder_ + ASSERT_EQ(writer.get_macro_block_write_ctx().get_macro_block_count(), 0); + data_macro_block_cnt_ = merge_root_index_builder_->roots_[0]->macro_metas_->count(); + ASSERT_GE(data_macro_block_cnt_, 0); + int64_t column_cnt = 0; + ObTabletID tablet_id(TestIndexBlockDataPrepare::tablet_id_); + ObLSID ls_id(ls_id_); + ObLSHandle ls_handle; + ObTabletHandle tablet_handle; + ObLSService *ls_svr = MTL(ObLSService*); + ObStorageSchema *storage_schema = nullptr; + ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD)); + ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle)); + ASSERT_EQ(OB_SUCCESS, tablet_handle.get_obj()->load_storage_schema(allocator_, storage_schema)); + ASSERT_EQ(OB_SUCCESS, storage_schema->get_stored_column_count_in_sstable(column_cnt)); + prepare_partial_sstable(column_cnt); + prepare_merge_ddl_kvs(); + ObTabletObjLoadHelper::free(allocator_, storage_schema); +} + void TestIndexBlockDataPrepare::prepare_partial_sstable(const int64_t column_cnt) { ObSSTableMergeRes res; @@ -987,13 +1040,13 @@ void TestIndexBlockDataPrepare::prepare_partial_sstable(const int64_t column_cnt param.nested_offset_ = res.nested_offset_; param.nested_size_ = res.nested_size_; param.compressor_type_ = ObCompressorType::NONE_COMPRESSOR; - param.data_block_ids_ = res.data_block_ids_; - param.other_block_ids_ = res.other_block_ids_; param.ddl_scn_.convert_from_ts(ObTimeUtility::current_time()); param.filled_tx_scn_.set_min(); param.contain_uncommitted_row_ = false; param.encrypt_id_ = res.encrypt_id_; param.master_key_id_ = res.master_key_id_; + ASSERT_EQ(OB_SUCCESS, param.data_block_ids_.assign(res.data_block_ids_)); + ASSERT_EQ(OB_SUCCESS, param.other_block_ids_.assign(res.other_block_ids_)); if (param.table_key_.is_co_sstable() && param.column_group_cnt_ <= 1) { param.column_group_cnt_ = column_cnt + 2; /* set column group_cnt to avoid return err, cnt is calculated as each + all + default*/ } diff --git a/mittest/mtlenv/storage/blocksstable/test_block_cache.cpp b/mittest/mtlenv/storage/blocksstable/test_block_cache.cpp index 43cffe7f45..075314816a 100644 --- a/mittest/mtlenv/storage/blocksstable/test_block_cache.cpp +++ b/mittest/mtlenv/storage/blocksstable/test_block_cache.cpp @@ -114,13 +114,9 @@ TEST_F(TestObMicroBlockCache, test_block_cache) ObIndexBlockRowScanner idx_row_scanner; ObMicroBlockData root_block; ObMicroIndexInfo micro_idx_info; - ObArray agg_projector; - ObArray agg_column_schema; ObArray micro_idx_infos; sstable_.get_index_tree_root(root_block); ASSERT_EQ(OB_SUCCESS, idx_row_scanner.init( - agg_projector, - agg_column_schema, tablet_handle_.get_obj()->get_rowkey_read_info().get_datum_utils(), allocator_, context_.query_flag_, diff --git a/mittest/mtlenv/storage/blocksstable/test_index_block_row_scanner.cpp b/mittest/mtlenv/storage/blocksstable/test_index_block_row_scanner.cpp index cb85c4953d..2cdde1b671 100755 --- a/mittest/mtlenv/storage/blocksstable/test_index_block_row_scanner.cpp +++ b/mittest/mtlenv/storage/blocksstable/test_index_block_row_scanner.cpp @@ -139,8 +139,6 @@ TEST_F(TestIndexBlockRowScanner, prefetch_and_scan) ObIndexBlockRowParser idx_row_parser; ObIndexBlockRowScanner idx_scanner; ObIndexBlockRowScanner raw_idx_scanner; - ObArray agg_projector; - ObArray agg_column_schema; ObQueryFlag query_flag; query_flag.set_use_block_cache(); ObIndexBlockDataTransformer transformer; @@ -154,9 +152,9 @@ TEST_F(TestIndexBlockRowScanner, prefetch_and_scan) ASSERT_TRUE(root_blk_header->is_valid()); ASSERT_EQ(OB_SUCCESS, idx_scanner.init( - agg_projector, agg_column_schema, tablet_handle_.get_obj()->get_rowkey_read_info().get_datum_utils(), allocator_, query_flag, 0)); + tablet_handle_.get_obj()->get_rowkey_read_info().get_datum_utils(), allocator_, query_flag, 0)); ASSERT_EQ(OB_SUCCESS, raw_idx_scanner.init( - agg_projector, agg_column_schema, tablet_handle_.get_obj()->get_rowkey_read_info().get_datum_utils(), allocator_, query_flag, 0)); + tablet_handle_.get_obj()->get_rowkey_read_info().get_datum_utils(), allocator_, query_flag, 0)); ObMacroBlockHandle macro_handle; const ObIndexBlockRowHeader *idx_row_header = nullptr; diff --git a/mittest/mtlenv/storage/blocksstable/test_merge_cg_scanner.cpp b/mittest/mtlenv/storage/blocksstable/test_merge_cg_scanner.cpp new file mode 100644 index 0000000000..a8452d16f3 --- /dev/null +++ b/mittest/mtlenv/storage/blocksstable/test_merge_cg_scanner.cpp @@ -0,0 +1,402 @@ +/** + * Copyright (c) 2023 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#include +#define private public +#define protected public + +#include "lib/random/ob_random.h" +#include "sql/engine/basic/ob_pushdown_filter.h" +#include "storage/tablet/ob_tablet.h" +#include "storage/column_store/ob_cg_prefetcher.h" +#include "storage/column_store/ob_cg_scanner.h" +#include "ob_index_block_data_prepare.h" + +namespace oceanbase +{ +using namespace storage; +using namespace common; +namespace blocksstable +{ +class TestCGScanner : public TestIndexBlockDataPrepare +{ +public: + TestCGScanner(); + virtual ~TestCGScanner(); + static void SetUpTestCase(); + static void TearDownTestCase(); + + virtual void SetUp(); + virtual void TearDown(); + void prepare_cg_query_param(const bool is_reverse_scan, const ObVersionRange &scan_version, const uint32_t cg_idx); + void destroy_cg_query_param(); + void check_data(ObCGRowScanner *cg_scanner, int64_t start, int64_t locate_count, sql::ObDatum *datums, const bool is_reverse); + + void test_border(const bool is_reverse); + void test_random(const bool is_reverse); + + +public: + ObArenaAllocator allocator_; + ObDatumRow start_row_; + ObDatumRow end_row_; + ObTableAccessParam access_param_; +}; + +TestCGScanner::TestCGScanner() + : TestIndexBlockDataPrepare("Test cg sstable row scanner", ObMergeType::MAJOR_MERGE) +{ + is_cg_data_ = true; + is_ddl_merge_data_ = true; + max_row_cnt_ = 150000; + max_partial_row_cnt_ = 78881; + partial_kv_start_idx_ = 3; +} + +TestCGScanner::~TestCGScanner() +{ +} + +void TestCGScanner::SetUpTestCase() +{ + TestIndexBlockDataPrepare::SetUpTestCase(); +} + +void TestCGScanner::TearDownTestCase() +{ + TestIndexBlockDataPrepare::TearDownTestCase(); +} + +void TestCGScanner::prepare_cg_query_param(const bool is_reverse_scan, const ObVersionRange &scan_version, const uint32_t cg_idx) +{ + schema_cols_.set_allocator(&allocator_); + schema_cols_.init(table_schema_.get_column_count()); + ASSERT_EQ(OB_SUCCESS, table_schema_.get_column_ids(schema_cols_)); + access_param_.iter_param_.table_id_ = table_schema_.get_table_id(); + access_param_.iter_param_.tablet_id_ = table_schema_.get_table_id(); + access_param_.iter_param_.is_same_schema_column_ = true; + access_param_.iter_param_.has_virtual_columns_ = false; + access_param_.iter_param_.vectorized_enabled_ = true; + access_param_.iter_param_.pd_storage_flag_.pd_blockscan_ = true; + access_param_.iter_param_.pd_storage_flag_.pd_filter_ = true; + access_param_.iter_param_.pd_storage_flag_.use_iter_pool_ = true; + access_param_.iter_param_.pd_storage_flag_.use_column_store_ = true; + /* + ObSEArray cg_col_descs; + ObTableReadInfo *cg_read_info = nullptr; + ASSERT_EQ(OB_SUCCESS, tablet_handle_.get_obj()->get_cg_col_descs(cg_idx, cg_col_descs)); + ASSERT_EQ(OB_SUCCESS, MTL(ObTenantCGReadInfoMgr *)->get_cg_read_info(cg_col_descs.at(0), + ObTabletID(tablet_id_), + cg_read_info)); + */ + access_param_.iter_param_.read_info_ = cg_read_info_handle_.get_read_info(); + + //jsut for test + ObQueryFlag query_flag(ObQueryFlag::Forward, + true, /*is daily merge scan*/ + true, /*is read multiple macro block*/ + true, /*sys task scan, read one macro block in single io*/ + false /*full row scan flag, obsoleted*/, + false,/*index back*/ + false); /*query_stat*/ + query_flag.set_not_use_row_cache(); + query_flag.set_use_block_cache(); + if (is_reverse_scan) { + query_flag.scan_order_ = ObQueryFlag::ScanOrder::Reverse; + } else { + query_flag.scan_order_ = ObQueryFlag::ScanOrder::Forward; + } + ASSERT_EQ(OB_SUCCESS, context_.init(query_flag, + store_ctx_, + allocator_, + allocator_, + scan_version)); + context_.ls_id_ = ls_id_; + context_.limit_param_ = nullptr; + context_.is_inited_ = true; +} + +void TestCGScanner::destroy_cg_query_param() +{ + access_param_.reset(); + schema_cols_.reset(); + context_.reset(); + read_info_.reset(); +} + +void TestCGScanner::SetUp() +{ + TestIndexBlockDataPrepare::SetUp(); + ASSERT_EQ(OB_SUCCESS, start_row_.init(allocator_, TEST_COLUMN_CNT)); + ASSERT_EQ(OB_SUCCESS, end_row_.init(allocator_, TEST_COLUMN_CNT)); + ObLSID ls_id(ls_id_); + ObTabletID tablet_id(tablet_id_); + ObLSHandle ls_handle; + ObLSService *ls_svr = MTL(ObLSService*); + ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, ls_handle, ObLSGetMod::STORAGE_MOD)); + + ASSERT_EQ(OB_SUCCESS, ls_handle.get_ls()->get_tablet(tablet_id, tablet_handle_)); + sstable_.key_.table_type_ = ObITable::TableType::NORMAL_COLUMN_GROUP_SSTABLE; +} + +void TestCGScanner::TearDown() +{ + tablet_handle_.reset(); + TestIndexBlockDataPrepare::TearDown(); +} + +void TestCGScanner::check_data( + ObCGRowScanner *cg_scanner, + int64_t start, + int64_t locate_count, + sql::ObDatum *datums, + const bool is_reverse) +{ + int ret = OB_SUCCESS; + uint64_t count = 0; + int64_t get_count = 0; + int64_t data_count = MIN(row_cnt_ - start, locate_count); + int64_t end = MIN(row_cnt_ - 1, start + locate_count - 1); + int64_t sql_batch_size = 256; + while (OB_SUCC(ret)) { + if (OB_FAIL(cg_scanner->get_next_rows(count, sql_batch_size))) { + if (OB_ITER_END != ret) { + STORAGE_LOG(WARN, "Fail to get next rows", K(ret)); + ASSERT_EQ(OB_ITER_END, ret); + } else { + for (int64_t i = 0; i < count; i++) { + //STORAGE_LOG(INFO, "get next row info", K(get_count), K(i), K(datums[i].get_int())); + if (!is_reverse) { + ASSERT_EQ(start + get_count, datums[i].get_int()) << i << " start " << start << " locate_count " << locate_count; + } else { + ASSERT_EQ(end - get_count, datums[i].get_int()) << i << " start " << start << " locate_count " << locate_count; + } + get_count++; + } + } + } else { + for (int64_t i = 0; i < count; i++) { + //STORAGE_LOG(INFO, "get next row info", K(get_count), K(i), K(datums[i].get_int())); + if (!is_reverse) { + ASSERT_EQ(start + get_count, datums[i].get_int()) << i << " start " << start << " locate_count " << locate_count; + } else { + ASSERT_EQ(end - get_count, datums[i].get_int()) << i << " start " << start << " locate_count " << locate_count; + } + get_count++; + } + } + } +} + +void TestCGScanner::test_random(const bool is_reverse) +{ + ObVersionRange scan_version; + scan_version.base_version_ = 1; + scan_version.multi_version_start_ = 1; + scan_version.snapshot_version_ = INT64_MAX; + // prepare query param + prepare_cg_query_param(is_reverse, scan_version, 0); + sql::ObExecContext exec_ctx(allocator_); + sql::ObEvalCtx eval_ctx(exec_ctx); + sql::ObPushdownExprSpec expr_spec(allocator_); + static int64_t array_cnt = 1024; + eval_ctx.batch_idx_ = 0; + eval_ctx.batch_size_ = 256; + expr_spec.max_batch_size_ = 256; + void *datum_buf = allocator_.alloc((sizeof(sql::ObDatum) + sizeof(int64_t)) * array_cnt * 2); + sql::ObDatum *datums = new (datum_buf) sql::ObDatum[array_cnt]; + eval_ctx.frames_ = (char **)(&datum_buf); + sql::ObPushdownOperator op(eval_ctx, expr_spec); + ObFixedArray out_cols_project; + out_cols_project.set_allocator(&allocator_); + out_cols_project.init(1); + out_cols_project.push_back(0); + sql::ExprFixedArray exprs; + exprs.set_allocator(&allocator_); + void *expr_buf = allocator_.alloc(sizeof(sql::ObExpr)); + ASSERT_NE(nullptr, expr_buf); + sql::ObExpr *expr = reinterpret_cast(expr_buf); + expr->reset(); + expr->res_buf_off_ = sizeof(sql::ObDatum) * array_cnt; + expr->res_buf_len_ = sizeof(int64_t); + char *ptr = (char *)datum_buf + expr->res_buf_off_; + for (int64_t i = 0; i < array_cnt; i++) { + datums[i].ptr_ = ptr; + ptr += expr->res_buf_len_; + } + + exprs.init(1); + exprs.push_back(expr); + access_param_.iter_param_.out_cols_project_ = &out_cols_project; + access_param_.iter_param_.output_exprs_ = &exprs; + access_param_.iter_param_.op_ = &op; + void *buf = allocator_.alloc(sizeof(ObBlockRowStore)); + ASSERT_NE(nullptr, buf); + ObBlockRowStore *block_row_store = new (buf) ObBlockRowStore(context_); + ASSERT_EQ(OB_SUCCESS, block_row_store->init(access_param_)); + context_.block_row_store_ = block_row_store; + + buf = allocator_.alloc(sizeof(ObCGRowScanner)); + ASSERT_NE(nullptr, buf); + ObCGRowScanner *cg_scanner = new (buf) ObCGRowScanner(); + ObSSTableWrapper wrapper; + wrapper.sstable_ = &sstable_; + ASSERT_EQ(OB_SUCCESS, cg_scanner->init(access_param_.iter_param_, context_, wrapper)); + + int retry_cnt = 15; + while (retry_cnt > 0) { + int64_t start = ObRandom::rand(0, row_cnt_ - 1); + int64_t locate_count = ObRandom::rand(1, row_cnt_); + + STORAGE_LOG(INFO, "start to locate random range", K(retry_cnt), K(start), K(locate_count), K(row_cnt_)); + ASSERT_EQ(OB_SUCCESS, cg_scanner->locate(ObCSRange(start, locate_count))); + check_data(cg_scanner, start, locate_count, datums, is_reverse); + + retry_cnt--; + } + + STORAGE_LOG(INFO, "test random finished"); + + allocator_.free(datum_buf); + datum_buf = nullptr; + expr->reset(); + allocator_.free(expr); + expr = nullptr; + out_cols_project.reset(); + allocator_.free(block_row_store); + block_row_store = nullptr; + cg_scanner->reset(); + allocator_.free(cg_scanner); + cg_scanner = nullptr; + destroy_cg_query_param(); +} + +void TestCGScanner::test_border(const bool is_reverse) +{ + ObVersionRange scan_version; + scan_version.base_version_ = 1; + scan_version.multi_version_start_ = 1; + scan_version.snapshot_version_ = INT64_MAX; + // prepare query param + prepare_cg_query_param(is_reverse, scan_version, 0); + sql::ObExecContext exec_ctx(allocator_); + sql::ObEvalCtx eval_ctx(exec_ctx); + sql::ObPushdownExprSpec expr_spec(allocator_); + static int64_t array_cnt = 1024; + eval_ctx.batch_idx_ = 0; + eval_ctx.batch_size_ = 256; + expr_spec.max_batch_size_ = 256; + void *datum_buf = allocator_.alloc((sizeof(sql::ObDatum) + sizeof(int64_t)) * array_cnt * 2); + sql::ObDatum *datums = new (datum_buf) sql::ObDatum[array_cnt]; + eval_ctx.frames_ = (char **)(&datum_buf); + sql::ObPushdownOperator op(eval_ctx, expr_spec); + ObFixedArray out_cols_project; + out_cols_project.set_allocator(&allocator_); + out_cols_project.init(1); + out_cols_project.push_back(0); + sql::ExprFixedArray exprs; + exprs.set_allocator(&allocator_); + void *expr_buf = allocator_.alloc(sizeof(sql::ObExpr)); + ASSERT_NE(nullptr, expr_buf); + sql::ObExpr *expr = reinterpret_cast(expr_buf); + expr->reset(); + expr->res_buf_off_ = sizeof(sql::ObDatum) * array_cnt; + expr->res_buf_len_ = sizeof(int64_t); + char *ptr = (char *)datum_buf + expr->res_buf_off_; + for (int64_t i = 0; i < array_cnt; i++) { + datums[i].ptr_ = ptr; + ptr += expr->res_buf_len_; + } + + exprs.init(1); + exprs.push_back(expr); + access_param_.iter_param_.out_cols_project_ = &out_cols_project; + access_param_.iter_param_.output_exprs_ = &exprs; + access_param_.iter_param_.op_ = &op; + void *buf = allocator_.alloc(sizeof(ObBlockRowStore)); + ASSERT_NE(nullptr, buf); + ObBlockRowStore *block_row_store = new (buf) ObBlockRowStore(context_); + ASSERT_EQ(OB_SUCCESS, block_row_store->init(access_param_)); + context_.block_row_store_ = block_row_store; + + buf = allocator_.alloc(sizeof(ObCGRowScanner)); + ASSERT_NE(nullptr, buf); + ObCGRowScanner *cg_scanner = new (buf) ObCGRowScanner(); + ObSSTableWrapper wrapper; + wrapper.sstable_ = &sstable_; + ASSERT_EQ(OB_SUCCESS, cg_scanner->init(access_param_.iter_param_, context_, wrapper)); + + int64_t start = 0; + int64_t locate_count = 0; + ASSERT_EQ(OB_INVALID_ARGUMENT, cg_scanner->locate(ObCSRange(start, locate_count))); + + start = row_cnt_; + locate_count = 1; + ASSERT_EQ(OB_ITER_END, cg_scanner->locate(ObCSRange(start, locate_count))); + + start = row_cnt_ - 99; + locate_count = 100; + ASSERT_EQ(OB_SUCCESS, cg_scanner->locate(ObCSRange(start, locate_count))); + check_data(cg_scanner, start, locate_count, datums, is_reverse); + + start = 0; + locate_count = row_cnt_ + 1; + ASSERT_EQ(OB_SUCCESS, cg_scanner->locate(ObCSRange(start, locate_count))); + check_data(cg_scanner, start, locate_count, datums, is_reverse); + + STORAGE_LOG(INFO, "test border finished"); + + allocator_.free(datum_buf); + datum_buf = nullptr; + expr->reset(); + allocator_.free(expr); + expr = nullptr; + out_cols_project.reset(); + allocator_.free(block_row_store); + block_row_store = nullptr; + cg_scanner->reset(); + allocator_.free(cg_scanner); + cg_scanner = nullptr; + destroy_cg_query_param(); +} + +TEST_F(TestCGScanner, test_border) +{ + test_border(false); +} + +TEST_F(TestCGScanner, test_random) +{ + test_random(false); +} + +TEST_F(TestCGScanner, test_border_reverse) +{ + test_border(true); +} + +TEST_F(TestCGScanner, test_random_reverse) +{ + test_random(true); +} + +} +} + +int main(int argc, char **argv) +{ + system("rm -f test_merge_cg_scanner.log*"); + OB_LOGGER.set_file_name("test_merge_cg_scanner.log", true, true); + oceanbase::common::ObLogger::get_logger().set_log_level("INFO"); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/mittest/mtlenv/storage/test_co_merge.cpp b/mittest/mtlenv/storage/test_co_merge.cpp index d10e5684b0..41ebb1dc2f 100644 --- a/mittest/mtlenv/storage/test_co_merge.cpp +++ b/mittest/mtlenv/storage/test_co_merge.cpp @@ -138,8 +138,8 @@ void close_builder_and_prepare_sstable( param.master_key_id_ = res.master_key_id_; param.nested_size_ = res.nested_size_; param.nested_offset_ = res.nested_offset_; - param.data_block_ids_ = res.data_block_ids_; - param.other_block_ids_ = res.other_block_ids_; + ASSERT_EQ(OB_SUCCESS, param.data_block_ids_.assign(res.data_block_ids_)); + ASSERT_EQ(OB_SUCCESS, param.other_block_ids_.assign(res.other_block_ids_)); param.nested_size_ = res.nested_size_; param.nested_offset_ = res.nested_offset_; if (is_major_merge_type(data_store_desc.get_merge_type())) { diff --git a/mittest/simple_server/CMakeLists.txt b/mittest/simple_server/CMakeLists.txt index 0cb6a2a3f4..857f578109 100644 --- a/mittest/simple_server/CMakeLists.txt +++ b/mittest/simple_server/CMakeLists.txt @@ -105,6 +105,7 @@ ob_unittest_observer(test_ls_replica test_ls_replica.cpp) ob_unittest_observer(test_create_clone_tenant_resource_pool test_create_clone_tenant_resource_pool.cpp) ob_unittest_observer(test_tablet_autoinc_mgr test_tablet_autoinc_mgr.cpp) ob_unittest_observer(test_tenant_snapshot_service test_tenant_snapshot_service.cpp) +ob_unittest_observer(test_callbacks_with_reverse_order test_callbacks_with_reverse_order.cpp) # TODO(muwei.ym): open later ob_ha_unittest_observer(test_transfer_handler storage_ha/test_transfer_handler.cpp) ob_ha_unittest_observer(test_transfer_and_restart_basic storage_ha/test_transfer_and_restart_basic.cpp) diff --git a/mittest/simple_server/test_callbacks_with_reverse_order.cpp b/mittest/simple_server/test_callbacks_with_reverse_order.cpp new file mode 100644 index 0000000000..72f6595aad --- /dev/null +++ b/mittest/simple_server/test_callbacks_with_reverse_order.cpp @@ -0,0 +1,336 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ +#include +#include +#include +#define protected public +#define private public +#include "env/ob_simple_cluster_test_base.h" +#include "storage/compaction/ob_compaction_diagnose.h" +#include "storage/compaction/ob_schedule_dag_func.h" +#include "storage/ls/ob_ls.h" +#include "storage/tx_storage/ob_ls_handle.h" +#include "storage/tx_storage/ob_ls_service.h" +#include "storage/tx/ob_tx_data_functor.h" +#include "storage/tablet/ob_tablet.h" +#include "storage/ob_relative_table.h" +#include "storage/ob_dml_running_ctx.h" +#include "storage/access/ob_rows_info.h" +static int qcc = 0; +static int qcc2 = 0; +static int qcc3 = 0; +namespace oceanbase +{ +namespace storage +{ +int ObLSTabletService::insert_tablet_rows( + const int64_t row_count, + ObTabletHandle &tablet_handle, + ObDMLRunningCtx &run_ctx, + ObStoreRow *rows, + ObRowsInfo &rows_info) +{ + int ret = OB_SUCCESS; + ObRelativeTable &table = run_ctx.relative_table_; + const bool check_exists = !table.is_storage_index_table() || table.is_unique_index(); + bool exists = false; + // // 1. Defensive checking of new rows. + // if (GCONF.enable_defensive_check()) { + // for (int64_t i = 0; OB_SUCC(ret) && i < row_count; i++) { + // ObStoreRow &tbl_row = rows[i]; + // if (OB_FAIL(check_new_row_legitimacy(run_ctx, tbl_row.row_val_))) { + // LOG_WARN("Failed to check new row legitimacy", K(ret), K_(tbl_row.row_val)); + // } + // } + // } + + // 2. Check uniqueness constraint in memetable only(active + frozen). + // It would be more efficient and elegant to completely merge the uniqueness constraint + // and write conflict checking, but the implementation currently is to minimize intrusion + // into the memtable. + // if (check_exists && OB_FAIL(tablet_handle.get_obj()->rowkeys_exists(run_ctx.store_ctx_, table, + // rows_info, exists))) { + // LOG_WARN("Failed to check the uniqueness constraint", K(ret), K(rows_info)); + // } else if (exists) { + // ret = OB_ERR_PRIMARY_KEY_DUPLICATE; + // blocksstable::ObDatumRowkey &duplicate_rowkey = rows_info.get_conflict_rowkey(); + // LOG_WARN("Rowkey already exist", K(ret), K(table), K(duplicate_rowkey)); + // } + + // 3. Insert rows with uniqueness constraint and write conflict checking. + // Check write conflict in memtable + sstable. + // Check uniqueness constraint in sstable only. + if (OB_SUCC(ret)) { + if (OB_FAIL(tablet_handle.get_obj()->insert_rows(table, run_ctx.store_ctx_, rows, rows_info, + check_exists, *run_ctx.col_descs_, row_count, run_ctx.dml_param_.encrypt_meta_))) { + if (OB_ERR_PRIMARY_KEY_DUPLICATE == ret) { + blocksstable::ObDatumRowkey &duplicate_rowkey = rows_info.get_conflict_rowkey(); + TRANS_LOG(WARN, "Rowkey already exist", K(ret), K(table), K(duplicate_rowkey), + K(rows_info.get_conflict_idx())); + } else if (OB_TRY_LOCK_ROW_CONFLICT != ret) { + TRANS_LOG(WARN, "Failed to insert rows to tablet", K(ret), K(rows_info)); + } + } + } + + // 4. Log user error message if rowkey is duplicate. + if (OB_ERR_PRIMARY_KEY_DUPLICATE == ret && !run_ctx.dml_param_.is_ignore_) { + int tmp_ret = OB_SUCCESS; + char rowkey_buffer[OB_TMP_BUF_SIZE_256]; + ObString index_name = "PRIMARY"; + if (OB_TMP_FAIL(extract_rowkey(table, rows_info.get_conflict_rowkey(), + rowkey_buffer, OB_TMP_BUF_SIZE_256, run_ctx.dml_param_.tz_info_))) { + TRANS_LOG(WARN, "Failed to extract rowkey", K(ret), K(tmp_ret)); + } + if (table.is_index_table()) { + if (OB_TMP_FAIL(table.get_index_name(index_name))) { + TRANS_LOG(WARN, "Failed to get index name", K(ret), K(tmp_ret)); + } + } else if (lib::is_oracle_mode() && OB_TMP_FAIL(table.get_primary_key_name(index_name))) { + TRANS_LOG(WARN, "Failed to get pk name", K(ret), K(tmp_ret)); + } + LOG_USER_ERROR(OB_ERR_PRIMARY_KEY_DUPLICATE, rowkey_buffer, index_name.length(), index_name.ptr()); + } + return ret; +} + +int ObStorageTableGuard::refresh_and_protect_table(ObRelativeTable &relative_table) +{ + int ret = OB_SUCCESS; + ObTabletTableIterator &iter = relative_table.tablet_iter_; + const share::ObLSID &ls_id = tablet_->get_tablet_meta().ls_id_; + const common::ObTabletID &tablet_id = tablet_->get_tablet_meta().tablet_id_; + bool need_print = false; + if (tablet_id.id() == 200001 && store_ctx_.mvcc_acc_ctx_.tx_id_.get_id() % 2 == 0 && qcc2 == 0) { + need_print = true; + qcc2++; + TRANS_LOG(INFO, "qc debug", K(store_ctx_.mvcc_acc_ctx_.tx_id_), KPC(iter.table_iter()->get_last_memtable())); + usleep(1 * 1000 * 1000); + TRANS_LOG(INFO, "qc debug", K(store_ctx_.mvcc_acc_ctx_.tx_id_), KPC(iter.table_iter()->get_last_memtable())); + } + if (tablet_id.id() == 200001 && store_ctx_.mvcc_acc_ctx_.tx_id_.get_id() % 2 == 1 && qcc3 == 0) { + while (qcc2 == 0) { + usleep(1000); + } + } + while (OB_SUCC(ret) && need_to_refresh_table(*iter.table_iter())) { + if (OB_FAIL(store_ctx_.ls_->get_tablet_svr()->get_read_tables( + tablet_id, + ObTabletCommon::DEFAULT_GET_TABLET_DURATION_US, + store_ctx_.mvcc_acc_ctx_.get_snapshot_version().get_val_for_tx(), + iter, + relative_table.allow_not_ready()))) { + TRANS_LOG(WARN, "fail to get", K(store_ctx_.mvcc_acc_ctx_.tx_id_), K(ret)); + } else { + // no worry. iter will hold tablet reference and its life cycle is longer than guard + tablet_ = iter.get_tablet(); + if (store_ctx_.timeout_ > 0) { + const int64_t query_left_time = store_ctx_.timeout_ - ObTimeUtility::current_time(); + if (query_left_time <= 0) { + ret = OB_TRANS_STMT_TIMEOUT; + } + } + } + } + if (need_print) { + TRANS_LOG(INFO, "qc debug", K(store_ctx_.mvcc_acc_ctx_.tx_id_), KPC(iter.table_iter()->get_last_memtable())); + } + if (OB_SUCC(ret)) { + if (tablet_id.id() == 200001 && store_ctx_.mvcc_acc_ctx_.tx_id_.get_id() % 2 == 1 && qcc3 == 0) { + qcc++; + qcc3++; + TRANS_LOG(INFO, "qc debug2", K(store_ctx_.mvcc_acc_ctx_.tx_id_), KPC(iter.table_iter()->get_last_memtable())); + usleep(2 * 1000 * 1000); + TRANS_LOG(INFO, "qc debug2", K(store_ctx_.mvcc_acc_ctx_.tx_id_), KPC(iter.table_iter()->get_last_memtable())); + } + } + return ret; +} +} +namespace unittest +{ +using namespace oceanbase::transaction; +using namespace oceanbase::storage; +using namespace oceanbase::memtable; +using namespace oceanbase::storage::checkpoint; +#define EXE_SQL(sql_str) \ + ASSERT_EQ(OB_SUCCESS, sql.assign(sql_str)); \ + ASSERT_EQ(OB_SUCCESS, sql_proxy.write(sql.ptr(), affected_rows)); +#define EXE_SQL_FMT(...) \ + ASSERT_EQ(OB_SUCCESS, sql.assign_fmt(__VA_ARGS__)); \ + ASSERT_EQ(OB_SUCCESS, sql_proxy.write(sql.ptr(), affected_rows)); +#define WRITE_SQL_BY_CONN(conn, sql_str) \ + ASSERT_EQ(OB_SUCCESS, sql.assign(sql_str)); \ + ASSERT_EQ(OB_SUCCESS, conn->execute_write(OB_SYS_TENANT_ID, sql.ptr(), affected_rows)); +#define WRITE_SQL_FMT_BY_CONN(conn, ...) \ + ASSERT_EQ(OB_SUCCESS, sql.assign_fmt(__VA_ARGS__)); \ + ASSERT_EQ(OB_SUCCESS, conn->execute_write(OB_SYS_TENANT_ID, sql.ptr(), affected_rows)); +#define READ_SQL_BY_CONN(conn, sql_str) \ + ASSERT_EQ(OB_SUCCESS, sql.assign(sql_str)); \ + ASSERT_EQ(OB_SUCCESS, conn->execute_read(OB_SYS_TENANT_ID, sql.ptr(), read_res)); +class ObCallbackReverseTest : public ObSimpleClusterTestBase +{ +public: + ObCallbackReverseTest() : ObSimpleClusterTestBase("callbacks_with_reverse_order", "200G", "40G") {} + void prepare_tenant_env() + { + common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2(); + int64_t affected_rows = 0; + ObSqlString sql; + sqlclient::ObISQLConnection *connection = nullptr; + ASSERT_EQ(OB_SUCCESS, sql_proxy.acquire(connection)); + ASSERT_NE(nullptr, connection); + WRITE_SQL_BY_CONN(connection, "set GLOBAL ob_trx_timeout = 10000000000"); + WRITE_SQL_BY_CONN(connection, "set GLOBAL ob_trx_idle_timeout = 10000000000"); + WRITE_SQL_BY_CONN(connection, "set GLOBAL ob_query_timeout = 10000000000"); + WRITE_SQL_BY_CONN(connection, "alter system set enable_early_lock_release = False;"); + WRITE_SQL_BY_CONN(connection, "alter system set undo_retention = 1800;"); + sleep(5); + } + void create_test_tenant(uint64_t &tenant_id) + { + TRANS_LOG(INFO, "create_tenant start"); + ASSERT_EQ(OB_SUCCESS, create_tenant("tt1", "20G", "100G")); + ASSERT_EQ(OB_SUCCESS, get_tenant_id(tenant_id)); + ASSERT_EQ(OB_SUCCESS, get_curr_simple_server().init_sql_proxy2()); + TRANS_LOG(INFO, "create_tenant end", K(tenant_id)); + } + // you should use single partition when using it + void get_tablet_id_with_table_name(const char *name, + ObTabletID &tablet) + { + common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy(); + int ret = OB_SUCCESS; + ObSqlString sql; + int64_t affected_rows = 0; + int64_t tablet_id = 0; + ASSERT_EQ(OB_SUCCESS, sql.assign_fmt("select tablet_id from oceanbase.__all_virtual_table where table_name=%s", name)); + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + ASSERT_EQ(OB_SUCCESS, sql_proxy.read(res, sql.ptr())); + sqlclient::ObMySQLResult *result = res.get_result(); + ASSERT_NE(nullptr, result); + ASSERT_EQ(OB_SUCCESS, result->next()); + ASSERT_EQ(OB_SUCCESS, result->get_int("tablet_id", tablet_id)); + } + tablet = (uint64_t)tablet_id; + } + void minor_freeze_data() + { + common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2(); + sqlclient::ObISQLConnection *connection = nullptr; + ASSERT_EQ(OB_SUCCESS, sql_proxy.acquire(connection)); + int ret = OB_SUCCESS; + ObSqlString sql; + int64_t affected_rows = 0; + WRITE_SQL_BY_CONN(connection, "alter system minor freeze;"); + } + void get_ls(uint64_t tenant_id, ObLS *&ls) + { + ls = nullptr; + share::ObTenantSwitchGuard tenant_guard; + ASSERT_EQ(OB_SUCCESS, tenant_guard.switch_to(tenant_id)); + ObLSService *ls_svr = MTL(ObLSService*); + ASSERT_NE(nullptr, ls_svr); + ObLSHandle handle; + share::ObLSID ls_id(1001); + ASSERT_EQ(OB_SUCCESS, ls_svr->get_ls(ls_id, handle, ObLSGetMod::STORAGE_MOD)); + ASSERT_NE(nullptr, ls = handle.get_ls()); + } + void get_memtable(const ObTabletID tablet_id, + ObTableHandleV2 &handle) + { + ObLS *ls = NULL; + get_ls(1002, ls); + ObTabletHandle tablet_handle; + ObTablet *tablet = nullptr; + ASSERT_EQ(OB_SUCCESS, ls->get_tablet_svr()->get_tablet(tablet_id, tablet_handle)); + tablet = tablet_handle.get_obj(); + ASSERT_EQ(OB_SUCCESS, tablet->get_active_memtable(handle)); + } +private: +}; +TEST_F(ObCallbackReverseTest, callback_reverse_test) +{ + ObSqlString sql; + int64_t affected_rows = 0; + // ============================== Phase1. create tenant and table ============================== + TRANS_LOG(INFO, "create tenant start"); + uint64_t tenant_id = 0; + create_test_tenant(tenant_id); + TRANS_LOG(INFO, "create tenant end"); + share::ObTenantSwitchGuard tenant_guard; + ASSERT_EQ(OB_SUCCESS, tenant_guard.switch_to(tenant_id)); + TRANS_LOG(INFO, "create table start"); + common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2(); + EXE_SQL("create table qcc (a int primary key)"); + usleep(10 * 1000 * 1000); + TRANS_LOG(INFO, "create_table end"); + prepare_tenant_env(); + std::thread t1( + [this]() { + ObSqlString sql; + int64_t affected_rows = 0; + common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2(); + sqlclient::ObISQLConnection *connection = nullptr; + ASSERT_EQ(OB_SUCCESS, sql_proxy.acquire(connection)); + ASSERT_NE(nullptr, connection); + WRITE_SQL_BY_CONN(connection, "set SESSION ob_trx_timeout = 10000000000"); + WRITE_SQL_BY_CONN(connection, "set SESSION ob_trx_idle_timeout = 10000000000"); + WRITE_SQL_BY_CONN(connection, "set SESSION ob_query_timeout = 10000000000"); + WRITE_SQL_BY_CONN(connection, "set SESSION ob_trx_lock_timeout = 0"); + TRANS_LOG(INFO, "insert data start1"); + WRITE_SQL_BY_CONN(connection, "begin;"); + WRITE_SQL_FMT_BY_CONN(connection, "insert into qcc values(1);"); + WRITE_SQL_BY_CONN(connection, "commit;"); + TRANS_LOG(INFO, "insert data end1"); + }); + std::thread t2( + [this]() { + ObSqlString sql; + int64_t affected_rows = 0; + common::ObMySQLProxy &sql_proxy = get_curr_simple_server().get_sql_proxy2(); + sqlclient::ObISQLConnection *connection = nullptr; + ASSERT_EQ(OB_SUCCESS, sql_proxy.acquire(connection)); + ASSERT_NE(nullptr, connection); + WRITE_SQL_BY_CONN(connection, "set SESSION ob_trx_timeout = 10000000000"); + WRITE_SQL_BY_CONN(connection, "set SESSION ob_trx_idle_timeout = 10000000000"); + WRITE_SQL_BY_CONN(connection, "set SESSION ob_query_timeout = 10000000000"); + WRITE_SQL_BY_CONN(connection, "set SESSION ob_trx_lock_timeout = 0"); + TRANS_LOG(INFO, "insert data start2"); + WRITE_SQL_BY_CONN(connection, "begin;"); + WRITE_SQL_FMT_BY_CONN(connection, "insert into qcc values(1);"); + WRITE_SQL_BY_CONN(connection, "commit;"); + TRANS_LOG(INFO, "insert data end2"); + }); + std::thread t3( + [this]() { + while (qcc == 0) { + TRANS_LOG(INFO, "qcc is not increased", K(qcc)); + usleep(100 * 1000); + } + minor_freeze_data(); + }); + t1.join(); + t2.join(); + t3.join(); + ASSERT_EQ(1, qcc); +} +} // namespace unittest +} // namespace oceanbase +int main(int argc, char **argv) +{ + using namespace oceanbase::unittest; + oceanbase::unittest::init_log_and_gtest(argc, argv); + OB_LOGGER.set_log_level("info"); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/src/libtable/test/ob_batch_execute_test.cpp b/src/libtable/test/ob_batch_execute_test.cpp index 010d08a8bc..2da78fb89f 100644 --- a/src/libtable/test/ob_batch_execute_test.cpp +++ b/src/libtable/test/ob_batch_execute_test.cpp @@ -9903,7 +9903,17 @@ TEST_F(TestBatchExecute, table_query_with_filter) ASSERT_EQ(OB_SUCCESS, query.set_scan_index(ObString::make_string("primary"))); ASSERT_EQ(OB_SUCCESS, query.set_filter(ObString::make_string("TableCompareFilter(<, 'C2:50')"))); int ret = the_table->execute_query(query, iter); - ASSERT_NE(OB_SUCCESS, ret); + ASSERT_EQ(OB_SUCCESS, ret); + int64_t result_cnt = 0; + while (OB_SUCC(iter->get_next_entity(result_entity))) { + result_cnt++; + ObObj v1, v3; + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C1, v1)); + ASSERT_EQ(OB_SUCCESS, result_entity->get_property(C3, v3)); + ASSERT_LE(v1.get_int(), 50); + // fprintf(stderr, "(%ld,%ld,%s)\n", v1.get_int(), v2.get_int(), S(v3)); + } + ASSERT_EQ(10, result_cnt); // fprintf(stderr, "query ret=%d\n", ret); } // end case 6 { diff --git a/src/logservice/data_dictionary/ob_data_dict_service.cpp b/src/logservice/data_dictionary/ob_data_dict_service.cpp index 419b450734..f19a956a58 100644 --- a/src/logservice/data_dictionary/ob_data_dict_service.cpp +++ b/src/logservice/data_dictionary/ob_data_dict_service.cpp @@ -415,6 +415,7 @@ int ObDataDictService::generate_dict_and_dump_(const share::SCN &snapshot_scn) int64_t schema_version = OB_INVALID_VERSION; ObArray database_ids; ObArray table_ids; + int64_t filter_table_count = 0; if (OB_FAIL(sql_client_.get_schema_version(tenant_id_, snapshot_scn, schema_version))) { ret = OB_SCHEMA_EAGAIN; @@ -424,7 +425,7 @@ int ObDataDictService::generate_dict_and_dump_(const share::SCN &snapshot_scn) DDLOG(WARN, "handle_tenant_meta_ failed", KR(ret), K(snapshot_scn)); } else if (OB_FAIL(handle_database_metas_(schema_version, database_ids))) { DDLOG(WARN, "handle_database_metas_ failed", KR(ret), K(snapshot_scn)); - } else if (OB_FAIL(handle_table_metas_(schema_version, table_ids))) { + } else if (OB_FAIL(handle_table_metas_(schema_version, table_ids, filter_table_count))) { DDLOG(WARN, "handle_table_metas_ failed", KR(ret), K(snapshot_scn), K(schema_version)); } @@ -433,7 +434,8 @@ int ObDataDictService::generate_dict_and_dump_(const share::SCN &snapshot_scn) K(snapshot_scn), K(schema_version), "database_count", database_ids.count(), - "table_count", table_ids.count()); + "table_count", table_ids.count(), + K(filter_table_count)); return ret; } @@ -602,12 +604,15 @@ int ObDataDictService::handle_database_metas_( int ObDataDictService::handle_table_metas_( const int64_t schema_version, - const ObIArray &table_ids) + const ObIArray &table_ids, + int64_t &filter_table_count) { int ret = OB_SUCCESS; - ObArenaAllocator tb_meta_allocator("ObDatDictTbMeta"); - schema::ObSchemaGetterGuard schema_guard; // will reset while getting schem_guard + lib::ObMemAttr mem_attr(tenant_id_, "ObDatDictTbMeta"); + ObArenaAllocator tb_meta_allocator(mem_attr); static const int64_t batch_table_meta_size = 200; + filter_table_count = 0; + schema::ObSchemaGetterGuard schema_guard; // will reset while getting schem_guard for (int i = 0; OB_SUCC(ret) && ! stop_flag_ && i < table_ids.count(); i++) { const ObTableSchema *table_schema = NULL; @@ -633,6 +638,7 @@ int ObDataDictService::handle_table_metas_( } else if (OB_FAIL(filter_table_(*table_schema, is_filtered))) { DDLOG(WARN, "filter_table_ failed", KR(ret), K(is_filtered), KPC(table_schema)); } else if (is_filtered) { + filter_table_count++; DDLOG(DEBUG, "filter_table_", K(schema_version), "table_id", table_schema->get_table_id(), diff --git a/src/logservice/data_dictionary/ob_data_dict_service.h b/src/logservice/data_dictionary/ob_data_dict_service.h index 129e98a0ad..1a62ccef45 100644 --- a/src/logservice/data_dictionary/ob_data_dict_service.h +++ b/src/logservice/data_dictionary/ob_data_dict_service.h @@ -104,7 +104,8 @@ private: const ObIArray &database_ids); int handle_table_metas_( const int64_t schema_version, - const ObIArray &table_ids); + const ObIArray &table_ids, + int64_t &filter_table_count); int filter_table_(const share::schema::ObTableSchema &table_schema, bool &is_filtered); private: static const int64_t TIMER_TASK_INTERVAL; diff --git a/src/logservice/libobcdc/src/ob_log_fetcher.cpp b/src/logservice/libobcdc/src/ob_log_fetcher.cpp index 73e8523fe4..795b37f851 100644 --- a/src/logservice/libobcdc/src/ob_log_fetcher.cpp +++ b/src/logservice/libobcdc/src/ob_log_fetcher.cpp @@ -132,7 +132,8 @@ int ObLogFetcher::init( cfg.blacklist_history_overdue_time_min, cfg.blacklist_history_clear_interval_min, is_tenant_mode, - TCTX.tenant_id_))) { + TCTX.tenant_id_, + OB_SERVER_TENANT_ID))) { LOG_ERROR("ObLogRouterService init failer", KR(ret), K(prefer_region), K(cluster_id)); } else if (OB_FAIL(progress_controller_.init(cfg.ls_count_upper_limit))) { LOG_ERROR("init progress controller fail", KR(ret)); diff --git a/src/logservice/logfetcher/ob_log_fetcher.cpp b/src/logservice/logfetcher/ob_log_fetcher.cpp index 1b73f88adf..cc516786c9 100755 --- a/src/logservice/logfetcher/ob_log_fetcher.cpp +++ b/src/logservice/logfetcher/ob_log_fetcher.cpp @@ -126,7 +126,8 @@ int ObLogFetcher::init( cfg.blacklist_history_overdue_time_min, cfg.blacklist_history_clear_interval_min, true/*is_tenant_mode*/, - source_tenant_id))) { + source_tenant_id, + self_tenant_id))) { LOG_ERROR("ObLogRouterService init failer", KR(ret), K(region), K(cluster_id), K(source_tenant_id)); } else if (OB_FAIL(progress_controller_.init(cfg.ls_count_upper_limit))) { LOG_ERROR("init progress controller fail", KR(ret)); diff --git a/src/logservice/logrouteservice/ob_log_route_service.cpp b/src/logservice/logrouteservice/ob_log_route_service.cpp index 2c28bb1ae4..fbaae36d46 100755 --- a/src/logservice/logrouteservice/ob_log_route_service.cpp +++ b/src/logservice/logrouteservice/ob_log_route_service.cpp @@ -30,6 +30,7 @@ ObLogRouteService::ObLogRouteService() : cluster_id_(OB_INVALID_CLUSTER_ID), is_tenant_mode_(false), source_tenant_id_(OB_INVALID_TENANT_ID), + self_tenant_id_(OB_INVALID_TENANT_ID), is_stopped_(true), ls_route_key_set_(), ls_router_map_(), @@ -72,13 +73,14 @@ int ObLogRouteService::init(ObISQLClient *proxy, const int64_t blacklist_history_overdue_time_min, const int64_t blacklist_history_clear_interval_min, const bool is_tenant_mode, - const uint64_t tenant_id) + const uint64_t source_tenant_id, + const uint64_t self_tenant_id) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; const int64_t size = sizeof(ObLSRouterValue); - lib::ObMemAttr log_router_mem_attr(OB_SYS_TENANT_ID, "LogRouter"); - lib::ObMemAttr asyn_task_mem_attr(OB_SYS_TENANT_ID, "RouterAsynTask"); + lib::ObMemAttr log_router_mem_attr(self_tenant_id, "LogRouter"); + lib::ObMemAttr asyn_task_mem_attr(self_tenant_id, "RouterAsynTask"); timer_.set_run_wrapper(MTL_CTX()); if (IS_INIT) { @@ -102,7 +104,7 @@ int ObLogRouteService::init(ObISQLClient *proxy, K(external_server_blacklist)); } else if (OB_FAIL(systable_queryer_.init(cluster_id, is_across_cluster, *proxy, err_handler))) { LOG_WARN("systable_queryer_ init failed", KR(ret), K(cluster_id), K(is_across_cluster)); - } else if (OB_FAIL(all_svr_cache_.init(systable_queryer_, is_tenant_mode, tenant_id, prefer_region, + } else if (OB_FAIL(all_svr_cache_.init(systable_queryer_, is_tenant_mode, source_tenant_id, prefer_region, all_server_cache_update_interval_sec, all_zone_cache_update_interval_sec))) { LOG_WARN("all_svr_cache_ init failed", KR(ret), K(is_tenant_mode), K(prefer_region), K(all_server_cache_update_interval_sec), K(all_zone_cache_update_interval_sec)); @@ -114,7 +116,8 @@ int ObLogRouteService::init(ObISQLClient *proxy, LOG_WARN("TG_SET_HANDLER_AND_START failed", KR(ret), K(tg_id_)); } else { cluster_id_ = cluster_id; - source_tenant_id_ = tenant_id; + source_tenant_id_ = source_tenant_id; + self_tenant_id_ = self_tenant_id; log_router_allocator_.set_nway(NWAY); asyn_task_allocator_.set_nway(NWAY); timer_id_ = lib::TGDefIDs::LogRouterTimer; @@ -201,9 +204,14 @@ void ObLogRouteService::destroy() systable_queryer_.destroy(); all_svr_cache_.destroy(); svr_blacklist_.destroy(); + + log_router_allocator_.destroy(); + asyn_task_allocator_.destroy(); + err_handler_ = NULL; cluster_id_ = OB_INVALID_CLUSTER_ID; + self_tenant_id_ = OB_INVALID_TENANT_ID; source_tenant_id_ = OB_INVALID_TENANT_ID; background_refresh_time_sec_ = 0; blacklist_survival_time_sec_ = 0; diff --git a/src/logservice/logrouteservice/ob_log_route_service.h b/src/logservice/logrouteservice/ob_log_route_service.h index a82baf36de..4327549553 100755 --- a/src/logservice/logrouteservice/ob_log_route_service.h +++ b/src/logservice/logrouteservice/ob_log_route_service.h @@ -86,7 +86,8 @@ public: const int64_t blacklist_history_overdue_time_min = 30, const int64_t blacklist_history_clear_interval_min = 20, const bool is_tenant_mode = false, - const uint64_t tenant_id = OB_INVALID_TENANT_ID); + const uint64_t tenant_id = OB_INVALID_TENANT_ID, + const uint64_t self_tenant_id = OB_SERVER_TENANT_ID); int start(); void stop(); void wait(); @@ -375,6 +376,7 @@ private: int64_t cluster_id_; bool is_tenant_mode_; int64_t source_tenant_id_; + uint64_t self_tenant_id_; volatile bool is_stopped_ CACHE_ALIGNED; LSRouteKeySet ls_route_key_set_; LSRouterMap ls_router_map_; diff --git a/src/logservice/palf/log_config_mgr.cpp b/src/logservice/palf/log_config_mgr.cpp index 7fab6242c1..b87493f26e 100755 --- a/src/logservice/palf/log_config_mgr.cpp +++ b/src/logservice/palf/log_config_mgr.cpp @@ -59,6 +59,7 @@ LogConfigMgr::LogConfigMgr() last_wait_barrier_time_us_(OB_INVALID_TIMESTAMP), last_wait_committed_end_lsn_(), last_sync_meta_for_arb_election_leader_time_us_(OB_INVALID_TIMESTAMP), + forwarding_config_proposal_id_(INVALID_PROPOSAL_ID), parent_lock_(common::ObLatchIds::PALF_CM_PARENT_LOCK), register_time_us_(OB_INVALID_TIMESTAMP), parent_(), @@ -172,6 +173,7 @@ void LogConfigMgr::destroy() all_learnerlist_.reset(); paxos_member_region_map_.destroy(); last_sync_meta_for_arb_election_leader_time_us_ = OB_INVALID_TIMESTAMP; + forwarding_config_proposal_id_ = INVALID_PROPOSAL_ID; region_ = DEFAULT_REGION_NAME; state_ = ConfigChangeState::INIT; reconfig_barrier_.reset(); @@ -229,7 +231,6 @@ int LogConfigMgr::set_initial_member_list(const common::ObMemberList &member_lis PALF_LOG(WARN, "LogConfigMgr not init", KR(ret)); } else if (!member_list.is_valid() || !arb_member.is_valid() || - (replica_num & 1) != 0 || replica_num <= 0 || replica_num > OB_MAX_MEMBER_NUMBER || INVALID_PROPOSAL_ID == proposal_id || @@ -245,6 +246,7 @@ int LogConfigMgr::set_initial_member_list(const common::ObMemberList &member_lis if (OB_FAIL(set_initial_config_info_(config_info, proposal_id, init_config_version))) { PALF_LOG(WARN, "set_initial_config_info failed", K(ret), K_(palf_id), K_(self), K(config_info), K(proposal_id)); } else { + forwarding_config_proposal_id_ = proposal_id; PALF_LOG(INFO, "set_initial_member_list success", K(ret), K_(palf_id), K_(self), K_(log_ms_meta), K(member_list), K(arb_member), K(replica_num), K(proposal_id)); } } @@ -2459,6 +2461,38 @@ int LogConfigMgr::sync_get_committed_end_lsn_(const LogConfigChangeArgs &args, return ret; } +// need rlock of PalfHandleImpl +// The arb server don't support set_initial_member_list, +// so we need to forward LogConfigMeta to arb member. otherwise, +// if only 1F1A are created successfully when creating PALF group, +// A will not vote for the F because of empty config meta. +int LogConfigMgr::forward_initial_config_meta_to_arb() +{ + int ret = OB_SUCCESS; + const common::ObMember &arb_member = log_ms_meta_.curr_.config_.arbitration_member_; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else if (INVALID_PROPOSAL_ID == forwarding_config_proposal_id_ || + false == arb_member.is_valid()) { + // skip + } else { + common::ObMemberList forward_list; + if (forwarding_config_proposal_id_ != state_mgr_->get_proposal_id() || + forwarding_config_proposal_id_ != log_ms_meta_.proposal_id_) { + forwarding_config_proposal_id_ = INVALID_PROPOSAL_ID; + PALF_LOG(INFO, "stop forward_initial_config_meta_to_arb", KR(ret), K_(palf_id), K_(self)); + } else if (OB_FAIL(forward_list.add_member(arb_member))) { + PALF_LOG(WARN, "add_member failed", KR(ret), K_(palf_id), K_(self), K(forward_list), K(arb_member)); + } else if (OB_FAIL(log_engine_->submit_change_config_meta_req(forward_list, + log_ms_meta_.proposal_id_, log_ms_meta_.prev_log_proposal_id_, + log_ms_meta_.prev_lsn_, log_ms_meta_.prev_mode_pid_, log_ms_meta_))) { + PALF_LOG(WARN, "submit_change_config_meta_req failed", KR(ret), K_(palf_id), K_(self), + K(arb_member), K_(log_ms_meta)); + } + } + return ret; +} + //================================ Config Change ================================ //================================ Child ================================ diff --git a/src/logservice/palf/log_config_mgr.h b/src/logservice/palf/log_config_mgr.h index ddabd879a7..c1f7d81184 100755 --- a/src/logservice/palf/log_config_mgr.h +++ b/src/logservice/palf/log_config_mgr.h @@ -477,6 +477,7 @@ public: int sync_meta_for_arb_election_leader(); void set_sync_to_degraded_learners(); bool is_sync_to_degraded_learners() const; + int forward_initial_config_meta_to_arb(); // ================ Config Change ================== // ==================== Child ======================== virtual int register_parent(); @@ -680,6 +681,7 @@ private: mutable int64_t last_wait_barrier_time_us_; mutable LSN last_wait_committed_end_lsn_; int64_t last_sync_meta_for_arb_election_leader_time_us_; + int64_t forwarding_config_proposal_id_; // ================= Config Change ================= // ==================== Child ======================== mutable common::ObSpinLock parent_lock_; diff --git a/src/logservice/palf/log_io_task.cpp b/src/logservice/palf/log_io_task.cpp index abe96c14c4..148d9e8cb9 100644 --- a/src/logservice/palf/log_io_task.cpp +++ b/src/logservice/palf/log_io_task.cpp @@ -696,7 +696,6 @@ int LogIOFlashbackTask::do_task_(int tg_id, IPalfHandleImplGuard &guard) } else if (OB_FAIL(push_task_into_cb_thread_pool_(tg_id, this))) { PALF_LOG(WARN, "push_flush_cb_to_thread_pool_ failed", K(ret)); } else { - PALF_LOG(INFO, "LogIOFlashbackTask do_task success", K(ret), K(palf_id_)); } return ret; } diff --git a/src/logservice/palf/palf_handle_impl.cpp b/src/logservice/palf/palf_handle_impl.cpp index 5513f9b598..5a852bc36c 100755 --- a/src/logservice/palf/palf_handle_impl.cpp +++ b/src/logservice/palf/palf_handle_impl.cpp @@ -2594,6 +2594,7 @@ int PalfHandleImpl::check_and_switch_state() } else if (OB_FAIL(mode_mgr_.leader_do_loop_work())) { PALF_LOG(WARN, "LogModeMgr::leader_do_loop_work failed", KR(ret), K_(self), K_(palf_id)); } + (void) config_mgr_.forward_initial_config_meta_to_arb(); } while (0); if (OB_UNLIKELY(config_state_changed)) { WLockGuard guard(lock_); diff --git a/src/logservice/restoreservice/ob_log_restore_net_driver.cpp b/src/logservice/restoreservice/ob_log_restore_net_driver.cpp index e02fb0a490..264ada2f33 100644 --- a/src/logservice/restoreservice/ob_log_restore_net_driver.cpp +++ b/src/logservice/restoreservice/ob_log_restore_net_driver.cpp @@ -381,13 +381,11 @@ int ObLogRestoreNetDriver::add_ls_if_needed_with_lock_(const share::ObLSID &id, int ObLogRestoreNetDriver::init_fetcher_if_needed_(const int64_t cluster_id, const uint64_t tenant_id) { int ret = OB_SUCCESS; - void *buffer = NULL; if (NULL != fetcher_) { // fetcher already exist - } else if (OB_ISNULL(buffer = mtl_malloc(sizeof(logfetcher::ObLogFetcher), "LogFetcher"))) { + } else if (OB_ISNULL(fetcher_ = MTL_NEW(logfetcher::ObLogFetcher, "LogFetcher"))) { ret = OB_ALLOCATE_MEMORY_FAILED; } else { - fetcher_ = new (buffer) logfetcher::ObLogFetcher(); const logfetcher::LogFetcherUser log_fetcher_user = logfetcher::LogFetcherUser::STANDBY; const bool is_loading_data_dict_baseline_data = false; const logfetcher::ClientFetchingMode fetching_mode = logfetcher::ClientFetchingMode::FETCHING_MODE_INTEGRATED; @@ -516,7 +514,7 @@ void ObLogRestoreNetDriver::destroy_fetcher_() } else { fetcher_->stop(); fetcher_->destroy(); - mtl_free(fetcher_); + MTL_DELETE(ObLogFetcher, "LogFetcher", fetcher_); fetcher_ = NULL; } } @@ -532,7 +530,7 @@ void ObLogRestoreNetDriver::destroy_fetcher_forcedly_() CLOG_LOG(INFO, "destroy_fetcher forcedly"); fetcher_->stop(); fetcher_->destroy(); - mtl_free(fetcher_); + MTL_DELETE(ObLogFetcher, "LogFetcher", fetcher_); fetcher_ = NULL; } diff --git a/src/objit/include/objit/common/ob_item_type.h b/src/objit/include/objit/common/ob_item_type.h index ac205c678f..d6eb9d1fae 100755 --- a/src/objit/include/objit/common/ob_item_type.h +++ b/src/objit/include/objit/common/ob_item_type.h @@ -843,6 +843,9 @@ typedef enum ObItemType T_FUN_SYS_ST_SYMDIFFERENCE = 1729, T_FUN_SYS_PRIV_ST_ASMVTGEOM = 1730, T_FUN_SYS_PRIV_ST_MAKE_VALID = 1731, + T_FUN_SYS_XML_CONCAT = 1732, + T_FUN_SYS_XML_FOREST = 1733, + T_FUN_SYS_XML_EXISTSNODE = 1734, ///< @note add new oracle only function type before this line T_FUN_SYS_TABLET_AUTOINC_NEXTVAL = 1801, // add only for heap table @@ -1246,7 +1249,6 @@ typedef enum ObItemType T_ROLLBACK, T_ANY_HOST_NAME, T_USER_WITH_HOST_NAME, - T_FLUSH_PRIVILEGES, T_INDEX_COLUMN_LIST, T_INDEX_USING_ALGORITHM, @@ -2381,6 +2383,11 @@ typedef enum ObItemType T_COLUMN_GROUP_DROP, T_ALTER_COLUMN_GROUP_OPTION, T_BLOCKING, + T_PQ_SUBQUERY, + T_FLUSH_PRIVILEGES, + T_SCHEMA_ID, + T_CANCEL_TRANSFER_PARTITION, + T_CANCEL_BALANCE_JOB, T_MAX //Attention: add a new type before T_MAX } ObItemType; diff --git a/src/observer/mysql/obmp_stmt_execute.cpp b/src/observer/mysql/obmp_stmt_execute.cpp index 9bd39c7891..99ec07b4cb 100644 --- a/src/observer/mysql/obmp_stmt_execute.cpp +++ b/src/observer/mysql/obmp_stmt_execute.cpp @@ -2544,6 +2544,7 @@ int ObMPStmtExecute::parse_param_value(ObIAllocator &allocator, LOG_WARN("failed to parse basic param value", K(ret)); } else { param.set_param_meta(); + param.set_length(param.get_val_len()); } } } else if (!support_send_long_data(type)) { @@ -2640,6 +2641,7 @@ int ObMPStmtExecute::parse_param_value(ObIAllocator &allocator, LOG_WARN("failed to parse basic param value", K(ret)); } else { param.set_param_meta(); + param.set_length(param.get_val_len()); } } piece->get_allocator()->free(tmp); diff --git a/src/observer/ob_rpc_processor_simple.cpp b/src/observer/ob_rpc_processor_simple.cpp index 83914582d9..93a3af766c 100644 --- a/src/observer/ob_rpc_processor_simple.cpp +++ b/src/observer/ob_rpc_processor_simple.cpp @@ -72,6 +72,7 @@ #include "sql/spm/ob_spm_controller.h" #endif #include "sql/plan_cache/ob_ps_cache.h" +#include "pl/pl_cache/ob_pl_cache_mgr.h" #include "rootserver/ob_primary_ls_service.h" // for ObPrimaryLSService #include "rootserver/ob_root_utils.h" #include "sql/session/ob_sql_session_info.h" @@ -1348,7 +1349,27 @@ int ObFlushCacheP::process() break; } case CACHE_TYPE_PL_OBJ: { - if (arg_.is_all_tenant_) { + if (arg_.is_fine_grained_) { // fine-grained plan cache evict + bool is_evict_by_schema_id = common::OB_INVALID_ID != arg_.schema_id_; + MTL_SWITCH(arg_.tenant_id_) { + ObPlanCache* plan_cache = MTL(ObPlanCache*); + if (arg_.db_ids_.count() == 0) { + if (is_evict_by_schema_id) { + ret = plan_cache->flush_pl_cache_single_cache_obj(OB_INVALID_ID, arg_.schema_id_); + } else { + ret = plan_cache->flush_pl_cache_single_cache_obj(OB_INVALID_ID, arg_.sql_id_); + } + } else { + for (uint64_t i=0; iflush_pl_cache_single_cache_obj(arg_.db_ids_.at(i), arg_.schema_id_); + } else { + ret = plan_cache->flush_pl_cache_single_cache_obj(arg_.db_ids_.at(i), arg_.sql_id_); + } + } + } + } + } else if (arg_.is_all_tenant_) { common::ObArray tenant_ids; if (OB_ISNULL(GCTX.omt_)) { ret = OB_ERR_UNEXPECTED; diff --git a/src/observer/ob_server.cpp b/src/observer/ob_server.cpp index 3d1deb228f..d8171699e3 100644 --- a/src/observer/ob_server.cpp +++ b/src/observer/ob_server.cpp @@ -877,6 +877,12 @@ int ObServer::start() FLOG_INFO("success to start multi tenant"); } + if (FAILEDx(wr_service_.start())) { + LOG_ERROR("failed to start wr service", K(ret)); + } else { + LOG_INFO("success to start wr service"); + } + if (FAILEDx(ObServerCheckpointSlogHandler::get_instance().start())) { LOG_ERROR("fail to start server checkpoint slog handler", KR(ret)); } else { @@ -889,12 +895,6 @@ int ObServer::start() FLOG_INFO("success to start log pool"); } - if (FAILEDx(wr_service_.start())) { - LOG_ERROR("failed to start wr service", K(ret)); - } else { - LOG_INFO("success to start wr service"); - } - if (FAILEDx(try_update_hidden_sys())) { LOG_ERROR("fail to update hidden sys tenant", KR(ret)); } else { diff --git a/src/observer/ob_srv_deliver.cpp b/src/observer/ob_srv_deliver.cpp index 23ef7f61e4..9bb7b92a3c 100644 --- a/src/observer/ob_srv_deliver.cpp +++ b/src/observer/ob_srv_deliver.cpp @@ -237,6 +237,13 @@ int get_user_tenant(ObRequest &req, char *user_name_buf, char *tenant_name_buf) return ret; } +static void set_sql_sock_mem_pool_tenant_id(ObRequest &req, int64_t tenant_id) +{ + if (req.get_nio_protocol() == ObRequest::TRANSPORT_PROTO_POC) { + obmysql::ObSqlSockSession* sess = (obmysql::ObSqlSockSession*)req.get_server_handle_context(); + sess->pool_.set_tenant_id(tenant_id); + } +} int dispatch_req(const uint64_t tenant_id, ObRequest &req, QueueThread *global_mysql_queue) { int ret = OB_SUCCESS; @@ -259,6 +266,7 @@ int dispatch_req(const uint64_t tenant_id, ObRequest &req, QueueThread *global_m } else if (OB_ISNULL(mysql_queue)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("mysql_queue is NULL", K(ret), K(tenant_id)); + } else if (FALSE_IT(set_sql_sock_mem_pool_tenant_id(req, tenant_id))) { } else if (!mysql_queue->queue_.push(&req, MAX_QUEUE_LEN)) { // MAX_QUEUE_LEN = 10000; ret = OB_QUEUE_OVERFLOW; EVENT_INC(MYSQL_DELIVER_FAIL); diff --git a/src/observer/omt/ob_multi_tenant.cpp b/src/observer/omt/ob_multi_tenant.cpp index 17f5f4f1fc..801843ea91 100644 --- a/src/observer/omt/ob_multi_tenant.cpp +++ b/src/observer/omt/ob_multi_tenant.cpp @@ -438,7 +438,7 @@ int ObMultiTenant::init(ObAddr myaddr, } if (OB_SUCC(ret) && mtl_bind_flag) { - MTL_BIND(ObTenantIOManager::mtl_init, ObTenantIOManager::mtl_destroy); + MTL_BIND2(nullptr, ObTenantIOManager::mtl_init, nullptr, nullptr, nullptr, ObTenantIOManager::mtl_destroy); // base mtl MTL_BIND2(mtl_new_default, storage::mds::ObTenantMdsService::mtl_init, storage::mds::ObTenantMdsService::mtl_start, storage::mds::ObTenantMdsService::mtl_stop, storage::mds::ObTenantMdsService::mtl_wait, mtl_destroy_default); @@ -522,29 +522,29 @@ int ObMultiTenant::init(ObAddr myaddr, MTL_BIND2(mtl_new_default, ObUDRMgr::mtl_init, nullptr, ObUDRMgr::mtl_stop, nullptr, mtl_destroy_default); MTL_BIND2(mtl_new_default, ObTenantCGReadInfoMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); MTL_BIND2(mtl_new_default, ObDecodeResourcePool::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); - - MTL_BIND2(nullptr, ObPxPools::mtl_init, nullptr, ObPxPools::mtl_stop, nullptr, ObPxPools::mtl_destroy); - MTL_BIND(ObTenantDfc::mtl_init, ObTenantDfc::mtl_destroy); - MTL_BIND(init_compat_mode, nullptr); + MTL_BIND2(mtl_new_default, ObPxPools::mtl_init, nullptr, ObPxPools::mtl_stop, nullptr, ObPxPools::mtl_destroy); + MTL_BIND2(ObTenantDfc::mtl_new, ObTenantDfc::mtl_init, nullptr, nullptr, nullptr, ObTenantDfc::mtl_destroy); + MTL_BIND2(nullptr, init_compat_mode, nullptr, nullptr, nullptr, nullptr); MTL_BIND2(ObMySQLRequestManager::mtl_new, ObMySQLRequestManager::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, ObMySQLRequestManager::mtl_destroy); MTL_BIND2(mtl_new_default, ObTenantWeakReadService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); - //MTL_BIND(ObTransAuditRecordMgr::mtl_init, ObTransAuditRecordMgr::mtl_destroy); - MTL_BIND(ObTenantSqlMemoryManager::mtl_init, ObTenantSqlMemoryManager::mtl_destroy); - MTL_BIND(ObPlanMonitorNodeList::mtl_init, ObPlanMonitorNodeList::mtl_destroy); + //MTL_BIND2(ObTransAuditRecordMgr::mtl_init, ObTransAuditRecordMgr::mtl_destroy); + MTL_BIND2(ObTenantSqlMemoryManager::mtl_new, ObTenantSqlMemoryManager::mtl_init, nullptr, nullptr, nullptr, ObTenantSqlMemoryManager::mtl_destroy); + MTL_BIND2(mtl_new_default, ObPlanMonitorNodeList::mtl_init, nullptr, nullptr, nullptr, ObPlanMonitorNodeList::mtl_destroy); MTL_BIND2(mtl_new_default, ObTableLoadService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); MTL_BIND2(mtl_new_default, ObSharedMacroBlockMgr::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); - MTL_BIND(ObFLTSpanMgr::mtl_init, ObFLTSpanMgr::mtl_destroy); - MTL_BIND(common::sqlclient::ObTenantOciEnvs::mtl_init, common::sqlclient::ObTenantOciEnvs::mtl_destroy); + MTL_BIND2(mtl_new_default, ObFLTSpanMgr::mtl_init, nullptr, nullptr, nullptr, ObFLTSpanMgr::mtl_destroy); + MTL_BIND2(common::sqlclient::ObTenantOciEnvs::mtl_new, common::sqlclient::ObTenantOciEnvs::mtl_init, + nullptr, nullptr, nullptr, common::sqlclient::ObTenantOciEnvs::mtl_destroy); MTL_BIND2(mtl_new_default, ObPlanCache::mtl_init, nullptr, ObPlanCache::mtl_stop, nullptr, mtl_destroy_default); MTL_BIND2(mtl_new_default, ObPsCache::mtl_init, nullptr, ObPsCache::mtl_stop, nullptr, mtl_destroy_default); MTL_BIND2(server_obj_pool_mtl_new, nullptr, nullptr, nullptr, nullptr, server_obj_pool_mtl_destroy); MTL_BIND2(server_obj_pool_mtl_new, nullptr, nullptr, nullptr, nullptr, server_obj_pool_mtl_destroy); MTL_BIND2(mtl_new_default, ObTenantDirectLoadMgr::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); - MTL_BIND(ObDetectManager::mtl_init, ObDetectManager::mtl_destroy); - MTL_BIND(ObTenantSQLSessionMgr::mtl_init, ObTenantSQLSessionMgr::mtl_destroy); + MTL_BIND2(ObDetectManager::mtl_new, ObDetectManager::mtl_init, nullptr, nullptr, nullptr, ObDetectManager::mtl_destroy); + MTL_BIND2(ObTenantSQLSessionMgr::mtl_new, ObTenantSQLSessionMgr::mtl_init, nullptr, nullptr, nullptr, ObTenantSQLSessionMgr::mtl_destroy); MTL_BIND2(mtl_new_default, ObDTLIntermResultManager::mtl_init, ObDTLIntermResultManager::mtl_start, ObDTLIntermResultManager::mtl_stop, ObDTLIntermResultManager::mtl_wait, ObDTLIntermResultManager::mtl_destroy); if (GCONF._enable_new_sql_nio && GCONF._enable_tenant_sql_net_thread) { diff --git a/src/observer/omt/ob_tenant.h b/src/observer/omt/ob_tenant.h index 03aed0ff5f..8691830eb3 100644 --- a/src/observer/omt/ob_tenant.h +++ b/src/observer/omt/ob_tenant.h @@ -138,10 +138,7 @@ public: { int ret = common::OB_SUCCESS; uint64_t tenant_id = MTL_ID(); - pools = OB_NEW(ObPxPools, ObMemAttr(tenant_id, "PxPools")); - if (OB_ISNULL(pools)) { - ret = common::OB_ALLOCATE_MEMORY_FAILED; - } else if (OB_FAIL(pools->init(tenant_id))) { + if (OB_FAIL(pools->init(tenant_id))) { } return ret; } diff --git a/src/observer/omt/ob_th_worker.cpp b/src/observer/omt/ob_th_worker.cpp index 9c9cd47377..5cfca08e5e 100644 --- a/src/observer/omt/ob_th_worker.cpp +++ b/src/observer/omt/ob_th_worker.cpp @@ -224,7 +224,7 @@ ObThWorker::Status ObThWorker::check_wait() } else if (curr_time > last_check_time_ + WORKER_CHECK_PERIOD) { st = check_throttle(); if (st != WS_OUT_OF_THROTTLE) { - if (OB_UNLIKELY(curr_time > get_query_start_time() + threshold)) { + if (OB_UNLIKELY(0 != threshold && curr_time > get_query_start_time() + threshold)) { tenant_->lq_yield(*this); } } diff --git a/src/observer/table/ob_table_cg_service.cpp b/src/observer/table/ob_table_cg_service.cpp index 0d68e30dbb..82eb9b60e4 100644 --- a/src/observer/table/ob_table_cg_service.cpp +++ b/src/observer/table/ob_table_cg_service.cpp @@ -2004,6 +2004,7 @@ int ObTableDmlCgService::generate_das_base_ctdef(uint64_t index_tid, base_ctdef.index_tid_ = index_tid; base_ctdef.is_ignore_ = false; // insert ignore base_ctdef.is_batch_stmt_ = false; + base_ctdef.is_table_api_ = true; int64_t binlog_row_image = share::ObBinlogRowImage::FULL; ObSQLSessionInfo &session = ctx.get_session_info(); diff --git a/src/observer/table/ob_table_context.cpp b/src/observer/table/ob_table_context.cpp index abbd0f22d1..309cbb46ea 100644 --- a/src/observer/table/ob_table_context.cpp +++ b/src/observer/table/ob_table_context.cpp @@ -545,7 +545,7 @@ int ObTableCtx::adjust_column_type(const ObExprResType &column_type, if (!is_autoincrement) { ret = OB_BAD_NULL_ERROR; } - } else if (obj.is_null()) { + } else if (obj.is_null() || is_inc()) { // continue } else if (column_type.get_type() != obj.get_type() && !(ob_is_string_type(column_type.get_type()) && ob_is_string_type(obj.get_type()))) { @@ -876,7 +876,9 @@ int ObTableCtx::init_scan(const ObTableQuery &query, int ret = OB_SUCCESS; const ObString &index_name = query.get_index_name(); const ObIArray &select_columns = query.get_select_columns(); - const bool select_all_columns = select_columns.empty() || query.is_aggregate_query() || is_ttl_table_; + bool has_filter = (query.get_htable_filter().is_valid() || query.get_filter_string().length() > 0); + const bool select_all_columns = select_columns.empty() || query.is_aggregate_query() || is_ttl_table_ + || (has_filter && !is_htable()); const ObColumnSchemaV2 *column_schema = nullptr; operation_type_ = ObTableOperationType::Type::SCAN; // init is_weak_read_,scan_order_ diff --git a/src/observer/table/ob_table_context.h b/src/observer/table/ob_table_context.h index a30f899e80..0a7f88172c 100644 --- a/src/observer/table/ob_table_context.h +++ b/src/observer/table/ob_table_context.h @@ -247,6 +247,10 @@ public: return ObTableOperationType::Type::APPEND == operation_type_ || ObTableOperationType::Type::INCREMENT == operation_type_; } + OB_INLINE bool is_inc() const + { + return ObTableOperationType::Type::INCREMENT == operation_type_; + } OB_INLINE bool is_dml() const { return ObTableOperationType::Type::GET != operation_type_ && !is_scan_; @@ -556,7 +560,6 @@ public: column_infos_(alloc), alloc_(alloc) { - das_ctdef_.is_table_api_ = true; } TO_STRING_KV(K_(das_ctdef), K_(related_ctdefs)); @@ -596,7 +599,6 @@ public: related_ins_ctdefs_(alloc), alloc_(alloc) { - das_ctdef_.is_table_api_ = true; } TO_STRING_KV(K_(full_row), K_(delta_row), @@ -653,7 +655,6 @@ public: related_ctdefs_(alloc), alloc_(alloc) { - das_ctdef_.is_table_api_ = true; } TO_STRING_KV(K_(das_ctdef), K_(related_ctdefs)); @@ -745,7 +746,6 @@ public: das_ctdef_(alloc), alloc_(alloc) { - das_ctdef_.is_table_api_ = true; } TO_STRING_KV(K_(das_ctdef)); ObDASLockCtDef das_ctdef_; diff --git a/src/observer/table/ob_table_execute_processor.cpp b/src/observer/table/ob_table_execute_processor.cpp index dcfc323586..e7402882e0 100644 --- a/src/observer/table/ob_table_execute_processor.cpp +++ b/src/observer/table/ob_table_execute_processor.cpp @@ -114,6 +114,7 @@ int ObTableApiExecuteP::init_tb_ctx() ObTableOperationType::Type op_type = arg_.table_operation_.type(); tb_ctx_.set_entity(&arg_.table_operation_.entity()); tb_ctx_.set_operation_type(op_type); + tb_ctx_.set_entity_type(arg_.entity_type_); if (tb_ctx_.is_init()) { LOG_INFO("tb ctx has been inited", K_(tb_ctx)); diff --git a/src/observer/table/ob_table_filter.cpp b/src/observer/table/ob_table_filter.cpp index b80634af30..198c033c19 100644 --- a/src/observer/table/ob_table_filter.cpp +++ b/src/observer/table/ob_table_filter.cpp @@ -350,6 +350,49 @@ int ObTableFilterOperator::check_limit_param() return ret; } +int ObTableFilterOperator::init_full_column_name(const ObIArray& col_arr) +{ + int ret = OB_SUCCESS; + bool is_select_column_empty = query_->get_select_columns().empty(); // query select column is empty when do queryAndMutate + if (is_aggregate_query()) { + // do nothing + } else if (OB_FAIL(full_column_name_.assign(col_arr))) { + LOG_WARN("fail to assign full column name", K(ret)); + } else if (!is_select_column_empty && OB_FAIL(one_result_->assign_property_names(query_->get_select_columns()))) { // normal query should reset select column + LOG_WARN("fail to assign query column name", K(ret)); + } + return ret; +} + +int ObTableFilterOperator::add_row(table::ObTableQueryResult *next_result, ObNewRow *row) +{ + int ret = OB_SUCCESS; + ObNewRow new_row; + const ObIArray &select_columns = query_->get_select_columns(); + if (!select_columns.empty()) { + size_t new_size = select_columns.count(); + size_t old_size = full_column_name_.count(); + ObObj cell_arr[new_size]; + new_row.assign(cell_arr, new_size); + for (size_t i = 0; i < old_size; i ++) { + int64_t idx = -1; + if (!has_exist_in_array(select_columns, full_column_name_.at(i), &idx)) { + // do nothing + } else { + cell_arr[idx] = row->get_cell(i); + } + } + if (OB_FAIL(next_result->add_row(new_row))) { + LOG_WARN("failed to add row", K(ret)); + } + } else { // query select column is empty when do queryAndMutate + if (OB_FAIL(next_result->add_row(*row))) { + LOG_WARN("failed to add row", K(ret)); + } + } + return ret; +} + int ObTableFilterOperator::get_next_result(ObTableQueryResult *&next_result) { int ret = OB_SUCCESS; @@ -453,7 +496,7 @@ int ObTableFilterOperator::get_normal_result(table::ObTableQueryResult *&next_re if (OB_SUCC(ret)) { if (NULL != last_row_) { - if (OB_FAIL(one_result_->add_row(*last_row_))) { + if (OB_FAIL(add_row(one_result_, last_row_))) { LOG_WARN("failed to add row", K(ret)); } else { row_idx_++; @@ -469,7 +512,7 @@ int ObTableFilterOperator::get_normal_result(table::ObTableQueryResult *&next_re bool has_reach_limit = (row_idx_ >= offset + limit); next_result = one_result_; ObNewRow *row = nullptr; - const ObIArray &select_columns = one_result_->get_select_columns(); + const ObIArray &select_columns = full_column_name_; const int64_t N = select_columns.count(); while (OB_SUCC(ret) && (!has_limit || !has_reach_limit) && @@ -490,7 +533,7 @@ int ObTableFilterOperator::get_normal_result(table::ObTableQueryResult *&next_re if (has_limit && row_idx_ < offset) { row_idx_++; - } else if (OB_FAIL(one_result_->add_row(*row))) { + } else if (OB_FAIL(add_row(one_result_, row))) { if (OB_BUF_NOT_ENOUGH == ret) { ret = OB_SUCCESS; last_row_ = row; diff --git a/src/observer/table/ob_table_filter.h b/src/observer/table/ob_table_filter.h index cb77859740..057fccbc9b 100644 --- a/src/observer/table/ob_table_filter.h +++ b/src/observer/table/ob_table_filter.h @@ -207,9 +207,12 @@ public: is_first_result_(true), has_more_rows_(true), row_idx_(0), - agg_calculator_(query) + agg_calculator_(query), + full_column_name_() {} - virtual ~ObTableFilterOperator() {} + virtual ~ObTableFilterOperator() { + full_column_name_.reset(); + } virtual int get_next_result(ObTableQueryResult *&next_result) override; virtual bool has_more_result() const override { return has_more_rows_; } virtual void set_one_result(ObTableQueryResult *result) override { one_result_ = result; } @@ -220,6 +223,8 @@ public: int get_aggregate_result(table::ObTableQueryResult *&next_result); int get_normal_result(table::ObTableQueryResult *&next_result); bool is_aggregate_query() { return agg_calculator_.is_exist(); } + int add_row(table::ObTableQueryResult *next_result, ObNewRow *row); + int init_full_column_name(const ObIArray& col_arr); private: int check_limit_param(); private: @@ -234,6 +239,7 @@ private: bool has_more_rows_; int64_t row_idx_; // not filtered row index ObTableAggCalculator agg_calculator_; + ObSEArray full_column_name_; }; } // end namespace table diff --git a/src/observer/table/ob_table_query_and_mutate_processor.cpp b/src/observer/table/ob_table_query_and_mutate_processor.cpp index 3a6068262d..11f91b44a5 100644 --- a/src/observer/table/ob_table_query_and_mutate_processor.cpp +++ b/src/observer/table/ob_table_query_and_mutate_processor.cpp @@ -220,6 +220,7 @@ int ObTableQueryAndMutateP::init_scan_tb_ctx(ObTableApiCacheGuard &cache_guard) const ObTableQuery &query = arg_.query_and_mutate_.get_query(); bool is_weak_read = false; tb_ctx_.set_scan(true); + tb_ctx_.set_entity_type(arg_.entity_type_); if (tb_ctx_.is_init()) { LOG_INFO("tb ctx has been inited", K_(tb_ctx)); diff --git a/src/observer/table/ob_table_query_common.cpp b/src/observer/table/ob_table_query_common.cpp index 3afc1282f6..11710fddfe 100644 --- a/src/observer/table/ob_table_query_common.cpp +++ b/src/observer/table/ob_table_query_common.cpp @@ -68,8 +68,8 @@ int ObTableQueryUtils::generate_query_result_iterator(ObIAllocator &allocator, bool has_filter = (query.get_htable_filter().is_valid() || query.get_filter_string().length() > 0); const ObString &kv_attributes = tb_ctx.get_table_schema()->get_kv_attributes(); - if (OB_FAIL(one_result.assign_property_names(tb_ctx.get_query_col_names()))) { - LOG_WARN("fail to assign property names to one result", K(ret), K(tb_ctx)); + if (OB_FAIL(one_result.deep_copy_property_names(tb_ctx.get_query_col_names()))) { + LOG_WARN("fail to deep copy property names to one result", K(ret), K(tb_ctx)); } else if (has_filter) { if (is_hkv) { ObHTableFilterOperator *htable_result_iter = nullptr; @@ -105,6 +105,8 @@ int ObTableQueryUtils::generate_query_result_iterator(ObIAllocator &allocator, one_result))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to alloc table query result iterator", K(ret)); + } else if (OB_FAIL(table_result_iter->init_full_column_name(tb_ctx.get_query_col_names()))) { + LOG_WARN("fail to int full column name", K(ret)); } else if (OB_FAIL(table_result_iter->parse_filter_string(&allocator))) { LOG_WARN("fail to parse table filter string", K(ret)); } else { diff --git a/src/observer/table/ob_table_query_processor.cpp b/src/observer/table/ob_table_query_processor.cpp index 55324b3ab7..1c76ba53c5 100644 --- a/src/observer/table/ob_table_query_processor.cpp +++ b/src/observer/table/ob_table_query_processor.cpp @@ -127,6 +127,7 @@ int ObTableQueryP::init_tb_ctx(ObTableApiCacheGuard &cache_guard) ObExprFrameInfo *expr_frame_info = nullptr; bool is_weak_read = arg_.consistency_level_ == ObTableConsistencyLevel::EVENTUAL; tb_ctx_.set_scan(true); + tb_ctx_.set_entity_type(arg_.entity_type_); if (tb_ctx_.is_init()) { LOG_INFO("tb ctx has been inited", K_(tb_ctx)); diff --git a/src/observer/table/ob_table_query_sync_processor.cpp b/src/observer/table/ob_table_query_sync_processor.cpp index c635242026..4f2f980d37 100644 --- a/src/observer/table/ob_table_query_sync_processor.cpp +++ b/src/observer/table/ob_table_query_sync_processor.cpp @@ -395,6 +395,7 @@ int ObTableQuerySyncP::init_tb_ctx(ObTableCtx &ctx) ObExprFrameInfo &expr_frame_info = query_ctx.expr_frame_info_; bool is_weak_read = arg_.consistency_level_ == ObTableConsistencyLevel::EVENTUAL; ctx.set_scan(true); + ctx.set_entity_type(arg_.entity_type_); if (ctx.is_init()) { LOG_INFO("tb ctx has been inited", K(ctx)); @@ -541,7 +542,7 @@ int ObTableQuerySyncP::query_scan_without_init() if (OB_ISNULL(result_iter)) { ret = OB_ERR_NULL_VALUE; LOG_WARN("unexpected null result iterator", K(ret)); - } else if (OB_FAIL(result_.assign_property_names(tb_ctx.get_query_col_names()))) { + } else if (OB_FAIL(result_.deep_copy_property_names(tb_ctx.get_query_col_names()))) { LOG_WARN("fail to assign property names to one result", K(ret), K(tb_ctx)); } else { ObTableQueryResult *query_result = nullptr; diff --git a/src/observer/table_load/client/ob_table_direct_load_rpc_executor.cpp b/src/observer/table_load/client/ob_table_direct_load_rpc_executor.cpp index d0eaeb185f..cec0fcc986 100644 --- a/src/observer/table_load/client/ob_table_direct_load_rpc_executor.cpp +++ b/src/observer/table_load/client/ob_table_direct_load_rpc_executor.cpp @@ -221,6 +221,7 @@ int ObTableDirectLoadBeginExecutor::create_table_ctx() start_arg.tenant_id_ = tenant_id; start_arg.table_id_ = table_id; start_arg.parallelism_ = arg_.parallel_; + start_arg.is_load_data_ = true; if (OB_FAIL(ObTableLoadRedefTable::start(start_arg, start_res, *client_task_->get_session_info()))) { LOG_WARN("fail to start redef table", KR(ret), K(start_arg)); diff --git a/src/observer/table_load/ob_table_load_mem_compactor.cpp b/src/observer/table_load/ob_table_load_mem_compactor.cpp index b921e9abef..de5360188a 100644 --- a/src/observer/table_load/ob_table_load_mem_compactor.cpp +++ b/src/observer/table_load/ob_table_load_mem_compactor.cpp @@ -575,9 +575,6 @@ int ObTableLoadMemCompactor::finish() LOG_WARN("fail to start parallel merge", KR(ret)); } } - if (OB_SUCC(ret)) { - mem_ctx_.reset(); // mem_ctx的tables已经copy，需要提前释放 - } return ret; } @@ -608,6 +605,9 @@ int ObTableLoadMemCompactor::build_result_for_heap_table() } } } + if (OB_SUCC(ret)) { + mem_ctx_.reset(); + } return ret; } @@ -625,6 +625,9 @@ int ObTableLoadMemCompactor::add_table_to_parallel_merge_ctx() LOG_WARN("fail to add tablet sstable", KR(ret)); } } + if (OB_SUCC(ret)) { + mem_ctx_.reset(); // mem_ctx的tables已经copy，需要提前释放 + } return ret; } diff --git a/src/observer/table_load/ob_table_load_merger.cpp b/src/observer/table_load/ob_table_load_merger.cpp index 94bbc12008..2181c99238 100644 --- a/src/observer/table_load/ob_table_load_merger.cpp +++ b/src/observer/table_load/ob_table_load_merger.cpp @@ -656,6 +656,11 @@ int ObTableLoadMerger::handle_merge_thread_finish(int ret_code) if (OB_UNLIKELY(is_stop_ || has_error_)) { } else { LOG_INFO("LOAD MERGE COMPLETED"); + // release tmpfile + // TODO(suzhi.yt) release all tables and merge tasks + if (!store_ctx_->is_fast_heap_table_) { + table_compact_ctx_.result_.release_all_table_data(); + } if (store_ctx_->ctx_->schema_.is_column_store_) { if (OB_FAIL(build_rescan_ctx())) { LOG_WARN("fail to build rescan ctx", KR(ret)); diff --git a/src/observer/table_load/ob_table_load_multiple_heap_table_compactor.cpp b/src/observer/table_load/ob_table_load_multiple_heap_table_compactor.cpp index 1aa30b4ac0..697e16df3a 100644 --- a/src/observer/table_load/ob_table_load_multiple_heap_table_compactor.cpp +++ b/src/observer/table_load/ob_table_load_multiple_heap_table_compactor.cpp @@ -496,9 +496,6 @@ int ObTableLoadMultipleHeapTableCompactor::finish() } else if (OB_FAIL(compact_ctx_->handle_table_compact_success())) { LOG_WARN("fail to handle_table_compact_success", KR(ret)); } - if (OB_SUCC(ret)) { - mem_ctx_.reset(); // mem_ctx的tables已经copy，需要提前释放 - } return ret; } @@ -529,6 +526,9 @@ int ObTableLoadMultipleHeapTableCompactor::build_result_for_heap_table() } } } + if (OB_SUCC(ret)) { + mem_ctx_.reset(); // mem_ctx的tables已经copy，需要提前释放 + } return ret; } diff --git a/src/observer/table_load/ob_table_load_service.cpp b/src/observer/table_load/ob_table_load_service.cpp index 5a2a15505b..92b33d5e86 100644 --- a/src/observer/table_load/ob_table_load_service.cpp +++ b/src/observer/table_load/ob_table_load_service.cpp @@ -407,17 +407,22 @@ int ObTableLoadService::check_support_direct_load(uint64_t table_id) ObTableLoadSchema::get_table_schema(tenant_id, table_id, schema_guard, table_schema))) { LOG_WARN("fail to get table schema", KR(ret), K(tenant_id), K(table_id)); } - // check if it is an oracle temporary table - else if (lib::is_oracle_mode() && table_schema->is_tmp_table()) { + // check if it is a user table + else if (!table_schema->is_user_table()) { ret = OB_NOT_SUPPORTED; - LOG_WARN("direct-load does not support oracle temporary table", KR(ret)); - FORWARD_USER_ERROR_MSG(ret, "direct-load does not support oracle temporary table"); - } - // check if it is a view - else if (table_schema->is_view_table()) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("direct-load does not support view table", KR(ret)); - FORWARD_USER_ERROR_MSG(ret, "direct-load does not support view table"); + if (lib::is_oracle_mode() && table_schema->is_tmp_table()) { + LOG_WARN("direct-load does not support oracle temporary table", KR(ret)); + FORWARD_USER_ERROR_MSG(ret, "direct-load does not support oracle temporary table"); + } else if (table_schema->is_view_table()) { + LOG_WARN("direct-load does not support view table", KR(ret)); + FORWARD_USER_ERROR_MSG(ret, "direct-load does not support view table"); + } else if (table_schema->is_mlog_table()) { + LOG_WARN("direct-load does not support materialized view log table", KR(ret)); + FORWARD_USER_ERROR_MSG(ret, "direct-load does not support materialized view log table"); + } else { + LOG_WARN("direct-load does not support non-user table", KR(ret)); + FORWARD_USER_ERROR_MSG(ret, "direct-load does not support non-user table"); + } } // check if exists generated column else if (OB_UNLIKELY(table_schema->has_generated_column())) { @@ -441,6 +446,12 @@ int ObTableLoadService::check_support_direct_load(uint64_t table_id) LOG_WARN("direct-load does not support table has udt column", KR(ret)); FORWARD_USER_ERROR_MSG(ret, "direct-load does not support table has udt column"); } + // check if table has mlog + else if (table_schema->has_mlog_table()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("direct-load does not support table with materialized view log", KR(ret)); + FORWARD_USER_ERROR_MSG(ret, "direct-load does not support table with materialized view log"); + } } return ret; } diff --git a/src/observer/table_load/ob_table_load_store.cpp b/src/observer/table_load/ob_table_load_store.cpp index 871e111d04..406c24588d 100644 --- a/src/observer/table_load/ob_table_load_store.cpp +++ b/src/observer/table_load/ob_table_load_store.cpp @@ -961,6 +961,29 @@ int ObTableLoadStore::px_finish_trans(const ObTableLoadTransId &trans_id) return ret; } +int ObTableLoadStore::px_check_for_write(const ObTabletID &tablet_id) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("ObTableLoadStore not init", KR(ret), KP(this)); + } else { + bool is_exist = false; + for (int64_t i = 0; i < store_ctx_->ls_partition_ids_.count(); ++i) { + const ObTableLoadLSIdAndPartitionId &ls_part_id = store_ctx_->ls_partition_ids_.at(i); + if (ls_part_id.part_tablet_id_.tablet_id_ == tablet_id) { + is_exist = true; + break; + } + } + if (OB_UNLIKELY(!is_exist)) { + ret = OB_NOT_MASTER; + LOG_WARN("not partition master", KR(ret), K(tablet_id), K(store_ctx_->ls_partition_ids_)); + } + } + return ret; +} + int ObTableLoadStore::px_write(const ObTableLoadTransId &trans_id, const ObTabletID &tablet_id, const ObIArray &row_array) { diff --git a/src/observer/table_load/ob_table_load_store.h b/src/observer/table_load/ob_table_load_store.h index 59d2cd28fe..84bdfb095c 100644 --- a/src/observer/table_load/ob_table_load_store.h +++ b/src/observer/table_load/ob_table_load_store.h @@ -85,6 +85,7 @@ private: public: int px_start_trans(const table::ObTableLoadTransId &trans_id); int px_finish_trans(const table::ObTableLoadTransId &trans_id); + int px_check_for_write(const ObTabletID &tablet_id); int px_write(const table::ObTableLoadTransId &trans_id, const ObTabletID &tablet_id, const common::ObIArray &row_array); diff --git a/src/observer/table_load/ob_table_load_table_compactor.cpp b/src/observer/table_load/ob_table_load_table_compactor.cpp index 390505f54f..ffcaf501ea 100644 --- a/src/observer/table_load/ob_table_load_table_compactor.cpp +++ b/src/observer/table_load/ob_table_load_table_compactor.cpp @@ -97,22 +97,26 @@ int ObTableLoadTableCompactResult::add_table(ObIDirectLoadPartitionTable *table) return ret; } +void ObTableLoadTableCompactResult::release_all_table_data() +{ + for (int64_t i = 0; i < all_table_array_.count(); ++i) { + ObIDirectLoadPartitionTable *table = all_table_array_.at(i); + table->release_data(); + } +} + /** * ObTableLoadTableCompactCtx */ ObTableLoadTableCompactCtx::ObTableLoadTableCompactCtx() - : allocator_("TLD_TCCtx"), store_ctx_(nullptr), merger_(nullptr), compactor_(nullptr) + : store_ctx_(nullptr), merger_(nullptr), compactor_(nullptr) { } ObTableLoadTableCompactCtx::~ObTableLoadTableCompactCtx() { - if (nullptr != compactor_) { - compactor_->~ObTableLoadTableCompactor(); - allocator_.free(compactor_); - compactor_ = nullptr; - } + release_compactor(); } int ObTableLoadTableCompactCtx::init(ObTableLoadStoreCtx *store_ctx, ObTableLoadMerger &merger) @@ -125,7 +129,6 @@ int ObTableLoadTableCompactCtx::init(ObTableLoadStoreCtx *store_ctx, ObTableLoad if (OB_FAIL(result_.init())) { LOG_WARN("fail to init result", KR(ret)); } else { - allocator_.set_tenant_id(MTL_ID()); store_ctx_ = store_ctx; merger_ = &merger; } @@ -138,37 +141,61 @@ bool ObTableLoadTableCompactCtx::is_valid() const return nullptr != store_ctx_ && nullptr != merger_; } -ObTableLoadTableCompactor *ObTableLoadTableCompactCtx::new_compactor() +int ObTableLoadTableCompactCtx::new_compactor() { - ObTableLoadTableCompactor *ret = nullptr; - if (store_ctx_->is_multiple_mode_) { - if (store_ctx_->table_data_desc_.is_heap_table_) { - ret = OB_NEWx(ObTableLoadMultipleHeapTableCompactor, (&allocator_)); - } else { - ret = OB_NEWx(ObTableLoadMemCompactor, (&allocator_)); - } + int ret = OB_SUCCESS; + obsys::ObWLockGuard guard(rwlock_); + if (OB_NOT_NULL(compactor_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected not null compactor", KR(ret), KP(compactor_)); } else { - ret = OB_NEWx(ObTableLoadGeneralTableCompactor, (&allocator_)); + ObMemAttr attr(MTL_ID(), "TLD_Compactor"); + if (store_ctx_->is_multiple_mode_) { + if (store_ctx_->table_data_desc_.is_heap_table_) { + compactor_ = OB_NEW(ObTableLoadMultipleHeapTableCompactor, attr); + } else { + compactor_ = OB_NEW(ObTableLoadMemCompactor, attr); + } + } else { + compactor_ = OB_NEW(ObTableLoadGeneralTableCompactor, attr); + } + if (OB_ISNULL(compactor_)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to new ObTableLoadTableCompactor", KR(ret)); + } } return ret; } +void ObTableLoadTableCompactCtx::release_compactor() +{ + obsys::ObWLockGuard guard(rwlock_); + if (nullptr != compactor_) { + ObMemAttr attr(MTL_ID(), "TLD_Compactor"); + OB_DELETE(ObTableLoadTableCompactor, attr, compactor_); + compactor_ = nullptr; + } +} + int ObTableLoadTableCompactCtx::start() { int ret = OB_SUCCESS; - if (OB_ISNULL(compactor_ = new_compactor())) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to new ObTableLoadGeneralTableCompactor", KR(ret)); - } else if (OB_FAIL(compactor_->init(this))) { - LOG_WARN("fail to init compactor", KR(ret)); - } else if (OB_FAIL(compactor_->start())) { - LOG_WARN("fail to start compactor", KR(ret)); + if (OB_FAIL(new_compactor())) { + LOG_WARN("fail to new compactor", KR(ret)); + } else { + obsys::ObRLockGuard guard(rwlock_); + if (OB_FAIL(compactor_->init(this))) { + LOG_WARN("fail to init compactor", KR(ret)); + } else if (OB_FAIL(compactor_->start())) { + LOG_WARN("fail to start compactor", KR(ret)); + } } return ret; } void ObTableLoadTableCompactCtx::stop() { + obsys::ObRLockGuard guard(rwlock_); if (OB_NOT_NULL(compactor_)) { compactor_->stop(); } @@ -177,9 +204,7 @@ void ObTableLoadTableCompactCtx::stop() int ObTableLoadTableCompactCtx::handle_table_compact_success() { // release compactor - compactor_->~ObTableLoadTableCompactor(); - allocator_.free(compactor_); - compactor_ = nullptr; + release_compactor(); // notify merger return merger_->handle_table_compact_success(); } diff --git a/src/observer/table_load/ob_table_load_table_compactor.h b/src/observer/table_load/ob_table_load_table_compactor.h index 3854755cbe..bc3ac4bb47 100644 --- a/src/observer/table_load/ob_table_load_table_compactor.h +++ b/src/observer/table_load/ob_table_load_table_compactor.h @@ -39,6 +39,7 @@ public: void reset(); int init(); int add_table(storage::ObIDirectLoadPartitionTable *table); + void release_all_table_data(); public: typedef common::ObLinkHashMap TabletResultMap; @@ -59,12 +60,13 @@ public: int handle_table_compact_success(); TO_STRING_KV(KP_(store_ctx), KP_(merger), KP_(compactor)); private: - ObTableLoadTableCompactor *new_compactor(); + int new_compactor(); + void release_compactor(); public: - common::ObArenaAllocator allocator_; ObTableLoadStoreCtx *store_ctx_; ObTableLoadMerger *merger_; + mutable obsys::ObRWLock rwlock_; ObTableLoadTableCompactor *compactor_; ObTableLoadTableCompactResult result_; }; diff --git a/src/observer/virtual_table/ob_all_virtual_tenant_snapshot_ls_replica.cpp b/src/observer/virtual_table/ob_all_virtual_tenant_snapshot_ls_replica.cpp index 7c04247901..354bfd5229 100644 --- a/src/observer/virtual_table/ob_all_virtual_tenant_snapshot_ls_replica.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tenant_snapshot_ls_replica.cpp @@ -110,13 +110,14 @@ int ObAllVirtualTenantSnapshotLSReplica::process_curr_tenant(ObNewRow *&row) SERVER_LOG(WARN, "row is null or column count mismatch", KR(ret), KP(r), K(cur_row_.count_)); } else { ObString ls_meta_package_str; - HEAP_VAR(ObLSMetaPackage, ls_meta_package) { - for (int64_t i = 0; OB_SUCC(ret) && i < cur_row_.count_; ++i) { - uint64_t col_id = output_column_ids_.at(i); - if (LS_META_PACKAGE == col_id) { // decode ls_meta_package column - int64_t length = 0; - EXTRACT_VARCHAR_FIELD_MYSQL(*result_, "ls_meta_package", ls_meta_package_str); - if (ls_meta_package_str.empty()) { + for (int64_t i = 0; OB_SUCC(ret) && i < cur_row_.count_; ++i) { + uint64_t col_id = output_column_ids_.at(i); + if (LS_META_PACKAGE == col_id) { // decode ls_meta_package column + int64_t length = 0; + HEAP_VAR(ObLSMetaPackage, ls_meta_package) { + EXTRACT_VARCHAR_FIELD_MYSQL_SKIP_RET(*result_, "ls_meta_package", ls_meta_package_str); + if (OB_FAIL(ret)) { + } else if (ls_meta_package_str.empty()) { cur_row_.cells_[i].reset(); } else if (OB_ISNULL(ls_meta_buf_)) { ret = OB_ERR_UNEXPECTED; @@ -128,17 +129,17 @@ int ObAllVirtualTenantSnapshotLSReplica::process_curr_tenant(ObNewRow *&row) SERVER_LOG(WARN, "fail to get ls_meta_package string", KR(ret), K(length)); } else { cur_row_.cells_[i].set_lob_value(ObObjType::ObLongTextType, ls_meta_buf_, - static_cast(length)); + static_cast(length)); cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( ObCharset::get_default_charset())); } - } else if (col_id - OB_APP_MIN_COLUMN_ID >= 0 && col_id - OB_APP_MIN_COLUMN_ID < r->count_) { - // direct copy other columns - cur_row_.cells_[i] = r->get_cell(col_id - OB_APP_MIN_COLUMN_ID); - } else { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "unexpected column id", KR(ret), K(col_id), K(output_column_ids_)); } + } else if (col_id - OB_APP_MIN_COLUMN_ID >= 0 && col_id - OB_APP_MIN_COLUMN_ID < r->count_) { + // direct copy other columns + cur_row_.cells_[i] = r->get_cell(col_id - OB_APP_MIN_COLUMN_ID); + } else { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "unexpected column id", KR(ret), K(col_id), K(output_column_ids_)); } } } diff --git a/src/observer/virtual_table/ob_all_virtual_tenant_snapshot_ls_replica_history.cpp b/src/observer/virtual_table/ob_all_virtual_tenant_snapshot_ls_replica_history.cpp index 8bee20e815..65e481aeda 100644 --- a/src/observer/virtual_table/ob_all_virtual_tenant_snapshot_ls_replica_history.cpp +++ b/src/observer/virtual_table/ob_all_virtual_tenant_snapshot_ls_replica_history.cpp @@ -110,13 +110,14 @@ int ObAllVirtualTenantSnapshotLSReplicaHistory::process_curr_tenant(ObNewRow *&r SERVER_LOG(WARN, "row is null or column count mismatch", KR(ret), KP(r), K(cur_row_.count_)); } else { ObString ls_meta_package_str; - HEAP_VAR(ObLSMetaPackage, ls_meta_package) { - for (int64_t i = 0; OB_SUCC(ret) && i < cur_row_.count_; ++i) { - uint64_t col_id = output_column_ids_.at(i); - if (LS_META_PACKAGE == col_id) { // decode ls_meta_package column - int64_t length = 0; - EXTRACT_VARCHAR_FIELD_MYSQL(*result_, "ls_meta_package", ls_meta_package_str); - if (ls_meta_package_str.empty()) { + for (int64_t i = 0; OB_SUCC(ret) && i < cur_row_.count_; ++i) { + uint64_t col_id = output_column_ids_.at(i); + if (LS_META_PACKAGE == col_id) { // decode ls_meta_package column + int64_t length = 0; + HEAP_VAR(ObLSMetaPackage, ls_meta_package) { + EXTRACT_VARCHAR_FIELD_MYSQL_SKIP_RET(*result_, "ls_meta_package", ls_meta_package_str); + if (OB_FAIL(ret)) { + } else if (ls_meta_package_str.empty()) { cur_row_.cells_[i].reset(); } else if (OB_ISNULL(ls_meta_buf_)) { ret = OB_ERR_UNEXPECTED; @@ -128,17 +129,17 @@ int ObAllVirtualTenantSnapshotLSReplicaHistory::process_curr_tenant(ObNewRow *&r SERVER_LOG(WARN, "fail to get ls_meta_package string", KR(ret), K(length)); } else { cur_row_.cells_[i].set_lob_value(ObObjType::ObLongTextType, ls_meta_buf_, - static_cast(length)); + static_cast(length)); cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( ObCharset::get_default_charset())); } - } else if (col_id - OB_APP_MIN_COLUMN_ID >= 0 && col_id - OB_APP_MIN_COLUMN_ID < r->count_) { - // direct copy other columns - cur_row_.cells_[i] = r->get_cell(col_id - OB_APP_MIN_COLUMN_ID); - } else { - ret = OB_ERR_UNEXPECTED; - SERVER_LOG(WARN, "unexpected column id", KR(ret), K(col_id), K(output_column_ids_)); } + } else if (col_id - OB_APP_MIN_COLUMN_ID >= 0 && col_id - OB_APP_MIN_COLUMN_ID < r->count_) { + // direct copy other columns + cur_row_.cells_[i] = r->get_cell(col_id - OB_APP_MIN_COLUMN_ID); + } else { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "unexpected column id", KR(ret), K(col_id), K(output_column_ids_)); } } } diff --git a/src/observer/virtual_table/ob_gv_sql.cpp b/src/observer/virtual_table/ob_gv_sql.cpp index efa0279acd..2b49e22de1 100644 --- a/src/observer/virtual_table/ob_gv_sql.cpp +++ b/src/observer/virtual_table/ob_gv_sql.cpp @@ -206,7 +206,17 @@ int ObGVSql::fill_cells(const ObILibCacheObject *cache_obj, const ObPlanCache &p //sql_id case share::ALL_VIRTUAL_PLAN_STAT_CDE::SQL_ID: { ObString sql_id; - if (!cache_obj->is_sql_crsr()) { + if (OB_NOT_NULL(pl_func)) { + if (OB_FAIL(ob_write_string(*allocator_, + pl_func->get_stat().sql_id_, + sql_id))) { + SERVER_LOG(ERROR, "copy sql_id failed", K(ret)); + } else { + cells[i].set_varchar(sql_id); + cells[i].set_collation_type(ObCharset::get_default_collation( + ObCharset::get_default_charset())); + } + } else if (!cache_obj->is_sql_crsr()) { cells[i].set_null(); } else if (OB_FAIL(ob_write_string(*allocator_, plan->stat_.sql_id_, diff --git a/src/pl/ob_pl.cpp b/src/pl/ob_pl.cpp index 98472e3adc..559436eb38 100644 --- a/src/pl/ob_pl.cpp +++ b/src/pl/ob_pl.cpp @@ -2098,7 +2098,14 @@ int ObPL::get_pl_function(ObExecContext &ctx, pc_ctx.schema_guard_ = ctx.get_sql_ctx()->schema_guard_; pc_ctx.cache_params_ = ¶ms; pc_ctx.raw_sql_ = sql; - MEMCPY(pc_ctx.sql_id_, ctx.get_sql_ctx()->sql_id_, (int32_t)sizeof(ctx.get_sql_ctx()->sql_id_)); + if (ctx.get_sql_ctx()->sql_id_[0] != '\0') { + MEMCPY(pc_ctx.sql_id_, ctx.get_sql_ctx()->sql_id_, (int32_t)sizeof(ctx.get_sql_ctx()->sql_id_)); + } else { + CK (!pc_ctx.raw_sql_.empty()); + OX ((void)ObSQLUtils::md5(pc_ctx.raw_sql_, + pc_ctx.sql_id_, + (int32_t)sizeof(pc_ctx.sql_id_))); + } pc_ctx.key_.namespace_ = ObLibCacheNameSpace::NS_ANON; pc_ctx.key_.db_id_ = database_id; @@ -2572,11 +2579,6 @@ int ObPLExecState::set_var(int64_t var_idx, const ObObjParam& value) OZ (ctx_.get_user_type(udt_id, user_type), K(udt_id)); CK (OB_NOT_NULL(user_type)); OZ (init_complex_obj(*get_allocator(), *user_type, params->at(var_idx))); - if (OB_SUCC(ret) && user_type->is_collection_type()) { - ObPLCollection *coll = reinterpret_cast(params->at(var_idx).get_ext()); - CK (OB_NOT_NULL(coll)); - OX (coll->set_count(OB_INVALID_COUNT)); - } } } else if (!copy_value.is_ext()) { bool is_ref_cursor = params->at(var_idx).is_ref_cursor_type(); @@ -2836,6 +2838,7 @@ int ObPLExecState::init_complex_obj(ObIAllocator &allocator, common::ObMySQLProxy *sql_proxy = NULL; ObPLPackageGuard *package_guard = NULL; const ObPLDataType *real_pl_type = &pl_type; + bool set_null = pl_type.is_record_type() ? true : (top_call_ && ctx_.exec_ctx_->get_sql_ctx()->is_execute_call_stmt_) ? false : true; CK (OB_NOT_NULL(session = ctx_.exec_ctx_->get_my_session())); CK (OB_NOT_NULL(schema_guard = ctx_.exec_ctx_->get_sql_ctx()->schema_guard_)); CK (OB_NOT_NULL(sql_proxy = ctx_.exec_ctx_->get_sql_proxy())); @@ -2876,12 +2879,12 @@ int ObPLExecState::init_complex_obj(ObIAllocator &allocator, OX (obj.set_is_ref_cursor_type(true)); } else if (real_pl_type->is_udt_type()) { ObPLUDTNS ns(*schema_guard); - OZ (ns.init_complex_obj(allocator, *real_pl_type, obj, false)); + OZ (ns.init_complex_obj(allocator, *real_pl_type, obj, false, set_null)); } else if (OB_NOT_NULL(session->get_pl_context()) && OB_NOT_NULL(session->get_pl_context()->get_current_ctx())) { pl::ObPLINS *ns = session->get_pl_context()->get_current_ctx(); CK (OB_NOT_NULL(ns)); - OZ (ns->init_complex_obj(allocator, *real_pl_type, obj, false)); + OZ (ns->init_complex_obj(allocator, *real_pl_type, obj, false, set_null)); } else { ObPLResolveCtx ns(allocator, *session, @@ -2889,7 +2892,7 @@ int ObPLExecState::init_complex_obj(ObIAllocator &allocator, *package_guard, *sql_proxy, false); - OZ (ns.init_complex_obj(allocator, *real_pl_type, obj, false)); + OZ (ns.init_complex_obj(allocator, *real_pl_type, obj, false, set_null)); } OX (obj.set_udt_id(real_pl_type->get_user_type_id())); return ret; @@ -4223,7 +4226,7 @@ int ObPLINS::init_complex_obj(ObIAllocator &allocator, const ObPLDataType &pl_type, common::ObObjParam &obj, bool set_allocator, - bool set_record_null) const + bool set_null) const { int ret = OB_SUCCESS; int64_t init_size = 0; @@ -4269,7 +4272,7 @@ int ObPLINS::init_complex_obj(ObIAllocator &allocator, } // f(self object_type, p1 out object_type), p1 will be init here, we have to set it null // but self can't be set to null. - if (OB_SUCC(ret) && user_type->is_object_type() && set_record_null) { + if (OB_SUCC(ret) && user_type->is_object_type() && set_null) { OX (record->set_is_null(true)); } } @@ -4309,10 +4312,12 @@ int ObPLINS::init_complex_obj(ObIAllocator &allocator, CK (OB_NOT_NULL(coll)); OX (set_allocator ? coll->set_allocator(&allocator) : coll->set_allocator(NULL)); if (OB_FAIL(ret)) { - } else if (user_type->is_associative_array_type()) { - coll->set_inited(); - } else { - OX ((obj.is_ext() && obj.get_ext() != 0) ? (void)NULL : coll->set_inited()); + } else if (!set_null) { + if (user_type->is_associative_array_type()) { + OX (coll->set_inited()); + } else { + OX ((obj.is_ext() && obj.get_ext() != 0) ? (void)NULL : coll->set_inited()); + } } OX (coll->set_type(pl_type.get_type())); OZ (get_element_data_type(pl_type, elem_desc, &allocator)); diff --git a/src/pl/ob_pl.h b/src/pl/ob_pl.h index 5433b31248..196c3bb289 100644 --- a/src/pl/ob_pl.h +++ b/src/pl/ob_pl.h @@ -134,7 +134,7 @@ public: const ObPLDataType &pl_type, common::ObObjParam &obj, bool set_allocator = false, - bool set_record_null = true) const; + bool set_null = true) const; }; class ObPLFunctionBase diff --git a/src/pl/ob_pl_code_generator.cpp b/src/pl/ob_pl_code_generator.cpp index 952ce47d4c..98d956a312 100644 --- a/src/pl/ob_pl_code_generator.cpp +++ b/src/pl/ob_pl_code_generator.cpp @@ -433,6 +433,11 @@ int ObPLCodeGenerateVisitor::visit(const ObPLAssignStmt &s) LOG_WARN("Unexpected const expr", K(const_expr->get_value()), K(ret)); } } else if (into_expr->is_obj_access_expr()) { + if (s.get_value_index(i) != PL_CONSTRUCT_COLLECTION + && ObObjAccessIdx::has_same_collection_access(s.get_value_expr(i), static_cast(into_expr))) { + ObLLVMValue p_result_obj; + OZ (generator_.generate_expr(s.get_value_index(i), s, OB_INVALID_INDEX, p_result_obj)); + } OZ (generator_.generate_expr(s.get_into_index(i), s, OB_INVALID_INDEX, into_address)); } diff --git a/src/pl/ob_pl_package_manager.cpp b/src/pl/ob_pl_package_manager.cpp index 280fe54f6c..ddfc43c579 100644 --- a/src/pl/ob_pl_package_manager.cpp +++ b/src/pl/ob_pl_package_manager.cpp @@ -849,6 +849,7 @@ int ObPLPackageManager::set_package_var_val(const ObPLResolveCtx &resolve_ctx, } if (!need_deserialize) { OZ (package_state->update_changed_vars(var_idx)); + OX (resolve_ctx.session_info_.set_pl_can_retry(false)); } return ret; } diff --git a/src/pl/ob_pl_resolver.cpp b/src/pl/ob_pl_resolver.cpp index cb8d1bc5d6..920be29e45 100644 --- a/src/pl/ob_pl_resolver.cpp +++ b/src/pl/ob_pl_resolver.cpp @@ -3750,17 +3750,17 @@ int ObPLResolver::resolve_question_mark_node( return ret; } -bool ObPLResolver::is_question_mark_value(ObRawExpr *into_expr) +bool ObPLResolver::is_question_mark_value(ObRawExpr *into_expr, ObPLBlockNS *ns) { bool ret = false; if (OB_NOT_NULL(into_expr) && T_QUESTIONMARK == into_expr->get_expr_type() && (static_cast(into_expr))->get_value().is_unknown() - && OB_NOT_NULL(current_block_) - && OB_NOT_NULL(current_block_->get_symbol_table())) { + && OB_NOT_NULL(ns) + && OB_NOT_NULL(ns->get_symbol_table())) { const ObPLVar *var = NULL; int64_t idx = (static_cast(into_expr))->get_value().get_unknown(); - if (OB_NOT_NULL(var = current_block_->get_symbol_table()->get_symbol(idx))) { + if (OB_NOT_NULL(var = ns->get_symbol_table()->get_symbol(idx))) { if (var->get_name().prefix_match(ANONYMOUS_ARG)) { ret = true; } @@ -3769,21 +3769,33 @@ bool ObPLResolver::is_question_mark_value(ObRawExpr *into_expr) return ret; } -int ObPLResolver::set_question_mark_type(ObRawExpr *into_expr, const ObPLDataType *type) +int ObPLResolver::set_question_mark_type(ObRawExpr *into_expr, ObPLBlockNS *ns, const ObPLDataType *type) { int ret = OB_SUCCESS; ObConstRawExpr *const_expr = NULL; const ObPLVar *var = NULL; + ObExprResType res_type; CK (OB_NOT_NULL(into_expr)); CK (OB_NOT_NULL(type)); + CK (OB_NOT_NULL(ns)); CK (T_QUESTIONMARK == into_expr->get_expr_type()); CK (OB_NOT_NULL(const_expr = static_cast(into_expr))); - CK (OB_NOT_NULL(current_block_->get_symbol_table())); - CK (OB_NOT_NULL(var = current_block_ - ->get_symbol_table()->get_symbol(const_expr->get_value().get_unknown()))); + CK (OB_NOT_NULL(ns->get_symbol_table())); + CK (OB_NOT_NULL(var = ns->get_symbol_table()->get_symbol(const_expr->get_value().get_unknown()))); CK (var->get_name().prefix_match(ANONYMOUS_ARG)); OX ((const_cast(var))->set_type(*type)); OX ((const_cast(var))->set_readonly(false)); + if (OB_FAIL(ret)) { + } else if (type->is_obj_type()) { + CK (OB_NOT_NULL(type->get_data_type())); + OX (res_type.set_meta(type->get_data_type()->get_meta_type())); + OX (res_type.set_accuracy(type->get_data_type()->get_accuracy())); + } else { + OX (res_type.set_ext()); + OX (res_type.set_extend_type(type->get_type())); + OX (res_type.set_udt_id(type->get_user_type_id())); + } + OX (into_expr->set_result_type(res_type)); return ret; } @@ -3943,19 +3955,19 @@ int ObPLResolver::resolve_assign(const ObStmtNodeTree *parse_tree, ObPLAssignStm } } // 目标是QuestionMark, 需要设置目标的类型, 因为QuestionMark默认是无类型的, 在赋值时确定类型 - if (OB_SUCC(ret) && is_question_mark_value(into_expr)) { + if (OB_SUCC(ret) && is_question_mark_value(into_expr, &(current_block_->get_namespace()))) { if (value_expr->get_result_type().is_ext()) { const ObUserDefinedType *user_type = NULL; OZ (current_block_->get_namespace().get_pl_data_type_by_id( value_expr->get_result_type().get_udt_id(), user_type)); - OZ (set_question_mark_type(into_expr, user_type)); + OZ (set_question_mark_type(into_expr, &(current_block_->get_namespace()), user_type)); } else { ObDataType data_type; ObPLDataType pl_type; OX (data_type.set_meta_type(value_expr->get_result_type())); OX (data_type.set_accuracy(value_expr->get_result_type().get_accuracy())); OX (pl_type.set_data_type(data_type)); - OX (set_question_mark_type(into_expr, &pl_type)); + OX (set_question_mark_type(into_expr, &(current_block_->get_namespace()), &pl_type)); } } } else { @@ -8303,8 +8315,9 @@ int ObPLResolver::resolve_fetch( !right->get_data_type()->get_meta_type().is_ext()) { if (right->get_data_type()->get_meta_type().is_null() && stmt->get_into().count() > i && - is_question_mark_value(func.get_expr(stmt->get_into(i)))) { - OZ (set_question_mark_type(func.get_expr(stmt->get_into(i)), left)); + is_question_mark_value(func.get_expr(stmt->get_into(i)), &(current_block_->get_namespace()))) { + OZ (set_question_mark_type( + func.get_expr(stmt->get_into(i)), &(current_block_->get_namespace()), left)); } else { CK (OB_NOT_NULL(left->get_data_type())); OX (is_compatible = cast_supported(left->get_data_type()->get_obj_type(), @@ -11966,18 +11979,17 @@ int ObPLResolver::resolve_udf_info( ObExprResType result_type; CK (OB_NOT_NULL(var = table->get_symbol(position))); if (OB_SUCC(ret) && var->is_readonly()) { - if (var->get_name().prefix_match(ANONYMOUS_ARG)) { - ObPLVar* shadow_var = const_cast(var); - ObIRoutineParam *iparam = NULL; - OX (shadow_var->set_readonly(false)); - CK (OB_NOT_NULL(routine_info)); - OZ (routine_info->get_routine_param(i, iparam)); - if (OB_SUCC(ret) && iparam->is_inout_param()) { - shadow_var->set_name(ANONYMOUS_INOUT_ARG); - } - } else { - ret = OB_ERR_VARIABLE_IS_READONLY; - LOG_WARN("variable is read only", K(ret), K(position), KPC(var)); + ret = OB_ERR_VARIABLE_IS_READONLY; + LOG_WARN("variable is read only", K(ret), K(position), KPC(var)); + } + if (OB_SUCC(ret) && var->get_name().prefix_match(ANONYMOUS_ARG)) { + ObPLVar* shadow_var = const_cast(var); + ObIRoutineParam *iparam = NULL; + OX (shadow_var->set_readonly(false)); + CK (OB_NOT_NULL(routine_info)); + OZ (routine_info->get_routine_param(i, iparam)); + if (OB_SUCC(ret) && iparam->is_inout_param()) { + shadow_var->set_name(ANONYMOUS_INOUT_ARG); } } if (OB_SUCC(ret)) { @@ -13459,7 +13471,7 @@ int ObPLResolver::resolve_routine(ObObjAccessIdent &access_ident, && OB_FAIL(resolve_composite_access(access_ident, access_idxs, ns, func))) { LOG_WARN("failed to access composite access", K(ret), K(access_ident), K(access_idxs)); } - if (OB_FAIL(ret)) { + if (OB_FAIL(ret) && OB_ERR_NOT_FUNC_NAME != ret) { LOG_INFO("failed to resolve routine", K(ret), K(database_name), K(package_name), K(routine_name), K(routine_type), K(access_ident), K(access_idxs)); ret = OB_ERR_FUNCTION_UNKNOWN; @@ -14087,6 +14099,19 @@ int ObPLResolver::resolve_access_ident(ObObjAccessIdent &access_ident, // 当前 access_ident.access_name_, pl_data_type, var_index); + if (ObPLExternalNS::PKG_VAR == type + && cnt > 0) { + if (ObObjAccessIdx::IS_PKG_NS == access_idxs.at(cnt - 1).access_type_ + && access_ident.has_brackets_ + && access_ident.params_.count() == 0) { + ObSqlString object_name; + ObString empty_str; + construct_name(empty_str, access_idxs.at(cnt - 1).var_name_, access_ident.access_name_, object_name); + ret = OB_ERR_NOT_FUNC_NAME; + LOG_USER_ERROR(OB_ERR_NOT_FUNC_NAME, object_name.string().length(), object_name.string().ptr()); + } + + } OZ (build_access_idx_sys_func(parent_id, access_idx)); OZ (access_idxs.push_back(access_idx), K(access_idx)); if (OB_FAIL(ret)) { @@ -15195,13 +15220,16 @@ int ObPLResolver::resolve_cursor(ObPLCompileUnitAST &func, } if (OB_SUCC(ret) && OB_ISNULL(package_info)) { ObSchemaChecker checker; - ObSEArray syn_id_array; + ObSynonymChecker synonym_checker; ObString new_package_name; + bool is_exist = false; OZ (checker.init(resolve_ctx_.schema_guard_, resolve_ctx_.session_info_.get_sessid())); - OZ (checker.get_obj_info_recursively_with_synonym( - tenant_id, database_id, package_name, database_id, new_package_name, syn_id_array, true)); - OZ (resolve_ctx_.schema_guard_.get_package_info( - tenant_id, database_id, new_package_name, PACKAGE_TYPE, compatible_mode, package_info)); + OZ (ObResolverUtils::resolve_synonym_object_recursively( + checker, synonym_checker, tenant_id, database_id, package_name, database_id, new_package_name, is_exist)); + if (OB_SUCC(ret) && is_exist) { + OZ (resolve_ctx_.schema_guard_.get_package_info( + tenant_id, database_id, new_package_name, PACKAGE_TYPE, compatible_mode, package_info)); + } if (OB_SUCC(ret) && OB_ISNULL(package_info) && OB_SYS_DATABASE_ID == database_id) { OZ (resolve_ctx_.schema_guard_.get_package_info( diff --git a/src/pl/ob_pl_resolver.h b/src/pl/ob_pl_resolver.h index c10ef3c264..5d3d2870e0 100644 --- a/src/pl/ob_pl_resolver.h +++ b/src/pl/ob_pl_resolver.h @@ -451,7 +451,22 @@ public: ObProcType &routine_type, const ObPLDataType &ret_type); static int build_pl_integer_type(ObPLIntegerType type, ObPLDataType &data_type); + static bool is_question_mark_value(ObRawExpr *into_expr, ObPLBlockNS *ns); + static int set_question_mark_type(ObRawExpr *into_expr, ObPLBlockNS *ns, const ObPLDataType *type); + static + int build_obj_access_func_name(const ObIArray &access_idxs, + ObRawExprFactory &expr_factory, + const sql::ObSQLSessionInfo *session_info, + ObSchemaGetterGuard *schema_guard, + bool for_write, + ObString &result); + static + int set_write_property(ObRawExpr *obj_expr, + ObRawExprFactory &expr_factory, + const ObSQLSessionInfo *session_info, + ObSchemaGetterGuard *schema_guard, + bool for_write); int get_caller_accessor_item( const ObPLStmtBlock *caller, AccessorItem &caller_item); int check_package_accessible( @@ -913,19 +928,6 @@ private: int get_subprogram_var( ObPLBlockNS &ns, uint64_t subprogram_id, int64_t var_idx, const ObPLVar *&var); static - int build_obj_access_func_name(const ObIArray &access_idxs, - ObRawExprFactory &expr_factory, - const sql::ObSQLSessionInfo *session_info, - ObSchemaGetterGuard *schema_guard, - bool for_write, - ObString &result); - static - int set_write_property(ObRawExpr *obj_expr, - ObRawExprFactory &expr_factory, - const ObSQLSessionInfo *session_info, - ObSchemaGetterGuard *schema_guard, - bool for_write); - static int make_var_from_access(const ObIArray &access_idxs, ObRawExprFactory &expr_factory, const sql::ObSQLSessionInfo *session_info, @@ -1060,8 +1062,6 @@ private: inline void set_item_type(ObItemType item_type) { item_type_ = item_type; } int resolve_question_mark_node(const ObStmtNodeTree *into_node, ObRawExpr *&into_expr); - bool is_question_mark_value(ObRawExpr *into_expr); - int set_question_mark_type(ObRawExpr *into_expr, const ObPLDataType *type); int check_cursor_formal_params(const ObIArray& formal_params, ObPLCursor &cursor, diff --git a/src/pl/ob_pl_stmt.cpp b/src/pl/ob_pl_stmt.cpp index a94181e877..a1ec27c886 100644 --- a/src/pl/ob_pl_stmt.cpp +++ b/src/pl/ob_pl_stmt.cpp @@ -2332,6 +2332,9 @@ int ObPLBlockNS::find_sub_attr_by_name(const ObUserDefinedType &user_type, if (OB_ISNULL(record_type.get_record_member_type(member_index))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("type is invalid", K(ret)); + } else if (access_ident.has_brackets_ && 0 == access_ident.params_.count()) { + ret = OB_ERR_NOT_FUNC_NAME; + LOG_USER_ERROR(OB_ERR_NOT_FUNC_NAME, attr_name.length(), attr_name.ptr()); } else { new(&access_idx)ObObjAccessIdx(*record_type.get_record_member_type(member_index), ObObjAccessIdx::IS_CONST, attr_name, *record_type.get_record_member_type(member_index), diff --git a/src/pl/ob_pl_type.cpp b/src/pl/ob_pl_type.cpp index b265bc1e1c..41270b0506 100644 --- a/src/pl/ob_pl_type.cpp +++ b/src/pl/ob_pl_type.cpp @@ -614,7 +614,7 @@ int ObPLDataType::get_datum_type(common::ObObjType obj_type, jit::ObLLVMHelper& } int ObPLDataType::generate_assign_with_null(ObPLCodeGenerator &generator, - const ObPLBlockNS &ns, + const ObPLINS &ns, jit::ObLLVMValue &allocator, jit::ObLLVMValue &dest) const { @@ -624,7 +624,7 @@ int ObPLDataType::generate_assign_with_null(ObPLCodeGenerator &generator, LOG_WARN("unexpected type to assign NULL", K(*this), K(ret)); } else { const ObUserDefinedType *user_type = NULL; - if (OB_FAIL(ns.get_pl_data_type_by_id(get_user_type_id(), user_type))) { + if (OB_FAIL(ns.get_user_type(get_user_type_id(), user_type))) { LOG_WARN("failed to get user type", K(*this), K(ret)); } else if (OB_ISNULL(user_type)) { ret = OB_ERR_UNEXPECTED; @@ -1554,7 +1554,6 @@ void ObObjAccessIdx::reset() bool ObObjAccessIdx::operator==(const ObObjAccessIdx &other) const { - int ret = OB_SUCCESS; // udf deterministic default value is false, we need display setting check_ctx.need_check_deterministic_ ObExprEqualCheckContext check_ctx; check_ctx.need_check_deterministic_ = false; @@ -1901,6 +1900,31 @@ int ObObjAccessIdx::get_package_id( return ret; } +bool ObObjAccessIdx::has_same_collection_access(const ObRawExpr *expr, const ObObjAccessRawExpr *access_expr) +{ + bool ret = false; + if (expr->is_obj_access_expr()) { + const ObIArray &left = static_cast(expr)->get_access_idxs(); + const ObIArray &right= access_expr->get_access_idxs(); + for (int64_t i = 0; i < left.count() && i < right.count(); ++i) { + if (!(left.at(i) == right.at(i))) { + break; + } else if (left.at(i).elem_type_.is_collection_type()) { + ret = true; + break; + } + } + } else { + for (int64_t i = 0; i < expr->get_param_count(); ++i) { + if (has_same_collection_access(expr->get_param_expr(i), access_expr)) { + ret = true; + break; + } + } + } + return ret; +} + bool ObObjAccessIdx::has_collection_access(const ObRawExpr *expr) { bool ret = false; diff --git a/src/pl/ob_pl_type.h b/src/pl/ob_pl_type.h index 994dc16e49..e6f8e3546f 100644 --- a/src/pl/ob_pl_type.h +++ b/src/pl/ob_pl_type.h @@ -41,6 +41,7 @@ namespace oceanbase namespace sql { class ObRawExpr; +class ObObjAccessRawExpr; class ObSqlExpression; class ObRawExprFactory; class ObExecContext; @@ -483,7 +484,7 @@ public: static int get_datum_type(common::ObObjType obj_type, jit::ObLLVMHelper& helper, ObPLADTService &adt_service, jit::ObLLVMType &type); virtual int generate_assign_with_null(ObPLCodeGenerator &generator, - const ObPLBlockNS &ns, + const ObPLINS &ns, jit::ObLLVMValue &allocator, jit::ObLLVMValue &dest) const; virtual int generate_default_value(ObPLCodeGenerator &generator, @@ -753,6 +754,7 @@ public: uint64_t &package_id, uint64_t &var_idx); static int get_package_id(const sql::ObRawExpr *expr, uint64_t& package_id, uint64_t *p_var_idx = NULL); + static bool has_same_collection_access(const sql::ObRawExpr *expr, const sql::ObObjAccessRawExpr *access_expr); static bool has_collection_access(const sql::ObRawExpr *expr); static int datum_need_copy(const sql::ObRawExpr *into, const sql::ObRawExpr *value, AccessType &alloc_scop); static int64_t get_local_variable_idx(const common::ObIArray &access_idxs); diff --git a/src/pl/ob_pl_user_type.cpp b/src/pl/ob_pl_user_type.cpp index 6b74594826..45df610462 100644 --- a/src/pl/ob_pl_user_type.cpp +++ b/src/pl/ob_pl_user_type.cpp @@ -51,7 +51,7 @@ const ObPLDataType *ObUserDefinedType::get_member(int64_t i) const int ObUserDefinedType::generate_assign_with_null( ObPLCodeGenerator &generator, - const ObPLBlockNS &ns, jit::ObLLVMValue &allocator, jit::ObLLVMValue &dest) const + const ObPLINS &ns, jit::ObLLVMValue &allocator, jit::ObLLVMValue &dest) const { UNUSEDx(generator, ns, allocator, dest); return OB_SUCCESS; } @@ -1016,7 +1016,7 @@ int ObRecordType::deep_copy( } int ObRecordType::generate_assign_with_null(ObPLCodeGenerator &generator, - const ObPLBlockNS &ns, + const ObPLINS &ns, jit::ObLLVMValue &allocator, jit::ObLLVMValue &dest) const { @@ -1194,27 +1194,58 @@ int ObRecordType::generate_default_value(ObPLCodeGenerator &generator, } if (OB_SUCC(ret)) { ptr_elem.reset(); - if (member->member_type_.is_obj_type() || OB_INVALID_INDEX != member->get_default()) { + OZ (generator.extract_element_ptr_from_record(value, + get_record_member_count(), + i, + ptr_elem)); + if (OB_FAIL(ret)) { + } else if (member->member_type_.is_obj_type() || OB_INVALID_INDEX != member->get_default()) { //不论基础类型还是复杂类型，如果有default，直接把default值存入即可 - OZ (generator.extract_element_ptr_from_record(value, - get_record_member_count(), - i, - ptr_elem)); OZ (generator.get_helper().create_store(obobj_res, ptr_elem)); OZ (generator.generate_check_not_null(*stmt, member->member_type_.get_not_null(), result)); - } else { //复杂类型如果没有default，调用generate_construct + if (OB_SUCC(ret) && !member->member_type_.is_obj_type()) { // process complex null value + ObLLVMBasicBlock null_branch; + ObLLVMBasicBlock final_branch; + ObLLVMValue p_type_value; + ObLLVMValue type_value; + ObLLVMValue is_null; + ObLLVMValue allocator; + ObLLVMValue extend_value; + ObLLVMValue init_value; + ObLLVMValue composite_value; + ObLLVMType ir_type; + ObLLVMType ir_pointer_type; + int64_t init_size = OB_INVALID_SIZE; + OZ (generator.get_helper().create_block(ObString("null_branch"), generator.get_func(), null_branch)); + OZ (generator.get_helper().create_block(ObString("final_branch"), generator.get_func(), final_branch)); + OZ (generator.extract_type_ptr_from_objparam(result, p_type_value)); + OZ (generator.get_helper().create_load(ObString("load_type"), p_type_value, type_value)); + OZ (generator.get_helper().create_icmp_eq(type_value, ObNullType, is_null)); + OZ (generator.get_helper().create_cond_br(is_null, null_branch, final_branch)); + // null branch + OZ (generator.set_current(null_branch)); + OZ (SMART_CALL(member->member_type_.generate_new(generator, ns, extend_value, stmt))); + OZ (generator.get_helper().get_int8(member->member_type_.get_type(), type_value)); + OZ (member->member_type_.get_size(ns, PL_TYPE_INIT_SIZE, init_size)); + OZ (generator.get_helper().get_int32(init_size, init_value)); + OZ (generator.generate_set_extend(ptr_elem, type_value, init_value, extend_value)); + OZ (generator.generate_null(ObIntType, allocator)); + OZ (generator.get_llvm_type(member->member_type_, ir_type)); + OZ (ir_type.get_pointer_to(ir_pointer_type)); + OZ (generator.get_helper().create_int_to_ptr(ObString("cast_extend_to_ptr"), extend_value, ir_pointer_type, composite_value)); + OZ (member->member_type_.generate_assign_with_null(generator, ns, allocator, composite_value)); + OZ (generator.get_helper().create_br(final_branch)); + // final branch + OZ (generator.set_current(final_branch)); + } + } else { //复杂类型如果没有default，调用generate_new ObLLVMValue extend_value; ObLLVMValue type_value; ObLLVMValue init_value; int64_t init_size = OB_INVALID_SIZE; - OZ (generator.extract_element_ptr_from_record(value, - get_record_member_count(), - i, - ptr_elem)); OZ (SMART_CALL(member->member_type_.generate_new(generator, ns, extend_value, stmt))); - OZ (generator.get_helper().get_int8(member->member_type_.get_type(), type_value)); OZ (member->member_type_.get_size(ns, PL_TYPE_INIT_SIZE, init_size)); OZ (generator.get_helper().get_int32(init_size, init_value)); @@ -1939,7 +1970,7 @@ int ObCollectionType::get_size(const ObPLINS &ns, ObPLTypeSize type, int64_t &si } int ObCollectionType::generate_assign_with_null(ObPLCodeGenerator &generator, - const ObPLBlockNS &ns, + const ObPLINS &ns, jit::ObLLVMValue &allocator, jit::ObLLVMValue &dest) const { @@ -1951,7 +1982,7 @@ int ObCollectionType::generate_assign_with_null(ObPLCodeGenerator &generator, ObLLVMType int_type; ObLLVMValue int_value; - if (generator.get_helper().get_llvm_type(ObIntType, int_type)) { + if (OB_FAIL(generator.get_helper().get_llvm_type(ObIntType, int_type))) { LOG_WARN("failed to get_llvm_type", K(ret)); } else if (OB_FAIL(generator.get_helper().create_ptr_to_int(ObString("cast_ptr_to_int64"), dest, int_type, int_value))) { @@ -2246,6 +2277,15 @@ int ObCollectionType::deserialize( OZ (ObSPIService::spi_extend_assoc_array( OB_INVALID_ID, &resolve_ctx, *(table->get_allocator()), *assoc_table, count)); } else { + if (table->get_count() > 0) { + for (int64_t i = 0; i < table->get_count(); ++i) { + int tmp_ret = OB_SUCCESS; + if ((tmp_ret = ObUserDefinedType::destruct_obj(table->get_data()[i], &resolve_ctx.session_info_)) != OB_SUCCESS) { + LOG_WARN("failed to destruct obj, memory may leak", K(ret), K(tmp_ret), K(i)); + } + } + } + OX (table->set_count(0)); OZ (ObSPIService::spi_set_collection( OB_INVALID_ID, &resolve_ctx, *table->get_allocator(), *table, count, true)); } diff --git a/src/pl/ob_pl_user_type.h b/src/pl/ob_pl_user_type.h index 7ee6b87a94..b8e4927147 100644 --- a/src/pl/ob_pl_user_type.h +++ b/src/pl/ob_pl_user_type.h @@ -64,7 +64,7 @@ public: virtual int64_t get_member_count() const; virtual const ObPLDataType *get_member(int64_t i) const; virtual int generate_assign_with_null( - ObPLCodeGenerator &generator, const ObPLBlockNS &ns, + ObPLCodeGenerator &generator, const ObPLINS &ns, jit::ObLLVMValue &allocator, jit::ObLLVMValue &dest) const; virtual int generate_default_value( ObPLCodeGenerator &generator,const ObPLINS &ns, @@ -240,9 +240,10 @@ public: virtual int64_t get_member_count() const { return 0; } virtual const ObPLDataType *get_member(int64_t i) const { UNUSED(i); return NULL; } virtual int generate_assign_with_null(ObPLCodeGenerator &generator, - jit::ObLLVMValue &allocator, - jit::ObLLVMValue &dest) const - { UNUSED(generator); UNUSED(allocator); UNUSED(dest); return OB_SUCCESS;} + ObPLINS &ns, + jit::ObLLVMValue &allocator, + jit::ObLLVMValue &dest) const + { UNUSED(generator); UNUSED(ns), UNUSED(allocator); UNUSED(dest); return OB_SUCCESS;} virtual int generate_construct(ObPLCodeGenerator &generator, const ObPLINS &ns, jit::ObLLVMValue &value, @@ -391,7 +392,7 @@ public: virtual const ObPLDataType *get_member(int64_t i) const { return get_record_member_type(i); } virtual int generate_assign_with_null(ObPLCodeGenerator &generator, - const ObPLBlockNS &ns, + const ObPLINS &ns, jit::ObLLVMValue &allocator, jit::ObLLVMValue &dest) const; @@ -549,13 +550,13 @@ public: jit::ObLLVMValue &value, const pl::ObPLStmt *stmt = NULL) const; virtual int generate_assign_with_null(ObPLCodeGenerator &generator, - const ObPLBlockNS &ns, + const ObPLINS &ns, jit::ObLLVMValue &allocator, jit::ObLLVMValue &dest) const; virtual int generate_new(ObPLCodeGenerator &generator, - const ObPLINS &ns, - jit::ObLLVMValue &value, - const pl::ObPLStmt *s = NULL) const; + const ObPLINS &ns, + jit::ObLLVMValue &value, + const pl::ObPLStmt *s = NULL) const; virtual int newx(common::ObIAllocator &allocator, const ObPLINS *ns, int64_t &ptr) const; diff --git a/src/pl/pl_cache/ob_pl_cache.cpp b/src/pl/pl_cache/ob_pl_cache.cpp index a2dabfc32d..e73d09bf5d 100644 --- a/src/pl/pl_cache/ob_pl_cache.cpp +++ b/src/pl/pl_cache/ob_pl_cache.cpp @@ -992,7 +992,19 @@ int ObPLObjectSet::inner_add_cache_obj(ObILibCacheCtx &ctx, LOG_WARN("failed to init pl function", K(ret)); } else { bool is_old_version = false; - if (pl_object_value->check_value_version(pc_ctx.schema_guard_, + if (cache_object->is_anon() || + cache_object->is_call_stmt()) { + common::ObString sql_id_org(common::OB_MAX_SQL_ID_LENGTH, (const char*)&pc_ctx.sql_id_); + if (sql_id_.empty()) { + if (OB_FAIL(ob_write_string(allocator_, sql_id_org, sql_id_))) { + LOG_WARN("failed to deep copy sql_id_", K(sql_id_org), K(ret)); + } + } else { + CK (sql_id_ == sql_id_org); + } + } + if (OB_FAIL(ret)) { + } else if (pl_object_value->check_value_version(pc_ctx.schema_guard_, true, schema_array, is_old_version)) { diff --git a/src/pl/pl_cache/ob_pl_cache.h b/src/pl/pl_cache/ob_pl_cache.h index 8f08ddcf46..3164b63fea 100644 --- a/src/pl/pl_cache/ob_pl_cache.h +++ b/src/pl/pl_cache/ob_pl_cache.h @@ -55,6 +55,7 @@ struct PCVPlSchemaObj schema_version_(0), schema_type_(share::schema::OB_MAX_SCHEMA), table_type_(share::schema::MAX_TABLE_TYPE), + table_name_(), is_tmp_table_(false), is_explicit_db_name_(false), inner_alloc_(nullptr) {} @@ -66,6 +67,7 @@ struct PCVPlSchemaObj schema_version_(0), schema_type_(share::schema::OB_MAX_SCHEMA), table_type_(share::schema::MAX_TABLE_TYPE), + table_name_(), is_tmp_table_(false), is_explicit_db_name_(false), inner_alloc_(alloc) {} @@ -298,6 +300,8 @@ public: void destroy(); + common::ObString &get_sql_id() { return sql_id_; } + int create_new_pl_object_value(ObPLObjectValue *&pl_object_value); void free_pl_object_value(ObPLObjectValue *pl_object_value); int64_t get_mem_size(); @@ -306,6 +310,7 @@ public: private: bool is_inited_; ObPLObjectKey key_; //used for manager key memory + common::ObString sql_id_; // a list of plan sets with different param types combination common::ObDList object_value_sets_; }; diff --git a/src/pl/pl_cache/ob_pl_cache_mgr.cpp b/src/pl/pl_cache/ob_pl_cache_mgr.cpp index 4bde114b1a..5d19674a27 100644 --- a/src/pl/pl_cache/ob_pl_cache_mgr.cpp +++ b/src/pl/pl_cache/ob_pl_cache_mgr.cpp @@ -204,5 +204,27 @@ int ObPLCacheMgr::cache_evict_all_pl(ObPlanCache *lib_cache) return ret; } +template +int ObPLCacheMgr::cache_evict_pl_cache_single(ObPlanCache *lib_cache, uint64_t db_id, EvictAttr &attr) +{ + int ret = OB_SUCCESS; + PL_CACHE_LOG(TRACE, "cache evict single plan start"); + if (OB_ISNULL(lib_cache)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("lib cache is null"); + } else { + LCKeyValueArray to_evict_keys; + GETPLKVEntryOp get_ids_op(db_id, attr, &to_evict_keys, PCV_GET_PL_KEY_HANDLE); + if (OB_FAIL(lib_cache->foreach_cache_evict(get_ids_op))) { + PL_CACHE_LOG(WARN, "failed to foreach cache evict", K(ret)); + } + } + PL_CACHE_LOG(TRACE, "cache evict single plan end"); + return ret; +} + +template int ObPLCacheMgr::cache_evict_pl_cache_single(ObPlanCache *lib_cache, uint64_t db_id, uint64_t &schema_id); +template int ObPLCacheMgr::cache_evict_pl_cache_single(ObPlanCache *lib_cache, uint64_t db_id, common::ObString &sql_id); + } } \ No newline at end of file diff --git a/src/pl/pl_cache/ob_pl_cache_mgr.h b/src/pl/pl_cache/ob_pl_cache_mgr.h index b919624bb6..59d4f886b5 100644 --- a/src/pl/pl_cache/ob_pl_cache_mgr.h +++ b/src/pl/pl_cache/ob_pl_cache_mgr.h @@ -53,6 +53,84 @@ struct ObGetPLKVEntryOp : public sql::ObKVEntryTraverseOp } }; +struct ObGetPLKVEntryBySchemaIdOp : public ObKVEntryTraverseOp +{ + explicit ObGetPLKVEntryBySchemaIdOp(uint64_t db_id, + uint64_t schema_id, + LCKeyValueArray *key_val_list, + const CacheRefHandleID ref_handle) + : ObKVEntryTraverseOp(key_val_list, ref_handle), + db_id_(db_id), + schema_id_(schema_id) + { + } + virtual int check_entry_match(LibCacheKVEntry &entry, bool &is_match) + { + int ret = OB_SUCCESS; + is_match = false; + if (ObLibCacheNameSpace::NS_PRCR == entry.first->namespace_ || + ObLibCacheNameSpace::NS_SFC == entry.first->namespace_ || + ObLibCacheNameSpace::NS_PKG == entry.first->namespace_) { + ObPLObjectKey *key = static_cast(entry.first); + if (db_id_ != common::OB_INVALID_ID && db_id_ != key->db_id_) { + // skip entry that has non-matched db_id + } else if (schema_id_ != key->key_id_) { + // skip entry which is not same schema id + } else { + is_match = true; + } + } + return ret; + } + + uint64_t db_id_; + uint64_t schema_id_; +}; + +struct ObGetPLKVEntryBySQLIDOp : public ObKVEntryTraverseOp +{ + explicit ObGetPLKVEntryBySQLIDOp(uint64_t db_id, + common::ObString sql_id, + LCKeyValueArray *key_val_list, + const CacheRefHandleID ref_handle) + : ObKVEntryTraverseOp(key_val_list, ref_handle), + db_id_(db_id), + sql_id_(sql_id) + { + } + virtual int check_entry_match(LibCacheKVEntry &entry, bool &is_match) + { + int ret = OB_SUCCESS; + is_match = false; + if (ObLibCacheNameSpace::NS_ANON == entry.first->namespace_ || + ObLibCacheNameSpace::NS_CALLSTMT == entry.first->namespace_) { + ObPLObjectKey *key = static_cast(entry.first); + ObPLObjectSet *node = static_cast(entry.second); + if (db_id_ != common::OB_INVALID_ID && db_id_ != key->db_id_) { + // skip entry that has non-matched db_id + } else if (sql_id_ != node->get_sql_id()) { + // skip entry which not contains same sql_id + } else { + is_match = true; + } + } + return ret; + } + bool contain_sql_id(common::ObIArray &sql_ids) + { + bool contains = false; + for (int64_t i = 0; !contains && i < sql_ids.count(); i++) { + if (sql_ids.at(i) == sql_id_) { + contains = true; + } + } + return contains; + } + + uint64_t db_id_; + common::ObString sql_id_; +}; + class ObPLCacheMgr { public: @@ -65,6 +143,8 @@ public: ObCacheObjGuard& guard, ObPLCacheCtx &pc_ctx); static int cache_evict_all_pl(ObPlanCache *lib_cache); + template + static int cache_evict_pl_cache_single(ObPlanCache *lib_cache, uint64_t db_id, EvictAttr &attr); private: static int add_pl_object(ObPlanCache *lib_cache, diff --git a/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp b/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp index c102797a20..96669ead51 100644 --- a/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp +++ b/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp @@ -410,7 +410,10 @@ int ObBackupSetTaskMgr::calc_task_turn_(const ObBackupDataTaskType &type, int64_ case ObBackupStatus::BACKUP_USER_META: { if (type.is_backup_meta()) { turn_id = set_task_attr_.meta_turn_id_; - } else if (type.is_backup_minor()) { + } + } + case ObBackupStatus::BACKUP_META_FINISH: { + if (type.is_backup_minor()) { turn_id = set_task_attr_.minor_turn_id_; } break; @@ -558,7 +561,7 @@ int ObBackupSetTaskMgr::backup_meta_finish_() { int ret = OB_SUCCESS; ObArray ls_task; - ObArray ls_ids; + ObArray new_ls_ids; share::SCN consistent_scn; DEBUG_SYNC(BEFORE_BACKUP_META_FINISH); @@ -568,11 +571,12 @@ int ObBackupSetTaskMgr::backup_meta_finish_() } else if (ls_task.empty()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("[DATA_BACKUP]no logstream task", K(ret), "job_id", job_attr_->job_id_, "tenant_id", job_attr_->tenant_id_); + } else if (OB_FALSE_IT(DEBUG_SYNC(BEFORE_MERGE_BACKUP_META_INFO))) { } else if (OB_FAIL(calc_consistent_scn_(ls_task, consistent_scn))) { LOG_WARN("failed to calc consistent scn", K(ret), K(ls_task)); } else if (OB_FAIL(merge_ls_meta_infos_(ls_task))) { LOG_WARN("fail to merge ls meta infos", K(ret), K(ls_task)); - } else if (OB_FAIL(merge_tablet_to_ls_info_(consistent_scn, ls_task, ls_ids))) { + } else if (OB_FAIL(merge_tablet_to_ls_info_(consistent_scn, ls_task, new_ls_ids))) { LOG_WARN("[DATA_BACKUP]failed to merge tablet to ls info", K(ret), K(ls_task)); } else if (OB_FALSE_IT(DEBUG_SYNC(BEFORE_BACKUP_DATA))) { } else if (OB_FAIL(trans_.start(sql_proxy_, meta_tenant_id_))) { @@ -584,8 +588,8 @@ int ObBackupSetTaskMgr::backup_meta_finish_() LOG_WARN("[DATA_BACKUP]fail to update task type to backup data", K(ret)); } else if (OB_FAIL(advance_status_(trans_, next_status))) { LOG_WARN("[DATA_BACKUP]failed to advance status to BACKUP_DATA_MINOR", K(ret), K(next_status)); - } else if (OB_FAIL(generate_ls_tasks_(ls_ids, type))) { - LOG_WARN("failed to generate ls tasks", K(ret), K(ls_ids), K(type)); + } else if (OB_FAIL(generate_ls_tasks_(new_ls_ids, type))) { + LOG_WARN("failed to generate ls tasks", K(ret), K(new_ls_ids), K(type)); } else { ROOTSERVICE_EVENT_ADD("backup_data", "after_backup_consistent_scn"); } diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index 2a05d6ea77..5a3bbd22bb 100755 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -13345,8 +13345,6 @@ int ObDDLService::create_hidden_table( allocator, tenant_data_version))) { LOG_WARN("fail to create hidden table", K(ret)); - } else if (OB_FAIL(ddl_operator.update_table_attribute(new_table_schema, trans, OB_DDL_ALTER_TABLE))) { - LOG_WARN("failed to update data table schema attribute", K(ret)); } else { LOG_INFO("create hidden table success!", K(table_id), K(new_table_schema)); } @@ -13680,8 +13678,6 @@ int ObDDLService::recover_restore_table_ddl_task( false/*bind_tablets*/, *src_tenant_schema_guard, *dst_tenant_schema_guard, ddl_operator, dst_tenant_trans, allocator, tenant_data_version))) { LOG_WARN("create user hidden table failed", K(ret), K(arg), K(tenant_data_version)); - } else if (OB_FAIL(ddl_operator.update_table_attribute(dst_table_schema, dst_tenant_trans, OB_DDL_ALTER_TABLE))) { - LOG_WARN("failed to update data table schema attribute", K(ret), K(arg)); } else { ObPrepareAlterTableArgParam param; if (OB_FAIL(param.init(arg.consumer_group_id_, session_id, 0/*sql_mode, unused*/, arg.ddl_stmt_str_, @@ -16442,7 +16438,6 @@ int ObDDLService::create_user_hidden_table(const ObTableSchema &orig_table_schem } if (OB_SUCC(ret)) { - bool need_sync_schema_version = false; ObTableCreator table_creator( tenant_id, frozen_scn, @@ -16460,7 +16455,8 @@ int ObDDLService::create_user_hidden_table(const ObTableSchema &orig_table_schem } for (int64_t i = 0; OB_SUCC(ret) && i < schemas.count(); i++) { share::schema::ObTableSchema *table_schema = const_cast(schemas.at(i)); - if (OB_FAIL(ddl_operator.create_table(*table_schema, trans, NULL, need_sync_schema_version))) { + if (OB_FAIL(ddl_operator.create_table(*table_schema, trans, NULL, + i == schemas.count() - 1/*need_sync_schema_version, to update data table schema version*/))) { LOG_WARN("failed to create table schema", K(ret)); } else if (OB_FAIL(ddl_operator.insert_temp_table_info(trans, *table_schema))) { LOG_WARN("failed to insert temp table info", K(ret), KPC(table_schema)); @@ -16468,12 +16464,20 @@ int ObDDLService::create_user_hidden_table(const ObTableSchema &orig_table_schem } int64_t last_schema_version = OB_INVALID_VERSION; + if (FAILEDx(get_last_schema_version(last_schema_version))) { + LOG_WARN("get last schema version failed", K(ret), K(last_schema_version)); + } else if (OB_UNLIKELY(last_schema_version < schemas.at(schemas.count() - 1)->get_schema_version())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected last schema version", K(ret), K(last_schema_version), + "table schema version", schemas.at(schemas.count() - 1)->get_schema_version()); + } else { + // update schema version after sync version by creating lob table. + hidden_table_schema.set_schema_version(last_schema_version); + } + for (int64_t i = 0; OB_SUCC(ret) && i < schemas.count(); i++) { share::schema::ObTableSchema *table_schema = const_cast(schemas.at(i)); - if (OB_INVALID_VERSION == last_schema_version - && OB_FAIL(get_last_schema_version(last_schema_version))) { - LOG_WARN("get last schema version failed", K(ret), K(last_schema_version)); - } else if (OB_FAIL(ddl_operator.insert_ori_schema_version(trans, tenant_id, + if (OB_FAIL(ddl_operator.insert_ori_schema_version(trans, tenant_id, table_schema->get_table_id(), last_schema_version))) { LOG_WARN("failed to insert_ori_schema_version!", K(ret), KPC(table_schema), K(last_schema_version)); } diff --git a/src/rootserver/ob_system_admin_util.cpp b/src/rootserver/ob_system_admin_util.cpp index 35096ddaf4..641b28a72d 100644 --- a/src/rootserver/ob_system_admin_util.cpp +++ b/src/rootserver/ob_system_admin_util.cpp @@ -1963,6 +1963,7 @@ int ObAdminFlushCache::execute(const obrpc::ObAdminFlushCacheArg &arg) if (arg.is_fine_grained_) { fc_arg.sql_id_ = arg.sql_id_; fc_arg.is_fine_grained_ = arg.is_fine_grained_; + fc_arg.schema_id_ = arg.schema_id_; for(int64_t j=0; OB_SUCC(ret) && jlocate_datum_for_write(eval_ctx_); - // aggr_info.expr_->set_evaluated_projected(eval_ctx_); - // uint64_t init_val = - // aggr_info.get_expr_type() == T_FUN_SYS_BIT_AND ? UINT_MAX_VAL[ObUInt64Type] : 0; - // result.set_uint(init_val); - ret = OB_NOT_IMPLEMENT; - LOG_WARN("not implemented", K(ret)); + uint64_t init_val = + (aggr_info.get_expr_type() == T_FUN_SYS_BIT_AND ? UINT_MAX_VAL[ObUInt64Type] : 0); + res_vec->set_uint(output_idx, init_val); break; } default: { diff --git a/src/share/allocator/ob_mds_allocator.cpp b/src/share/allocator/ob_mds_allocator.cpp index f5e4e04994..b2b55beaf7 100644 --- a/src/share/allocator/ob_mds_allocator.cpp +++ b/src/share/allocator/ob_mds_allocator.cpp @@ -51,7 +51,8 @@ void ObTenantMdsAllocator::init_throttle_config(int64_t &resource_limit, int64_t max_duration = MDS_THROTTLE_MAX_DURATION; } } -void ObTenantMdsAllocator::adaptive_update_limit(const int64_t holding_size, +void ObTenantMdsAllocator::adaptive_update_limit(const int64_t tenant_id, + const int64_t holding_size, const int64_t config_specify_resource_limit, int64_t &resource_limit, int64_t &last_update_limit_ts, diff --git a/src/share/allocator/ob_memstore_allocator.cpp b/src/share/allocator/ob_memstore_allocator.cpp index fe853fea00..97b3169233 100644 --- a/src/share/allocator/ob_memstore_allocator.cpp +++ b/src/share/allocator/ob_memstore_allocator.cpp @@ -234,7 +234,8 @@ void ObMemstoreAllocator::init_throttle_config(int64_t &resource_limit, } } -void ObMemstoreAllocator::adaptive_update_limit(const int64_t holding_size, +void ObMemstoreAllocator::adaptive_update_limit(const int64_t tenant_id, + const int64_t holding_size, const int64_t config_specify_resource_limit, int64_t &resource_limit, int64_t &last_update_limit_ts, diff --git a/src/share/allocator/ob_shared_memory_allocator_mgr.cpp b/src/share/allocator/ob_shared_memory_allocator_mgr.cpp index 6cf6fe4ffe..91e06403b0 100644 --- a/src/share/allocator/ob_shared_memory_allocator_mgr.cpp +++ b/src/share/allocator/ob_shared_memory_allocator_mgr.cpp @@ -31,8 +31,12 @@ namespace share { void ObSharedMemAllocMgr::update_throttle_config() { - int64_t tenant_id = MTL_ID(); - int64_t total_memory = lib::get_tenant_memory_limit(tenant_id); + if (MTL_ID() != tenant_id_) { + SHARE_LOG_RET(ERROR, OB_ERR_UNEXPECTED, "update throttle config in an invalid tenant", K(MTL_ID()), K(tenant_id_)); + return; + } + + int64_t total_memory = lib::get_tenant_memory_limit(tenant_id_); omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID())); if (tenant_config.is_valid()) { int64_t share_mem_limit_percentage = tenant_config->_tx_share_memory_limit_percentage; @@ -46,10 +50,10 @@ void ObSharedMemAllocMgr::update_throttle_config() share_mem_limit_percentage = memstore_limit_percentage + 10; } - int64_t share_mem_limit = total_memory * share_mem_limit_percentage / 100LL; - int64_t memstore_limit = total_memory * memstore_limit_percentage / 100LL; - int64_t tx_data_limit = total_memory * tx_data_limit_percentage / 100LL; - int64_t mds_limit = total_memory * mds_limit_percentage / 100LL; + int64_t share_mem_limit = total_memory / 100 * share_mem_limit_percentage; + int64_t memstore_limit = total_memory / 100 * memstore_limit_percentage; + int64_t tx_data_limit = total_memory / 100 * tx_data_limit_percentage; + int64_t mds_limit = total_memory / 100 * mds_limit_percentage; bool share_config_changed = false; (void)share_resource_throttle_tool_.update_throttle_config( @@ -70,39 +74,22 @@ void ObSharedMemAllocMgr::update_throttle_config() if (share_config_changed || memstore_config_changed || tx_data_config_changed || mds_config_changed) { SHARE_LOG(INFO, "[Throttle] Update Config", + K(tenant_id_), + K(total_memory), K(share_mem_limit_percentage), + K(share_mem_limit), K(memstore_limit_percentage), + K(memstore_limit), K(tx_data_limit_percentage), + K(tx_data_limit), K(mds_limit_percentage), + K(mds_limit), K(trigger_percentage), K(max_duration)); - if (share_config_changed) { - SHARE_LOG(INFO, - "[Throttle] Update Config", - THROTTLE_CONFIG_LOG(FakeAllocatorForTxShare, share_mem_limit, trigger_percentage, max_duration)); - } - - if (memstore_config_changed) { - SHARE_LOG(INFO, - "[Throttle] Update Config", - THROTTLE_CONFIG_LOG(ObMemstoreAllocator, memstore_limit, trigger_percentage, max_duration)); - } - - if (tx_data_config_changed) { - SHARE_LOG(INFO, - "[Throttle] Update Config", - THROTTLE_CONFIG_LOG(ObTenantTxDataAllocator, tx_data_limit, trigger_percentage, max_duration)); - } - - if (mds_config_changed) { - SHARE_LOG(INFO, - "[Throttle] Update Config", - THROTTLE_CONFIG_LOG(ObTenantMdsAllocator, mds_limit, trigger_percentage, max_duration)); - } } } else { - SHARE_LOG_RET(WARN, OB_INVALID_CONFIG, "invalid tenant config", K(tenant_id), K(total_memory)); + SHARE_LOG_RET(WARN, OB_INVALID_CONFIG, "invalid tenant config", K(tenant_id_), K(total_memory)); } } diff --git a/src/share/allocator/ob_shared_memory_allocator_mgr.h b/src/share/allocator/ob_shared_memory_allocator_mgr.h index f79f9a5bff..5502edb15e 100644 --- a/src/share/allocator/ob_shared_memory_allocator_mgr.h +++ b/src/share/allocator/ob_shared_memory_allocator_mgr.h @@ -23,8 +23,7 @@ namespace share { class ObSharedMemAllocMgr { public: - ObSharedMemAllocMgr() - : share_resource_throttle_tool_(), + ObSharedMemAllocMgr(): share_resource_throttle_tool_(), memstore_allocator_(), tx_data_allocator_(), mds_allocator_() {} @@ -47,8 +46,9 @@ public: share_resource_throttle_tool_.init(&memstore_allocator_, &tx_data_allocator_, &mds_allocator_))) { SHARE_LOG(ERROR, "init share resource throttle tool failed", KR(ret)); } else { + tenant_id_ = MTL_ID(); share_resource_throttle_tool_.enable_adaptive_limit(); - SHARE_LOG(INFO, "finish init mtl share mem allocator mgr", K(MTL_ID()), KP(this)); + SHARE_LOG(INFO, "finish init mtl share mem allocator mgr", K(tenant_id_), KP(this)); } return ret; } @@ -71,6 +71,7 @@ private: void update_mds_throttle_config_(const int64_t total_memory, omt::ObTenantConfigGuard &config); private: + int64_t tenant_id_; TxShareThrottleTool share_resource_throttle_tool_; ObMemstoreAllocator memstore_allocator_; ObTenantTxDataAllocator tx_data_allocator_; diff --git a/src/share/allocator/ob_tx_data_allocator.cpp b/src/share/allocator/ob_tx_data_allocator.cpp index 39f025f39b..e51a839854 100644 --- a/src/share/allocator/ob_tx_data_allocator.cpp +++ b/src/share/allocator/ob_tx_data_allocator.cpp @@ -48,7 +48,8 @@ void ObTenantTxDataAllocator::init_throttle_config(int64_t &resource_limit, max_duration = TX_DATA_THROTTLE_MAX_DURATION; } } -void ObTenantTxDataAllocator::adaptive_update_limit(const int64_t holding_size, +void ObTenantTxDataAllocator::adaptive_update_limit(const int64_t tenant_id, + const int64_t holding_size, const int64_t config_specify_resource_limit, int64_t &resource_limit, int64_t &last_update_limit_ts, diff --git a/src/share/datum/ob_datum_funcs.cpp b/src/share/datum/ob_datum_funcs.cpp index e3e4ed1a27..9e4b65d494 100644 --- a/src/share/datum/ob_datum_funcs.cpp +++ b/src/share/datum/ob_datum_funcs.cpp @@ -471,6 +471,19 @@ bool ObDatumFuncs::is_geometry(const ObObjType type) return (tc == ObGeometryTC); } +/** + * This function is primarily responsible for handling inconsistent hash computations + * for null types and the null values of those types, such as string, float, double, etc. + * It ensures that the hashing process treats null values and null type representations + * consistently across such data types, avoiding discrepancies in hash results. + */ +bool ObDatumFuncs::is_null_aware_hash_type(const ObObjType type) +{ + const ObObjTypeClass tc = OBJ_TYPE_TO_CLASS[type]; + return is_string_type(type) || is_json(type) || is_geometry(type) || + (tc == ObUserDefinedSQLTC) || (tc == ObFloatTC) || (tc == ObDoubleTC); +} + OB_SERIALIZE_MEMBER(ObCmpFunc, ser_cmp_func_); OB_SERIALIZE_MEMBER(ObHashFunc, ser_hash_func_, ser_batch_hash_func_); diff --git a/src/share/datum/ob_datum_funcs.h b/src/share/datum/ob_datum_funcs.h index e98b2456ae..4e7726fff2 100644 --- a/src/share/datum/ob_datum_funcs.h +++ b/src/share/datum/ob_datum_funcs.h @@ -44,6 +44,7 @@ public: static bool is_varying_len_char_type(const ObObjType type, const ObCollationType cs_type) { return (type == ObNVarchar2Type || (type == ObVarcharType && cs_type != CS_TYPE_BINARY)); } + static bool is_null_aware_hash_type(const ObObjType type); static ObScale max_scale(const ObScale s1, const ObScale s2) { ObScale max_scale = SCALE_UNKNOWN_YET; diff --git a/src/share/detect/ob_detect_callback.cpp b/src/share/detect/ob_detect_callback.cpp index 3abcd5d3d1..26befd59e1 100644 --- a/src/share/detect/ob_detect_callback.cpp +++ b/src/share/detect/ob_detect_callback.cpp @@ -261,9 +261,15 @@ int ObSingleDfoDetectCB::do_callback() int ret = OB_SUCCESS; int clean_ret = OB_E(EventTable::EN_ENABLE_CLEAN_INTERM_RES) OB_SUCCESS; if (OB_SUCC(clean_ret)) { - ret = MTL(sql::dtl::ObDTLIntermResultManager*)->erase_interm_result_info(key_, false); - ret = ret == OB_HASH_NOT_EXIST ? OB_SUCCESS : ret; - LIB_LOG(WARN, "[DM] single dfo erase_interm_result_info", K(ret), K(key_), K_(trace_id)); + dtl::ObDTLIntermResultManager *interm_res_manager = MTL(dtl::ObDTLIntermResultManager*); + if (OB_ISNULL(interm_res_manager)) { + LIB_LOG(WARN, "[DM] single dfo erase_interm_result_info, but interm_res_manager is null", + K(ret), K(key_), K_(trace_id)); + } else { + ret = interm_res_manager->erase_interm_result_info(key_, false); + ret = ret == OB_HASH_NOT_EXIST ? OB_SUCCESS : ret; + LIB_LOG(WARN, "[DM] single dfo erase_interm_result_info", K(ret), K(key_), K_(trace_id)); + } } return ret; } @@ -271,9 +277,15 @@ int ObSingleDfoDetectCB::do_callback() int ObTempTableDetectCB::do_callback() { int ret = OB_SUCCESS; - ret = MTL(sql::dtl::ObDTLIntermResultManager*)->erase_interm_result_info(key_, false); - ret = ret == OB_HASH_NOT_EXIST ? OB_SUCCESS : ret; - LIB_LOG(WARN, "[DM] temp table erase_interm_result_info", K(ret), K(key_), K_(trace_id)); + dtl::ObDTLIntermResultManager *interm_res_manager = MTL(dtl::ObDTLIntermResultManager*); + if (OB_ISNULL(interm_res_manager)) { + LIB_LOG(WARN, "[DM] temp table erase_interm_result_info, but interm_res_manager is null", + K(ret), K(key_), K_(trace_id)); + } else { + ret = interm_res_manager->erase_interm_result_info(key_, false); + ret = ret == OB_HASH_NOT_EXIST ? OB_SUCCESS : ret; + LIB_LOG(WARN, "[DM] temp table erase_interm_result_info", K(ret), K(key_), K_(trace_id)); + } return ret; } diff --git a/src/share/detect/ob_detect_manager.cpp b/src/share/detect/ob_detect_manager.cpp index 86d9707f6f..22d682f3e0 100644 --- a/src/share/detect/ob_detect_manager.cpp +++ b/src/share/detect/ob_detect_manager.cpp @@ -155,6 +155,18 @@ void ObDMMultiDlist::pop_active_node( } } +int ObDetectManager::mtl_new(ObDetectManager *&dm) +{ + int ret = OB_SUCCESS; + uint64_t tenant_id = MTL_ID(); + dm = OB_NEW(ObDetectManager, ObMemAttr(tenant_id, "DetectManager"), tenant_id); + if (OB_ISNULL(dm)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LIB_LOG(WARN, "[DM] failed to alloc detect manager", K(ret)); + } + return ret; +} + int ObDetectManager::mtl_init(ObDetectManager *&dm) { int ret = OB_SUCCESS; @@ -165,11 +177,7 @@ int ObDetectManager::mtl_init(ObDetectManager *&dm) if (is_meta_tenant(tenant_id)) { mem_factor = mem_factor * 0.01; } - dm = OB_NEW(ObDetectManager, ObMemAttr(tenant_id, "DetectManager"), tenant_id); - if (OB_ISNULL(dm)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LIB_LOG(WARN, "[DM] failed to alloc detect manager", K(ret)); - } else if (OB_FAIL(dm->init(GCTX.self_addr(), mem_factor))) { + if (OB_FAIL(dm->init(GCTX.self_addr(), mem_factor))) { LIB_LOG(WARN, "[DM] failed to init detect manager", K(ret)); } return ret; diff --git a/src/share/detect/ob_detect_manager.h b/src/share/detect/ob_detect_manager.h index 2085c3419d..713d1786e2 100644 --- a/src/share/detect/ob_detect_manager.h +++ b/src/share/detect/ob_detect_manager.h @@ -110,6 +110,7 @@ private: static const uint64_t ACTIVATE_DELAY_TIME = 5 * 1000L * 1000L; // dm only detects checkitems that have been present for at least "ACTIVATE_DELAY_TIME" seconds public: + static int mtl_new(ObDetectManager *&dm); static int mtl_init(ObDetectManager *&dm); static void mtl_destroy(ObDetectManager *&dm); public: diff --git a/src/share/diagnosis/ob_sql_plan_monitor_node_list.cpp b/src/share/diagnosis/ob_sql_plan_monitor_node_list.cpp index 4f4f612d61..825e7d6f99 100644 --- a/src/share/diagnosis/ob_sql_plan_monitor_node_list.cpp +++ b/src/share/diagnosis/ob_sql_plan_monitor_node_list.cpp @@ -110,20 +110,9 @@ int ObPlanMonitorNodeList::mtl_init(ObPlanMonitorNodeList* &node_list) { int ret = OB_SUCCESS; uint64_t tenant_id = lib::current_resource_owner_id(); - node_list = OB_NEW(ObPlanMonitorNodeList, ObMemAttr(tenant_id, MOD_LABEL)); - if (nullptr == node_list) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memory for ObPlanMonitorNodeList", K(ret)); - } else { - int64_t mem_limit = get_tenant_memory_limit(tenant_id); - if (OB_FAIL(node_list->init(tenant_id, mem_limit))) { - LOG_WARN("failed to init event list", K(ret)); - } - } - if (OB_FAIL(ret) && node_list != nullptr) { - // cleanup - common::ob_delete(node_list); - node_list = nullptr; + int64_t mem_limit = get_tenant_memory_limit(tenant_id); + if (OB_FAIL(node_list->init(tenant_id, mem_limit))) { + LOG_WARN("failed to init event list", K(ret)); } return ret; } diff --git a/src/share/external_table/ob_external_table_file_task.h b/src/share/external_table/ob_external_table_file_task.h index c53155ed3c..cd65fbf3b8 100644 --- a/src/share/external_table/ob_external_table_file_task.h +++ b/src/share/external_table/ob_external_table_file_task.h @@ -53,7 +53,8 @@ public: ObString location_; ObString pattern_; sql::ObExprRegexpSessionVariables regexp_vars_; - TO_STRING_KV(K_(location)); +public: + TO_STRING_KV(K_(location), K_(pattern), K_(regexp_vars)); }; class ObLoadExternalFileListRes diff --git a/src/share/inner_table/ob_inner_table_schema.12201_12250.cpp b/src/share/inner_table/ob_inner_table_schema.12201_12250.cpp index b822836342..969bf26784 100644 --- a/src/share/inner_table/ob_inner_table_schema.12201_12250.cpp +++ b/src/share/inner_table/ob_inner_table_schema.12201_12250.cpp @@ -580,6 +580,25 @@ int ObInnerTableSchema::all_virtual_ddl_task_status_schema(ObTableSchema &table_ true, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ObObj consensus_schema_version_default; + consensus_schema_version_default.set_int(-1); + ADD_COLUMN_SCHEMA_T("consensus_schema_version", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + consensus_schema_version_default, + consensus_schema_version_default); //default_value + } table_schema.set_index_using_type(USING_BTREE); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); diff --git a/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp b/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp index 300197c153..a4428f744e 100644 --- a/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp +++ b/src/share/inner_table/ob_inner_table_schema.12251_12300.cpp @@ -2730,6 +2730,25 @@ int ObInnerTableSchema::all_virtual_ddl_error_message_schema(ObTableSchema &tabl true, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ObObj consensus_schema_version_default; + consensus_schema_version_default.set_int(-1); + ADD_COLUMN_SCHEMA_T("consensus_schema_version", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + consensus_schema_version_default, + consensus_schema_version_default); //default_value + } table_schema.set_index_using_type(USING_BTREE); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); diff --git a/src/share/inner_table/ob_inner_table_schema.21501_21550.cpp b/src/share/inner_table/ob_inner_table_schema.21501_21550.cpp index 4ca1f6ef0e..4b6ecd5f36 100644 --- a/src/share/inner_table/ob_inner_table_schema.21501_21550.cpp +++ b/src/share/inner_table/ob_inner_table_schema.21501_21550.cpp @@ -325,6 +325,806 @@ int ObInnerTableSchema::dba_ob_clone_history_schema(ObTableSchema &table_schema) return ret; } +int ObInnerTableSchema::cdb_mview_logs_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_CDB_MVIEW_LOGS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_CDB_MVIEW_LOGS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT B.TENANT_ID AS TENANT_ID, CAST(A.DATABASE_NAME AS CHAR(128)) AS LOG_OWNER, CAST(D.TABLE_NAME AS CHAR(128)) AS MASTER, CAST(B.TABLE_NAME AS CHAR(128)) AS LOG_TABLE, CAST(NULL AS CHAR(128)) AS LOG_TRIGGER, CAST(IF(D.TABLE_MODE & 66048 = 66048, 'YES', 'NO') AS CHAR(3)) AS ROWIDS, CAST(IF(D.TABLE_MODE & 66048 = 0, 'YES', 'NO') AS CHAR(3)) AS PRIMARY_KEY, CAST('NO' AS CHAR(3)) AS OBJECT_ID, CAST( IF(( SELECT COUNT(*) FROM oceanbase.__all_virtual_column C1, oceanbase.__all_virtual_column C2 WHERE B.TENANT_ID = C1.TENANT_ID AND B.TABLE_ID = C1.TABLE_ID AND C1.COLUMN_ID >= 16 AND C1.COLUMN_ID < 65520 AND D.TENANT_ID = C2.TENANT_ID AND D.TABLE_ID = C2.TABLE_ID AND C2.ROWKEY_POSITION != 0 AND C1.COLUMN_ID != C2.COLUMN_ID ) = 0, 'NO', 'YES') AS CHAR(3) ) AS FILTER_COLUMNS, CAST('YES' AS CHAR(3)) AS SEQUENCE, CAST('YES' AS CHAR(3)) AS INCLUDE_NEW_VALUES, CAST(IF(C.PURGE_MODE = 1, 'YES', 'NO') AS CHAR(3)) AS PURGE_ASYNCHRONOUS, CAST(IF(C.PURGE_MODE = 2, 'YES', 'NO') AS CHAR(3)) AS PURGE_DEFERRED, CAST(C.PURGE_START AS DATETIME) AS PURGE_START, CAST(C.PURGE_NEXT AS CHAR(200)) AS PURGE_INTERVAL, CAST(C.LAST_PURGE_DATE AS DATETIME) AS LAST_PURGE_DATE, CAST(0 AS SIGNED) AS LAST_PURGE_STATUS, C.LAST_PURGE_ROWS AS NUM_ROWS_PURGED, CAST('YES' AS CHAR(3)) AS COMMIT_SCN_BASED, CAST('NO' AS CHAR(3)) AS STAGING_LOG FROM oceanbase.__all_virtual_database A, oceanbase.__all_virtual_table B, oceanbase.__all_virtual_mlog C, oceanbase.__all_virtual_table D WHERE A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND B.TENANT_ID = D.TENANT_ID AND C.TENANT_ID = D.TENANT_ID AND B.TABLE_ID = C.MLOG_ID AND B.TABLE_TYPE = 15 AND B.DATA_TABLE_ID = D.TABLE_ID )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::dba_mview_logs_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_MVIEW_LOGS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_MVIEW_LOGS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.DATABASE_NAME AS CHAR(128)) AS LOG_OWNER, CAST(D.TABLE_NAME AS CHAR(128)) AS MASTER, CAST(B.TABLE_NAME AS CHAR(128)) AS LOG_TABLE, CAST(NULL AS CHAR(128)) AS LOG_TRIGGER, CAST(IF(D.TABLE_MODE & 66048 = 66048, 'YES', 'NO') AS CHAR(3)) AS ROWIDS, CAST(IF(D.TABLE_MODE & 66048 = 0, 'YES', 'NO') AS CHAR(3)) AS PRIMARY_KEY, CAST('NO' AS CHAR(3)) AS OBJECT_ID, CAST( IF(( SELECT COUNT(*) FROM oceanbase.__all_column C1, oceanbase.__all_column C2 WHERE B.TENANT_ID = C1.TENANT_ID AND B.TABLE_ID = C1.TABLE_ID AND C1.COLUMN_ID >= 16 AND C1.COLUMN_ID < 65520 AND D.TENANT_ID = C2.TENANT_ID AND D.TABLE_ID = C2.TABLE_ID AND C2.ROWKEY_POSITION != 0 AND C1.COLUMN_ID != C2.COLUMN_ID ) = 0, 'NO', 'YES') AS CHAR(3) ) AS FILTER_COLUMNS, CAST('YES' AS CHAR(3)) AS SEQUENCE, CAST('YES' AS CHAR(3)) AS INCLUDE_NEW_VALUES, CAST(IF(C.PURGE_MODE = 1, 'YES', 'NO') AS CHAR(3)) AS PURGE_ASYNCHRONOUS, CAST(IF(C.PURGE_MODE = 2, 'YES', 'NO') AS CHAR(3)) AS PURGE_DEFERRED, CAST(C.PURGE_START AS DATETIME) AS PURGE_START, CAST(C.PURGE_NEXT AS CHAR(200)) AS PURGE_INTERVAL, CAST(C.LAST_PURGE_DATE AS DATETIME) AS LAST_PURGE_DATE, CAST(0 AS SIGNED) AS LAST_PURGE_STATUS, C.LAST_PURGE_ROWS AS NUM_ROWS_PURGED, CAST('YES' AS CHAR(3)) AS COMMIT_SCN_BASED, CAST('NO' AS CHAR(3)) AS STAGING_LOG FROM oceanbase.__all_database A, oceanbase.__all_table B, oceanbase.__all_mlog C, oceanbase.__all_table D WHERE A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND B.TENANT_ID = D.TENANT_ID AND C.TENANT_ID = D.TENANT_ID AND B.TABLE_ID = C.MLOG_ID AND B.TABLE_TYPE = 15 AND B.DATA_TABLE_ID = D.TABLE_ID )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::cdb_mviews_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_CDB_MVIEWS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_CDB_MVIEWS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT B.TENANT_ID AS TENANT_ID, CAST(A.DATABASE_NAME AS CHAR(128)) AS OWNER, CAST(B.TABLE_NAME AS CHAR(128)) AS MVIEW_NAME, CAST(D.TABLE_NAME AS CHAR(128)) AS CONTAINER_NAME, B.VIEW_DEFINITION AS QUERY, CAST(LENGTH(B.VIEW_DEFINITION) AS SIGNED) AS QUERY_LEN, CAST('N' AS CHAR(1)) AS UPDATABLE, CAST(NULL AS CHAR(128)) AS UPDATE_LOG, CAST(NULL AS CHAR(128)) AS MASTER_ROLLBACK_SEG, CAST(NULL AS CHAR(128)) AS MASTER_LINK, CAST('N' AS CHAR(1)) AS REWRITE_ENABLED, CAST(NULL AS CHAR(9)) AS REWRITE_CAPABILITY, CAST( CASE C.REFRESH_MODE WHEN 0 THEN 'NEVER' WHEN 1 THEN 'DEMAND' WHEN 2 THEN 'COMMIT' WHEN 3 THEN 'STATEMENT' ELSE NULL END AS CHAR(6) ) AS REFRESH_MODE, CAST( CASE C.REFRESH_METHOD WHEN 0 THEN 'NEVER' WHEN 1 THEN 'COMPLETE' WHEN 2 THEN 'FAST' WHEN 3 THEN 'FORCE' ELSE NULL END AS CHAR(8) ) AS REFRESH_METHOD, CAST( CASE C.BUILD_MODE WHEN 0 THEN 'IMMEDIATE' WHEN 1 THEN 'DEFERRED' WHEN 2 THEN 'PERBUILT' ELSE NULL END AS CHAR(9) ) AS BUILD_MODE, CAST(NULL AS CHAR(18)) AS FAST_REFRESHABLE, CAST( CASE C.LAST_REFRESH_TYPE WHEN 0 THEN 'COMPLETE' WHEN 1 THEN 'FAST' ELSE 'NA' END AS CHAR(8) ) AS LAST_REFRESH_TYPE, CAST(C.LAST_REFRESH_DATE AS DATETIME) AS LAST_REFRESH_DATE, CAST(C.LAST_REFRESH_DATE + C.LAST_REFRESH_TIME * 1000 * 1000 AS DATETIME) AS LAST_REFRESH_END_TIME, CAST(NULL AS CHAR(19)) AS STALENESS, CAST(NULL AS CHAR(19)) AS AFTER_FAST_REFRESH, CAST(IF(C.BUILD_MODE = 2, 'Y', 'N') AS CHAR(1)) AS UNKNOWN_PREBUILT, CAST('N' AS CHAR(1)) AS UNKNOWN_PLSQL_FUNC, CAST('N' AS CHAR(1)) AS UNKNOWN_EXTERNAL_TABLE, CAST('N' AS CHAR(1)) AS UNKNOWN_CONSIDER_FRESH, CAST('N' AS CHAR(1)) AS UNKNOWN_IMPORT, CAST('N' AS CHAR(1)) AS UNKNOWN_TRUSTED_FD, CAST(NULL AS CHAR(19)) AS COMPILE_STATE, CAST('Y' AS CHAR(1)) AS USE_NO_INDEX, CAST(NULL AS DATETIME) AS STALE_SINCE, CAST(NULL AS SIGNED) AS NUM_PCT_TABLES, CAST(NULL AS SIGNED) AS NUM_FRESH_PCT_REGIONS, CAST(NULL AS SIGNED) AS NUM_STALE_PCT_REGIONS, CAST('NO' AS CHAR(3)) AS SEGMENT_CREATED, CAST(NULL AS CHAR(128)) AS EVALUATION_EDITION, CAST(NULL AS CHAR(128)) AS UNUSABLE_BEFORE, CAST(NULL AS CHAR(128)) AS UNUSABLE_BEGINNING, CAST(NULL AS CHAR(100)) AS DEFAULT_COLLATION, CAST('N' AS CHAR(1)) AS ON_QUERY_COMPUTATION FROM oceanbase.__all_virtual_database A, oceanbase.__all_virtual_table B, oceanbase.__all_virtual_mview C, oceanbase.__all_virtual_table D WHERE A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND B.TENANT_ID = C.TENANT_ID AND B.TABLE_ID = C.MVIEW_ID AND B.TABLE_TYPE = 7 AND B.TENANT_ID = D.TENANT_ID AND B.DATA_TABLE_ID = D.TABLE_ID )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::dba_mviews_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_MVIEWS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_MVIEWS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.DATABASE_NAME AS CHAR(128)) AS OWNER, CAST(B.TABLE_NAME AS CHAR(128)) AS MVIEW_NAME, CAST(D.TABLE_NAME AS CHAR(128)) AS CONTAINER_NAME, B.VIEW_DEFINITION AS QUERY, CAST(LENGTH(B.VIEW_DEFINITION) AS SIGNED) AS QUERY_LEN, CAST('N' AS CHAR(1)) AS UPDATABLE, CAST(NULL AS CHAR(128)) AS UPDATE_LOG, CAST(NULL AS CHAR(128)) AS MASTER_ROLLBACK_SEG, CAST(NULL AS CHAR(128)) AS MASTER_LINK, CAST('N' AS CHAR(1)) AS REWRITE_ENABLED, CAST(NULL AS CHAR(9)) AS REWRITE_CAPABILITY, CAST( CASE C.REFRESH_MODE WHEN 0 THEN 'NEVER' WHEN 1 THEN 'DEMAND' WHEN 2 THEN 'COMMIT' WHEN 3 THEN 'STATEMENT' ELSE NULL END AS CHAR(6) ) AS REFRESH_MODE, CAST( CASE C.REFRESH_METHOD WHEN 0 THEN 'NEVER' WHEN 1 THEN 'COMPLETE' WHEN 2 THEN 'FAST' WHEN 3 THEN 'FORCE' ELSE NULL END AS CHAR(8) ) AS REFRESH_METHOD, CAST( CASE C.BUILD_MODE WHEN 0 THEN 'IMMEDIATE' WHEN 1 THEN 'DEFERRED' WHEN 2 THEN 'PERBUILT' ELSE NULL END AS CHAR(9) ) AS BUILD_MODE, CAST(NULL AS CHAR(18)) AS FAST_REFRESHABLE, CAST( CASE C.LAST_REFRESH_TYPE WHEN 0 THEN 'COMPLETE' WHEN 1 THEN 'FAST' ELSE 'NA' END AS CHAR(8) ) AS LAST_REFRESH_TYPE, CAST(C.LAST_REFRESH_DATE AS DATETIME) AS LAST_REFRESH_DATE, CAST(C.LAST_REFRESH_DATE + C.LAST_REFRESH_TIME * 1000 * 1000 AS DATETIME) AS LAST_REFRESH_END_TIME, CAST(NULL AS CHAR(19)) AS STALENESS, CAST(NULL AS CHAR(19)) AS AFTER_FAST_REFRESH, CAST(IF(C.BUILD_MODE = 2, 'Y', 'N') AS CHAR(1)) AS UNKNOWN_PREBUILT, CAST('N' AS CHAR(1)) AS UNKNOWN_PLSQL_FUNC, CAST('N' AS CHAR(1)) AS UNKNOWN_EXTERNAL_TABLE, CAST('N' AS CHAR(1)) AS UNKNOWN_CONSIDER_FRESH, CAST('N' AS CHAR(1)) AS UNKNOWN_IMPORT, CAST('N' AS CHAR(1)) AS UNKNOWN_TRUSTED_FD, CAST(NULL AS CHAR(19)) AS COMPILE_STATE, CAST('Y' AS CHAR(1)) AS USE_NO_INDEX, CAST(NULL AS DATETIME) AS STALE_SINCE, CAST(NULL AS SIGNED) AS NUM_PCT_TABLES, CAST(NULL AS SIGNED) AS NUM_FRESH_PCT_REGIONS, CAST(NULL AS SIGNED) AS NUM_STALE_PCT_REGIONS, CAST('NO' AS CHAR(3)) AS SEGMENT_CREATED, CAST(NULL AS CHAR(128)) AS EVALUATION_EDITION, CAST(NULL AS CHAR(128)) AS UNUSABLE_BEFORE, CAST(NULL AS CHAR(128)) AS UNUSABLE_BEGINNING, CAST(NULL AS CHAR(100)) AS DEFAULT_COLLATION, CAST('N' AS CHAR(1)) AS ON_QUERY_COMPUTATION FROM oceanbase.__all_database A, oceanbase.__all_table B, oceanbase.__all_mview C, oceanbase.__all_table D WHERE A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND B.TENANT_ID = C.TENANT_ID AND B.TABLE_ID = C.MVIEW_ID AND B.TABLE_TYPE = 7 AND B.TENANT_ID = D.TENANT_ID AND B.DATA_TABLE_ID = D.TABLE_ID )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::cdb_mvref_stats_sys_defaults_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_CDB_MVREF_STATS_SYS_DEFAULTS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_CDB_MVREF_STATS_SYS_DEFAULTS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT TENANT_ID, CAST(PARAMETER_NAME AS CHAR(16)) AS PARAMETER_NAME, CAST(VALUE AS CHAR(40)) AS VALUE FROM ( /* COLLECTION_LEVEL */ SELECT TENANT_ID, 'COLLECTION_LEVEL' PARAMETER_NAME, CASE IFNULL(MAX(COLLECTION_LEVEL), 1) WHEN 0 THEN 'NONE' WHEN 1 THEN 'TYPICAL' WHEN 2 THEN 'ADVANCED' ELSE NULL END VALUE FROM oceanbase.__all_virtual_mview_refresh_stats_sys_defaults RIGHT OUTER JOIN (SELECT TENANT_ID FROM oceanbase.__all_tenant WHERE TENANT_ID = 1 OR (TENANT_ID & 0x1) = 0) USING (TENANT_ID) GROUP BY TENANT_ID UNION ALL /* RETENTION_PERIOD */ SELECT TENANT_ID, 'RETENTION_PERIOD' PARAMETER_NAME, CAST(IFNULL(MAX(RETENTION_PERIOD), 31) AS CHAR) VALUE FROM oceanbase.__all_virtual_mview_refresh_stats_sys_defaults RIGHT OUTER JOIN (SELECT TENANT_ID FROM oceanbase.__all_tenant WHERE TENANT_ID = 1 OR (TENANT_ID & 0x1) = 0) USING (TENANT_ID) GROUP BY TENANT_ID ) ORDER BY TENANT_ID )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::dba_mvref_stats_sys_defaults_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_MVREF_STATS_SYS_DEFAULTS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_MVREF_STATS_SYS_DEFAULTS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(PARAMETER_NAME AS CHAR(16)) AS PARAMETER_NAME, CAST(VALUE AS CHAR(40)) AS VALUE FROM ( /* COLLECTION_LEVEL */ SELECT 'COLLECTION_LEVEL' PARAMETER_NAME, CASE IFNULL(MAX(COLLECTION_LEVEL), 1) WHEN 0 THEN 'NONE' WHEN 1 THEN 'TYPICAL' WHEN 2 THEN 'ADVANCED' ELSE NULL END VALUE FROM oceanbase.__all_mview_refresh_stats_sys_defaults UNION ALL /* RETENTION_PERIOD */ SELECT 'RETENTION_PERIOD' PARAMETER_NAME, CAST(IFNULL(MAX(RETENTION_PERIOD), 31) AS CHAR) VALUE FROM oceanbase.__all_mview_refresh_stats_sys_defaults ) )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::cdb_mvref_stats_params_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_CDB_MVREF_STATS_PARAMS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_CDB_MVREF_STATS_PARAMS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT TENANT_ID, CAST(MV_OWNER AS CHAR(128)) AS MV_OWNER, CAST(MV_NAME AS CHAR(128)) AS MV_NAME, CAST( CASE COLLECTION_LEVEL WHEN 0 THEN 'NONE' WHEN 1 THEN 'TYPICAL' WHEN 2 THEN 'ADVANCED' ELSE NULL END AS CHAR(8) ) AS COLLECTION_LEVEL, RETENTION_PERIOD FROM ( WITH DEFVALS AS ( SELECT TENANT_ID, IFNULL(MAX(COLLECTION_LEVEL), 1) AS COLLECTION_LEVEL, IFNULL(MAX(RETENTION_PERIOD), 31) AS RETENTION_PERIOD FROM oceanbase.__all_virtual_mview_refresh_stats_sys_defaults RIGHT OUTER JOIN (SELECT TENANT_ID FROM oceanbase.__all_tenant WHERE TENANT_ID = 1 OR (TENANT_ID & 0x1) = 0) USING (TENANT_ID) GROUP BY TENANT_ID ) SELECT B.TENANT_ID TENANT_ID, A.DATABASE_NAME MV_OWNER, B.TABLE_NAME MV_NAME, IFNULL(C.COLLECTION_LEVEL, D.COLLECTION_LEVEL) COLLECTION_LEVEL, IFNULL(C.RETENTION_PERIOD, D.RETENTION_PERIOD) RETENTION_PERIOD FROM oceanbase.__all_virtual_database A, oceanbase.__all_virtual_table B, ( SELECT TENANT_ID, MVIEW_ID, COLLECTION_LEVEL, RETENTION_PERIOD FROM oceanbase.__all_virtual_mview_refresh_stats_params RIGHT OUTER JOIN ( SELECT TENANT_ID, MVIEW_ID FROM oceanbase.__all_virtual_mview ) USING (TENANT_ID, MVIEW_ID) ) C, DEFVALS D WHERE A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND B.TENANT_ID = C.TENANT_ID AND B.TABLE_ID = C.MVIEW_ID AND B.TABLE_TYPE = 7 AND C.TENANT_ID = D.TENANT_ID ) )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::dba_mvref_stats_params_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_MVREF_STATS_PARAMS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_MVREF_STATS_PARAMS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(MV_OWNER AS CHAR(128)) AS MV_OWNER, CAST(MV_NAME AS CHAR(128)) AS MV_NAME, CAST( CASE COLLECTION_LEVEL WHEN 0 THEN 'NONE' WHEN 1 THEN 'TYPICAL' WHEN 2 THEN 'ADVANCED' ELSE NULL END AS CHAR(8) ) AS COLLECTION_LEVEL, RETENTION_PERIOD FROM ( WITH DEFVALS AS ( SELECT IFNULL(MAX(COLLECTION_LEVEL), 1) AS COLLECTION_LEVEL, IFNULL(MAX(RETENTION_PERIOD), 31) AS RETENTION_PERIOD FROM oceanbase.__all_mview_refresh_stats_sys_defaults ) SELECT A.DATABASE_NAME MV_OWNER, B.TABLE_NAME MV_NAME, IFNULL(C.COLLECTION_LEVEL, D.COLLECTION_LEVEL) COLLECTION_LEVEL, IFNULL(C.RETENTION_PERIOD, D.RETENTION_PERIOD) RETENTION_PERIOD FROM oceanbase.__all_database A, oceanbase.__all_table B, ( SELECT TENANT_ID, MVIEW_ID, COLLECTION_LEVEL, RETENTION_PERIOD FROM oceanbase.__all_mview_refresh_stats_params RIGHT OUTER JOIN ( SELECT TENANT_ID, MVIEW_ID FROM oceanbase.__all_mview ) USING (TENANT_ID, MVIEW_ID) ) C, DEFVALS D WHERE A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND B.TENANT_ID = C.TENANT_ID AND B.TABLE_ID = C.MVIEW_ID AND B.TABLE_TYPE = 7 ) )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::cdb_mvref_run_stats_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_CDB_MVREF_RUN_STATS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_CDB_MVREF_RUN_STATS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT B.TENANT_ID AS TENANT_ID, CAST(A.USER_NAME AS CHAR(128)) AS RUN_OWNER, B.REFRESH_ID AS REFRESH_ID, B.NUM_MVS_TOTAL AS NUM_MVS, CAST(B.MVIEWS AS CHAR(4000)) AS MVIEWS, CAST(B.BASE_TABLES AS CHAR(4000)) AS BASE_TABLES, CAST(B.METHOD AS CHAR(4000)) AS METHOD, CAST(B.ROLLBACK_SEG AS CHAR(4000)) AS ROLLBACK_SEG, CAST(IF(B.PUSH_DEFERRED_RPC = 1, 'Y', 'N') AS CHAR(1)) AS PUSH_DEFERRED_RPC, CAST(IF(B.REFRESH_AFTER_ERRORS = 1, 'Y', 'N') AS CHAR(1)) AS REFRESH_AFTER_ERRORS, B.PURGE_OPTION AS PURGE_OPTION, B.PARALLELISM AS PARALLELISM, B.HEAP_SIZE AS HEAP_SIZE, CAST(IF(B.ATOMIC_REFRESH = 1, 'Y', 'N') AS CHAR(1)) AS ATOMIC_REFRESH, CAST(IF(B.NESTED = 1, 'Y', 'N') AS CHAR(1)) AS NESTED, CAST(IF(B.OUT_OF_PLACE = 1, 'Y', 'N') AS CHAR(1)) AS OUT_OF_PLACE, B.NUMBER_OF_FAILURES AS NUMBER_OF_FAILURES, CAST(B.START_TIME AS DATETIME) AS START_TIME, CAST(B.END_TIME AS DATETIME) AS END_TIME, B.ELAPSED_TIME AS ELAPSED_TIME, CAST(0 AS SIGNED) AS LOG_SETUP_TIME, B.LOG_PURGE_TIME AS LOG_PURGE_TIME, CAST(IF(B.COMPLETE_STATS_AVALIABLE = 1, 'Y', 'N') AS CHAR(1)) AS COMPLETE_STATS_AVAILABLE FROM oceanbase.__all_virtual_user A, oceanbase.__all_virtual_mview_refresh_run_stats B WHERE A.TENANT_ID = B.TENANT_ID AND A.USER_ID = B.RUN_USER_ID )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::dba_mvref_run_stats_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_MVREF_RUN_STATS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_MVREF_RUN_STATS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.USER_NAME AS CHAR(128)) AS RUN_OWNER, B.REFRESH_ID AS REFRESH_ID, B.NUM_MVS_TOTAL AS NUM_MVS, CAST(B.MVIEWS AS CHAR(4000)) AS MVIEWS, CAST(B.BASE_TABLES AS CHAR(4000)) AS BASE_TABLES, CAST(B.METHOD AS CHAR(4000)) AS METHOD, CAST(B.ROLLBACK_SEG AS CHAR(4000)) AS ROLLBACK_SEG, CAST(IF(B.PUSH_DEFERRED_RPC = 1, 'Y', 'N') AS CHAR(1)) AS PUSH_DEFERRED_RPC, CAST(IF(B.REFRESH_AFTER_ERRORS = 1, 'Y', 'N') AS CHAR(1)) AS REFRESH_AFTER_ERRORS, B.PURGE_OPTION AS PURGE_OPTION, B.PARALLELISM AS PARALLELISM, B.HEAP_SIZE AS HEAP_SIZE, CAST(IF(B.ATOMIC_REFRESH = 1, 'Y', 'N') AS CHAR(1)) AS ATOMIC_REFRESH, CAST(IF(B.NESTED = 1, 'Y', 'N') AS CHAR(1)) AS NESTED, CAST(IF(B.OUT_OF_PLACE = 1, 'Y', 'N') AS CHAR(1)) AS OUT_OF_PLACE, B.NUMBER_OF_FAILURES AS NUMBER_OF_FAILURES, CAST(B.START_TIME AS DATETIME) AS START_TIME, CAST(B.END_TIME AS DATETIME) AS END_TIME, B.ELAPSED_TIME AS ELAPSED_TIME, CAST(0 AS SIGNED) AS LOG_SETUP_TIME, B.LOG_PURGE_TIME AS LOG_PURGE_TIME, CAST(IF(B.COMPLETE_STATS_AVALIABLE = 1, 'Y', 'N') AS CHAR(1)) AS COMPLETE_STATS_AVAILABLE FROM oceanbase.__all_user A, oceanbase.__all_mview_refresh_run_stats B WHERE A.TENANT_ID = B.TENANT_ID AND A.USER_ID = B.RUN_USER_ID )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::cdb_mvref_stats_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_CDB_MVREF_STATS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_CDB_MVREF_STATS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT B.TENANT_ID AS TENANT_ID, CAST(A.DATABASE_NAME AS CHAR(128)) AS MV_OWNER, CAST(B.TABLE_NAME AS CHAR(128)) AS MV_NAME, C.REFRESH_ID AS REFRESH_ID, CAST( CASE C.REFRESH_TYPE WHEN 0 THEN 'COMPLETE' WHEN 1 THEN 'FAST' ELSE NULL END AS CHAR(30) ) AS REFRESH_METHOD, CAST(NULL AS CHAR(4000)) AS REFRESH_OPTIMIZATIONS, CAST(NULL AS CHAR(4000)) AS ADDITIONAL_EXECUTIONS, CAST(C.START_TIME AS DATETIME) AS START_TIME, CAST(C.END_TIME AS DATETIME) AS END_TIME, C.ELAPSED_TIME AS ELAPSED_TIME, CAST(0 AS SIGNED) AS LOG_SETUP_TIME, C.LOG_PURGE_TIME AS LOG_PURGE_TIME, C.INITIAL_NUM_ROWS AS INITIAL_NUM_ROWS, C.FINAL_NUM_ROWS AS FINAL_NUM_ROWS FROM oceanbase.__all_virtual_database A, oceanbase.__all_virtual_table B, oceanbase.__all_virtual_mview_refresh_stats C WHERE A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND B.TENANT_ID = C.TENANT_ID AND B.TABLE_ID = C.MVIEW_ID AND B.TABLE_TYPE = 7 )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::dba_mvref_stats_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_MVREF_STATS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_MVREF_STATS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.DATABASE_NAME AS CHAR(128)) AS MV_OWNER, CAST(B.TABLE_NAME AS CHAR(128)) AS MV_NAME, C.REFRESH_ID AS REFRESH_ID, CAST( CASE C.REFRESH_TYPE WHEN 0 THEN 'COMPLETE' WHEN 1 THEN 'FAST' ELSE NULL END AS CHAR(30) ) AS REFRESH_METHOD, CAST(NULL AS CHAR(4000)) AS REFRESH_OPTIMIZATIONS, CAST(NULL AS CHAR(4000)) AS ADDITIONAL_EXECUTIONS, CAST(C.START_TIME AS DATETIME) AS START_TIME, CAST(C.END_TIME AS DATETIME) AS END_TIME, C.ELAPSED_TIME AS ELAPSED_TIME, CAST(0 AS SIGNED) AS LOG_SETUP_TIME, C.LOG_PURGE_TIME AS LOG_PURGE_TIME, C.INITIAL_NUM_ROWS AS INITIAL_NUM_ROWS, C.FINAL_NUM_ROWS AS FINAL_NUM_ROWS FROM oceanbase.__all_database A, oceanbase.__all_table B, oceanbase.__all_mview_refresh_stats C WHERE A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND B.TENANT_ID = C.TENANT_ID AND B.TABLE_ID = C.MVIEW_ID AND B.TABLE_TYPE = 7 )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::cdb_mvref_change_stats_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_CDB_MVREF_CHANGE_STATS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_CDB_MVREF_CHANGE_STATS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT E.TENANT_ID AS TENANT_ID, CAST(C.DATABASE_NAME AS CHAR(128)) AS TBL_OWNER, CAST(D.TABLE_NAME AS CHAR(128)) AS TBL_NAME, CAST(A.DATABASE_NAME AS CHAR(128)) AS MV_OWNER, CAST(B.TABLE_NAME AS CHAR(128)) AS MV_NAME, E.REFRESH_ID AS REFRESH_ID, E.NUM_ROWS_INS AS NUM_ROWS_INS, E.NUM_ROWS_UPD AS NUM_ROWS_UPD, E.NUM_ROWS_DEL AS NUM_ROWS_DEL, CAST(0 AS SIGNED) AS NUM_ROWS_DL_INS, CAST('N' AS CHAR(1)) AS PMOPS_OCCURRED, CAST(NULL AS CHAR(4000)) AS PMOP_DETAILS, E.NUM_ROWS AS NUM_ROWS FROM oceanbase.__all_virtual_database A, oceanbase.__all_virtual_table B, oceanbase.__all_virtual_database C, oceanbase.__all_virtual_table D, oceanbase.__all_virtual_mview_refresh_change_stats E WHERE A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND C.TENANT_ID = D.TENANT_ID AND C.DATABASE_ID = D.DATABASE_ID AND E.TENANT_ID = B.TENANT_ID AND E.MVIEW_ID = B.TABLE_ID AND E.TENANT_ID = D.TENANT_ID AND E.DETAIL_TABLE_ID = D.TABLE_ID )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::dba_mvref_change_stats_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_MVREF_CHANGE_STATS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_MVREF_CHANGE_STATS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(C.DATABASE_NAME AS CHAR(128)) AS TBL_OWNER, CAST(D.TABLE_NAME AS CHAR(128)) AS TBL_NAME, CAST(A.DATABASE_NAME AS CHAR(128)) AS MV_OWNER, CAST(B.TABLE_NAME AS CHAR(128)) AS MV_NAME, E.REFRESH_ID AS REFRESH_ID, E.NUM_ROWS_INS AS NUM_ROWS_INS, E.NUM_ROWS_UPD AS NUM_ROWS_UPD, E.NUM_ROWS_DEL AS NUM_ROWS_DEL, CAST(0 AS SIGNED) AS NUM_ROWS_DL_INS, CAST('N' AS CHAR(1)) AS PMOPS_OCCURRED, CAST(NULL AS CHAR(4000)) AS PMOP_DETAILS, E.NUM_ROWS AS NUM_ROWS FROM oceanbase.__all_database A, oceanbase.__all_table B, oceanbase.__all_database C, oceanbase.__all_table D, oceanbase.__all_mview_refresh_change_stats E WHERE A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND C.TENANT_ID = D.TENANT_ID AND C.DATABASE_ID = D.DATABASE_ID AND E.TENANT_ID = B.TENANT_ID AND E.MVIEW_ID = B.TABLE_ID AND E.TENANT_ID = D.TENANT_ID AND E.DETAIL_TABLE_ID = D.TABLE_ID )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::cdb_mvref_stmt_stats_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_CDB_MVREF_STMT_STATS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_CDB_MVREF_STMT_STATS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT C.TENANT_ID AS TENANT_ID, CAST(A.DATABASE_NAME AS CHAR(128)) AS MV_OWNER, CAST(B.TABLE_NAME AS CHAR(128)) AS MV_NAME, C.REFRESH_ID AS REFRESH_ID, C.STEP AS STEP, CAST(C.SQLID AS CHAR(32)) AS SQLID, C.STMT AS STMT, C.EXECUTION_TIME AS EXECUTION_TIME, C.EXECUTION_PLAN AS EXECUTION_PLAN FROM oceanbase.__all_virtual_database A, oceanbase.__all_virtual_table B, oceanbase.__all_virtual_mview_refresh_stmt_stats C WHERE A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND B.TENANT_ID = C.TENANT_ID AND B.TABLE_ID = C.MVIEW_ID )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::dba_mvref_stmt_stats_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_MVREF_STMT_STATS_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_MVREF_STMT_STATS_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.DATABASE_NAME AS CHAR(128)) AS MV_OWNER, CAST(B.TABLE_NAME AS CHAR(128)) AS MV_NAME, C.REFRESH_ID AS REFRESH_ID, C.STEP AS STEP, CAST(C.SQLID AS CHAR(32)) AS SQLID, C.STMT AS STMT, C.EXECUTION_TIME AS EXECUTION_TIME, C.EXECUTION_PLAN AS EXECUTION_PLAN FROM oceanbase.__all_database A, oceanbase.__all_table B, oceanbase.__all_mview_refresh_stmt_stats C WHERE A.TENANT_ID = B.TENANT_ID AND A.DATABASE_ID = B.DATABASE_ID AND B.TENANT_ID = C.TENANT_ID AND B.TABLE_ID = C.MVIEW_ID )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + } // end namespace share } // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.25251_25300.cpp b/src/share/inner_table/ob_inner_table_schema.25251_25300.cpp index d6505a9486..ba3c65b42d 100644 --- a/src/share/inner_table/ob_inner_table_schema.25251_25300.cpp +++ b/src/share/inner_table/ob_inner_table_schema.25251_25300.cpp @@ -975,6 +975,856 @@ int ObInnerTableSchema::user_users_schema(ObTableSchema &table_schema) return ret; } +int ObInnerTableSchema::dba_mview_logs_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_MVIEW_LOGS_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_MVIEW_LOGS_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS LOG_OWNER, CAST(D.TABLE_NAME AS VARCHAR2(128)) AS MASTER, CAST(B.TABLE_NAME AS VARCHAR2(128)) AS LOG_TABLE, CAST(NULL AS VARCHAR2(128)) AS LOG_TRIGGER, CAST(DECODE(bitand(D.TABLE_MODE, 66048), 66048, 'YES', 'NO') AS VARCHAR2(3)) AS ROWIDS, CAST(DECODE(bitand(D.TABLE_MODE, 66048), 0, 'YES', 'NO') AS VARCHAR2(3)) AS PRIMARY_KEY, CAST('NO' AS VARCHAR2(3)) AS OBJECT_ID, CAST( DECODE(( SELECT COUNT(*) FROM SYS.ALL_VIRTUAL_COLUMN_REAL_AGENT C1, SYS.ALL_VIRTUAL_COLUMN_REAL_AGENT C2 WHERE B.TABLE_ID = C1.TABLE_ID AND C1.COLUMN_ID >= 16 AND C1.COLUMN_ID < 65520 AND D.TABLE_ID = C2.TABLE_ID AND C2.ROWKEY_POSITION != 0 AND C1.COLUMN_ID != C2.COLUMN_ID AND C1.TENANT_ID = EFFECTIVE_TENANT_ID() AND C2.TENANT_ID = EFFECTIVE_TENANT_ID() ), 0, 'NO', 'YES') AS VARCHAR2(3) ) AS FILTER_COLUMNS, CAST('YES' AS VARCHAR2(3)) AS SEQUENCE, CAST('YES' AS VARCHAR2(3)) AS INCLUDE_NEW_VALUES, CAST(DECODE(C.PURGE_MODE, 1, 'YES', 'NO') AS VARCHAR2(3)) AS PURGE_ASYNCHRONOUS, CAST(DECODE(C.PURGE_MODE, 2, 'YES', 'NO') AS VARCHAR2(3)) AS PURGE_DEFERRED, CAST(C.PURGE_START AS DATE) AS PURGE_START /* TODO: DD-MON-YYYY */, CAST(C.PURGE_NEXT AS VARCHAR2(200)) AS PURGE_INTERVAL, CAST(C.LAST_PURGE_DATE AS DATE) AS LAST_PURGE_DATE /* TODO: DD-MON-YYYY */, CAST(0 AS NUMBER) AS LAST_PURGE_STATUS, CAST(C.LAST_PURGE_ROWS AS NUMBER) AS NUM_ROWS_PURGED, CAST('YES' AS VARCHAR2(3)) AS COMMIT_SCN_BASED, CAST('NO' AS VARCHAR2(3)) AS STAGING_LOG FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, SYS.ALL_VIRTUAL_MLOG_REAL_AGENT C, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT D WHERE A.DATABASE_ID = B.DATABASE_ID AND B.TABLE_ID = C.MLOG_ID AND B.TABLE_TYPE = 15 AND B.DATA_TABLE_ID = D.TABLE_ID AND A.TENANT_ID = EFFECTIVE_TENANT_ID() AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND C.TENANT_ID = EFFECTIVE_TENANT_ID() AND D.TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::all_mview_logs_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_MVIEW_LOGS_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_MVIEW_LOGS_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS LOG_OWNER, CAST(D.TABLE_NAME AS VARCHAR2(128)) AS MASTER, CAST(B.TABLE_NAME AS VARCHAR2(128)) AS LOG_TABLE, CAST(NULL AS VARCHAR2(128)) AS LOG_TRIGGER, CAST(DECODE(bitand(D.TABLE_MODE, 66048), 66048, 'YES', 'NO') AS VARCHAR2(3)) AS ROWIDS, CAST(DECODE(bitand(D.TABLE_MODE, 66048), 0, 'YES', 'NO') AS VARCHAR2(3)) AS PRIMARY_KEY, CAST('NO' AS VARCHAR2(3)) AS OBJECT_ID, CAST( DECODE(( SELECT COUNT(*) FROM SYS.ALL_VIRTUAL_COLUMN_REAL_AGENT C1, SYS.ALL_VIRTUAL_COLUMN_REAL_AGENT C2 WHERE B.TABLE_ID = C1.TABLE_ID AND C1.COLUMN_ID >= 16 AND C1.COLUMN_ID < 65520 AND D.TABLE_ID = C2.TABLE_ID AND C2.ROWKEY_POSITION != 0 AND C1.COLUMN_ID != C2.COLUMN_ID AND C1.TENANT_ID = EFFECTIVE_TENANT_ID() AND C2.TENANT_ID = EFFECTIVE_TENANT_ID() ), 0, 'NO', 'YES') AS VARCHAR2(3) ) AS FILTER_COLUMNS, CAST('YES' AS VARCHAR2(3)) AS SEQUENCE, CAST('YES' AS VARCHAR2(3)) AS INCLUDE_NEW_VALUES, CAST(DECODE(C.PURGE_MODE, 1, 'YES', 'NO') AS VARCHAR2(3)) AS PURGE_ASYNCHRONOUS, CAST(DECODE(C.PURGE_MODE, 2, 'YES', 'NO') AS VARCHAR2(3)) AS PURGE_DEFERRED, CAST(C.PURGE_START AS DATE) AS PURGE_START /* TODO: DD-MON-YYYY */, CAST(C.PURGE_NEXT AS VARCHAR2(200)) AS PURGE_INTERVAL, CAST(C.LAST_PURGE_DATE AS DATE) AS LAST_PURGE_DATE /* TODO: DD-MON-YYYY */, CAST(0 AS NUMBER) AS LAST_PURGE_STATUS, CAST(C.LAST_PURGE_ROWS AS NUMBER) AS NUM_ROWS_PURGED, CAST('YES' AS VARCHAR2(3)) AS COMMIT_SCN_BASED, CAST('NO' AS VARCHAR2(3)) AS STAGING_LOG FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, SYS.ALL_VIRTUAL_MLOG_REAL_AGENT C, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT D WHERE A.DATABASE_ID = B.DATABASE_ID AND B.TABLE_ID = C.MLOG_ID AND B.TABLE_TYPE = 15 AND B.DATA_TABLE_ID = D.TABLE_ID AND A.TENANT_ID = EFFECTIVE_TENANT_ID() AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND C.TENANT_ID = EFFECTIVE_TENANT_ID() AND D.TENANT_ID = EFFECTIVE_TENANT_ID() AND (A.DATABASE_ID = USERENV('SCHEMAID') OR USER_CAN_ACCESS_OBJ(1, B.TABLE_ID, B.DATABASE_ID) = 1) )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::user_mview_logs_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_USER_MVIEW_LOGS_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_USER_MVIEW_LOGS_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS LOG_OWNER, CAST(D.TABLE_NAME AS VARCHAR2(128)) AS MASTER, CAST(B.TABLE_NAME AS VARCHAR2(128)) AS LOG_TABLE, CAST(NULL AS VARCHAR2(128)) AS LOG_TRIGGER, CAST(DECODE(bitand(D.TABLE_MODE, 66048), 66048, 'YES', 'NO') AS VARCHAR2(3)) AS ROWIDS, CAST(DECODE(bitand(D.TABLE_MODE, 66048), 0, 'YES', 'NO') AS VARCHAR2(3)) AS PRIMARY_KEY, CAST('NO' AS VARCHAR2(3)) AS OBJECT_ID, CAST( DECODE(( SELECT COUNT(*) FROM SYS.ALL_VIRTUAL_COLUMN_REAL_AGENT C1, SYS.ALL_VIRTUAL_COLUMN_REAL_AGENT C2 WHERE B.TABLE_ID = C1.TABLE_ID AND C1.COLUMN_ID >= 16 AND C1.COLUMN_ID < 65520 AND D.TABLE_ID = C2.TABLE_ID AND C2.ROWKEY_POSITION != 0 AND C1.COLUMN_ID != C2.COLUMN_ID AND C1.TENANT_ID = EFFECTIVE_TENANT_ID() AND C2.TENANT_ID = EFFECTIVE_TENANT_ID() ), 0, 'NO', 'YES') AS VARCHAR2(3) ) AS FILTER_COLUMNS, CAST('YES' AS VARCHAR2(3)) AS SEQUENCE, CAST('YES' AS VARCHAR2(3)) AS INCLUDE_NEW_VALUES, CAST(DECODE(C.PURGE_MODE, 1, 'YES', 'NO') AS VARCHAR2(3)) AS PURGE_ASYNCHRONOUS, CAST(DECODE(C.PURGE_MODE, 2, 'YES', 'NO') AS VARCHAR2(3)) AS PURGE_DEFERRED, CAST(C.PURGE_START AS DATE) AS PURGE_START /* TODO: DD-MON-YYYY */, CAST(C.PURGE_NEXT AS VARCHAR2(200)) AS PURGE_INTERVAL, CAST(C.LAST_PURGE_DATE AS DATE) AS LAST_PURGE_DATE /* TODO: DD-MON-YYYY */, CAST(0 AS NUMBER) AS LAST_PURGE_STATUS, CAST(C.LAST_PURGE_ROWS AS NUMBER) AS NUM_ROWS_PURGED, CAST('YES' AS VARCHAR2(3)) AS COMMIT_SCN_BASED, CAST('NO' AS VARCHAR2(3)) AS STAGING_LOG FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, SYS.ALL_VIRTUAL_MLOG_REAL_AGENT C, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT D WHERE A.DATABASE_ID = B.DATABASE_ID AND B.TABLE_ID = C.MLOG_ID AND B.TABLE_TYPE = 15 AND B.DATA_TABLE_ID = D.TABLE_ID AND A.TENANT_ID = EFFECTIVE_TENANT_ID() AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND C.TENANT_ID = EFFECTIVE_TENANT_ID() AND D.TENANT_ID = EFFECTIVE_TENANT_ID() AND A.DATABASE_NAME = SYS_CONTEXT('USERENV','CURRENT_USER') )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::dba_mviews_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_MVIEWS_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_MVIEWS_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS OWNER, CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MVIEW_NAME, CAST(D.TABLE_NAME AS VARCHAR2(128)) AS CONTAINER_NAME, B.VIEW_DEFINITION AS QUERY /* TODO: LONG */, CAST(LENGTH(B.VIEW_DEFINITION) AS NUMBER) AS QUERY_LEN, CAST('N' AS VARCHAR2(1)) AS UPDATABLE, CAST(NULL AS VARCHAR2(128)) AS UPDATE_LOG, CAST(NULL AS VARCHAR2(128)) AS MASTER_ROLLBACK_SEG, CAST(NULL AS VARCHAR2(128)) AS MASTER_LINK, CAST('N' AS VARCHAR2(1)) AS REWRITE_ENABLED, CAST(NULL AS VARCHAR2(9)) AS REWRITE_CAPABILITY, CAST( DECODE(C.REFRESH_MODE, 0, 'NEVER', 1, 'DEMAND', 2, 'COMMIT', 3, 'STATEMENT', NULL ) AS VARCHAR2(6) ) AS REFRESH_MODE, CAST( DECODE(C.REFRESH_METHOD, 0, 'NEVER', 1, 'COMPLETE', 2, 'FAST', 3, 'FORCE', NULL ) AS VARCHAR2(8) ) AS REFRESH_METHOD, CAST( DECODE(C.BUILD_MODE, 0, 'IMMEDIATE', 1, 'DEFERRED', 2, 'PERBUILT', NULL ) AS VARCHAR2(9) ) AS BUILD_MODE, CAST(NULL AS VARCHAR2(18)) AS FAST_REFRESHABLE, CAST( DECODE(C.LAST_REFRESH_TYPE, 0, 'COMPLETE', 1, 'FAST', 'NA' ) AS VARCHAR2(8) ) AS LAST_REFRESH_TYPE, CAST(C.LAST_REFRESH_DATE AS DATE) AS LAST_REFRESH_DATE /* TODO: DD-MON-YYYY */, CAST(C.LAST_REFRESH_DATE + C.LAST_REFRESH_TIME * 1000 * 1000 AS DATE) AS LAST_REFRESH_END_TIME /* TODO: DD-MON-YYYY */, CAST(NULL AS VARCHAR2(19)) AS STALENESS, CAST(NULL AS VARCHAR2(19)) AS AFTER_FAST_REFRESH, CAST(DECODE(C.BUILD_MODE, 2, 'Y', 'N') AS VARCHAR2(1)) AS UNKNOWN_PREBUILT, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_PLSQL_FUNC, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_EXTERNAL_TABLE, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_CONSIDER_FRESH, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_IMPORT, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_TRUSTED_FD, CAST(NULL AS VARCHAR2(19)) AS COMPILE_STATE, CAST('Y' AS VARCHAR2(1)) AS USE_NO_INDEX, CAST(NULL AS DATE) AS STALE_SINCE, CAST(NULL AS NUMBER) AS NUM_PCT_TABLES, CAST(NULL AS NUMBER) AS NUM_FRESH_PCT_REGIONS, CAST(NULL AS NUMBER) AS NUM_STALE_PCT_REGIONS, CAST('NO' AS VARCHAR2(3)) AS SEGMENT_CREATED, CAST(NULL AS VARCHAR2(128)) AS EVALUATION_EDITION, CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEFORE, CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEGINNING, CAST(NULL AS VARCHAR2(100)) AS DEFAULT_COLLATION, CAST('N' AS VARCHAR2(1)) AS ON_QUERY_COMPUTATION FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, SYS.ALL_VIRTUAL_MVIEW_REAL_AGENT C, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT D WHERE A.DATABASE_ID = B.DATABASE_ID AND B.TABLE_ID = C.MVIEW_ID AND B.TABLE_TYPE = 7 AND B.DATA_TABLE_ID = D.TABLE_ID AND A.TENANT_ID = EFFECTIVE_TENANT_ID() AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND C.TENANT_ID = EFFECTIVE_TENANT_ID() AND D.TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::all_mviews_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_ALL_MVIEWS_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_ALL_MVIEWS_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS OWNER, CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MVIEW_NAME, CAST(D.TABLE_NAME AS VARCHAR2(128)) AS CONTAINER_NAME, B.VIEW_DEFINITION AS QUERY /* TODO: LONG */, CAST(LENGTH(B.VIEW_DEFINITION) AS NUMBER) AS QUERY_LEN, CAST('N' AS VARCHAR2(1)) AS UPDATABLE, CAST(NULL AS VARCHAR2(128)) AS UPDATE_LOG, CAST(NULL AS VARCHAR2(128)) AS MASTER_ROLLBACK_SEG, CAST(NULL AS VARCHAR2(128)) AS MASTER_LINK, CAST('N' AS VARCHAR2(1)) AS REWRITE_ENABLED, CAST(NULL AS VARCHAR2(9)) AS REWRITE_CAPABILITY, CAST( DECODE(C.REFRESH_MODE, 0, 'NEVER', 1, 'DEMAND', 2, 'COMMIT', 3, 'STATEMENT', NULL ) AS VARCHAR2(6) ) AS REFRESH_MODE, CAST( DECODE(C.REFRESH_METHOD, 0, 'NEVER', 1, 'COMPLETE', 2, 'FAST', 3, 'FORCE', NULL ) AS VARCHAR2(8) ) AS REFRESH_METHOD, CAST( DECODE(C.BUILD_MODE, 0, 'IMMEDIATE', 1, 'DEFERRED', 2, 'PERBUILT', NULL ) AS VARCHAR2(9) ) AS BUILD_MODE, CAST(NULL AS VARCHAR2(18)) AS FAST_REFRESHABLE, CAST( DECODE(C.LAST_REFRESH_TYPE, 0, 'COMPLETE', 1, 'FAST', 'NA' ) AS VARCHAR2(8) ) AS LAST_REFRESH_TYPE, CAST(C.LAST_REFRESH_DATE AS DATE) AS LAST_REFRESH_DATE /* TODO: DD-MON-YYYY */, CAST(C.LAST_REFRESH_DATE + C.LAST_REFRESH_TIME * 1000 * 1000 AS DATE) AS LAST_REFRESH_END_TIME /* TODO: DD-MON-YYYY */, CAST(NULL AS VARCHAR2(19)) AS STALENESS, CAST(NULL AS VARCHAR2(19)) AS AFTER_FAST_REFRESH, CAST(DECODE(C.BUILD_MODE, 2, 'Y', 'N') AS VARCHAR2(1)) AS UNKNOWN_PREBUILT, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_PLSQL_FUNC, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_EXTERNAL_TABLE, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_CONSIDER_FRESH, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_IMPORT, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_TRUSTED_FD, CAST(NULL AS VARCHAR2(19)) AS COMPILE_STATE, CAST('Y' AS VARCHAR2(1)) AS USE_NO_INDEX, CAST(NULL AS DATE) AS STALE_SINCE, CAST(NULL AS NUMBER) AS NUM_PCT_TABLES, CAST(NULL AS NUMBER) AS NUM_FRESH_PCT_REGIONS, CAST(NULL AS NUMBER) AS NUM_STALE_PCT_REGIONS, CAST('NO' AS VARCHAR2(3)) AS SEGMENT_CREATED, CAST(NULL AS VARCHAR2(128)) AS EVALUATION_EDITION, CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEFORE, CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEGINNING, CAST(NULL AS VARCHAR2(100)) AS DEFAULT_COLLATION, CAST('N' AS VARCHAR2(1)) AS ON_QUERY_COMPUTATION FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, SYS.ALL_VIRTUAL_MVIEW_REAL_AGENT C, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT D WHERE A.DATABASE_ID = B.DATABASE_ID AND B.TABLE_ID = C.MVIEW_ID AND B.TABLE_TYPE = 7 AND B.DATA_TABLE_ID = D.TABLE_ID AND A.TENANT_ID = EFFECTIVE_TENANT_ID() AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND C.TENANT_ID = EFFECTIVE_TENANT_ID() AND D.TENANT_ID = EFFECTIVE_TENANT_ID() AND (A.DATABASE_ID = USERENV('SCHEMAID') OR USER_CAN_ACCESS_OBJ(1, B.TABLE_ID, B.DATABASE_ID) = 1) )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::user_mviews_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_USER_MVIEWS_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_USER_MVIEWS_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS OWNER, CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MVIEW_NAME, CAST(D.TABLE_NAME AS VARCHAR2(128)) AS CONTAINER_NAME, B.VIEW_DEFINITION AS QUERY /* TODO: LONG */, CAST(LENGTH(B.VIEW_DEFINITION) AS NUMBER) AS QUERY_LEN, CAST('N' AS VARCHAR2(1)) AS UPDATABLE, CAST(NULL AS VARCHAR2(128)) AS UPDATE_LOG, CAST(NULL AS VARCHAR2(128)) AS MASTER_ROLLBACK_SEG, CAST(NULL AS VARCHAR2(128)) AS MASTER_LINK, CAST('N' AS VARCHAR2(1)) AS REWRITE_ENABLED, CAST(NULL AS VARCHAR2(9)) AS REWRITE_CAPABILITY, CAST( DECODE(C.REFRESH_MODE, 0, 'NEVER', 1, 'DEMAND', 2, 'COMMIT', 3, 'STATEMENT', NULL ) AS VARCHAR2(6) ) AS REFRESH_MODE, CAST( DECODE(C.REFRESH_METHOD, 0, 'NEVER', 1, 'COMPLETE', 2, 'FAST', 3, 'FORCE', NULL ) AS VARCHAR2(8) ) AS REFRESH_METHOD, CAST( DECODE(C.BUILD_MODE, 0, 'IMMEDIATE', 1, 'DEFERRED', 2, 'PERBUILT', NULL ) AS VARCHAR2(9) ) AS BUILD_MODE, CAST(NULL AS VARCHAR2(18)) AS FAST_REFRESHABLE, CAST( DECODE(C.LAST_REFRESH_TYPE, 0, 'COMPLETE', 1, 'FAST', 'NA' ) AS VARCHAR2(8) ) AS LAST_REFRESH_TYPE, CAST(C.LAST_REFRESH_DATE AS DATE) AS LAST_REFRESH_DATE /* TODO: DD-MON-YYYY */, CAST(C.LAST_REFRESH_DATE + C.LAST_REFRESH_TIME * 1000 * 1000 AS DATE) AS LAST_REFRESH_END_TIME /* TODO: DD-MON-YYYY */, CAST(NULL AS VARCHAR2(19)) AS STALENESS, CAST(NULL AS VARCHAR2(19)) AS AFTER_FAST_REFRESH, CAST(DECODE(C.BUILD_MODE, 2, 'Y', 'N') AS VARCHAR2(1)) AS UNKNOWN_PREBUILT, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_PLSQL_FUNC, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_EXTERNAL_TABLE, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_CONSIDER_FRESH, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_IMPORT, CAST('N' AS VARCHAR2(1)) AS UNKNOWN_TRUSTED_FD, CAST(NULL AS VARCHAR2(19)) AS COMPILE_STATE, CAST('Y' AS VARCHAR2(1)) AS USE_NO_INDEX, CAST(NULL AS DATE) AS STALE_SINCE, CAST(NULL AS NUMBER) AS NUM_PCT_TABLES, CAST(NULL AS NUMBER) AS NUM_FRESH_PCT_REGIONS, CAST(NULL AS NUMBER) AS NUM_STALE_PCT_REGIONS, CAST('NO' AS VARCHAR2(3)) AS SEGMENT_CREATED, CAST(NULL AS VARCHAR2(128)) AS EVALUATION_EDITION, CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEFORE, CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEGINNING, CAST(NULL AS VARCHAR2(100)) AS DEFAULT_COLLATION, CAST('N' AS VARCHAR2(1)) AS ON_QUERY_COMPUTATION FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, SYS.ALL_VIRTUAL_MVIEW_REAL_AGENT C, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT D WHERE A.DATABASE_ID = B.DATABASE_ID AND B.TABLE_ID = C.MVIEW_ID AND B.TABLE_TYPE = 7 AND B.DATA_TABLE_ID = D.TABLE_ID AND A.TENANT_ID = EFFECTIVE_TENANT_ID() AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND C.TENANT_ID = EFFECTIVE_TENANT_ID() AND D.TENANT_ID = EFFECTIVE_TENANT_ID() AND A.DATABASE_NAME = SYS_CONTEXT('USERENV','CURRENT_USER') )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::dba_mvref_stats_sys_defaults_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_MVREF_STATS_SYS_DEFAULTS_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_MVREF_STATS_SYS_DEFAULTS_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(PARAMETER_NAME AS CHAR(16)) AS PARAMETER_NAME, CAST(VALUE AS VARCHAR2(40)) AS VALUE FROM ( /* COLLECTION_LEVEL */ SELECT 'COLLECTION_LEVEL' PARAMETER_NAME, DECODE(NVL(MAX(COLLECTION_LEVEL), 1), 0, 'NONE', 1, 'TYPICAL', 2, 'ADVANCED', NULL) VALUE FROM SYS.ALL_VIRTUAL_MVIEW_REFRESH_STATS_SYS_DEFAULTS_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL /* RETENTION_PERIOD */ SELECT 'RETENTION_PERIOD' PARAMETER_NAME, TO_CHAR(NVL(MAX(RETENTION_PERIOD), 31)) VALUE FROM SYS.ALL_VIRTUAL_MVIEW_REFRESH_STATS_SYS_DEFAULTS_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() ) )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::user_mvref_stats_sys_defaults_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_USER_MVREF_STATS_SYS_DEFAULTS_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_USER_MVREF_STATS_SYS_DEFAULTS_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(PARAMETER_NAME AS CHAR(16)) AS PARAMETER_NAME, CAST(VALUE AS VARCHAR2(40)) AS VALUE FROM ( /* COLLECTION_LEVEL */ SELECT 'COLLECTION_LEVEL' PARAMETER_NAME, DECODE(NVL(MAX(COLLECTION_LEVEL), 1), 0, 'NONE', 1, 'TYPICAL', 2, 'ADVANCED', NULL) VALUE FROM SYS.ALL_VIRTUAL_MVIEW_REFRESH_STATS_SYS_DEFAULTS_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() UNION ALL /* RETENTION_PERIOD */ SELECT 'RETENTION_PERIOD' PARAMETER_NAME, TO_CHAR(NVL(MAX(RETENTION_PERIOD), 31)) VALUE FROM SYS.ALL_VIRTUAL_MVIEW_REFRESH_STATS_SYS_DEFAULTS_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() ) )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::dba_mvref_stats_params_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_MVREF_STATS_PARAMS_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_MVREF_STATS_PARAMS_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(MV_OWNER AS VARCHAR2(128)) AS MV_OWNER, CAST(MV_NAME AS VARCHAR2(128)) AS MV_NAME, CAST( DECODE(COLLECTION_LEVEL, 0, 'NONE', 1, 'TYPICAL', 2, 'ADVANCED', NULL ) AS VARCHAR2(8) ) AS COLLECTION_LEVEL, CAST(RETENTION_PERIOD AS NUMBER) AS RETENTION_PERIOD FROM ( WITH DEFVALS AS ( SELECT NVL(MAX(COLLECTION_LEVEL), 1) AS COLLECTION_LEVEL, NVL(MAX(RETENTION_PERIOD), 31) AS RETENTION_PERIOD FROM SYS.ALL_VIRTUAL_MVIEW_REFRESH_STATS_SYS_DEFAULTS_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() ) SELECT A.DATABASE_NAME MV_OWNER, B.TABLE_NAME MV_NAME, NVL(C.COLLECTION_LEVEL, D.COLLECTION_LEVEL) COLLECTION_LEVEL, NVL(C.RETENTION_PERIOD, D.RETENTION_PERIOD) RETENTION_PERIOD FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, ( SELECT TENANT_ID, MVIEW_ID, COLLECTION_LEVEL, RETENTION_PERIOD FROM SYS.ALL_VIRTUAL_MVIEW_REFRESH_STATS_PARAMS_REAL_AGENT RIGHT OUTER JOIN ( SELECT TENANT_ID, MVIEW_ID FROM SYS.ALL_VIRTUAL_MVIEW_REAL_AGENT ) USING (TENANT_ID, MVIEW_ID) ) C, DEFVALS D WHERE A.DATABASE_ID = B.DATABASE_ID AND B.TABLE_ID = C.MVIEW_ID AND B.TABLE_TYPE = 7 AND A.TENANT_ID = EFFECTIVE_TENANT_ID() AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND C.TENANT_ID = EFFECTIVE_TENANT_ID() ) )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::user_mvref_stats_params_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_USER_MVREF_STATS_PARAMS_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_USER_MVREF_STATS_PARAMS_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(MV_OWNER AS VARCHAR2(128)) AS MV_OWNER, CAST(MV_NAME AS VARCHAR2(128)) AS MV_NAME, CAST( DECODE(COLLECTION_LEVEL, 0, 'NONE', 1, 'TYPICAL', 2, 'ADVANCED', NULL ) AS VARCHAR2(8) ) AS COLLECTION_LEVEL, CAST(RETENTION_PERIOD AS NUMBER) AS RETENTION_PERIOD FROM ( WITH DEFVALS AS ( SELECT NVL(MAX(COLLECTION_LEVEL), 1) AS COLLECTION_LEVEL, NVL(MAX(RETENTION_PERIOD), 31) AS RETENTION_PERIOD FROM SYS.ALL_VIRTUAL_MVIEW_REFRESH_STATS_SYS_DEFAULTS_REAL_AGENT WHERE TENANT_ID = EFFECTIVE_TENANT_ID() ) SELECT A.DATABASE_NAME MV_OWNER, B.TABLE_NAME MV_NAME, NVL(C.COLLECTION_LEVEL, D.COLLECTION_LEVEL) COLLECTION_LEVEL, NVL(C.RETENTION_PERIOD, D.RETENTION_PERIOD) RETENTION_PERIOD FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, ( SELECT TENANT_ID, MVIEW_ID, COLLECTION_LEVEL, RETENTION_PERIOD FROM SYS.ALL_VIRTUAL_MVIEW_REFRESH_STATS_PARAMS_REAL_AGENT RIGHT OUTER JOIN ( SELECT TENANT_ID, MVIEW_ID FROM SYS.ALL_VIRTUAL_MVIEW_REAL_AGENT ) USING (TENANT_ID, MVIEW_ID) ) C, DEFVALS D WHERE A.DATABASE_ID = B.DATABASE_ID AND B.TABLE_ID = C.MVIEW_ID AND B.TABLE_TYPE = 7 AND A.TENANT_ID = EFFECTIVE_TENANT_ID() AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND C.TENANT_ID = EFFECTIVE_TENANT_ID() AND A.DATABASE_NAME = SYS_CONTEXT('USERENV','CURRENT_USER') ) )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::dba_mvref_run_stats_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_MVREF_RUN_STATS_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_MVREF_RUN_STATS_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.USER_NAME AS VARCHAR2(128)) AS RUN_OWNER, CAST(B.REFRESH_ID AS NUMBER) AS REFRESH_ID, CAST(B.NUM_MVS_TOTAL AS NUMBER) AS NUM_MVS, CAST(B.MVIEWS AS VARCHAR2(4000)) AS MVIEWS, CAST(B.BASE_TABLES AS VARCHAR2(4000)) AS BASE_TABLES, CAST(B.METHOD AS VARCHAR2(4000)) AS METHOD, CAST(B.ROLLBACK_SEG AS VARCHAR2(4000)) AS ROLLBACK_SEG, CAST(DECODE(B.PUSH_DEFERRED_RPC, 1, 'Y', 'N') AS CHAR(1)) AS PUSH_DEFERRED_RPC, CAST(DECODE(B.REFRESH_AFTER_ERRORS, 1, 'Y', 'N') AS CHAR(1)) AS REFRESH_AFTER_ERRORS, CAST(B.PURGE_OPTION AS NUMBER) AS PURGE_OPTION, CAST(B.PARALLELISM AS NUMBER) AS PARALLELISM, CAST(B.HEAP_SIZE AS NUMBER) AS HEAP_SIZE, CAST(DECODE(B.ATOMIC_REFRESH, 1, 'Y', 'N') AS CHAR(1)) AS ATOMIC_REFRESH, CAST(DECODE(B.NESTED, 1, 'Y', 'N') AS CHAR(1)) AS NESTED, CAST(DECODE(B.OUT_OF_PLACE, 1, 'Y', 'N') AS CHAR(1)) AS OUT_OF_PLACE, CAST(B.NUMBER_OF_FAILURES AS NUMBER) AS NUMBER_OF_FAILURES, CAST(B.START_TIME AS TIMESTAMP(6)) AS START_TIME, CAST(B.END_TIME AS TIMESTAMP(6)) AS END_TIME, CAST(B.ELAPSED_TIME AS NUMBER) AS ELAPSED_TIME, CAST(0 AS NUMBER) AS LOG_SETUP_TIME, CAST(B.LOG_PURGE_TIME AS NUMBER) AS LOG_PURGE_TIME, CAST(DECODE(B.COMPLETE_STATS_AVALIABLE, 1, 'Y', 'N') AS CHAR(1)) AS COMPLETE_STATS_AVAILABLE FROM SYS.ALL_VIRTUAL_USER_REAL_AGENT A, SYS.ALL_VIRTUAL_MVIEW_REFRESH_RUN_STATS_REAL_AGENT B WHERE A.USER_ID = B.RUN_USER_ID AND A.TENANT_ID = EFFECTIVE_TENANT_ID() AND B.TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::user_mvref_run_stats_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_USER_MVREF_RUN_STATS_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_USER_MVREF_RUN_STATS_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(B.REFRESH_ID AS NUMBER) AS REFRESH_ID, CAST(B.NUM_MVS_TOTAL AS NUMBER) AS NUM_MVS, CAST(B.MVIEWS AS VARCHAR2(4000)) AS MVIEWS, CAST(B.BASE_TABLES AS VARCHAR2(4000)) AS BASE_TABLES, CAST(B.METHOD AS VARCHAR2(4000)) AS METHOD, CAST(B.ROLLBACK_SEG AS VARCHAR2(4000)) AS ROLLBACK_SEG, CAST(DECODE(B.PUSH_DEFERRED_RPC, 1, 'Y', 'N') AS CHAR(1)) AS PUSH_DEFERRED_RPC, CAST(DECODE(B.REFRESH_AFTER_ERRORS, 1, 'Y', 'N') AS CHAR(1)) AS REFRESH_AFTER_ERRORS, CAST(B.PURGE_OPTION AS NUMBER) AS PURGE_OPTION, CAST(B.PARALLELISM AS NUMBER) AS PARALLELISM, CAST(B.HEAP_SIZE AS NUMBER) AS HEAP_SIZE, CAST(DECODE(B.ATOMIC_REFRESH, 1, 'Y', 'N') AS CHAR(1)) AS ATOMIC_REFRESH, CAST(DECODE(B.NESTED, 1, 'Y', 'N') AS CHAR(1)) AS NESTED, CAST(DECODE(B.OUT_OF_PLACE, 1, 'Y', 'N') AS CHAR(1)) AS OUT_OF_PLACE, CAST(B.NUMBER_OF_FAILURES AS NUMBER) AS NUMBER_OF_FAILURES, CAST(B.START_TIME AS TIMESTAMP(6)) AS START_TIME, CAST(B.END_TIME AS TIMESTAMP(6)) AS END_TIME, CAST(B.ELAPSED_TIME AS NUMBER) AS ELAPSED_TIME, CAST(0 AS NUMBER) AS LOG_SETUP_TIME, CAST(B.LOG_PURGE_TIME AS NUMBER) AS LOG_PURGE_TIME, CAST(DECODE(B.COMPLETE_STATS_AVALIABLE, 1, 'Y', 'N') AS CHAR(1)) AS COMPLETE_STATS_AVAILABLE FROM SYS.ALL_VIRTUAL_USER_REAL_AGENT A, SYS.ALL_VIRTUAL_MVIEW_REFRESH_RUN_STATS_REAL_AGENT B WHERE A.USER_ID = B.RUN_USER_ID AND A.TENANT_ID = EFFECTIVE_TENANT_ID() AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND A.USER_NAME = SYS_CONTEXT('USERENV','CURRENT_USER') )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::dba_mvref_stats_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_MVREF_STATS_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_MVREF_STATS_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS MV_OWNER, CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MV_NAME, CAST(C.REFRESH_ID AS NUMBER) AS REFRESH_ID, CAST( DECODE(C.REFRESH_TYPE, 0, 'COMPLETE', 1, 'FAST', NULL ) AS VARCHAR2(30) ) AS REFRESH_METHOD, CAST(NULL AS VARCHAR2(4000)) AS REFRESH_OPTIMIZATIONS, CAST(NULL AS VARCHAR2(4000)) AS ADDITIONAL_EXECUTIONS, CAST(C.START_TIME AS TIMESTAMP(6)) AS START_TIME, CAST(C.END_TIME AS TIMESTAMP(6)) AS END_TIME, CAST(C.ELAPSED_TIME AS NUMBER) AS ELAPSED_TIME, CAST(0 AS NUMBER) AS LOG_SETUP_TIME, CAST(C.LOG_PURGE_TIME AS NUMBER) AS LOG_PURGE_TIME, CAST(C.INITIAL_NUM_ROWS AS NUMBER) AS INITIAL_NUM_ROWS, CAST(C.FINAL_NUM_ROWS AS NUMBER) AS FINAL_NUM_ROWS FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, SYS.ALL_VIRTUAL_MVIEW_REFRESH_STATS_REAL_AGENT C WHERE A.DATABASE_ID = B.DATABASE_ID AND B.TABLE_ID = C.MVIEW_ID AND B.TABLE_TYPE = 7 AND A.TENANT_ID = EFFECTIVE_TENANT_ID() AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND C.TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::user_mvref_stats_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_USER_MVREF_STATS_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_USER_MVREF_STATS_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MV_NAME, CAST(C.REFRESH_ID AS NUMBER) AS REFRESH_ID, CAST( DECODE(C.REFRESH_TYPE, 0, 'COMPLETE', 1, 'FAST', NULL ) AS VARCHAR2(30) ) AS REFRESH_METHOD, CAST(NULL AS VARCHAR2(4000)) AS REFRESH_OPTIMIZATIONS, CAST(NULL AS VARCHAR2(4000)) AS ADDITIONAL_EXECUTIONS, CAST(C.START_TIME AS TIMESTAMP(6)) AS START_TIME, CAST(C.END_TIME AS TIMESTAMP(6)) AS END_TIME, CAST(C.ELAPSED_TIME AS NUMBER) AS ELAPSED_TIME, CAST(0 AS NUMBER) AS LOG_SETUP_TIME, CAST(C.LOG_PURGE_TIME AS NUMBER) AS LOG_PURGE_TIME, CAST(C.INITIAL_NUM_ROWS AS NUMBER) AS INITIAL_NUM_ROWS, CAST(C.FINAL_NUM_ROWS AS NUMBER) AS FINAL_NUM_ROWS FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, SYS.ALL_VIRTUAL_MVIEW_REFRESH_STATS_REAL_AGENT C WHERE A.DATABASE_ID = B.DATABASE_ID AND B.TABLE_ID = C.MVIEW_ID AND B.TABLE_TYPE = 7 AND A.TENANT_ID = EFFECTIVE_TENANT_ID() AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND C.TENANT_ID = EFFECTIVE_TENANT_ID() AND A.DATABASE_NAME = SYS_CONTEXT('USERENV','CURRENT_USER') )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::dba_mvref_change_stats_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_MVREF_CHANGE_STATS_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_MVREF_CHANGE_STATS_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(C.DATABASE_NAME AS VARCHAR2(128)) AS TBL_OWNER, CAST(D.TABLE_NAME AS VARCHAR2(128)) AS TBL_NAME, CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS MV_OWNER, CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MV_NAME, CAST(E.REFRESH_ID AS NUMBER) AS REFRESH_ID, CAST(E.NUM_ROWS_INS AS NUMBER) AS NUM_ROWS_INS, CAST(E.NUM_ROWS_UPD AS NUMBER) AS NUM_ROWS_UPD, CAST(E.NUM_ROWS_DEL AS NUMBER) AS NUM_ROWS_DEL, CAST(0 AS NUMBER) AS NUM_ROWS_DL_INS, CAST('N' AS CHAR(1)) AS PMOPS_OCCURRED, CAST(NULL AS VARCHAR2(4000)) AS PMOP_DETAILS, CAST(E.NUM_ROWS AS NUMBER) AS NUM_ROWS FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT C, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT D, SYS.ALL_VIRTUAL_MVIEW_REFRESH_CHANGE_STATS_REAL_AGENT E WHERE A.DATABASE_ID = B.DATABASE_ID AND C.DATABASE_ID = D.DATABASE_ID AND E.MVIEW_ID = B.TABLE_ID AND E.DETAIL_TABLE_ID = D.TABLE_ID AND A.TENANT_ID = EFFECTIVE_TENANT_ID() AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND C.TENANT_ID = EFFECTIVE_TENANT_ID() AND D.TENANT_ID = EFFECTIVE_TENANT_ID() AND E.TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::user_mvref_change_stats_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_USER_MVREF_CHANGE_STATS_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_USER_MVREF_CHANGE_STATS_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(C.DATABASE_NAME AS VARCHAR2(128)) AS TBL_OWNER, CAST(D.TABLE_NAME AS VARCHAR2(128)) AS TBL_NAME, CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS MV_OWNER, CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MV_NAME, CAST(E.REFRESH_ID AS NUMBER) AS REFRESH_ID, CAST(E.NUM_ROWS_INS AS NUMBER) AS NUM_ROWS_INS, CAST(E.NUM_ROWS_UPD AS NUMBER) AS NUM_ROWS_UPD, CAST(E.NUM_ROWS_DEL AS NUMBER) AS NUM_ROWS_DEL, CAST(0 AS NUMBER) AS NUM_ROWS_DL_INS, CAST('N' AS CHAR(1)) AS PMOPS_OCCURRED, CAST(NULL AS VARCHAR2(4000)) AS PMOP_DETAILS, CAST(E.NUM_ROWS AS NUMBER) AS NUM_ROWS FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT C, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT D, SYS.ALL_VIRTUAL_MVIEW_REFRESH_CHANGE_STATS_REAL_AGENT E WHERE A.DATABASE_ID = B.DATABASE_ID AND C.DATABASE_ID = D.DATABASE_ID AND E.MVIEW_ID = B.TABLE_ID AND E.DETAIL_TABLE_ID = D.TABLE_ID AND A.TENANT_ID = EFFECTIVE_TENANT_ID() AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND C.TENANT_ID = EFFECTIVE_TENANT_ID() AND D.TENANT_ID = EFFECTIVE_TENANT_ID() AND E.TENANT_ID = EFFECTIVE_TENANT_ID() AND A.DATABASE_NAME = SYS_CONTEXT('USERENV','CURRENT_USER') )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + +int ObInnerTableSchema::dba_mvref_stmt_stats_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_DBA_MVREF_STMT_STATS_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_DBA_MVREF_STMT_STATS_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS MV_OWNER, CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MV_NAME, CAST(C.REFRESH_ID AS NUMBER) AS REFRESH_ID, CAST(C.STEP AS NUMBER) AS STEP, CAST(C.SQLID AS VARCHAR2(32)) AS SQLID /* TODO: VARCHAR2(14) */, C.STMT AS STMT /* TODO: CLOB */, CAST(C.EXECUTION_TIME AS NUMBER) AS EXECUTION_TIME, C.EXECUTION_PLAN AS EXECUTION_PLAN /* TODO: XMLTYPE STORAGE BINARY */ FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, SYS.ALL_VIRTUAL_MVIEW_REFRESH_STMT_STATS_REAL_AGENT C WHERE A.DATABASE_ID = B.DATABASE_ID AND B.TABLE_ID = C.MVIEW_ID AND A.TENANT_ID = EFFECTIVE_TENANT_ID() AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND C.TENANT_ID = EFFECTIVE_TENANT_ID() )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + } // end namespace share } // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.25301_25350.cpp b/src/share/inner_table/ob_inner_table_schema.25301_25350.cpp new file mode 100644 index 0000000000..1c1c7f44b2 --- /dev/null +++ b/src/share/inner_table/ob_inner_table_schema.25301_25350.cpp @@ -0,0 +1,80 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX SHARE_SCHEMA +#include "ob_inner_table_schema.h" + +#include "share/schema/ob_schema_macro_define.h" +#include "share/schema/ob_schema_service_sql_impl.h" +#include "share/schema/ob_table_schema.h" +#include "share/scn.h" + +namespace oceanbase +{ +using namespace share::schema; +using namespace common; +namespace share +{ + +int ObInnerTableSchema::user_mvref_stmt_stats_ora_schema(ObTableSchema &table_schema) +{ + int ret = OB_SUCCESS; + uint64_t column_id = OB_APP_MIN_COLUMN_ID - 1; + + //generated fields: + table_schema.set_tenant_id(OB_SYS_TENANT_ID); + table_schema.set_tablegroup_id(OB_INVALID_ID); + table_schema.set_database_id(OB_ORA_SYS_DATABASE_ID); + table_schema.set_table_id(OB_USER_MVREF_STMT_STATS_ORA_TID); + table_schema.set_rowkey_split_pos(0); + table_schema.set_is_use_bloomfilter(false); + table_schema.set_progressive_merge_num(0); + table_schema.set_rowkey_column_num(0); + table_schema.set_load_type(TABLE_LOAD_TYPE_IN_DISK); + table_schema.set_table_type(SYSTEM_VIEW); + table_schema.set_index_type(INDEX_TYPE_IS_NOT); + table_schema.set_def_type(TABLE_DEF_TYPE_INTERNAL); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_table_name(OB_USER_MVREF_STMT_STATS_ORA_TNAME))) { + LOG_ERROR("fail to set table_name", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_compress_func_name(OB_DEFAULT_COMPRESS_FUNC_NAME))) { + LOG_ERROR("fail to set compress_func_name", K(ret)); + } + } + table_schema.set_part_level(PARTITION_LEVEL_ZERO); + table_schema.set_charset_type(ObCharset::get_default_charset()); + table_schema.set_collation_type(ObCharset::get_default_collation(ObCharset::get_default_charset())); + + if (OB_SUCC(ret)) { + if (OB_FAIL(table_schema.set_view_definition(R"__( SELECT CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS MV_OWNER, CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MV_NAME, CAST(C.REFRESH_ID AS NUMBER) AS REFRESH_ID, CAST(C.STEP AS NUMBER) AS STEP, CAST(C.SQLID AS VARCHAR2(14)) AS SQLID /* TODO: VARCHAR2(14) */, C.STMT AS STMT /* TODO: CLOB */, CAST(C.EXECUTION_TIME AS NUMBER) AS EXECUTION_TIME, C.EXECUTION_PLAN AS EXECUTION_PLAN /* TODO: XMLTYPE STORAGE BINARY */ FROM SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, SYS.ALL_VIRTUAL_MVIEW_REFRESH_STMT_STATS_REAL_AGENT C WHERE A.DATABASE_ID = B.DATABASE_ID AND B.TABLE_ID = C.MVIEW_ID AND A.TENANT_ID = EFFECTIVE_TENANT_ID() AND B.TENANT_ID = EFFECTIVE_TENANT_ID() AND C.TENANT_ID = EFFECTIVE_TENANT_ID() AND A.DATABASE_NAME = SYS_CONTEXT('USERENV','CURRENT_USER') )__"))) { + LOG_ERROR("fail to set view_definition", K(ret)); + } + } + table_schema.set_index_using_type(USING_BTREE); + table_schema.set_row_store_type(ENCODING_ROW_STORE); + table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); + table_schema.set_progressive_merge_round(1); + table_schema.set_storage_format_version(3); + table_schema.set_tablet_id(0); + + table_schema.set_max_used_column_id(column_id); + return ret; +} + + +} // end namespace share +} // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema.301_350.cpp b/src/share/inner_table/ob_inner_table_schema.301_350.cpp index 3220e93514..a7692570ec 100644 --- a/src/share/inner_table/ob_inner_table_schema.301_350.cpp +++ b/src/share/inner_table/ob_inner_table_schema.301_350.cpp @@ -869,6 +869,25 @@ int ObInnerTableSchema::all_ddl_error_message_schema(ObTableSchema &table_schema true, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ObObj consensus_schema_version_default; + consensus_schema_version_default.set_int(-1); + ADD_COLUMN_SCHEMA_T("consensus_schema_version", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + consensus_schema_version_default, + consensus_schema_version_default); //default_value + } table_schema.set_index_using_type(USING_BTREE); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); @@ -2400,6 +2419,25 @@ int ObInnerTableSchema::all_ddl_task_status_schema(ObTableSchema &table_schema) true, //is_nullable false); //is_autoincrement } + + if (OB_SUCC(ret)) { + ObObj consensus_schema_version_default; + consensus_schema_version_default.set_int(-1); + ADD_COLUMN_SCHEMA_T("consensus_schema_version", //column_name + ++column_id, //column_id + 0, //rowkey_id + 0, //index_id + 0, //part_key_pos + ObIntType, //column_type + CS_TYPE_INVALID, //column_collation_type + sizeof(int64_t), //column_length + -1, //column_precision + -1, //column_scale + false, //is_nullable + false, //is_autoincrement + consensus_schema_version_default, + consensus_schema_version_default); //default_value + } table_schema.set_index_using_type(USING_BTREE); table_schema.set_row_store_type(ENCODING_ROW_STORE); table_schema.set_store_format(OB_STORE_FORMAT_DYNAMIC_MYSQL); diff --git a/src/share/inner_table/ob_inner_table_schema.h b/src/share/inner_table/ob_inner_table_schema.h index ade0ce9cc1..dbbe744df9 100644 --- a/src/share/inner_table/ob_inner_table_schema.h +++ b/src/share/inner_table/ob_inner_table_schema.h @@ -1653,6 +1653,22 @@ public: static int gv_ob_ls_snapshots_schema(share::schema::ObTableSchema &table_schema); static int v_ob_ls_snapshots_schema(share::schema::ObTableSchema &table_schema); static int dba_ob_clone_history_schema(share::schema::ObTableSchema &table_schema); + static int cdb_mview_logs_schema(share::schema::ObTableSchema &table_schema); + static int dba_mview_logs_schema(share::schema::ObTableSchema &table_schema); + static int cdb_mviews_schema(share::schema::ObTableSchema &table_schema); + static int dba_mviews_schema(share::schema::ObTableSchema &table_schema); + static int cdb_mvref_stats_sys_defaults_schema(share::schema::ObTableSchema &table_schema); + static int dba_mvref_stats_sys_defaults_schema(share::schema::ObTableSchema &table_schema); + static int cdb_mvref_stats_params_schema(share::schema::ObTableSchema &table_schema); + static int dba_mvref_stats_params_schema(share::schema::ObTableSchema &table_schema); + static int cdb_mvref_run_stats_schema(share::schema::ObTableSchema &table_schema); + static int dba_mvref_run_stats_schema(share::schema::ObTableSchema &table_schema); + static int cdb_mvref_stats_schema(share::schema::ObTableSchema &table_schema); + static int dba_mvref_stats_schema(share::schema::ObTableSchema &table_schema); + static int cdb_mvref_change_stats_schema(share::schema::ObTableSchema &table_schema); + static int dba_mvref_change_stats_schema(share::schema::ObTableSchema &table_schema); + static int cdb_mvref_stmt_stats_schema(share::schema::ObTableSchema &table_schema); + static int dba_mvref_stmt_stats_schema(share::schema::ObTableSchema &table_schema); static int dba_synonyms_schema(share::schema::ObTableSchema &table_schema); static int dba_objects_ora_schema(share::schema::ObTableSchema &table_schema); static int all_objects_schema(share::schema::ObTableSchema &table_schema); @@ -1916,6 +1932,24 @@ public: static int dba_ob_import_table_tasks_ora_schema(share::schema::ObTableSchema &table_schema); static int dba_ob_import_table_task_history_ora_schema(share::schema::ObTableSchema &table_schema); static int user_users_schema(share::schema::ObTableSchema &table_schema); + static int dba_mview_logs_ora_schema(share::schema::ObTableSchema &table_schema); + static int all_mview_logs_ora_schema(share::schema::ObTableSchema &table_schema); + static int user_mview_logs_ora_schema(share::schema::ObTableSchema &table_schema); + static int dba_mviews_ora_schema(share::schema::ObTableSchema &table_schema); + static int all_mviews_ora_schema(share::schema::ObTableSchema &table_schema); + static int user_mviews_ora_schema(share::schema::ObTableSchema &table_schema); + static int dba_mvref_stats_sys_defaults_ora_schema(share::schema::ObTableSchema &table_schema); + static int user_mvref_stats_sys_defaults_ora_schema(share::schema::ObTableSchema &table_schema); + static int dba_mvref_stats_params_ora_schema(share::schema::ObTableSchema &table_schema); + static int user_mvref_stats_params_ora_schema(share::schema::ObTableSchema &table_schema); + static int dba_mvref_run_stats_ora_schema(share::schema::ObTableSchema &table_schema); + static int user_mvref_run_stats_ora_schema(share::schema::ObTableSchema &table_schema); + static int dba_mvref_stats_ora_schema(share::schema::ObTableSchema &table_schema); + static int user_mvref_stats_ora_schema(share::schema::ObTableSchema &table_schema); + static int dba_mvref_change_stats_ora_schema(share::schema::ObTableSchema &table_schema); + static int user_mvref_change_stats_ora_schema(share::schema::ObTableSchema &table_schema); + static int dba_mvref_stmt_stats_ora_schema(share::schema::ObTableSchema &table_schema); + static int user_mvref_stmt_stats_ora_schema(share::schema::ObTableSchema &table_schema); static int gv_ob_sql_audit_ora_schema(share::schema::ObTableSchema &table_schema); static int v_ob_sql_audit_ora_schema(share::schema::ObTableSchema &table_schema); static int gv_instance_schema(share::schema::ObTableSchema &table_schema); @@ -4319,6 +4353,22 @@ const schema_create_func sys_view_schema_creators [] = { ObInnerTableSchema::gv_ob_ls_snapshots_schema, ObInnerTableSchema::v_ob_ls_snapshots_schema, ObInnerTableSchema::dba_ob_clone_history_schema, + ObInnerTableSchema::cdb_mview_logs_schema, + ObInnerTableSchema::dba_mview_logs_schema, + ObInnerTableSchema::cdb_mviews_schema, + ObInnerTableSchema::dba_mviews_schema, + ObInnerTableSchema::cdb_mvref_stats_sys_defaults_schema, + ObInnerTableSchema::dba_mvref_stats_sys_defaults_schema, + ObInnerTableSchema::cdb_mvref_stats_params_schema, + ObInnerTableSchema::dba_mvref_stats_params_schema, + ObInnerTableSchema::cdb_mvref_run_stats_schema, + ObInnerTableSchema::dba_mvref_run_stats_schema, + ObInnerTableSchema::cdb_mvref_stats_schema, + ObInnerTableSchema::dba_mvref_stats_schema, + ObInnerTableSchema::cdb_mvref_change_stats_schema, + ObInnerTableSchema::dba_mvref_change_stats_schema, + ObInnerTableSchema::cdb_mvref_stmt_stats_schema, + ObInnerTableSchema::dba_mvref_stmt_stats_schema, ObInnerTableSchema::dba_synonyms_schema, ObInnerTableSchema::dba_objects_ora_schema, ObInnerTableSchema::all_objects_schema, @@ -4582,6 +4632,24 @@ const schema_create_func sys_view_schema_creators [] = { ObInnerTableSchema::dba_ob_import_table_tasks_ora_schema, ObInnerTableSchema::dba_ob_import_table_task_history_ora_schema, ObInnerTableSchema::user_users_schema, + ObInnerTableSchema::dba_mview_logs_ora_schema, + ObInnerTableSchema::all_mview_logs_ora_schema, + ObInnerTableSchema::user_mview_logs_ora_schema, + ObInnerTableSchema::dba_mviews_ora_schema, + ObInnerTableSchema::all_mviews_ora_schema, + ObInnerTableSchema::user_mviews_ora_schema, + ObInnerTableSchema::dba_mvref_stats_sys_defaults_ora_schema, + ObInnerTableSchema::user_mvref_stats_sys_defaults_ora_schema, + ObInnerTableSchema::dba_mvref_stats_params_ora_schema, + ObInnerTableSchema::user_mvref_stats_params_ora_schema, + ObInnerTableSchema::dba_mvref_run_stats_ora_schema, + ObInnerTableSchema::user_mvref_run_stats_ora_schema, + ObInnerTableSchema::dba_mvref_stats_ora_schema, + ObInnerTableSchema::user_mvref_stats_ora_schema, + ObInnerTableSchema::dba_mvref_change_stats_ora_schema, + ObInnerTableSchema::user_mvref_change_stats_ora_schema, + ObInnerTableSchema::dba_mvref_stmt_stats_ora_schema, + ObInnerTableSchema::user_mvref_stmt_stats_ora_schema, ObInnerTableSchema::gv_ob_sql_audit_ora_schema, ObInnerTableSchema::v_ob_sql_audit_ora_schema, ObInnerTableSchema::gv_instance_schema, @@ -5897,6 +5965,14 @@ const uint64_t tenant_space_tables [] = { OB_DBA_INDEX_USAGE_TID, OB_GV_OB_LS_SNAPSHOTS_TID, OB_V_OB_LS_SNAPSHOTS_TID, + OB_DBA_MVIEW_LOGS_TID, + OB_DBA_MVIEWS_TID, + OB_DBA_MVREF_STATS_SYS_DEFAULTS_TID, + OB_DBA_MVREF_STATS_PARAMS_TID, + OB_DBA_MVREF_RUN_STATS_TID, + OB_DBA_MVREF_STATS_TID, + OB_DBA_MVREF_CHANGE_STATS_TID, + OB_DBA_MVREF_STMT_STATS_TID, OB_DBA_SYNONYMS_TID, OB_DBA_OBJECTS_ORA_TID, OB_ALL_OBJECTS_TID, @@ -6160,6 +6236,24 @@ const uint64_t tenant_space_tables [] = { OB_DBA_OB_IMPORT_TABLE_TASKS_ORA_TID, OB_DBA_OB_IMPORT_TABLE_TASK_HISTORY_ORA_TID, OB_USER_USERS_TID, + OB_DBA_MVIEW_LOGS_ORA_TID, + OB_ALL_MVIEW_LOGS_ORA_TID, + OB_USER_MVIEW_LOGS_ORA_TID, + OB_DBA_MVIEWS_ORA_TID, + OB_ALL_MVIEWS_ORA_TID, + OB_USER_MVIEWS_ORA_TID, + OB_DBA_MVREF_STATS_SYS_DEFAULTS_ORA_TID, + OB_USER_MVREF_STATS_SYS_DEFAULTS_ORA_TID, + OB_DBA_MVREF_STATS_PARAMS_ORA_TID, + OB_USER_MVREF_STATS_PARAMS_ORA_TID, + OB_DBA_MVREF_RUN_STATS_ORA_TID, + OB_USER_MVREF_RUN_STATS_ORA_TID, + OB_DBA_MVREF_STATS_ORA_TID, + OB_USER_MVREF_STATS_ORA_TID, + OB_DBA_MVREF_CHANGE_STATS_ORA_TID, + OB_USER_MVREF_CHANGE_STATS_ORA_TID, + OB_DBA_MVREF_STMT_STATS_ORA_TID, + OB_USER_MVREF_STMT_STATS_ORA_TID, OB_GV_OB_SQL_AUDIT_ORA_TID, OB_V_OB_SQL_AUDIT_ORA_TID, OB_GV_INSTANCE_TID, @@ -8341,6 +8435,14 @@ const char* const tenant_space_table_names [] = { OB_DBA_INDEX_USAGE_TNAME, OB_GV_OB_LS_SNAPSHOTS_TNAME, OB_V_OB_LS_SNAPSHOTS_TNAME, + OB_DBA_MVIEW_LOGS_TNAME, + OB_DBA_MVIEWS_TNAME, + OB_DBA_MVREF_STATS_SYS_DEFAULTS_TNAME, + OB_DBA_MVREF_STATS_PARAMS_TNAME, + OB_DBA_MVREF_RUN_STATS_TNAME, + OB_DBA_MVREF_STATS_TNAME, + OB_DBA_MVREF_CHANGE_STATS_TNAME, + OB_DBA_MVREF_STMT_STATS_TNAME, OB_DBA_SYNONYMS_TNAME, OB_DBA_OBJECTS_ORA_TNAME, OB_ALL_OBJECTS_TNAME, @@ -8604,6 +8706,24 @@ const char* const tenant_space_table_names [] = { OB_DBA_OB_IMPORT_TABLE_TASKS_ORA_TNAME, OB_DBA_OB_IMPORT_TABLE_TASK_HISTORY_ORA_TNAME, OB_USER_USERS_TNAME, + OB_DBA_MVIEW_LOGS_ORA_TNAME, + OB_ALL_MVIEW_LOGS_ORA_TNAME, + OB_USER_MVIEW_LOGS_ORA_TNAME, + OB_DBA_MVIEWS_ORA_TNAME, + OB_ALL_MVIEWS_ORA_TNAME, + OB_USER_MVIEWS_ORA_TNAME, + OB_DBA_MVREF_STATS_SYS_DEFAULTS_ORA_TNAME, + OB_USER_MVREF_STATS_SYS_DEFAULTS_ORA_TNAME, + OB_DBA_MVREF_STATS_PARAMS_ORA_TNAME, + OB_USER_MVREF_STATS_PARAMS_ORA_TNAME, + OB_DBA_MVREF_RUN_STATS_ORA_TNAME, + OB_USER_MVREF_RUN_STATS_ORA_TNAME, + OB_DBA_MVREF_STATS_ORA_TNAME, + OB_USER_MVREF_STATS_ORA_TNAME, + OB_DBA_MVREF_CHANGE_STATS_ORA_TNAME, + OB_USER_MVREF_CHANGE_STATS_ORA_TNAME, + OB_DBA_MVREF_STMT_STATS_ORA_TNAME, + OB_USER_MVREF_STMT_STATS_ORA_TNAME, OB_GV_OB_SQL_AUDIT_ORA_TNAME, OB_V_OB_SQL_AUDIT_ORA_TNAME, OB_GV_INSTANCE_TNAME, @@ -12243,10 +12363,10 @@ static inline int get_sys_table_lob_aux_schema(const uint64_t tid, const int64_t OB_CORE_TABLE_COUNT = 4; const int64_t OB_SYS_TABLE_COUNT = 279; const int64_t OB_VIRTUAL_TABLE_COUNT = 776; -const int64_t OB_SYS_VIEW_COUNT = 830; -const int64_t OB_SYS_TENANT_TABLE_COUNT = 1890; +const int64_t OB_SYS_VIEW_COUNT = 864; +const int64_t OB_SYS_TENANT_TABLE_COUNT = 1924; const int64_t OB_CORE_SCHEMA_VERSION = 1; -const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 1893; +const int64_t OB_BOOTSTRAP_SCHEMA_VERSION = 1927; } // end namespace share } // end namespace oceanbase diff --git a/src/share/inner_table/ob_inner_table_schema_constants.h b/src/share/inner_table/ob_inner_table_schema_constants.h index 697b8877b7..7b385bfb9d 100644 --- a/src/share/inner_table/ob_inner_table_schema_constants.h +++ b/src/share/inner_table/ob_inner_table_schema_constants.h @@ -1389,6 +1389,22 @@ const uint64_t OB_CDB_INDEX_USAGE_TID = 21513; // "CDB_INDEX_USAGE" const uint64_t OB_GV_OB_LS_SNAPSHOTS_TID = 21517; // "GV$OB_LS_SNAPSHOTS" const uint64_t OB_V_OB_LS_SNAPSHOTS_TID = 21518; // "V$OB_LS_SNAPSHOTS" const uint64_t OB_DBA_OB_CLONE_HISTORY_TID = 21519; // "DBA_OB_CLONE_HISTORY" +const uint64_t OB_CDB_MVIEW_LOGS_TID = 21525; // "CDB_MVIEW_LOGS" +const uint64_t OB_DBA_MVIEW_LOGS_TID = 21526; // "DBA_MVIEW_LOGS" +const uint64_t OB_CDB_MVIEWS_TID = 21527; // "CDB_MVIEWS" +const uint64_t OB_DBA_MVIEWS_TID = 21528; // "DBA_MVIEWS" +const uint64_t OB_CDB_MVREF_STATS_SYS_DEFAULTS_TID = 21529; // "CDB_MVREF_STATS_SYS_DEFAULTS" +const uint64_t OB_DBA_MVREF_STATS_SYS_DEFAULTS_TID = 21530; // "DBA_MVREF_STATS_SYS_DEFAULTS" +const uint64_t OB_CDB_MVREF_STATS_PARAMS_TID = 21531; // "CDB_MVREF_STATS_PARAMS" +const uint64_t OB_DBA_MVREF_STATS_PARAMS_TID = 21532; // "DBA_MVREF_STATS_PARAMS" +const uint64_t OB_CDB_MVREF_RUN_STATS_TID = 21533; // "CDB_MVREF_RUN_STATS" +const uint64_t OB_DBA_MVREF_RUN_STATS_TID = 21534; // "DBA_MVREF_RUN_STATS" +const uint64_t OB_CDB_MVREF_STATS_TID = 21535; // "CDB_MVREF_STATS" +const uint64_t OB_DBA_MVREF_STATS_TID = 21536; // "DBA_MVREF_STATS" +const uint64_t OB_CDB_MVREF_CHANGE_STATS_TID = 21537; // "CDB_MVREF_CHANGE_STATS" +const uint64_t OB_DBA_MVREF_CHANGE_STATS_TID = 21538; // "DBA_MVREF_CHANGE_STATS" +const uint64_t OB_CDB_MVREF_STMT_STATS_TID = 21539; // "CDB_MVREF_STMT_STATS" +const uint64_t OB_DBA_MVREF_STMT_STATS_TID = 21540; // "DBA_MVREF_STMT_STATS" const uint64_t OB_DBA_SYNONYMS_TID = 25001; // "DBA_SYNONYMS" const uint64_t OB_DBA_OBJECTS_ORA_TID = 25002; // "DBA_OBJECTS_ORA" const uint64_t OB_ALL_OBJECTS_TID = 25003; // "ALL_OBJECTS" @@ -1652,6 +1668,24 @@ const uint64_t OB_DBA_OB_IMPORT_TABLE_JOB_HISTORY_ORA_TID = 25265; // "DBA_OB_IM const uint64_t OB_DBA_OB_IMPORT_TABLE_TASKS_ORA_TID = 25266; // "DBA_OB_IMPORT_TABLE_TASKS_ORA" const uint64_t OB_DBA_OB_IMPORT_TABLE_TASK_HISTORY_ORA_TID = 25267; // "DBA_OB_IMPORT_TABLE_TASK_HISTORY_ORA" const uint64_t OB_USER_USERS_TID = 25278; // "USER_USERS" +const uint64_t OB_DBA_MVIEW_LOGS_ORA_TID = 25283; // "DBA_MVIEW_LOGS_ORA" +const uint64_t OB_ALL_MVIEW_LOGS_ORA_TID = 25284; // "ALL_MVIEW_LOGS_ORA" +const uint64_t OB_USER_MVIEW_LOGS_ORA_TID = 25285; // "USER_MVIEW_LOGS_ORA" +const uint64_t OB_DBA_MVIEWS_ORA_TID = 25286; // "DBA_MVIEWS_ORA" +const uint64_t OB_ALL_MVIEWS_ORA_TID = 25287; // "ALL_MVIEWS_ORA" +const uint64_t OB_USER_MVIEWS_ORA_TID = 25288; // "USER_MVIEWS_ORA" +const uint64_t OB_DBA_MVREF_STATS_SYS_DEFAULTS_ORA_TID = 25289; // "DBA_MVREF_STATS_SYS_DEFAULTS_ORA" +const uint64_t OB_USER_MVREF_STATS_SYS_DEFAULTS_ORA_TID = 25290; // "USER_MVREF_STATS_SYS_DEFAULTS_ORA" +const uint64_t OB_DBA_MVREF_STATS_PARAMS_ORA_TID = 25291; // "DBA_MVREF_STATS_PARAMS_ORA" +const uint64_t OB_USER_MVREF_STATS_PARAMS_ORA_TID = 25292; // "USER_MVREF_STATS_PARAMS_ORA" +const uint64_t OB_DBA_MVREF_RUN_STATS_ORA_TID = 25293; // "DBA_MVREF_RUN_STATS_ORA" +const uint64_t OB_USER_MVREF_RUN_STATS_ORA_TID = 25294; // "USER_MVREF_RUN_STATS_ORA" +const uint64_t OB_DBA_MVREF_STATS_ORA_TID = 25295; // "DBA_MVREF_STATS_ORA" +const uint64_t OB_USER_MVREF_STATS_ORA_TID = 25296; // "USER_MVREF_STATS_ORA" +const uint64_t OB_DBA_MVREF_CHANGE_STATS_ORA_TID = 25297; // "DBA_MVREF_CHANGE_STATS_ORA" +const uint64_t OB_USER_MVREF_CHANGE_STATS_ORA_TID = 25298; // "USER_MVREF_CHANGE_STATS_ORA" +const uint64_t OB_DBA_MVREF_STMT_STATS_ORA_TID = 25299; // "DBA_MVREF_STMT_STATS_ORA" +const uint64_t OB_USER_MVREF_STMT_STATS_ORA_TID = 25300; // "USER_MVREF_STMT_STATS_ORA" const uint64_t OB_GV_OB_SQL_AUDIT_ORA_TID = 28002; // "GV$OB_SQL_AUDIT_ORA" const uint64_t OB_V_OB_SQL_AUDIT_ORA_TID = 28003; // "V$OB_SQL_AUDIT_ORA" const uint64_t OB_GV_INSTANCE_TID = 28004; // "GV$INSTANCE" @@ -3945,6 +3979,22 @@ const char *const OB_CDB_INDEX_USAGE_TNAME = "CDB_INDEX_USAGE"; const char *const OB_GV_OB_LS_SNAPSHOTS_TNAME = "GV$OB_LS_SNAPSHOTS"; const char *const OB_V_OB_LS_SNAPSHOTS_TNAME = "V$OB_LS_SNAPSHOTS"; const char *const OB_DBA_OB_CLONE_HISTORY_TNAME = "DBA_OB_CLONE_HISTORY"; +const char *const OB_CDB_MVIEW_LOGS_TNAME = "CDB_MVIEW_LOGS"; +const char *const OB_DBA_MVIEW_LOGS_TNAME = "DBA_MVIEW_LOGS"; +const char *const OB_CDB_MVIEWS_TNAME = "CDB_MVIEWS"; +const char *const OB_DBA_MVIEWS_TNAME = "DBA_MVIEWS"; +const char *const OB_CDB_MVREF_STATS_SYS_DEFAULTS_TNAME = "CDB_MVREF_STATS_SYS_DEFAULTS"; +const char *const OB_DBA_MVREF_STATS_SYS_DEFAULTS_TNAME = "DBA_MVREF_STATS_SYS_DEFAULTS"; +const char *const OB_CDB_MVREF_STATS_PARAMS_TNAME = "CDB_MVREF_STATS_PARAMS"; +const char *const OB_DBA_MVREF_STATS_PARAMS_TNAME = "DBA_MVREF_STATS_PARAMS"; +const char *const OB_CDB_MVREF_RUN_STATS_TNAME = "CDB_MVREF_RUN_STATS"; +const char *const OB_DBA_MVREF_RUN_STATS_TNAME = "DBA_MVREF_RUN_STATS"; +const char *const OB_CDB_MVREF_STATS_TNAME = "CDB_MVREF_STATS"; +const char *const OB_DBA_MVREF_STATS_TNAME = "DBA_MVREF_STATS"; +const char *const OB_CDB_MVREF_CHANGE_STATS_TNAME = "CDB_MVREF_CHANGE_STATS"; +const char *const OB_DBA_MVREF_CHANGE_STATS_TNAME = "DBA_MVREF_CHANGE_STATS"; +const char *const OB_CDB_MVREF_STMT_STATS_TNAME = "CDB_MVREF_STMT_STATS"; +const char *const OB_DBA_MVREF_STMT_STATS_TNAME = "DBA_MVREF_STMT_STATS"; const char *const OB_DBA_SYNONYMS_TNAME = "DBA_SYNONYMS"; const char *const OB_DBA_OBJECTS_ORA_TNAME = "DBA_OBJECTS"; const char *const OB_ALL_OBJECTS_TNAME = "ALL_OBJECTS"; @@ -4208,6 +4258,24 @@ const char *const OB_DBA_OB_IMPORT_TABLE_JOB_HISTORY_ORA_TNAME = "DBA_OB_IMPORT_ const char *const OB_DBA_OB_IMPORT_TABLE_TASKS_ORA_TNAME = "DBA_OB_IMPORT_TABLE_TASKS"; const char *const OB_DBA_OB_IMPORT_TABLE_TASK_HISTORY_ORA_TNAME = "DBA_OB_IMPORT_TABLE_TASK_HISTORY"; const char *const OB_USER_USERS_TNAME = "USER_USERS"; +const char *const OB_DBA_MVIEW_LOGS_ORA_TNAME = "DBA_MVIEW_LOGS"; +const char *const OB_ALL_MVIEW_LOGS_ORA_TNAME = "ALL_MVIEW_LOGS"; +const char *const OB_USER_MVIEW_LOGS_ORA_TNAME = "USER_MVIEW_LOGS"; +const char *const OB_DBA_MVIEWS_ORA_TNAME = "DBA_MVIEWS"; +const char *const OB_ALL_MVIEWS_ORA_TNAME = "ALL_MVIEWS"; +const char *const OB_USER_MVIEWS_ORA_TNAME = "USER_MVIEWS"; +const char *const OB_DBA_MVREF_STATS_SYS_DEFAULTS_ORA_TNAME = "DBA_MVREF_STATS_SYS_DEFAULTS"; +const char *const OB_USER_MVREF_STATS_SYS_DEFAULTS_ORA_TNAME = "USER_MVREF_STATS_SYS_DEFAULTS"; +const char *const OB_DBA_MVREF_STATS_PARAMS_ORA_TNAME = "DBA_MVREF_STATS_PARAMS"; +const char *const OB_USER_MVREF_STATS_PARAMS_ORA_TNAME = "USER_MVREF_STATS_PARAMS"; +const char *const OB_DBA_MVREF_RUN_STATS_ORA_TNAME = "DBA_MVREF_RUN_STATS"; +const char *const OB_USER_MVREF_RUN_STATS_ORA_TNAME = "USER_MVREF_RUN_STATS"; +const char *const OB_DBA_MVREF_STATS_ORA_TNAME = "DBA_MVREF_STATS"; +const char *const OB_USER_MVREF_STATS_ORA_TNAME = "USER_MVREF_STATS"; +const char *const OB_DBA_MVREF_CHANGE_STATS_ORA_TNAME = "DBA_MVREF_CHANGE_STATS"; +const char *const OB_USER_MVREF_CHANGE_STATS_ORA_TNAME = "USER_MVREF_CHANGE_STATS"; +const char *const OB_DBA_MVREF_STMT_STATS_ORA_TNAME = "DBA_MVREF_STMT_STATS"; +const char *const OB_USER_MVREF_STMT_STATS_ORA_TNAME = "USER_MVREF_STMT_STATS"; const char *const OB_GV_OB_SQL_AUDIT_ORA_TNAME = "GV$OB_SQL_AUDIT"; const char *const OB_V_OB_SQL_AUDIT_ORA_TNAME = "V$OB_SQL_AUDIT"; const char *const OB_GV_INSTANCE_TNAME = "GV$INSTANCE"; diff --git a/src/share/inner_table/ob_inner_table_schema_def.py b/src/share/inner_table/ob_inner_table_schema_def.py index 2ae67db6c1..e2aec11b9c 100644 --- a/src/share/inner_table/ob_inner_table_schema_def.py +++ b/src/share/inner_table/ob_inner_table_schema_def.py @@ -2804,7 +2804,8 @@ def_table_schema( ('user_message', 'longtext', 'true'), ('dba_message', 'varchar:OB_MAX_ERROR_MSG_LEN', 'true'), ('parent_task_id', 'int', 'false', 0), - ('trace_id', 'varchar:OB_MAX_TRACE_ID_BUFFER_SIZE', 'true') + ('trace_id', 'varchar:OB_MAX_TRACE_ID_BUFFER_SIZE', 'true'), + ('consensus_schema_version', 'int', 'false', '-1') ], ) @@ -2955,6 +2956,7 @@ def_table_schema( ('ddl_stmt_str', 'longtext', 'true'), ('ret_code', 'int', 'false', '0'), ('message', 'longtext', 'true'), + ('consensus_schema_version', 'int', 'false', '-1'), ], ) @@ -31796,22 +31798,829 @@ FROM oceanbase.__all_clone_job_history ORDER BY CLONE_START_TIME #21523 DBA_OB_LS_REPLICA_TASK_HISTORY #21524 CDB_OB_LS_REPLICA_TASK_HISTORY -# 21525: CDB_MVIEW_LOGS -# 21526: DBA_MVIEW_LOGS -# 21527: CDB_MVIEWS -# 21528: DBA_MVIEWS -# 21529: CDB_MVREF_STATS_SYS_DEFAULTS -# 21530: DBA_MVREF_STATS_SYS_DEFAULTS -# 21531: CDB_MVREF_STATS_PARAMS -# 21532: DBA_MVREF_STATS_PARAMS -# 21533: CDB_MVREF_RUN_STATS -# 21534: DBA_MVREF_RUN_STATS -# 21535: CDB_MVREF_STATS -# 21536: DBA_MVREF_STATS -# 21537: CDB_MVREF_CHANGE_STATS -# 21538: DBA_MVREF_CHANGE_STATS -# 21539: CDB_MVREF_STMT_STATS -# 21540: DBA_MVREF_STMT_STATS +def_table_schema( + owner = 'suzhi.yt', + table_name = 'CDB_MVIEW_LOGS', + table_id = '21525', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + view_definition = """ + SELECT + B.TENANT_ID AS TENANT_ID, + CAST(A.DATABASE_NAME AS CHAR(128)) AS LOG_OWNER, + CAST(D.TABLE_NAME AS CHAR(128)) AS MASTER, + CAST(B.TABLE_NAME AS CHAR(128)) AS LOG_TABLE, + CAST(NULL AS CHAR(128)) AS LOG_TRIGGER, + CAST(IF(D.TABLE_MODE & 66048 = 66048, 'YES', 'NO') AS CHAR(3)) AS ROWIDS, + CAST(IF(D.TABLE_MODE & 66048 = 0, 'YES', 'NO') AS CHAR(3)) AS PRIMARY_KEY, + CAST('NO' AS CHAR(3)) AS OBJECT_ID, + CAST( + IF(( + SELECT COUNT(*) + FROM oceanbase.__all_virtual_column C1, + oceanbase.__all_virtual_column C2 + WHERE B.TENANT_ID = C1.TENANT_ID + AND B.TABLE_ID = C1.TABLE_ID + AND C1.COLUMN_ID >= 16 + AND C1.COLUMN_ID < 65520 + AND D.TENANT_ID = C2.TENANT_ID + AND D.TABLE_ID = C2.TABLE_ID + AND C2.ROWKEY_POSITION != 0 + AND C1.COLUMN_ID != C2.COLUMN_ID + ) = 0, 'NO', 'YES') AS CHAR(3) + ) AS FILTER_COLUMNS, + CAST('YES' AS CHAR(3)) AS SEQUENCE, + CAST('YES' AS CHAR(3)) AS INCLUDE_NEW_VALUES, + CAST(IF(C.PURGE_MODE = 1, 'YES', 'NO') AS CHAR(3)) AS PURGE_ASYNCHRONOUS, + CAST(IF(C.PURGE_MODE = 2, 'YES', 'NO') AS CHAR(3)) AS PURGE_DEFERRED, + CAST(C.PURGE_START AS DATETIME) AS PURGE_START, + CAST(C.PURGE_NEXT AS CHAR(200)) AS PURGE_INTERVAL, + CAST(C.LAST_PURGE_DATE AS DATETIME) AS LAST_PURGE_DATE, + CAST(0 AS SIGNED) AS LAST_PURGE_STATUS, + C.LAST_PURGE_ROWS AS NUM_ROWS_PURGED, + CAST('YES' AS CHAR(3)) AS COMMIT_SCN_BASED, + CAST('NO' AS CHAR(3)) AS STAGING_LOG + FROM + oceanbase.__all_virtual_database A, + oceanbase.__all_virtual_table B, + oceanbase.__all_virtual_mlog C, + oceanbase.__all_virtual_table D + WHERE A.TENANT_ID = B.TENANT_ID + AND A.DATABASE_ID = B.DATABASE_ID + AND B.TENANT_ID = D.TENANT_ID + AND C.TENANT_ID = D.TENANT_ID + AND B.TABLE_ID = C.MLOG_ID + AND B.TABLE_TYPE = 15 + AND B.DATA_TABLE_ID = D.TABLE_ID +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'DBA_MVIEW_LOGS', + table_id = '21526', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(A.DATABASE_NAME AS CHAR(128)) AS LOG_OWNER, + CAST(D.TABLE_NAME AS CHAR(128)) AS MASTER, + CAST(B.TABLE_NAME AS CHAR(128)) AS LOG_TABLE, + CAST(NULL AS CHAR(128)) AS LOG_TRIGGER, + CAST(IF(D.TABLE_MODE & 66048 = 66048, 'YES', 'NO') AS CHAR(3)) AS ROWIDS, + CAST(IF(D.TABLE_MODE & 66048 = 0, 'YES', 'NO') AS CHAR(3)) AS PRIMARY_KEY, + CAST('NO' AS CHAR(3)) AS OBJECT_ID, + CAST( + IF(( + SELECT COUNT(*) + FROM oceanbase.__all_column C1, + oceanbase.__all_column C2 + WHERE B.TENANT_ID = C1.TENANT_ID + AND B.TABLE_ID = C1.TABLE_ID + AND C1.COLUMN_ID >= 16 + AND C1.COLUMN_ID < 65520 + AND D.TENANT_ID = C2.TENANT_ID + AND D.TABLE_ID = C2.TABLE_ID + AND C2.ROWKEY_POSITION != 0 + AND C1.COLUMN_ID != C2.COLUMN_ID + ) = 0, 'NO', 'YES') AS CHAR(3) + ) AS FILTER_COLUMNS, + CAST('YES' AS CHAR(3)) AS SEQUENCE, + CAST('YES' AS CHAR(3)) AS INCLUDE_NEW_VALUES, + CAST(IF(C.PURGE_MODE = 1, 'YES', 'NO') AS CHAR(3)) AS PURGE_ASYNCHRONOUS, + CAST(IF(C.PURGE_MODE = 2, 'YES', 'NO') AS CHAR(3)) AS PURGE_DEFERRED, + CAST(C.PURGE_START AS DATETIME) AS PURGE_START, + CAST(C.PURGE_NEXT AS CHAR(200)) AS PURGE_INTERVAL, + CAST(C.LAST_PURGE_DATE AS DATETIME) AS LAST_PURGE_DATE, + CAST(0 AS SIGNED) AS LAST_PURGE_STATUS, + C.LAST_PURGE_ROWS AS NUM_ROWS_PURGED, + CAST('YES' AS CHAR(3)) AS COMMIT_SCN_BASED, + CAST('NO' AS CHAR(3)) AS STAGING_LOG + FROM + oceanbase.__all_database A, + oceanbase.__all_table B, + oceanbase.__all_mlog C, + oceanbase.__all_table D + WHERE A.TENANT_ID = B.TENANT_ID + AND A.DATABASE_ID = B.DATABASE_ID + AND B.TENANT_ID = D.TENANT_ID + AND C.TENANT_ID = D.TENANT_ID + AND B.TABLE_ID = C.MLOG_ID + AND B.TABLE_TYPE = 15 + AND B.DATA_TABLE_ID = D.TABLE_ID +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'CDB_MVIEWS', + table_id = '21527', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + view_definition = """ + SELECT + B.TENANT_ID AS TENANT_ID, + CAST(A.DATABASE_NAME AS CHAR(128)) AS OWNER, + CAST(B.TABLE_NAME AS CHAR(128)) AS MVIEW_NAME, + CAST(D.TABLE_NAME AS CHAR(128)) AS CONTAINER_NAME, + B.VIEW_DEFINITION AS QUERY, + CAST(LENGTH(B.VIEW_DEFINITION) AS SIGNED) AS QUERY_LEN, + CAST('N' AS CHAR(1)) AS UPDATABLE, + CAST(NULL AS CHAR(128)) AS UPDATE_LOG, + CAST(NULL AS CHAR(128)) AS MASTER_ROLLBACK_SEG, + CAST(NULL AS CHAR(128)) AS MASTER_LINK, + CAST('N' AS CHAR(1)) AS REWRITE_ENABLED, + CAST(NULL AS CHAR(9)) AS REWRITE_CAPABILITY, + CAST( + CASE C.REFRESH_MODE + WHEN 0 THEN 'NEVER' + WHEN 1 THEN 'DEMAND' + WHEN 2 THEN 'COMMIT' + WHEN 3 THEN 'STATEMENT' + ELSE NULL + END AS CHAR(6) + ) AS REFRESH_MODE, + CAST( + CASE C.REFRESH_METHOD + WHEN 0 THEN 'NEVER' + WHEN 1 THEN 'COMPLETE' + WHEN 2 THEN 'FAST' + WHEN 3 THEN 'FORCE' + ELSE NULL + END AS CHAR(8) + ) AS REFRESH_METHOD, + CAST( + CASE C.BUILD_MODE + WHEN 0 THEN 'IMMEDIATE' + WHEN 1 THEN 'DEFERRED' + WHEN 2 THEN 'PERBUILT' + ELSE NULL + END AS CHAR(9) + ) AS BUILD_MODE, + CAST(NULL AS CHAR(18)) AS FAST_REFRESHABLE, + CAST( + CASE C.LAST_REFRESH_TYPE + WHEN 0 THEN 'COMPLETE' + WHEN 1 THEN 'FAST' + ELSE 'NA' + END AS CHAR(8) + ) AS LAST_REFRESH_TYPE, + CAST(C.LAST_REFRESH_DATE AS DATETIME) AS LAST_REFRESH_DATE, + CAST(C.LAST_REFRESH_DATE + C.LAST_REFRESH_TIME * 1000 * 1000 AS DATETIME) AS LAST_REFRESH_END_TIME, + CAST(NULL AS CHAR(19)) AS STALENESS, + CAST(NULL AS CHAR(19)) AS AFTER_FAST_REFRESH, + CAST(IF(C.BUILD_MODE = 2, 'Y', 'N') AS CHAR(1)) AS UNKNOWN_PREBUILT, + CAST('N' AS CHAR(1)) AS UNKNOWN_PLSQL_FUNC, + CAST('N' AS CHAR(1)) AS UNKNOWN_EXTERNAL_TABLE, + CAST('N' AS CHAR(1)) AS UNKNOWN_CONSIDER_FRESH, + CAST('N' AS CHAR(1)) AS UNKNOWN_IMPORT, + CAST('N' AS CHAR(1)) AS UNKNOWN_TRUSTED_FD, + CAST(NULL AS CHAR(19)) AS COMPILE_STATE, + CAST('Y' AS CHAR(1)) AS USE_NO_INDEX, + CAST(NULL AS DATETIME) AS STALE_SINCE, + CAST(NULL AS SIGNED) AS NUM_PCT_TABLES, + CAST(NULL AS SIGNED) AS NUM_FRESH_PCT_REGIONS, + CAST(NULL AS SIGNED) AS NUM_STALE_PCT_REGIONS, + CAST('NO' AS CHAR(3)) AS SEGMENT_CREATED, + CAST(NULL AS CHAR(128)) AS EVALUATION_EDITION, + CAST(NULL AS CHAR(128)) AS UNUSABLE_BEFORE, + CAST(NULL AS CHAR(128)) AS UNUSABLE_BEGINNING, + CAST(NULL AS CHAR(100)) AS DEFAULT_COLLATION, + CAST('N' AS CHAR(1)) AS ON_QUERY_COMPUTATION + FROM + oceanbase.__all_virtual_database A, + oceanbase.__all_virtual_table B, + oceanbase.__all_virtual_mview C, + oceanbase.__all_virtual_table D + WHERE A.TENANT_ID = B.TENANT_ID + AND A.DATABASE_ID = B.DATABASE_ID + AND B.TENANT_ID = C.TENANT_ID + AND B.TABLE_ID = C.MVIEW_ID + AND B.TABLE_TYPE = 7 + AND B.TENANT_ID = D.TENANT_ID + AND B.DATA_TABLE_ID = D.TABLE_ID +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'DBA_MVIEWS', + table_id = '21528', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(A.DATABASE_NAME AS CHAR(128)) AS OWNER, + CAST(B.TABLE_NAME AS CHAR(128)) AS MVIEW_NAME, + CAST(D.TABLE_NAME AS CHAR(128)) AS CONTAINER_NAME, + B.VIEW_DEFINITION AS QUERY, + CAST(LENGTH(B.VIEW_DEFINITION) AS SIGNED) AS QUERY_LEN, + CAST('N' AS CHAR(1)) AS UPDATABLE, + CAST(NULL AS CHAR(128)) AS UPDATE_LOG, + CAST(NULL AS CHAR(128)) AS MASTER_ROLLBACK_SEG, + CAST(NULL AS CHAR(128)) AS MASTER_LINK, + CAST('N' AS CHAR(1)) AS REWRITE_ENABLED, + CAST(NULL AS CHAR(9)) AS REWRITE_CAPABILITY, + CAST( + CASE C.REFRESH_MODE + WHEN 0 THEN 'NEVER' + WHEN 1 THEN 'DEMAND' + WHEN 2 THEN 'COMMIT' + WHEN 3 THEN 'STATEMENT' + ELSE NULL + END AS CHAR(6) + ) AS REFRESH_MODE, + CAST( + CASE C.REFRESH_METHOD + WHEN 0 THEN 'NEVER' + WHEN 1 THEN 'COMPLETE' + WHEN 2 THEN 'FAST' + WHEN 3 THEN 'FORCE' + ELSE NULL + END AS CHAR(8) + ) AS REFRESH_METHOD, + CAST( + CASE C.BUILD_MODE + WHEN 0 THEN 'IMMEDIATE' + WHEN 1 THEN 'DEFERRED' + WHEN 2 THEN 'PERBUILT' + ELSE NULL + END AS CHAR(9) + ) AS BUILD_MODE, + CAST(NULL AS CHAR(18)) AS FAST_REFRESHABLE, + CAST( + CASE C.LAST_REFRESH_TYPE + WHEN 0 THEN 'COMPLETE' + WHEN 1 THEN 'FAST' + ELSE 'NA' + END AS CHAR(8) + ) AS LAST_REFRESH_TYPE, + CAST(C.LAST_REFRESH_DATE AS DATETIME) AS LAST_REFRESH_DATE, + CAST(C.LAST_REFRESH_DATE + C.LAST_REFRESH_TIME * 1000 * 1000 AS DATETIME) AS LAST_REFRESH_END_TIME, + CAST(NULL AS CHAR(19)) AS STALENESS, + CAST(NULL AS CHAR(19)) AS AFTER_FAST_REFRESH, + CAST(IF(C.BUILD_MODE = 2, 'Y', 'N') AS CHAR(1)) AS UNKNOWN_PREBUILT, + CAST('N' AS CHAR(1)) AS UNKNOWN_PLSQL_FUNC, + CAST('N' AS CHAR(1)) AS UNKNOWN_EXTERNAL_TABLE, + CAST('N' AS CHAR(1)) AS UNKNOWN_CONSIDER_FRESH, + CAST('N' AS CHAR(1)) AS UNKNOWN_IMPORT, + CAST('N' AS CHAR(1)) AS UNKNOWN_TRUSTED_FD, + CAST(NULL AS CHAR(19)) AS COMPILE_STATE, + CAST('Y' AS CHAR(1)) AS USE_NO_INDEX, + CAST(NULL AS DATETIME) AS STALE_SINCE, + CAST(NULL AS SIGNED) AS NUM_PCT_TABLES, + CAST(NULL AS SIGNED) AS NUM_FRESH_PCT_REGIONS, + CAST(NULL AS SIGNED) AS NUM_STALE_PCT_REGIONS, + CAST('NO' AS CHAR(3)) AS SEGMENT_CREATED, + CAST(NULL AS CHAR(128)) AS EVALUATION_EDITION, + CAST(NULL AS CHAR(128)) AS UNUSABLE_BEFORE, + CAST(NULL AS CHAR(128)) AS UNUSABLE_BEGINNING, + CAST(NULL AS CHAR(100)) AS DEFAULT_COLLATION, + CAST('N' AS CHAR(1)) AS ON_QUERY_COMPUTATION + FROM + oceanbase.__all_database A, + oceanbase.__all_table B, + oceanbase.__all_mview C, + oceanbase.__all_table D + WHERE A.TENANT_ID = B.TENANT_ID + AND A.DATABASE_ID = B.DATABASE_ID + AND B.TENANT_ID = C.TENANT_ID + AND B.TABLE_ID = C.MVIEW_ID + AND B.TABLE_TYPE = 7 + AND B.TENANT_ID = D.TENANT_ID + AND B.DATA_TABLE_ID = D.TABLE_ID +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'CDB_MVREF_STATS_SYS_DEFAULTS', + table_id = '21529', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + view_definition = """ + SELECT + TENANT_ID, + CAST(PARAMETER_NAME AS CHAR(16)) AS PARAMETER_NAME, + CAST(VALUE AS CHAR(40)) AS VALUE + FROM + ( + /* COLLECTION_LEVEL */ + SELECT + TENANT_ID, + 'COLLECTION_LEVEL' PARAMETER_NAME, + CASE IFNULL(MAX(COLLECTION_LEVEL), 1) + WHEN 0 THEN 'NONE' + WHEN 1 THEN 'TYPICAL' + WHEN 2 THEN 'ADVANCED' + ELSE NULL + END VALUE + FROM + oceanbase.__all_virtual_mview_refresh_stats_sys_defaults + RIGHT OUTER JOIN + (SELECT TENANT_ID FROM oceanbase.__all_tenant WHERE TENANT_ID = 1 OR (TENANT_ID & 0x1) = 0) + USING (TENANT_ID) + GROUP BY TENANT_ID + + UNION ALL + + /* RETENTION_PERIOD */ + SELECT + TENANT_ID, + 'RETENTION_PERIOD' PARAMETER_NAME, + CAST(IFNULL(MAX(RETENTION_PERIOD), 31) AS CHAR) VALUE + FROM + oceanbase.__all_virtual_mview_refresh_stats_sys_defaults + RIGHT OUTER JOIN + (SELECT TENANT_ID FROM oceanbase.__all_tenant WHERE TENANT_ID = 1 OR (TENANT_ID & 0x1) = 0) + USING (TENANT_ID) + GROUP BY TENANT_ID + ) + ORDER BY TENANT_ID +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'DBA_MVREF_STATS_SYS_DEFAULTS', + table_id = '21530', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(PARAMETER_NAME AS CHAR(16)) AS PARAMETER_NAME, + CAST(VALUE AS CHAR(40)) AS VALUE + FROM + ( + /* COLLECTION_LEVEL */ + SELECT + 'COLLECTION_LEVEL' PARAMETER_NAME, + CASE IFNULL(MAX(COLLECTION_LEVEL), 1) + WHEN 0 THEN 'NONE' + WHEN 1 THEN 'TYPICAL' + WHEN 2 THEN 'ADVANCED' + ELSE NULL + END VALUE + FROM + oceanbase.__all_mview_refresh_stats_sys_defaults + + UNION ALL + + /* RETENTION_PERIOD */ + SELECT + 'RETENTION_PERIOD' PARAMETER_NAME, + CAST(IFNULL(MAX(RETENTION_PERIOD), 31) AS CHAR) VALUE + FROM + oceanbase.__all_mview_refresh_stats_sys_defaults + ) +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'CDB_MVREF_STATS_PARAMS', + table_id = '21531', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + view_definition = """ + SELECT + TENANT_ID, + CAST(MV_OWNER AS CHAR(128)) AS MV_OWNER, + CAST(MV_NAME AS CHAR(128)) AS MV_NAME, + CAST( + CASE COLLECTION_LEVEL + WHEN 0 THEN 'NONE' + WHEN 1 THEN 'TYPICAL' + WHEN 2 THEN 'ADVANCED' + ELSE NULL + END AS CHAR(8) + ) AS COLLECTION_LEVEL, + RETENTION_PERIOD + FROM + ( + WITH DEFVALS AS + ( + SELECT + TENANT_ID, + IFNULL(MAX(COLLECTION_LEVEL), 1) AS COLLECTION_LEVEL, + IFNULL(MAX(RETENTION_PERIOD), 31) AS RETENTION_PERIOD + FROM + oceanbase.__all_virtual_mview_refresh_stats_sys_defaults + RIGHT OUTER JOIN + (SELECT TENANT_ID FROM oceanbase.__all_tenant WHERE TENANT_ID = 1 OR (TENANT_ID & 0x1) = 0) + USING (TENANT_ID) + GROUP BY TENANT_ID + ) + + SELECT + B.TENANT_ID TENANT_ID, + A.DATABASE_NAME MV_OWNER, + B.TABLE_NAME MV_NAME, + IFNULL(C.COLLECTION_LEVEL, D.COLLECTION_LEVEL) COLLECTION_LEVEL, + IFNULL(C.RETENTION_PERIOD, D.RETENTION_PERIOD) RETENTION_PERIOD + FROM + oceanbase.__all_virtual_database A, + oceanbase.__all_virtual_table B, + ( + SELECT TENANT_ID, MVIEW_ID, COLLECTION_LEVEL, RETENTION_PERIOD FROM oceanbase.__all_virtual_mview_refresh_stats_params + RIGHT OUTER JOIN + ( + SELECT TENANT_ID, MVIEW_ID FROM oceanbase.__all_virtual_mview + ) + USING (TENANT_ID, MVIEW_ID) + ) C, + DEFVALS D + WHERE A.TENANT_ID = B.TENANT_ID + AND A.DATABASE_ID = B.DATABASE_ID + AND B.TENANT_ID = C.TENANT_ID + AND B.TABLE_ID = C.MVIEW_ID + AND B.TABLE_TYPE = 7 + AND C.TENANT_ID = D.TENANT_ID + ) +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'DBA_MVREF_STATS_PARAMS', + table_id = '21532', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(MV_OWNER AS CHAR(128)) AS MV_OWNER, + CAST(MV_NAME AS CHAR(128)) AS MV_NAME, + CAST( + CASE COLLECTION_LEVEL + WHEN 0 THEN 'NONE' + WHEN 1 THEN 'TYPICAL' + WHEN 2 THEN 'ADVANCED' + ELSE NULL + END AS CHAR(8) + ) AS COLLECTION_LEVEL, + RETENTION_PERIOD + FROM + ( + WITH DEFVALS AS + ( + SELECT + IFNULL(MAX(COLLECTION_LEVEL), 1) AS COLLECTION_LEVEL, + IFNULL(MAX(RETENTION_PERIOD), 31) AS RETENTION_PERIOD + FROM + oceanbase.__all_mview_refresh_stats_sys_defaults + ) + + SELECT + A.DATABASE_NAME MV_OWNER, + B.TABLE_NAME MV_NAME, + IFNULL(C.COLLECTION_LEVEL, D.COLLECTION_LEVEL) COLLECTION_LEVEL, + IFNULL(C.RETENTION_PERIOD, D.RETENTION_PERIOD) RETENTION_PERIOD + FROM + oceanbase.__all_database A, + oceanbase.__all_table B, + ( + SELECT TENANT_ID, MVIEW_ID, COLLECTION_LEVEL, RETENTION_PERIOD FROM oceanbase.__all_mview_refresh_stats_params + RIGHT OUTER JOIN + ( + SELECT TENANT_ID, MVIEW_ID FROM oceanbase.__all_mview + ) + USING (TENANT_ID, MVIEW_ID) + ) C, + DEFVALS D + WHERE A.TENANT_ID = B.TENANT_ID + AND A.DATABASE_ID = B.DATABASE_ID + AND B.TENANT_ID = C.TENANT_ID + AND B.TABLE_ID = C.MVIEW_ID + AND B.TABLE_TYPE = 7 + ) +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'CDB_MVREF_RUN_STATS', + table_id = '21533', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + view_definition = """ + SELECT + B.TENANT_ID AS TENANT_ID, + CAST(A.USER_NAME AS CHAR(128)) AS RUN_OWNER, + B.REFRESH_ID AS REFRESH_ID, + B.NUM_MVS_TOTAL AS NUM_MVS, + CAST(B.MVIEWS AS CHAR(4000)) AS MVIEWS, + CAST(B.BASE_TABLES AS CHAR(4000)) AS BASE_TABLES, + CAST(B.METHOD AS CHAR(4000)) AS METHOD, + CAST(B.ROLLBACK_SEG AS CHAR(4000)) AS ROLLBACK_SEG, + CAST(IF(B.PUSH_DEFERRED_RPC = 1, 'Y', 'N') AS CHAR(1)) AS PUSH_DEFERRED_RPC, + CAST(IF(B.REFRESH_AFTER_ERRORS = 1, 'Y', 'N') AS CHAR(1)) AS REFRESH_AFTER_ERRORS, + B.PURGE_OPTION AS PURGE_OPTION, + B.PARALLELISM AS PARALLELISM, + B.HEAP_SIZE AS HEAP_SIZE, + CAST(IF(B.ATOMIC_REFRESH = 1, 'Y', 'N') AS CHAR(1)) AS ATOMIC_REFRESH, + CAST(IF(B.NESTED = 1, 'Y', 'N') AS CHAR(1)) AS NESTED, + CAST(IF(B.OUT_OF_PLACE = 1, 'Y', 'N') AS CHAR(1)) AS OUT_OF_PLACE, + B.NUMBER_OF_FAILURES AS NUMBER_OF_FAILURES, + CAST(B.START_TIME AS DATETIME) AS START_TIME, + CAST(B.END_TIME AS DATETIME) AS END_TIME, + B.ELAPSED_TIME AS ELAPSED_TIME, + CAST(0 AS SIGNED) AS LOG_SETUP_TIME, + B.LOG_PURGE_TIME AS LOG_PURGE_TIME, + CAST(IF(B.COMPLETE_STATS_AVALIABLE = 1, 'Y', 'N') AS CHAR(1)) AS COMPLETE_STATS_AVAILABLE + FROM + oceanbase.__all_virtual_user A, + oceanbase.__all_virtual_mview_refresh_run_stats B + WHERE A.TENANT_ID = B.TENANT_ID + AND A.USER_ID = B.RUN_USER_ID +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'DBA_MVREF_RUN_STATS', + table_id = '21534', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(A.USER_NAME AS CHAR(128)) AS RUN_OWNER, + B.REFRESH_ID AS REFRESH_ID, + B.NUM_MVS_TOTAL AS NUM_MVS, + CAST(B.MVIEWS AS CHAR(4000)) AS MVIEWS, + CAST(B.BASE_TABLES AS CHAR(4000)) AS BASE_TABLES, + CAST(B.METHOD AS CHAR(4000)) AS METHOD, + CAST(B.ROLLBACK_SEG AS CHAR(4000)) AS ROLLBACK_SEG, + CAST(IF(B.PUSH_DEFERRED_RPC = 1, 'Y', 'N') AS CHAR(1)) AS PUSH_DEFERRED_RPC, + CAST(IF(B.REFRESH_AFTER_ERRORS = 1, 'Y', 'N') AS CHAR(1)) AS REFRESH_AFTER_ERRORS, + B.PURGE_OPTION AS PURGE_OPTION, + B.PARALLELISM AS PARALLELISM, + B.HEAP_SIZE AS HEAP_SIZE, + CAST(IF(B.ATOMIC_REFRESH = 1, 'Y', 'N') AS CHAR(1)) AS ATOMIC_REFRESH, + CAST(IF(B.NESTED = 1, 'Y', 'N') AS CHAR(1)) AS NESTED, + CAST(IF(B.OUT_OF_PLACE = 1, 'Y', 'N') AS CHAR(1)) AS OUT_OF_PLACE, + B.NUMBER_OF_FAILURES AS NUMBER_OF_FAILURES, + CAST(B.START_TIME AS DATETIME) AS START_TIME, + CAST(B.END_TIME AS DATETIME) AS END_TIME, + B.ELAPSED_TIME AS ELAPSED_TIME, + CAST(0 AS SIGNED) AS LOG_SETUP_TIME, + B.LOG_PURGE_TIME AS LOG_PURGE_TIME, + CAST(IF(B.COMPLETE_STATS_AVALIABLE = 1, 'Y', 'N') AS CHAR(1)) AS COMPLETE_STATS_AVAILABLE + FROM + oceanbase.__all_user A, + oceanbase.__all_mview_refresh_run_stats B + WHERE A.TENANT_ID = B.TENANT_ID + AND A.USER_ID = B.RUN_USER_ID +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'CDB_MVREF_STATS', + table_id = '21535', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + view_definition = """ + SELECT + B.TENANT_ID AS TENANT_ID, + CAST(A.DATABASE_NAME AS CHAR(128)) AS MV_OWNER, + CAST(B.TABLE_NAME AS CHAR(128)) AS MV_NAME, + C.REFRESH_ID AS REFRESH_ID, + CAST( + CASE C.REFRESH_TYPE + WHEN 0 THEN 'COMPLETE' + WHEN 1 THEN 'FAST' + ELSE NULL + END AS CHAR(30) + ) AS REFRESH_METHOD, + CAST(NULL AS CHAR(4000)) AS REFRESH_OPTIMIZATIONS, + CAST(NULL AS CHAR(4000)) AS ADDITIONAL_EXECUTIONS, + CAST(C.START_TIME AS DATETIME) AS START_TIME, + CAST(C.END_TIME AS DATETIME) AS END_TIME, + C.ELAPSED_TIME AS ELAPSED_TIME, + CAST(0 AS SIGNED) AS LOG_SETUP_TIME, + C.LOG_PURGE_TIME AS LOG_PURGE_TIME, + C.INITIAL_NUM_ROWS AS INITIAL_NUM_ROWS, + C.FINAL_NUM_ROWS AS FINAL_NUM_ROWS + FROM + oceanbase.__all_virtual_database A, + oceanbase.__all_virtual_table B, + oceanbase.__all_virtual_mview_refresh_stats C + WHERE A.TENANT_ID = B.TENANT_ID + AND A.DATABASE_ID = B.DATABASE_ID + AND B.TENANT_ID = C.TENANT_ID + AND B.TABLE_ID = C.MVIEW_ID + AND B.TABLE_TYPE = 7 +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'DBA_MVREF_STATS', + table_id = '21536', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(A.DATABASE_NAME AS CHAR(128)) AS MV_OWNER, + CAST(B.TABLE_NAME AS CHAR(128)) AS MV_NAME, + C.REFRESH_ID AS REFRESH_ID, + CAST( + CASE C.REFRESH_TYPE + WHEN 0 THEN 'COMPLETE' + WHEN 1 THEN 'FAST' + ELSE NULL + END AS CHAR(30) + ) AS REFRESH_METHOD, + CAST(NULL AS CHAR(4000)) AS REFRESH_OPTIMIZATIONS, + CAST(NULL AS CHAR(4000)) AS ADDITIONAL_EXECUTIONS, + CAST(C.START_TIME AS DATETIME) AS START_TIME, + CAST(C.END_TIME AS DATETIME) AS END_TIME, + C.ELAPSED_TIME AS ELAPSED_TIME, + CAST(0 AS SIGNED) AS LOG_SETUP_TIME, + C.LOG_PURGE_TIME AS LOG_PURGE_TIME, + C.INITIAL_NUM_ROWS AS INITIAL_NUM_ROWS, + C.FINAL_NUM_ROWS AS FINAL_NUM_ROWS + FROM + oceanbase.__all_database A, + oceanbase.__all_table B, + oceanbase.__all_mview_refresh_stats C + WHERE A.TENANT_ID = B.TENANT_ID + AND A.DATABASE_ID = B.DATABASE_ID + AND B.TENANT_ID = C.TENANT_ID + AND B.TABLE_ID = C.MVIEW_ID + AND B.TABLE_TYPE = 7 +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'CDB_MVREF_CHANGE_STATS', + table_id = '21537', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + view_definition = """ + SELECT + E.TENANT_ID AS TENANT_ID, + CAST(C.DATABASE_NAME AS CHAR(128)) AS TBL_OWNER, + CAST(D.TABLE_NAME AS CHAR(128)) AS TBL_NAME, + CAST(A.DATABASE_NAME AS CHAR(128)) AS MV_OWNER, + CAST(B.TABLE_NAME AS CHAR(128)) AS MV_NAME, + E.REFRESH_ID AS REFRESH_ID, + E.NUM_ROWS_INS AS NUM_ROWS_INS, + E.NUM_ROWS_UPD AS NUM_ROWS_UPD, + E.NUM_ROWS_DEL AS NUM_ROWS_DEL, + CAST(0 AS SIGNED) AS NUM_ROWS_DL_INS, + CAST('N' AS CHAR(1)) AS PMOPS_OCCURRED, + CAST(NULL AS CHAR(4000)) AS PMOP_DETAILS, + E.NUM_ROWS AS NUM_ROWS + FROM + oceanbase.__all_virtual_database A, + oceanbase.__all_virtual_table B, + oceanbase.__all_virtual_database C, + oceanbase.__all_virtual_table D, + oceanbase.__all_virtual_mview_refresh_change_stats E + WHERE A.TENANT_ID = B.TENANT_ID + AND A.DATABASE_ID = B.DATABASE_ID + AND C.TENANT_ID = D.TENANT_ID + AND C.DATABASE_ID = D.DATABASE_ID + AND E.TENANT_ID = B.TENANT_ID + AND E.MVIEW_ID = B.TABLE_ID + AND E.TENANT_ID = D.TENANT_ID + AND E.DETAIL_TABLE_ID = D.TABLE_ID +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'DBA_MVREF_CHANGE_STATS', + table_id = '21538', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(C.DATABASE_NAME AS CHAR(128)) AS TBL_OWNER, + CAST(D.TABLE_NAME AS CHAR(128)) AS TBL_NAME, + CAST(A.DATABASE_NAME AS CHAR(128)) AS MV_OWNER, + CAST(B.TABLE_NAME AS CHAR(128)) AS MV_NAME, + E.REFRESH_ID AS REFRESH_ID, + E.NUM_ROWS_INS AS NUM_ROWS_INS, + E.NUM_ROWS_UPD AS NUM_ROWS_UPD, + E.NUM_ROWS_DEL AS NUM_ROWS_DEL, + CAST(0 AS SIGNED) AS NUM_ROWS_DL_INS, + CAST('N' AS CHAR(1)) AS PMOPS_OCCURRED, + CAST(NULL AS CHAR(4000)) AS PMOP_DETAILS, + E.NUM_ROWS AS NUM_ROWS + FROM + oceanbase.__all_database A, + oceanbase.__all_table B, + oceanbase.__all_database C, + oceanbase.__all_table D, + oceanbase.__all_mview_refresh_change_stats E + WHERE A.TENANT_ID = B.TENANT_ID + AND A.DATABASE_ID = B.DATABASE_ID + AND C.TENANT_ID = D.TENANT_ID + AND C.DATABASE_ID = D.DATABASE_ID + AND E.TENANT_ID = B.TENANT_ID + AND E.MVIEW_ID = B.TABLE_ID + AND E.TENANT_ID = D.TENANT_ID + AND E.DETAIL_TABLE_ID = D.TABLE_ID +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'CDB_MVREF_STMT_STATS', + table_id = '21539', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + view_definition = """ + SELECT + C.TENANT_ID AS TENANT_ID, + CAST(A.DATABASE_NAME AS CHAR(128)) AS MV_OWNER, + CAST(B.TABLE_NAME AS CHAR(128)) AS MV_NAME, + C.REFRESH_ID AS REFRESH_ID, + C.STEP AS STEP, + CAST(C.SQLID AS CHAR(32)) AS SQLID, + C.STMT AS STMT, + C.EXECUTION_TIME AS EXECUTION_TIME, + C.EXECUTION_PLAN AS EXECUTION_PLAN + FROM + oceanbase.__all_virtual_database A, + oceanbase.__all_virtual_table B, + oceanbase.__all_virtual_mview_refresh_stmt_stats C + WHERE A.TENANT_ID = B.TENANT_ID + AND A.DATABASE_ID = B.DATABASE_ID + AND B.TENANT_ID = C.TENANT_ID + AND B.TABLE_ID = C.MVIEW_ID +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'DBA_MVREF_STMT_STATS', + table_id = '21540', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(A.DATABASE_NAME AS CHAR(128)) AS MV_OWNER, + CAST(B.TABLE_NAME AS CHAR(128)) AS MV_NAME, + C.REFRESH_ID AS REFRESH_ID, + C.STEP AS STEP, + CAST(C.SQLID AS CHAR(32)) AS SQLID, + C.STMT AS STMT, + C.EXECUTION_TIME AS EXECUTION_TIME, + C.EXECUTION_PLAN AS EXECUTION_PLAN + FROM + oceanbase.__all_database A, + oceanbase.__all_table B, + oceanbase.__all_mview_refresh_stmt_stats C + WHERE A.TENANT_ID = B.TENANT_ID + AND A.DATABASE_ID = B.DATABASE_ID + AND B.TENANT_ID = C.TENANT_ID + AND B.TABLE_ID = C.MVIEW_ID +""".replace("\n", " ") +) # 余留位置 @@ -50829,24 +51638,1021 @@ def_table_schema( # 25281: DBA_UNUSED_COL_TABS # 25282: USER_UNUSED_COL_TABS -# 25283: DBA_MVIEW_LOGS -# 25284: ALL_MVIEW_LOGS -# 25285: USER_MVIEW_LOGS -# 25286: DBA_MVIEWS -# 25287: ALL_MVIEWS -# 25288: USER_MVIEWS -# 25289: DBA_MVREF_STATS_SYS_DEFAULTS -# 25290: USER_MVREF_STATS_SYS_DEFAULTS -# 25291: DBA_MVREF_STATS_PARAMS -# 25292: USER_MVREF_STATS_PARAMS -# 25293: DBA_MVREF_RUN_STATS -# 25294: USER_MVREF_RUN_STATS -# 25295: DBA_MVREF_STATS -# 25296: USER_MVREF_STATS -# 25297: DBA_MVREF_CHANGE_STATS -# 25298: USER_MVREF_CHANGE_STATS -# 25299: DBA_MVREF_STMT_STATS -# 25300: USER_MVREF_STMT_STATS +def_table_schema( + owner = 'suzhi.yt', + table_name = 'DBA_MVIEW_LOGS', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '25283', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS LOG_OWNER, + CAST(D.TABLE_NAME AS VARCHAR2(128)) AS MASTER, + CAST(B.TABLE_NAME AS VARCHAR2(128)) AS LOG_TABLE, + CAST(NULL AS VARCHAR2(128)) AS LOG_TRIGGER, + CAST(DECODE(bitand(D.TABLE_MODE, 66048), 66048, 'YES', 'NO') AS VARCHAR2(3)) AS ROWIDS, + CAST(DECODE(bitand(D.TABLE_MODE, 66048), 0, 'YES', 'NO') AS VARCHAR2(3)) AS PRIMARY_KEY, + CAST('NO' AS VARCHAR2(3)) AS OBJECT_ID, + CAST( + DECODE(( + SELECT COUNT(*) + FROM SYS.ALL_VIRTUAL_COLUMN_REAL_AGENT C1, + SYS.ALL_VIRTUAL_COLUMN_REAL_AGENT C2 + WHERE B.TABLE_ID = C1.TABLE_ID + AND C1.COLUMN_ID >= 16 + AND C1.COLUMN_ID < 65520 + AND D.TABLE_ID = C2.TABLE_ID + AND C2.ROWKEY_POSITION != 0 + AND C1.COLUMN_ID != C2.COLUMN_ID + AND C1.TENANT_ID = EFFECTIVE_TENANT_ID() + AND C2.TENANT_ID = EFFECTIVE_TENANT_ID() + ), 0, 'NO', 'YES') AS VARCHAR2(3) + ) AS FILTER_COLUMNS, + CAST('YES' AS VARCHAR2(3)) AS SEQUENCE, + CAST('YES' AS VARCHAR2(3)) AS INCLUDE_NEW_VALUES, + CAST(DECODE(C.PURGE_MODE, 1, 'YES', 'NO') AS VARCHAR2(3)) AS PURGE_ASYNCHRONOUS, + CAST(DECODE(C.PURGE_MODE, 2, 'YES', 'NO') AS VARCHAR2(3)) AS PURGE_DEFERRED, + CAST(C.PURGE_START AS DATE) AS PURGE_START /* TODO: DD-MON-YYYY */, + CAST(C.PURGE_NEXT AS VARCHAR2(200)) AS PURGE_INTERVAL, + CAST(C.LAST_PURGE_DATE AS DATE) AS LAST_PURGE_DATE /* TODO: DD-MON-YYYY */, + CAST(0 AS NUMBER) AS LAST_PURGE_STATUS, + CAST(C.LAST_PURGE_ROWS AS NUMBER) AS NUM_ROWS_PURGED, + CAST('YES' AS VARCHAR2(3)) AS COMMIT_SCN_BASED, + CAST('NO' AS VARCHAR2(3)) AS STAGING_LOG + FROM + SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, + SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, + SYS.ALL_VIRTUAL_MLOG_REAL_AGENT C, + SYS.ALL_VIRTUAL_TABLE_REAL_AGENT D + WHERE A.DATABASE_ID = B.DATABASE_ID + AND B.TABLE_ID = C.MLOG_ID + AND B.TABLE_TYPE = 15 + AND B.DATA_TABLE_ID = D.TABLE_ID + AND A.TENANT_ID = EFFECTIVE_TENANT_ID() + AND B.TENANT_ID = EFFECTIVE_TENANT_ID() + AND C.TENANT_ID = EFFECTIVE_TENANT_ID() + AND D.TENANT_ID = EFFECTIVE_TENANT_ID() +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'ALL_MVIEW_LOGS', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '25284', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS LOG_OWNER, + CAST(D.TABLE_NAME AS VARCHAR2(128)) AS MASTER, + CAST(B.TABLE_NAME AS VARCHAR2(128)) AS LOG_TABLE, + CAST(NULL AS VARCHAR2(128)) AS LOG_TRIGGER, + CAST(DECODE(bitand(D.TABLE_MODE, 66048), 66048, 'YES', 'NO') AS VARCHAR2(3)) AS ROWIDS, + CAST(DECODE(bitand(D.TABLE_MODE, 66048), 0, 'YES', 'NO') AS VARCHAR2(3)) AS PRIMARY_KEY, + CAST('NO' AS VARCHAR2(3)) AS OBJECT_ID, + CAST( + DECODE(( + SELECT COUNT(*) + FROM SYS.ALL_VIRTUAL_COLUMN_REAL_AGENT C1, + SYS.ALL_VIRTUAL_COLUMN_REAL_AGENT C2 + WHERE B.TABLE_ID = C1.TABLE_ID + AND C1.COLUMN_ID >= 16 + AND C1.COLUMN_ID < 65520 + AND D.TABLE_ID = C2.TABLE_ID + AND C2.ROWKEY_POSITION != 0 + AND C1.COLUMN_ID != C2.COLUMN_ID + AND C1.TENANT_ID = EFFECTIVE_TENANT_ID() + AND C2.TENANT_ID = EFFECTIVE_TENANT_ID() + ), 0, 'NO', 'YES') AS VARCHAR2(3) + ) AS FILTER_COLUMNS, + CAST('YES' AS VARCHAR2(3)) AS SEQUENCE, + CAST('YES' AS VARCHAR2(3)) AS INCLUDE_NEW_VALUES, + CAST(DECODE(C.PURGE_MODE, 1, 'YES', 'NO') AS VARCHAR2(3)) AS PURGE_ASYNCHRONOUS, + CAST(DECODE(C.PURGE_MODE, 2, 'YES', 'NO') AS VARCHAR2(3)) AS PURGE_DEFERRED, + CAST(C.PURGE_START AS DATE) AS PURGE_START /* TODO: DD-MON-YYYY */, + CAST(C.PURGE_NEXT AS VARCHAR2(200)) AS PURGE_INTERVAL, + CAST(C.LAST_PURGE_DATE AS DATE) AS LAST_PURGE_DATE /* TODO: DD-MON-YYYY */, + CAST(0 AS NUMBER) AS LAST_PURGE_STATUS, + CAST(C.LAST_PURGE_ROWS AS NUMBER) AS NUM_ROWS_PURGED, + CAST('YES' AS VARCHAR2(3)) AS COMMIT_SCN_BASED, + CAST('NO' AS VARCHAR2(3)) AS STAGING_LOG + FROM + SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, + SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, + SYS.ALL_VIRTUAL_MLOG_REAL_AGENT C, + SYS.ALL_VIRTUAL_TABLE_REAL_AGENT D + WHERE A.DATABASE_ID = B.DATABASE_ID + AND B.TABLE_ID = C.MLOG_ID + AND B.TABLE_TYPE = 15 + AND B.DATA_TABLE_ID = D.TABLE_ID + AND A.TENANT_ID = EFFECTIVE_TENANT_ID() + AND B.TENANT_ID = EFFECTIVE_TENANT_ID() + AND C.TENANT_ID = EFFECTIVE_TENANT_ID() + AND D.TENANT_ID = EFFECTIVE_TENANT_ID() + AND (A.DATABASE_ID = USERENV('SCHEMAID') + OR USER_CAN_ACCESS_OBJ(1, B.TABLE_ID, B.DATABASE_ID) = 1) +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'USER_MVIEW_LOGS', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '25285', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS LOG_OWNER, + CAST(D.TABLE_NAME AS VARCHAR2(128)) AS MASTER, + CAST(B.TABLE_NAME AS VARCHAR2(128)) AS LOG_TABLE, + CAST(NULL AS VARCHAR2(128)) AS LOG_TRIGGER, + CAST(DECODE(bitand(D.TABLE_MODE, 66048), 66048, 'YES', 'NO') AS VARCHAR2(3)) AS ROWIDS, + CAST(DECODE(bitand(D.TABLE_MODE, 66048), 0, 'YES', 'NO') AS VARCHAR2(3)) AS PRIMARY_KEY, + CAST('NO' AS VARCHAR2(3)) AS OBJECT_ID, + CAST( + DECODE(( + SELECT COUNT(*) + FROM SYS.ALL_VIRTUAL_COLUMN_REAL_AGENT C1, + SYS.ALL_VIRTUAL_COLUMN_REAL_AGENT C2 + WHERE B.TABLE_ID = C1.TABLE_ID + AND C1.COLUMN_ID >= 16 + AND C1.COLUMN_ID < 65520 + AND D.TABLE_ID = C2.TABLE_ID + AND C2.ROWKEY_POSITION != 0 + AND C1.COLUMN_ID != C2.COLUMN_ID + AND C1.TENANT_ID = EFFECTIVE_TENANT_ID() + AND C2.TENANT_ID = EFFECTIVE_TENANT_ID() + ), 0, 'NO', 'YES') AS VARCHAR2(3) + ) AS FILTER_COLUMNS, + CAST('YES' AS VARCHAR2(3)) AS SEQUENCE, + CAST('YES' AS VARCHAR2(3)) AS INCLUDE_NEW_VALUES, + CAST(DECODE(C.PURGE_MODE, 1, 'YES', 'NO') AS VARCHAR2(3)) AS PURGE_ASYNCHRONOUS, + CAST(DECODE(C.PURGE_MODE, 2, 'YES', 'NO') AS VARCHAR2(3)) AS PURGE_DEFERRED, + CAST(C.PURGE_START AS DATE) AS PURGE_START /* TODO: DD-MON-YYYY */, + CAST(C.PURGE_NEXT AS VARCHAR2(200)) AS PURGE_INTERVAL, + CAST(C.LAST_PURGE_DATE AS DATE) AS LAST_PURGE_DATE /* TODO: DD-MON-YYYY */, + CAST(0 AS NUMBER) AS LAST_PURGE_STATUS, + CAST(C.LAST_PURGE_ROWS AS NUMBER) AS NUM_ROWS_PURGED, + CAST('YES' AS VARCHAR2(3)) AS COMMIT_SCN_BASED, + CAST('NO' AS VARCHAR2(3)) AS STAGING_LOG + FROM + SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, + SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, + SYS.ALL_VIRTUAL_MLOG_REAL_AGENT C, + SYS.ALL_VIRTUAL_TABLE_REAL_AGENT D + WHERE A.DATABASE_ID = B.DATABASE_ID + AND B.TABLE_ID = C.MLOG_ID + AND B.TABLE_TYPE = 15 + AND B.DATA_TABLE_ID = D.TABLE_ID + AND A.TENANT_ID = EFFECTIVE_TENANT_ID() + AND B.TENANT_ID = EFFECTIVE_TENANT_ID() + AND C.TENANT_ID = EFFECTIVE_TENANT_ID() + AND D.TENANT_ID = EFFECTIVE_TENANT_ID() + AND A.DATABASE_NAME = SYS_CONTEXT('USERENV','CURRENT_USER') +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'DBA_MVIEWS', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '25286', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS OWNER, + CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MVIEW_NAME, + CAST(D.TABLE_NAME AS VARCHAR2(128)) AS CONTAINER_NAME, + B.VIEW_DEFINITION AS QUERY /* TODO: LONG */, + CAST(LENGTH(B.VIEW_DEFINITION) AS NUMBER) AS QUERY_LEN, + CAST('N' AS VARCHAR2(1)) AS UPDATABLE, + CAST(NULL AS VARCHAR2(128)) AS UPDATE_LOG, + CAST(NULL AS VARCHAR2(128)) AS MASTER_ROLLBACK_SEG, + CAST(NULL AS VARCHAR2(128)) AS MASTER_LINK, + CAST('N' AS VARCHAR2(1)) AS REWRITE_ENABLED, + CAST(NULL AS VARCHAR2(9)) AS REWRITE_CAPABILITY, + CAST( + DECODE(C.REFRESH_MODE, 0, 'NEVER', + 1, 'DEMAND', + 2, 'COMMIT', + 3, 'STATEMENT', + NULL + ) AS VARCHAR2(6) + ) AS REFRESH_MODE, + CAST( + DECODE(C.REFRESH_METHOD, 0, 'NEVER', + 1, 'COMPLETE', + 2, 'FAST', + 3, 'FORCE', + NULL + + ) AS VARCHAR2(8) + ) AS REFRESH_METHOD, + CAST( + DECODE(C.BUILD_MODE, 0, 'IMMEDIATE', + 1, 'DEFERRED', + 2, 'PERBUILT', + NULL + ) AS VARCHAR2(9) + ) AS BUILD_MODE, + CAST(NULL AS VARCHAR2(18)) AS FAST_REFRESHABLE, + CAST( + DECODE(C.LAST_REFRESH_TYPE, 0, 'COMPLETE', + 1, 'FAST', + 'NA' + ) AS VARCHAR2(8) + ) AS LAST_REFRESH_TYPE, + CAST(C.LAST_REFRESH_DATE AS DATE) AS LAST_REFRESH_DATE /* TODO: DD-MON-YYYY */, + CAST(C.LAST_REFRESH_DATE + C.LAST_REFRESH_TIME * 1000 * 1000 AS DATE) AS LAST_REFRESH_END_TIME /* TODO: DD-MON-YYYY */, + CAST(NULL AS VARCHAR2(19)) AS STALENESS, + CAST(NULL AS VARCHAR2(19)) AS AFTER_FAST_REFRESH, + CAST(DECODE(C.BUILD_MODE, 2, 'Y', 'N') AS VARCHAR2(1)) AS UNKNOWN_PREBUILT, + CAST('N' AS VARCHAR2(1)) AS UNKNOWN_PLSQL_FUNC, + CAST('N' AS VARCHAR2(1)) AS UNKNOWN_EXTERNAL_TABLE, + CAST('N' AS VARCHAR2(1)) AS UNKNOWN_CONSIDER_FRESH, + CAST('N' AS VARCHAR2(1)) AS UNKNOWN_IMPORT, + CAST('N' AS VARCHAR2(1)) AS UNKNOWN_TRUSTED_FD, + CAST(NULL AS VARCHAR2(19)) AS COMPILE_STATE, + CAST('Y' AS VARCHAR2(1)) AS USE_NO_INDEX, + CAST(NULL AS DATE) AS STALE_SINCE, + CAST(NULL AS NUMBER) AS NUM_PCT_TABLES, + CAST(NULL AS NUMBER) AS NUM_FRESH_PCT_REGIONS, + CAST(NULL AS NUMBER) AS NUM_STALE_PCT_REGIONS, + CAST('NO' AS VARCHAR2(3)) AS SEGMENT_CREATED, + CAST(NULL AS VARCHAR2(128)) AS EVALUATION_EDITION, + CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEFORE, + CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEGINNING, + CAST(NULL AS VARCHAR2(100)) AS DEFAULT_COLLATION, + CAST('N' AS VARCHAR2(1)) AS ON_QUERY_COMPUTATION + FROM + SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, + SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, + SYS.ALL_VIRTUAL_MVIEW_REAL_AGENT C, + SYS.ALL_VIRTUAL_TABLE_REAL_AGENT D + WHERE A.DATABASE_ID = B.DATABASE_ID + AND B.TABLE_ID = C.MVIEW_ID + AND B.TABLE_TYPE = 7 + AND B.DATA_TABLE_ID = D.TABLE_ID + AND A.TENANT_ID = EFFECTIVE_TENANT_ID() + AND B.TENANT_ID = EFFECTIVE_TENANT_ID() + AND C.TENANT_ID = EFFECTIVE_TENANT_ID() + AND D.TENANT_ID = EFFECTIVE_TENANT_ID() +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'ALL_MVIEWS', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '25287', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS OWNER, + CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MVIEW_NAME, + CAST(D.TABLE_NAME AS VARCHAR2(128)) AS CONTAINER_NAME, + B.VIEW_DEFINITION AS QUERY /* TODO: LONG */, + CAST(LENGTH(B.VIEW_DEFINITION) AS NUMBER) AS QUERY_LEN, + CAST('N' AS VARCHAR2(1)) AS UPDATABLE, + CAST(NULL AS VARCHAR2(128)) AS UPDATE_LOG, + CAST(NULL AS VARCHAR2(128)) AS MASTER_ROLLBACK_SEG, + CAST(NULL AS VARCHAR2(128)) AS MASTER_LINK, + CAST('N' AS VARCHAR2(1)) AS REWRITE_ENABLED, + CAST(NULL AS VARCHAR2(9)) AS REWRITE_CAPABILITY, + CAST( + DECODE(C.REFRESH_MODE, 0, 'NEVER', + 1, 'DEMAND', + 2, 'COMMIT', + 3, 'STATEMENT', + NULL + ) AS VARCHAR2(6) + ) AS REFRESH_MODE, + CAST( + DECODE(C.REFRESH_METHOD, 0, 'NEVER', + 1, 'COMPLETE', + 2, 'FAST', + 3, 'FORCE', + NULL + + ) AS VARCHAR2(8) + ) AS REFRESH_METHOD, + CAST( + DECODE(C.BUILD_MODE, 0, 'IMMEDIATE', + 1, 'DEFERRED', + 2, 'PERBUILT', + NULL + ) AS VARCHAR2(9) + ) AS BUILD_MODE, + CAST(NULL AS VARCHAR2(18)) AS FAST_REFRESHABLE, + CAST( + DECODE(C.LAST_REFRESH_TYPE, 0, 'COMPLETE', + 1, 'FAST', + 'NA' + ) AS VARCHAR2(8) + ) AS LAST_REFRESH_TYPE, + CAST(C.LAST_REFRESH_DATE AS DATE) AS LAST_REFRESH_DATE /* TODO: DD-MON-YYYY */, + CAST(C.LAST_REFRESH_DATE + C.LAST_REFRESH_TIME * 1000 * 1000 AS DATE) AS LAST_REFRESH_END_TIME /* TODO: DD-MON-YYYY */, + CAST(NULL AS VARCHAR2(19)) AS STALENESS, + CAST(NULL AS VARCHAR2(19)) AS AFTER_FAST_REFRESH, + CAST(DECODE(C.BUILD_MODE, 2, 'Y', 'N') AS VARCHAR2(1)) AS UNKNOWN_PREBUILT, + CAST('N' AS VARCHAR2(1)) AS UNKNOWN_PLSQL_FUNC, + CAST('N' AS VARCHAR2(1)) AS UNKNOWN_EXTERNAL_TABLE, + CAST('N' AS VARCHAR2(1)) AS UNKNOWN_CONSIDER_FRESH, + CAST('N' AS VARCHAR2(1)) AS UNKNOWN_IMPORT, + CAST('N' AS VARCHAR2(1)) AS UNKNOWN_TRUSTED_FD, + CAST(NULL AS VARCHAR2(19)) AS COMPILE_STATE, + CAST('Y' AS VARCHAR2(1)) AS USE_NO_INDEX, + CAST(NULL AS DATE) AS STALE_SINCE, + CAST(NULL AS NUMBER) AS NUM_PCT_TABLES, + CAST(NULL AS NUMBER) AS NUM_FRESH_PCT_REGIONS, + CAST(NULL AS NUMBER) AS NUM_STALE_PCT_REGIONS, + CAST('NO' AS VARCHAR2(3)) AS SEGMENT_CREATED, + CAST(NULL AS VARCHAR2(128)) AS EVALUATION_EDITION, + CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEFORE, + CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEGINNING, + CAST(NULL AS VARCHAR2(100)) AS DEFAULT_COLLATION, + CAST('N' AS VARCHAR2(1)) AS ON_QUERY_COMPUTATION + FROM + SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, + SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, + SYS.ALL_VIRTUAL_MVIEW_REAL_AGENT C, + SYS.ALL_VIRTUAL_TABLE_REAL_AGENT D + WHERE A.DATABASE_ID = B.DATABASE_ID + AND B.TABLE_ID = C.MVIEW_ID + AND B.TABLE_TYPE = 7 + AND B.DATA_TABLE_ID = D.TABLE_ID + AND A.TENANT_ID = EFFECTIVE_TENANT_ID() + AND B.TENANT_ID = EFFECTIVE_TENANT_ID() + AND C.TENANT_ID = EFFECTIVE_TENANT_ID() + AND D.TENANT_ID = EFFECTIVE_TENANT_ID() + AND (A.DATABASE_ID = USERENV('SCHEMAID') + OR USER_CAN_ACCESS_OBJ(1, B.TABLE_ID, B.DATABASE_ID) = 1) +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'USER_MVIEWS', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '25288', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS OWNER, + CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MVIEW_NAME, + CAST(D.TABLE_NAME AS VARCHAR2(128)) AS CONTAINER_NAME, + B.VIEW_DEFINITION AS QUERY /* TODO: LONG */, + CAST(LENGTH(B.VIEW_DEFINITION) AS NUMBER) AS QUERY_LEN, + CAST('N' AS VARCHAR2(1)) AS UPDATABLE, + CAST(NULL AS VARCHAR2(128)) AS UPDATE_LOG, + CAST(NULL AS VARCHAR2(128)) AS MASTER_ROLLBACK_SEG, + CAST(NULL AS VARCHAR2(128)) AS MASTER_LINK, + CAST('N' AS VARCHAR2(1)) AS REWRITE_ENABLED, + CAST(NULL AS VARCHAR2(9)) AS REWRITE_CAPABILITY, + CAST( + DECODE(C.REFRESH_MODE, 0, 'NEVER', + 1, 'DEMAND', + 2, 'COMMIT', + 3, 'STATEMENT', + NULL + ) AS VARCHAR2(6) + ) AS REFRESH_MODE, + CAST( + DECODE(C.REFRESH_METHOD, 0, 'NEVER', + 1, 'COMPLETE', + 2, 'FAST', + 3, 'FORCE', + NULL + + ) AS VARCHAR2(8) + ) AS REFRESH_METHOD, + CAST( + DECODE(C.BUILD_MODE, 0, 'IMMEDIATE', + 1, 'DEFERRED', + 2, 'PERBUILT', + NULL + ) AS VARCHAR2(9) + ) AS BUILD_MODE, + CAST(NULL AS VARCHAR2(18)) AS FAST_REFRESHABLE, + CAST( + DECODE(C.LAST_REFRESH_TYPE, 0, 'COMPLETE', + 1, 'FAST', + 'NA' + ) AS VARCHAR2(8) + ) AS LAST_REFRESH_TYPE, + CAST(C.LAST_REFRESH_DATE AS DATE) AS LAST_REFRESH_DATE /* TODO: DD-MON-YYYY */, + CAST(C.LAST_REFRESH_DATE + C.LAST_REFRESH_TIME * 1000 * 1000 AS DATE) AS LAST_REFRESH_END_TIME /* TODO: DD-MON-YYYY */, + CAST(NULL AS VARCHAR2(19)) AS STALENESS, + CAST(NULL AS VARCHAR2(19)) AS AFTER_FAST_REFRESH, + CAST(DECODE(C.BUILD_MODE, 2, 'Y', 'N') AS VARCHAR2(1)) AS UNKNOWN_PREBUILT, + CAST('N' AS VARCHAR2(1)) AS UNKNOWN_PLSQL_FUNC, + CAST('N' AS VARCHAR2(1)) AS UNKNOWN_EXTERNAL_TABLE, + CAST('N' AS VARCHAR2(1)) AS UNKNOWN_CONSIDER_FRESH, + CAST('N' AS VARCHAR2(1)) AS UNKNOWN_IMPORT, + CAST('N' AS VARCHAR2(1)) AS UNKNOWN_TRUSTED_FD, + CAST(NULL AS VARCHAR2(19)) AS COMPILE_STATE, + CAST('Y' AS VARCHAR2(1)) AS USE_NO_INDEX, + CAST(NULL AS DATE) AS STALE_SINCE, + CAST(NULL AS NUMBER) AS NUM_PCT_TABLES, + CAST(NULL AS NUMBER) AS NUM_FRESH_PCT_REGIONS, + CAST(NULL AS NUMBER) AS NUM_STALE_PCT_REGIONS, + CAST('NO' AS VARCHAR2(3)) AS SEGMENT_CREATED, + CAST(NULL AS VARCHAR2(128)) AS EVALUATION_EDITION, + CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEFORE, + CAST(NULL AS VARCHAR2(128)) AS UNUSABLE_BEGINNING, + CAST(NULL AS VARCHAR2(100)) AS DEFAULT_COLLATION, + CAST('N' AS VARCHAR2(1)) AS ON_QUERY_COMPUTATION + FROM + SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, + SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, + SYS.ALL_VIRTUAL_MVIEW_REAL_AGENT C, + SYS.ALL_VIRTUAL_TABLE_REAL_AGENT D + WHERE A.DATABASE_ID = B.DATABASE_ID + AND B.TABLE_ID = C.MVIEW_ID + AND B.TABLE_TYPE = 7 + AND B.DATA_TABLE_ID = D.TABLE_ID + AND A.TENANT_ID = EFFECTIVE_TENANT_ID() + AND B.TENANT_ID = EFFECTIVE_TENANT_ID() + AND C.TENANT_ID = EFFECTIVE_TENANT_ID() + AND D.TENANT_ID = EFFECTIVE_TENANT_ID() + AND A.DATABASE_NAME = SYS_CONTEXT('USERENV','CURRENT_USER') +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'DBA_MVREF_STATS_SYS_DEFAULTS', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '25289', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(PARAMETER_NAME AS CHAR(16)) AS PARAMETER_NAME, + CAST(VALUE AS VARCHAR2(40)) AS VALUE + FROM + ( + /* COLLECTION_LEVEL */ + SELECT + 'COLLECTION_LEVEL' PARAMETER_NAME, + DECODE(NVL(MAX(COLLECTION_LEVEL), 1), + 0, 'NONE', + 1, 'TYPICAL', + 2, 'ADVANCED', + NULL) VALUE + FROM + SYS.ALL_VIRTUAL_MVIEW_REFRESH_STATS_SYS_DEFAULTS_REAL_AGENT + WHERE + TENANT_ID = EFFECTIVE_TENANT_ID() + + UNION ALL + + /* RETENTION_PERIOD */ + SELECT + 'RETENTION_PERIOD' PARAMETER_NAME, + TO_CHAR(NVL(MAX(RETENTION_PERIOD), 31)) VALUE + FROM + SYS.ALL_VIRTUAL_MVIEW_REFRESH_STATS_SYS_DEFAULTS_REAL_AGENT + WHERE + TENANT_ID = EFFECTIVE_TENANT_ID() + ) +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'USER_MVREF_STATS_SYS_DEFAULTS', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '25290', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(PARAMETER_NAME AS CHAR(16)) AS PARAMETER_NAME, + CAST(VALUE AS VARCHAR2(40)) AS VALUE + FROM + ( + /* COLLECTION_LEVEL */ + SELECT + 'COLLECTION_LEVEL' PARAMETER_NAME, + DECODE(NVL(MAX(COLLECTION_LEVEL), 1), + 0, 'NONE', + 1, 'TYPICAL', + 2, 'ADVANCED', + NULL) VALUE + FROM + SYS.ALL_VIRTUAL_MVIEW_REFRESH_STATS_SYS_DEFAULTS_REAL_AGENT + WHERE + TENANT_ID = EFFECTIVE_TENANT_ID() + + UNION ALL + + /* RETENTION_PERIOD */ + SELECT + 'RETENTION_PERIOD' PARAMETER_NAME, + TO_CHAR(NVL(MAX(RETENTION_PERIOD), 31)) VALUE + FROM + SYS.ALL_VIRTUAL_MVIEW_REFRESH_STATS_SYS_DEFAULTS_REAL_AGENT + WHERE + TENANT_ID = EFFECTIVE_TENANT_ID() + ) +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'DBA_MVREF_STATS_PARAMS', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '25291', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(MV_OWNER AS VARCHAR2(128)) AS MV_OWNER, + CAST(MV_NAME AS VARCHAR2(128)) AS MV_NAME, + CAST( + DECODE(COLLECTION_LEVEL, 0, 'NONE', + 1, 'TYPICAL', + 2, 'ADVANCED', + NULL + ) AS VARCHAR2(8) + ) AS COLLECTION_LEVEL, + CAST(RETENTION_PERIOD AS NUMBER) AS RETENTION_PERIOD + FROM + ( + WITH DEFVALS AS + ( + SELECT + NVL(MAX(COLLECTION_LEVEL), 1) AS COLLECTION_LEVEL, + NVL(MAX(RETENTION_PERIOD), 31) AS RETENTION_PERIOD + FROM + SYS.ALL_VIRTUAL_MVIEW_REFRESH_STATS_SYS_DEFAULTS_REAL_AGENT + WHERE + TENANT_ID = EFFECTIVE_TENANT_ID() + ) + + SELECT + A.DATABASE_NAME MV_OWNER, + B.TABLE_NAME MV_NAME, + NVL(C.COLLECTION_LEVEL, D.COLLECTION_LEVEL) COLLECTION_LEVEL, + NVL(C.RETENTION_PERIOD, D.RETENTION_PERIOD) RETENTION_PERIOD + FROM + SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, + SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, + ( + SELECT TENANT_ID, MVIEW_ID, COLLECTION_LEVEL, RETENTION_PERIOD FROM SYS.ALL_VIRTUAL_MVIEW_REFRESH_STATS_PARAMS_REAL_AGENT + RIGHT OUTER JOIN + ( + SELECT TENANT_ID, MVIEW_ID FROM SYS.ALL_VIRTUAL_MVIEW_REAL_AGENT + ) + USING (TENANT_ID, MVIEW_ID) + ) C, + DEFVALS D + WHERE A.DATABASE_ID = B.DATABASE_ID + AND B.TABLE_ID = C.MVIEW_ID + AND B.TABLE_TYPE = 7 + AND A.TENANT_ID = EFFECTIVE_TENANT_ID() + AND B.TENANT_ID = EFFECTIVE_TENANT_ID() + AND C.TENANT_ID = EFFECTIVE_TENANT_ID() + ) +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'USER_MVREF_STATS_PARAMS', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '25292', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(MV_OWNER AS VARCHAR2(128)) AS MV_OWNER, + CAST(MV_NAME AS VARCHAR2(128)) AS MV_NAME, + CAST( + DECODE(COLLECTION_LEVEL, 0, 'NONE', + 1, 'TYPICAL', + 2, 'ADVANCED', + NULL + ) AS VARCHAR2(8) + ) AS COLLECTION_LEVEL, + CAST(RETENTION_PERIOD AS NUMBER) AS RETENTION_PERIOD + FROM + ( + WITH DEFVALS AS + ( + SELECT + NVL(MAX(COLLECTION_LEVEL), 1) AS COLLECTION_LEVEL, + NVL(MAX(RETENTION_PERIOD), 31) AS RETENTION_PERIOD + FROM + SYS.ALL_VIRTUAL_MVIEW_REFRESH_STATS_SYS_DEFAULTS_REAL_AGENT + WHERE + TENANT_ID = EFFECTIVE_TENANT_ID() + ) + + SELECT + A.DATABASE_NAME MV_OWNER, + B.TABLE_NAME MV_NAME, + NVL(C.COLLECTION_LEVEL, D.COLLECTION_LEVEL) COLLECTION_LEVEL, + NVL(C.RETENTION_PERIOD, D.RETENTION_PERIOD) RETENTION_PERIOD + FROM + SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, + SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, + ( + SELECT TENANT_ID, MVIEW_ID, COLLECTION_LEVEL, RETENTION_PERIOD FROM SYS.ALL_VIRTUAL_MVIEW_REFRESH_STATS_PARAMS_REAL_AGENT + RIGHT OUTER JOIN + ( + SELECT TENANT_ID, MVIEW_ID FROM SYS.ALL_VIRTUAL_MVIEW_REAL_AGENT + ) + USING (TENANT_ID, MVIEW_ID) + ) C, + DEFVALS D + WHERE A.DATABASE_ID = B.DATABASE_ID + AND B.TABLE_ID = C.MVIEW_ID + AND B.TABLE_TYPE = 7 + AND A.TENANT_ID = EFFECTIVE_TENANT_ID() + AND B.TENANT_ID = EFFECTIVE_TENANT_ID() + AND C.TENANT_ID = EFFECTIVE_TENANT_ID() + AND A.DATABASE_NAME = SYS_CONTEXT('USERENV','CURRENT_USER') + ) +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'DBA_MVREF_RUN_STATS', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '25293', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(A.USER_NAME AS VARCHAR2(128)) AS RUN_OWNER, + CAST(B.REFRESH_ID AS NUMBER) AS REFRESH_ID, + CAST(B.NUM_MVS_TOTAL AS NUMBER) AS NUM_MVS, + CAST(B.MVIEWS AS VARCHAR2(4000)) AS MVIEWS, + CAST(B.BASE_TABLES AS VARCHAR2(4000)) AS BASE_TABLES, + CAST(B.METHOD AS VARCHAR2(4000)) AS METHOD, + CAST(B.ROLLBACK_SEG AS VARCHAR2(4000)) AS ROLLBACK_SEG, + CAST(DECODE(B.PUSH_DEFERRED_RPC, 1, 'Y', 'N') AS CHAR(1)) AS PUSH_DEFERRED_RPC, + CAST(DECODE(B.REFRESH_AFTER_ERRORS, 1, 'Y', 'N') AS CHAR(1)) AS REFRESH_AFTER_ERRORS, + CAST(B.PURGE_OPTION AS NUMBER) AS PURGE_OPTION, + CAST(B.PARALLELISM AS NUMBER) AS PARALLELISM, + CAST(B.HEAP_SIZE AS NUMBER) AS HEAP_SIZE, + CAST(DECODE(B.ATOMIC_REFRESH, 1, 'Y', 'N') AS CHAR(1)) AS ATOMIC_REFRESH, + CAST(DECODE(B.NESTED, 1, 'Y', 'N') AS CHAR(1)) AS NESTED, + CAST(DECODE(B.OUT_OF_PLACE, 1, 'Y', 'N') AS CHAR(1)) AS OUT_OF_PLACE, + CAST(B.NUMBER_OF_FAILURES AS NUMBER) AS NUMBER_OF_FAILURES, + CAST(B.START_TIME AS TIMESTAMP(6)) AS START_TIME, + CAST(B.END_TIME AS TIMESTAMP(6)) AS END_TIME, + CAST(B.ELAPSED_TIME AS NUMBER) AS ELAPSED_TIME, + CAST(0 AS NUMBER) AS LOG_SETUP_TIME, + CAST(B.LOG_PURGE_TIME AS NUMBER) AS LOG_PURGE_TIME, + CAST(DECODE(B.COMPLETE_STATS_AVALIABLE, 1, 'Y', 'N') AS CHAR(1)) AS COMPLETE_STATS_AVAILABLE + FROM + SYS.ALL_VIRTUAL_USER_REAL_AGENT A, + SYS.ALL_VIRTUAL_MVIEW_REFRESH_RUN_STATS_REAL_AGENT B + WHERE A.USER_ID = B.RUN_USER_ID + AND A.TENANT_ID = EFFECTIVE_TENANT_ID() + AND B.TENANT_ID = EFFECTIVE_TENANT_ID() +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'USER_MVREF_RUN_STATS', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '25294', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(B.REFRESH_ID AS NUMBER) AS REFRESH_ID, + CAST(B.NUM_MVS_TOTAL AS NUMBER) AS NUM_MVS, + CAST(B.MVIEWS AS VARCHAR2(4000)) AS MVIEWS, + CAST(B.BASE_TABLES AS VARCHAR2(4000)) AS BASE_TABLES, + CAST(B.METHOD AS VARCHAR2(4000)) AS METHOD, + CAST(B.ROLLBACK_SEG AS VARCHAR2(4000)) AS ROLLBACK_SEG, + CAST(DECODE(B.PUSH_DEFERRED_RPC, 1, 'Y', 'N') AS CHAR(1)) AS PUSH_DEFERRED_RPC, + CAST(DECODE(B.REFRESH_AFTER_ERRORS, 1, 'Y', 'N') AS CHAR(1)) AS REFRESH_AFTER_ERRORS, + CAST(B.PURGE_OPTION AS NUMBER) AS PURGE_OPTION, + CAST(B.PARALLELISM AS NUMBER) AS PARALLELISM, + CAST(B.HEAP_SIZE AS NUMBER) AS HEAP_SIZE, + CAST(DECODE(B.ATOMIC_REFRESH, 1, 'Y', 'N') AS CHAR(1)) AS ATOMIC_REFRESH, + CAST(DECODE(B.NESTED, 1, 'Y', 'N') AS CHAR(1)) AS NESTED, + CAST(DECODE(B.OUT_OF_PLACE, 1, 'Y', 'N') AS CHAR(1)) AS OUT_OF_PLACE, + CAST(B.NUMBER_OF_FAILURES AS NUMBER) AS NUMBER_OF_FAILURES, + CAST(B.START_TIME AS TIMESTAMP(6)) AS START_TIME, + CAST(B.END_TIME AS TIMESTAMP(6)) AS END_TIME, + CAST(B.ELAPSED_TIME AS NUMBER) AS ELAPSED_TIME, + CAST(0 AS NUMBER) AS LOG_SETUP_TIME, + CAST(B.LOG_PURGE_TIME AS NUMBER) AS LOG_PURGE_TIME, + CAST(DECODE(B.COMPLETE_STATS_AVALIABLE, 1, 'Y', 'N') AS CHAR(1)) AS COMPLETE_STATS_AVAILABLE + FROM + SYS.ALL_VIRTUAL_USER_REAL_AGENT A, + SYS.ALL_VIRTUAL_MVIEW_REFRESH_RUN_STATS_REAL_AGENT B + WHERE A.USER_ID = B.RUN_USER_ID + AND A.TENANT_ID = EFFECTIVE_TENANT_ID() + AND B.TENANT_ID = EFFECTIVE_TENANT_ID() + AND A.USER_NAME = SYS_CONTEXT('USERENV','CURRENT_USER') +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'DBA_MVREF_STATS', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '25295', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS MV_OWNER, + CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MV_NAME, + CAST(C.REFRESH_ID AS NUMBER) AS REFRESH_ID, + CAST( + DECODE(C.REFRESH_TYPE, 0, 'COMPLETE', + 1, 'FAST', + NULL + ) AS VARCHAR2(30) + ) AS REFRESH_METHOD, + CAST(NULL AS VARCHAR2(4000)) AS REFRESH_OPTIMIZATIONS, + CAST(NULL AS VARCHAR2(4000)) AS ADDITIONAL_EXECUTIONS, + CAST(C.START_TIME AS TIMESTAMP(6)) AS START_TIME, + CAST(C.END_TIME AS TIMESTAMP(6)) AS END_TIME, + CAST(C.ELAPSED_TIME AS NUMBER) AS ELAPSED_TIME, + CAST(0 AS NUMBER) AS LOG_SETUP_TIME, + CAST(C.LOG_PURGE_TIME AS NUMBER) AS LOG_PURGE_TIME, + CAST(C.INITIAL_NUM_ROWS AS NUMBER) AS INITIAL_NUM_ROWS, + CAST(C.FINAL_NUM_ROWS AS NUMBER) AS FINAL_NUM_ROWS + FROM + SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, + SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, + SYS.ALL_VIRTUAL_MVIEW_REFRESH_STATS_REAL_AGENT C + WHERE A.DATABASE_ID = B.DATABASE_ID + AND B.TABLE_ID = C.MVIEW_ID + AND B.TABLE_TYPE = 7 + AND A.TENANT_ID = EFFECTIVE_TENANT_ID() + AND B.TENANT_ID = EFFECTIVE_TENANT_ID() + AND C.TENANT_ID = EFFECTIVE_TENANT_ID() +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'USER_MVREF_STATS', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '25296', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MV_NAME, + CAST(C.REFRESH_ID AS NUMBER) AS REFRESH_ID, + CAST( + DECODE(C.REFRESH_TYPE, 0, 'COMPLETE', + 1, 'FAST', + NULL + ) AS VARCHAR2(30) + ) AS REFRESH_METHOD, + CAST(NULL AS VARCHAR2(4000)) AS REFRESH_OPTIMIZATIONS, + CAST(NULL AS VARCHAR2(4000)) AS ADDITIONAL_EXECUTIONS, + CAST(C.START_TIME AS TIMESTAMP(6)) AS START_TIME, + CAST(C.END_TIME AS TIMESTAMP(6)) AS END_TIME, + CAST(C.ELAPSED_TIME AS NUMBER) AS ELAPSED_TIME, + CAST(0 AS NUMBER) AS LOG_SETUP_TIME, + CAST(C.LOG_PURGE_TIME AS NUMBER) AS LOG_PURGE_TIME, + CAST(C.INITIAL_NUM_ROWS AS NUMBER) AS INITIAL_NUM_ROWS, + CAST(C.FINAL_NUM_ROWS AS NUMBER) AS FINAL_NUM_ROWS + FROM + SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, + SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, + SYS.ALL_VIRTUAL_MVIEW_REFRESH_STATS_REAL_AGENT C + WHERE A.DATABASE_ID = B.DATABASE_ID + AND B.TABLE_ID = C.MVIEW_ID + AND B.TABLE_TYPE = 7 + AND A.TENANT_ID = EFFECTIVE_TENANT_ID() + AND B.TENANT_ID = EFFECTIVE_TENANT_ID() + AND C.TENANT_ID = EFFECTIVE_TENANT_ID() + AND A.DATABASE_NAME = SYS_CONTEXT('USERENV','CURRENT_USER') +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'DBA_MVREF_CHANGE_STATS', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '25297', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(C.DATABASE_NAME AS VARCHAR2(128)) AS TBL_OWNER, + CAST(D.TABLE_NAME AS VARCHAR2(128)) AS TBL_NAME, + CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS MV_OWNER, + CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MV_NAME, + CAST(E.REFRESH_ID AS NUMBER) AS REFRESH_ID, + CAST(E.NUM_ROWS_INS AS NUMBER) AS NUM_ROWS_INS, + CAST(E.NUM_ROWS_UPD AS NUMBER) AS NUM_ROWS_UPD, + CAST(E.NUM_ROWS_DEL AS NUMBER) AS NUM_ROWS_DEL, + CAST(0 AS NUMBER) AS NUM_ROWS_DL_INS, + CAST('N' AS CHAR(1)) AS PMOPS_OCCURRED, + CAST(NULL AS VARCHAR2(4000)) AS PMOP_DETAILS, + CAST(E.NUM_ROWS AS NUMBER) AS NUM_ROWS + FROM + SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, + SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, + SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT C, + SYS.ALL_VIRTUAL_TABLE_REAL_AGENT D, + SYS.ALL_VIRTUAL_MVIEW_REFRESH_CHANGE_STATS_REAL_AGENT E + WHERE A.DATABASE_ID = B.DATABASE_ID + AND C.DATABASE_ID = D.DATABASE_ID + AND E.MVIEW_ID = B.TABLE_ID + AND E.DETAIL_TABLE_ID = D.TABLE_ID + AND A.TENANT_ID = EFFECTIVE_TENANT_ID() + AND B.TENANT_ID = EFFECTIVE_TENANT_ID() + AND C.TENANT_ID = EFFECTIVE_TENANT_ID() + AND D.TENANT_ID = EFFECTIVE_TENANT_ID() + AND E.TENANT_ID = EFFECTIVE_TENANT_ID() +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'USER_MVREF_CHANGE_STATS', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '25298', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(C.DATABASE_NAME AS VARCHAR2(128)) AS TBL_OWNER, + CAST(D.TABLE_NAME AS VARCHAR2(128)) AS TBL_NAME, + CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS MV_OWNER, + CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MV_NAME, + CAST(E.REFRESH_ID AS NUMBER) AS REFRESH_ID, + CAST(E.NUM_ROWS_INS AS NUMBER) AS NUM_ROWS_INS, + CAST(E.NUM_ROWS_UPD AS NUMBER) AS NUM_ROWS_UPD, + CAST(E.NUM_ROWS_DEL AS NUMBER) AS NUM_ROWS_DEL, + CAST(0 AS NUMBER) AS NUM_ROWS_DL_INS, + CAST('N' AS CHAR(1)) AS PMOPS_OCCURRED, + CAST(NULL AS VARCHAR2(4000)) AS PMOP_DETAILS, + CAST(E.NUM_ROWS AS NUMBER) AS NUM_ROWS + FROM + SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, + SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, + SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT C, + SYS.ALL_VIRTUAL_TABLE_REAL_AGENT D, + SYS.ALL_VIRTUAL_MVIEW_REFRESH_CHANGE_STATS_REAL_AGENT E + WHERE A.DATABASE_ID = B.DATABASE_ID + AND C.DATABASE_ID = D.DATABASE_ID + AND E.MVIEW_ID = B.TABLE_ID + AND E.DETAIL_TABLE_ID = D.TABLE_ID + AND A.TENANT_ID = EFFECTIVE_TENANT_ID() + AND B.TENANT_ID = EFFECTIVE_TENANT_ID() + AND C.TENANT_ID = EFFECTIVE_TENANT_ID() + AND D.TENANT_ID = EFFECTIVE_TENANT_ID() + AND E.TENANT_ID = EFFECTIVE_TENANT_ID() + AND A.DATABASE_NAME = SYS_CONTEXT('USERENV','CURRENT_USER') +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'DBA_MVREF_STMT_STATS', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '25299', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS MV_OWNER, + CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MV_NAME, + CAST(C.REFRESH_ID AS NUMBER) AS REFRESH_ID, + CAST(C.STEP AS NUMBER) AS STEP, + CAST(C.SQLID AS VARCHAR2(32)) AS SQLID /* TODO: VARCHAR2(14) */, + C.STMT AS STMT /* TODO: CLOB */, + CAST(C.EXECUTION_TIME AS NUMBER) AS EXECUTION_TIME, + C.EXECUTION_PLAN AS EXECUTION_PLAN /* TODO: XMLTYPE STORAGE BINARY */ + FROM + SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, + SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, + SYS.ALL_VIRTUAL_MVIEW_REFRESH_STMT_STATS_REAL_AGENT C + WHERE A.DATABASE_ID = B.DATABASE_ID + AND B.TABLE_ID = C.MVIEW_ID + AND A.TENANT_ID = EFFECTIVE_TENANT_ID() + AND B.TENANT_ID = EFFECTIVE_TENANT_ID() + AND C.TENANT_ID = EFFECTIVE_TENANT_ID() +""".replace("\n", " ") +) + +def_table_schema( + owner = 'suzhi.yt', + table_name = 'USER_MVREF_STMT_STATS', + name_postfix = '_ORA', + database_id = 'OB_ORA_SYS_DATABASE_ID', + table_id = '25300', + table_type = 'SYSTEM_VIEW', + rowkey_columns = [], + normal_columns = [], + gm_columns = [], + in_tenant_space = True, + view_definition = """ + SELECT + CAST(A.DATABASE_NAME AS VARCHAR2(128)) AS MV_OWNER, + CAST(B.TABLE_NAME AS VARCHAR2(128)) AS MV_NAME, + CAST(C.REFRESH_ID AS NUMBER) AS REFRESH_ID, + CAST(C.STEP AS NUMBER) AS STEP, + CAST(C.SQLID AS VARCHAR2(14)) AS SQLID /* TODO: VARCHAR2(14) */, + C.STMT AS STMT /* TODO: CLOB */, + CAST(C.EXECUTION_TIME AS NUMBER) AS EXECUTION_TIME, + C.EXECUTION_PLAN AS EXECUTION_PLAN /* TODO: XMLTYPE STORAGE BINARY */ + FROM + SYS.ALL_VIRTUAL_DATABASE_REAL_AGENT A, + SYS.ALL_VIRTUAL_TABLE_REAL_AGENT B, + SYS.ALL_VIRTUAL_MVIEW_REFRESH_STMT_STATS_REAL_AGENT C + WHERE A.DATABASE_ID = B.DATABASE_ID + AND B.TABLE_ID = C.MVIEW_ID + AND A.TENANT_ID = EFFECTIVE_TENANT_ID() + AND B.TENANT_ID = EFFECTIVE_TENANT_ID() + AND C.TENANT_ID = EFFECTIVE_TENANT_ID() + AND A.DATABASE_NAME = SYS_CONTEXT('USERENV','CURRENT_USER') +""".replace("\n", " ") +) # 余留位置 diff --git a/src/share/ls/ob_ls_creator.cpp b/src/share/ls/ob_ls_creator.cpp index 4d5b8c3196..95116bb14b 100644 --- a/src/share/ls/ob_ls_creator.cpp +++ b/src/share/ls/ob_ls_creator.cpp @@ -434,6 +434,8 @@ int ObLSCreator::create_ls_(const ObILSAddr &addrs, common::GlobalLearnerList &learner_list) { int ret = OB_SUCCESS; + bool need_create_arb_replica = false; + int64_t arb_replica_count = 0; if (OB_UNLIKELY(!is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret)); @@ -472,20 +474,10 @@ int ObLSCreator::create_ls_(const ObILSAddr &addrs, K(arg), K(tenant_id_)); } } - //wait all - if (OB_TMP_FAIL(create_ls_proxy_.wait_all(return_code_array))) { - ret = OB_SUCC(ret) ? tmp_ret : ret; - LOG_WARN("failed to wait all async rpc", KR(ret), KR(tmp_ret)); - } - if (FAILEDx(check_create_ls_result_(paxos_replica_num, return_code_array, member_list, learner_list))) { - LOG_WARN("failed to check ls result", KR(ret), K(paxos_replica_num), K(return_code_array), K(learner_list)); - } - #ifdef OB_BUILD_ARBITRATION // try to create A-replica if needed - // (1) ignore any erros because majority of F-replica is successfully created, + // (1) ignore any erros // arb replica task will generated by back groud process later to add A-replica - bool need_create_arb_replica = false; ObAddr arbitration_service_addr; if (OB_FAIL(ret)) { } else if (OB_SUCCESS != (tmp_ret = check_need_create_arb_replica_( @@ -502,8 +494,19 @@ int ObLSCreator::create_ls_(const ObILSAddr &addrs, } else { int64_t timestamp = 1; arbitration_service = ObMember(arbitration_service_addr, timestamp); + arb_replica_count = 1; } #endif + //wait all + if (OB_TMP_FAIL(create_ls_proxy_.wait_all(return_code_array))) { + ret = OB_SUCC(ret) ? tmp_ret : ret; + LOG_WARN("failed to wait all async rpc", KR(ret), KR(tmp_ret)); + } + if (FAILEDx(check_create_ls_result_(paxos_replica_num, return_code_array, member_list, learner_list, + need_create_arb_replica, arb_replica_count))) { + LOG_WARN("failed to check ls result", KR(ret), K(paxos_replica_num), K(return_code_array), K(learner_list), + K(need_create_arb_replica), K(arb_replica_count)); + } } } return ret; @@ -610,7 +613,9 @@ int ObLSCreator::check_create_ls_result_( const int64_t paxos_replica_num, const ObIArray &return_code_array, common::ObMemberList &member_list, - common::GlobalLearnerList &learner_list) + common::GlobalLearnerList &learner_list, + const bool with_arbitration_service, + const int64_t arb_replica_num) { int ret = OB_SUCCESS; member_list.reset(); @@ -663,10 +668,18 @@ int ObLSCreator::check_create_ls_result_( } } if (OB_FAIL(ret)) { - } else if (rootserver::majority(paxos_replica_num) > member_list.get_member_number()) { - ret = OB_REPLICA_NUM_NOT_ENOUGH; - LOG_WARN("success count less than majority", KR(ret), K(paxos_replica_num), - K(member_list)); + } else if (!id_.is_sys_ls() && with_arbitration_service) { + if (rootserver::majority(paxos_replica_num/*F-replica*/ + 1/*A-replica*/) > member_list.get_member_number() + arb_replica_num) { + ret = OB_REPLICA_NUM_NOT_ENOUGH; + LOG_WARN("success count less than majority with arb-replica", KR(ret), K(paxos_replica_num), + K(member_list), K(with_arbitration_service), K(arb_replica_num)); + } + } else { + // for sys log stream and non-arb user log stream + if (rootserver::majority(paxos_replica_num) > member_list.get_member_number()) { + ret = OB_REPLICA_NUM_NOT_ENOUGH; + LOG_WARN("success count less than majority", KR(ret), K(paxos_replica_num), K(member_list)); + } } LS_EVENT_ADD(tenant_id_, id_, "create_ls", ret, paxos_replica_num, member_list); } @@ -763,23 +776,89 @@ int ObLSCreator::check_member_list_and_learner_list_all_in_meta_table_( return ret; } +int ObLSCreator::construct_paxos_replica_number_to_persist_( + const int64_t paxos_replica_num, + const int64_t arb_replica_num, + const common::ObMemberList &member_list, + int64_t &paxos_replica_number_to_persist) +{ + int ret = OB_SUCCESS; + paxos_replica_number_to_persist = 0; + + if (OB_UNLIKELY(!is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret)); + } else if (OB_UNLIKELY(!member_list.is_valid()) + || OB_UNLIKELY(0 >= paxos_replica_num)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(member_list), K(paxos_replica_num)); + } else if (!id_.is_user_ls()) { + // for sys log stream + if (member_list.get_member_number() >= rootserver::majority(paxos_replica_num)) { + // good, majority of F-replica created successfully, set paxos_replica_num equal to locality + paxos_replica_number_to_persist = paxos_replica_num; + } else { + // sys log stream needs majority of F-replicas created success + ret = OB_INVALID_ARGUMENT; + LOG_WARN("majority is not satisfied", KR(ret), K_(tenant_id), K_(id), + K(member_list), K(paxos_replica_num), K(arb_replica_num)); + } + } else { + // for user log stream + if (member_list.get_member_number() >= rootserver::majority(paxos_replica_num)) { + // good, majority of F-replica created successfully, set paxos_replica_num equal to locality + paxos_replica_number_to_persist = paxos_replica_num; +#ifdef OB_BUILD_ARBITRATION + } else if (member_list.get_member_number() + arb_replica_num >= rootserver::majority(paxos_replica_num)) { + // check majority with arb-replica for user log stream + if (OB_UNLIKELY(1 != arb_replica_num)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("arb replica number must equals to 1 after check of majority with arb_replica_num", + KR(ret), K(paxos_replica_num), K(arb_replica_num), K(member_list)); + } else if (paxos_replica_num != 2 && paxos_replica_num != 4) { + // with arb-replica, paxos_replica_number in locality should be 2F1A or 4F1A + ret = OB_STATE_NOT_MATCH; + LOG_WARN("tenant with arbitration service should has paxos_replica_num is 2 or 4", KR(ret), K(paxos_replica_num)); + } else { + // when tenant's locality is 2F1A, but only 1F1A created successfully, paxos_replica_number should set 1 + // when tenant's locality is 4F1A, but only 2F1A created successfully, paxos_replica_number should set 3 + paxos_replica_number_to_persist = paxos_replica_num - 1; + } +#endif + } else { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("majority is not satisfied", KR(ret), K_(tenant_id), K_(id), + K(member_list), K(paxos_replica_num), K(arb_replica_num)); + } + } + return ret; +} + int ObLSCreator::set_member_list_(const common::ObMemberList &member_list, - const common::ObMember &arbitration_service, + const common::ObMember &arb_replica, const int64_t paxos_replica_num, const common::GlobalLearnerList &learner_list) { int ret = OB_SUCCESS; ObArray server_list; int tmp_ret = OB_SUCCESS; + int64_t paxos_replica_number_to_persist = 0; + int64_t arb_replica_count = arb_replica.is_valid() ? 1 : 0; if (OB_UNLIKELY(!is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret)); - } else if (OB_UNLIKELY(!member_list.is_valid() - || member_list.get_member_number() < rootserver::majority(paxos_replica_num))) { + } else if (OB_UNLIKELY(!member_list.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", KR(ret), K(member_list), K(paxos_replica_num)); + LOG_WARN("invalid argument", KR(ret), K(member_list)); } else if (!is_sys_tenant(tenant_id_) && OB_FAIL(check_member_list_and_learner_list_all_in_meta_table_(member_list, learner_list))) { LOG_WARN("fail to check member_list all in meta table", KR(ret), K(member_list), K(learner_list), K_(tenant_id), K_(id)); + } else if (OB_FAIL(construct_paxos_replica_number_to_persist_( + paxos_replica_num, + arb_replica_count, + member_list, + paxos_replica_number_to_persist))) { + LOG_WARN("fail to construct paxos replica number to set", KR(ret), K_(tenant_id), K_(id), + K(paxos_replica_num), K(arb_replica_count), K(member_list)); } else { ObTimeoutCtx ctx; if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, GCONF.rpc_timeout))) { @@ -789,9 +868,9 @@ int ObLSCreator::set_member_list_(const common::ObMemberList &member_list, for (int64_t i = 0; OB_SUCC(ret) && i < member_list.get_member_number(); ++i) { ObAddr addr; ObSetMemberListArgV2 arg; - if (OB_FAIL(arg.init(tenant_id_, id_, paxos_replica_num, member_list, arbitration_service, learner_list))) { + if (OB_FAIL(arg.init(tenant_id_, id_, paxos_replica_number_to_persist, member_list, arb_replica, learner_list))) { LOG_WARN("failed to init set member list arg", KR(ret), K_(id), K_(tenant_id), - K(paxos_replica_num), K(member_list), K(arbitration_service), K(learner_list)); + K(paxos_replica_number_to_persist), K(member_list), K(arb_replica), K(learner_list)); } else if (OB_FAIL(member_list.get_server_by_index(i, addr))) { LOG_WARN("failed to get member by index", KR(ret), K(i), K(member_list)); } else if (OB_TMP_FAIL(set_member_list_proxy_.call(addr, ctx.get_timeout(), @@ -808,7 +887,7 @@ int ObLSCreator::set_member_list_(const common::ObMemberList &member_list, LOG_WARN("failed to wait all async rpc", KR(ret), KR(tmp_ret)); } - if (FAILEDx(check_set_memberlist_result_(return_code_array, paxos_replica_num))) { + if (FAILEDx(check_set_memberlist_result_(return_code_array, paxos_replica_number_to_persist))) { LOG_WARN("failed to check set member liset result", KR(ret), K(paxos_replica_num), K(return_code_array)); } diff --git a/src/share/ls/ob_ls_creator.h b/src/share/ls/ob_ls_creator.h index 8652461fed..a65c6c85b2 100644 --- a/src/share/ls/ob_ls_creator.h +++ b/src/share/ls/ob_ls_creator.h @@ -125,8 +125,13 @@ private: int check_member_list_and_learner_list_all_in_meta_table_( const common::ObMemberList &member_list, const common::GlobalLearnerList &learner_list); + int construct_paxos_replica_number_to_persist_( + const int64_t paxos_replica_num, + const int64_t arb_replica_num, + const common::ObMemberList &member_list, + int64_t &paxos_replica_number_to_persist); int set_member_list_(const common::ObMemberList &member_list, - const common::ObMember &arbitration_service, + const common::ObMember &arb_replica, const int64_t paxos_replica_num, const common::GlobalLearnerList &learner_list); #ifdef OB_BUILD_ARBITRATION @@ -172,7 +177,9 @@ private: int check_create_ls_result_(const int64_t paxos_replica_num, const ObIArray &return_code_array, common::ObMemberList &member_list, - common::GlobalLearnerList &learner_list); + common::GlobalLearnerList &learner_list, + const bool with_arbitration_service, + const int64_t arb_replica_num); int check_set_memberlist_result_(const ObIArray &return_code_array, const int64_t paxos_replica_num); diff --git a/src/share/ob_debug_sync_point.h b/src/share/ob_debug_sync_point.h index e23430b649..c66bc6aea1 100755 --- a/src/share/ob_debug_sync_point.h +++ b/src/share/ob_debug_sync_point.h @@ -574,6 +574,7 @@ class ObString; ACT(REPLAY_SWITCH_TO_FOLLOWER_BEFORE_PUSH_SUBMIT_TASK,)\ ACT(BEFORE_CHECK_TTL_TASK_FINISH,)\ ACT(BEFORE_TTL_SCHEDULER_RUN,)\ + ACT(BEFORE_MERGE_BACKUP_META_INFO,)\ ACT(MAX_DEBUG_SYNC_POINT,) DECLARE_ENUM(ObDebugSyncPoint, debug_sync_point, OB_DEBUG_SYNC_POINT_DEF); diff --git a/src/share/ob_errno.cpp b/src/share/ob_errno.cpp index 7f682cd63f..6138449eb6 100644 --- a/src/share/ob_errno.cpp +++ b/src/share/ob_errno.cpp @@ -17739,11 +17739,11 @@ static const _error _error_OB_ERR_DBLINK_NO_LIB = { .error_solution = "Contact OceanBase Support", .mysql_errno = -1, .sqlstate = "HY000", - .str_error = "oci lib not founded", - .str_user_error = "oci lib not founded", + .str_error = "can not find the expected version of OCI LIB: %.*s", + .str_user_error = "can not find the expected version of OCI LIB: %.*s", .oracle_errno = 600, - .oracle_str_error = "ORA-00600: internal error code, arguments: -5976, oci lib not founded", - .oracle_str_user_error = "ORA-00600: internal error code, arguments: -5976, oci lib not founded" + .oracle_str_error = "ORA-00600: internal error code, arguments: -5976, can not find the expected version of OCI LIB: %.*s", + .oracle_str_user_error = "ORA-00600: internal error code, arguments: -5976, can not find the expected version of OCI LIB: %.*s" }; static const _error _error_OB_ERR_PARTITION_EXTENDED_ON_VIEW = { .error_name = "OB_ERR_PARTITION_EXTENDED_ON_VIEW", @@ -26049,6 +26049,18 @@ static const _error _error_OB_ERR_TIME_EARLIER_THAN_SYSDATE = { .oracle_str_error = "ORA-23420: the parameter must evaluate to a time in the future", .oracle_str_user_error = "ORA-23420: the parameter %s must evaluate to a time in the future" }; +static const _error _error_OB_ERR_NOT_FUNC_NAME = { + .error_name = "OB_ERR_NOT_FUNC_NAME", + .error_cause = "Internal Error", + .error_solution = "Contact OceanBase Support", + .mysql_errno = -1, + .sqlstate = "HY000", + .str_error = "object 'string' must be of type function or array to be used this way", + .str_user_error = "object '%.*s' must be of type function or array to be used this way", + .oracle_errno = 224, + .oracle_str_error = "PLS-00224: object 'string' must be of type function or array to be used this way", + .oracle_str_user_error = "PLS-00224: object '%.*s' must be of type function or array to be used this way" +}; static const _error _error_OB_ERR_KV_GLOBAL_INDEX_ROUTE = { .error_name = "OB_ERR_KV_GLOBAL_INDEX_ROUTE", .error_cause = "Internal Error", @@ -28717,6 +28729,7 @@ struct ObStrErrorInit _errors[-OB_ERR_CLIENT_LOCAL_FILES_DISABLED] = &_error_OB_ERR_CLIENT_LOCAL_FILES_DISABLED; _errors[-OB_ERR_OUT_PARAM_NOT_BIND_VAR] = &_error_OB_ERR_OUT_PARAM_NOT_BIND_VAR; _errors[-OB_ERR_TIME_EARLIER_THAN_SYSDATE] = &_error_OB_ERR_TIME_EARLIER_THAN_SYSDATE; + _errors[-OB_ERR_NOT_FUNC_NAME] = &_error_OB_ERR_NOT_FUNC_NAME; _errors[-OB_ERR_KV_GLOBAL_INDEX_ROUTE] = &_error_OB_ERR_KV_GLOBAL_INDEX_ROUTE; _errors[-OB_TTL_NOT_ENABLE] = &_error_OB_TTL_NOT_ENABLE; _errors[-OB_TTL_COLUMN_NOT_EXIST] = &_error_OB_TTL_COLUMN_NOT_EXIST; @@ -28792,7 +28805,7 @@ namespace oceanbase { namespace common { -int g_all_ob_errnos[2209] = {0, -4000, -4001, -4002, -4003, -4004, -4005, -4006, -4007, -4008, -4009, -4010, -4011, -4012, -4013, -4014, -4015, -4016, -4017, -4018, -4019, -4020, -4021, -4022, -4023, -4024, -4025, -4026, -4027, -4028, -4029, -4030, -4031, -4032, -4033, -4034, -4035, -4036, -4037, -4038, -4039, -4041, -4042, -4043, -4044, -4045, -4046, -4047, -4048, -4049, -4050, -4051, -4052, -4053, -4054, -4055, -4057, -4058, -4060, -4061, -4062, -4063, -4064, -4065, -4066, -4067, -4068, -4070, -4071, -4072, -4073, -4074, -4075, -4076, -4077, -4078, -4080, -4081, -4084, -4085, -4090, -4097, -4098, -4099, -4100, -4101, -4102, -4103, -4104, -4105, -4106, -4107, -4108, -4109, -4110, -4111, -4112, -4113, -4114, -4115, -4116, -4117, -4118, -4119, -4120, -4121, -4122, -4123, -4124, -4125, -4126, -4127, -4128, -4133, -4138, -4139, -4142, -4143, -4144, -4146, -4147, -4149, -4150, -4151, -4152, -4153, -4154, -4155, -4156, -4157, -4158, -4159, -4160, -4161, -4162, -4163, -4164, -4165, -4166, -4167, -4168, -4169, -4170, -4171, -4172, -4173, -4174, -4175, -4176, -4177, -4178, -4179, -4180, -4181, -4182, -4183, -4184, -4185, -4186, -4187, -4188, -4189, -4190, -4191, -4192, -4200, -4201, -4204, -4205, -4206, -4207, -4208, -4209, -4210, -4211, -4212, -4213, -4214, -4215, -4216, -4217, -4218, -4219, -4220, -4221, -4222, -4223, -4224, -4225, -4226, -4227, -4228, -4229, -4230, -4231, -4232, -4233, -4234, -4235, -4236, -4237, -4238, -4239, -4240, -4241, -4242, -4243, -4244, -4245, -4246, -4247, -4248, -4249, -4250, -4251, -4252, -4253, -4254, -4255, -4256, -4257, -4258, -4260, -4261, -4262, -4263, -4264, -4265, -4266, -4267, -4268, -4269, -4270, -4271, -4273, -4274, -4275, -4276, -4277, -4278, -4279, -4280, -4281, -4282, -4283, -4284, -4285, -4286, -4287, -4288, -4289, -4290, -4291, -4292, -4293, -4294, -4295, -4296, -4297, -4298, -4299, -4300, -4301, -4302, -4303, -4304, -4305, -4306, -4307, -4308, -4309, -4310, -4311, -4312, -4313, -4314, -4315, -4316, -4317, -4318, -4319, -4320, -4321, -4322, -4323, -4324, -4325, -4326, -4327, -4328, -4329, -4330, -4331, -4332, -4333, -4334, -4335, -4336, -4337, -4338, -4339, -4340, -4341, -4342, -4343, -4344, -4345, -4346, -4347, -4348, -4349, -4350, -4351, -4352, -4353, -4354, -4355, -4356, -4357, -4358, -4359, -4360, -4361, -4362, -4363, -4364, -4365, -4366, -4367, -4368, -4369, -4370, -4371, -4372, -4373, -4374, -4375, -4376, -4377, -4378, -4379, -4380, -4381, -4382, -4383, -4385, -4386, -4387, -4388, -4389, -4390, -4391, -4392, -4393, -4394, -4395, -4396, -4397, -4398, -4399, -4400, -4401, -4402, -4403, -4505, -4507, -4510, -4512, -4515, -4517, -4518, -4519, -4523, -4524, -4525, -4526, -4527, -4528, -4529, -4530, -4531, -4532, -4533, -4537, -4538, -4539, -4540, -4541, -4542, -4543, -4544, -4545, -4546, -4547, -4548, -4549, -4550, -4551, -4552, -4553, -4554, -4600, -4601, -4602, -4603, -4604, -4605, -4606, -4607, -4608, -4609, -4610, -4611, -4613, -4614, -4615, -4620, -4621, -4622, -4623, -4624, -4625, -4626, -4628, -4629, -4630, -4631, -4632, -4633, -4634, -4636, -4637, -4638, -4639, -4640, -4641, -4642, -4643, -4644, -4645, -4646, -4647, -4648, -4649, -4650, -4651, -4652, -4653, -4654, -4655, -4656, -4657, -4658, -4659, -4660, -4661, -4662, -4663, -4664, -4665, -4666, -4667, -4668, -4669, -4670, -4671, -4672, -4673, -4674, -4675, -4676, -4677, -4678, -4679, -4680, -4681, -4682, -4683, -4684, -4685, -4686, -4687, -4688, -4689, -4690, -4691, -4692, -4693, -4694, -4695, -4696, -4697, -4698, -4699, -4700, -4701, -4702, -4703, -4704, -4705, -4706, -4707, -4708, -4709, -4710, -4711, -4712, -4713, -4714, -4715, -4716, -4717, -4718, -4719, -4720, -4721, -4722, -4723, -4724, -4725, -4726, -4727, -4728, -4729, -4730, -4731, -4732, -4733, -4734, -4735, -4736, -4737, -4738, -4739, -4740, -4741, -4742, -4743, -4744, -4745, -4746, -4747, -4748, -4749, -4750, -4751, -4752, -4753, -4754, -4755, -4756, -4757, -4758, -4759, -4760, -4761, -4762, -4763, -4764, -4765, -4766, -4767, -4768, -4769, -4770, -4771, -5000, -5001, -5002, -5003, -5006, -5007, -5008, -5010, -5011, -5012, -5014, -5015, -5016, -5017, -5018, -5019, -5020, -5022, -5023, -5024, -5025, -5026, -5027, -5028, -5029, -5030, -5031, -5032, -5034, -5035, -5036, -5037, -5038, -5039, -5040, -5041, -5042, -5043, -5044, -5046, -5047, -5050, -5051, -5052, -5053, -5054, -5055, -5056, -5057, -5058, -5059, -5061, -5063, -5064, -5065, -5066, -5067, -5068, -5069, -5070, -5071, -5072, -5073, -5074, -5080, -5081, -5083, -5084, -5085, -5086, -5087, -5088, -5089, -5090, -5091, -5092, -5093, -5094, -5095, -5096, -5097, -5098, -5099, -5100, -5101, -5102, -5103, -5104, -5105, -5106, -5107, -5108, -5109, -5110, -5111, -5112, -5113, -5114, -5115, -5116, -5117, -5118, -5119, -5120, -5121, -5122, -5123, -5124, -5125, -5130, -5131, -5133, -5134, -5135, -5136, -5137, -5138, -5139, -5140, -5142, -5143, -5144, -5145, -5146, -5147, -5148, -5149, -5150, -5151, -5153, -5154, -5155, -5156, -5157, -5158, -5159, -5160, -5161, -5162, -5163, -5164, -5165, -5166, -5167, -5168, -5169, -5170, -5171, -5172, -5173, -5174, -5175, -5176, -5177, -5178, -5179, -5180, -5181, -5182, -5183, -5184, -5185, -5187, -5188, -5189, -5190, -5191, -5192, -5193, -5194, -5195, -5196, -5197, -5198, -5199, -5200, -5201, -5202, -5203, -5204, -5205, -5206, -5207, -5208, -5209, -5210, -5211, -5212, -5213, -5214, -5215, -5216, -5217, -5218, -5219, -5220, -5221, -5222, -5223, -5224, -5225, -5226, -5227, -5228, -5229, -5230, -5231, -5233, -5234, -5235, -5236, -5237, -5238, -5239, -5240, -5241, -5242, -5243, -5244, -5245, -5246, -5247, -5248, -5249, -5250, -5251, -5252, -5253, -5254, -5255, -5256, -5257, -5258, -5259, -5260, -5261, -5262, -5263, -5264, -5265, -5266, -5267, -5268, -5269, -5270, -5271, -5272, -5273, -5274, -5275, -5276, -5277, -5278, -5279, -5280, -5281, -5282, -5283, -5284, -5285, -5286, -5287, -5288, -5289, -5290, -5291, -5292, -5293, -5294, -5295, -5296, -5297, -5298, -5299, -5300, -5301, -5302, -5303, -5304, -5305, -5306, -5307, -5308, -5309, -5310, -5311, -5312, -5313, -5314, -5315, -5316, -5317, -5318, -5319, -5320, -5321, -5322, -5323, -5324, -5325, -5326, -5327, -5328, -5329, -5330, -5331, -5332, -5333, -5334, -5335, -5336, -5337, -5338, -5339, -5340, -5341, -5342, -5343, -5344, -5345, -5346, -5347, -5348, -5349, -5350, -5351, -5352, -5353, -5354, -5355, -5356, -5357, -5358, -5359, -5360, -5361, -5362, -5363, -5364, -5365, -5366, -5367, -5368, -5369, -5370, -5371, -5372, -5373, -5374, -5375, -5376, -5377, -5378, -5379, -5380, -5381, -5382, -5383, -5384, -5385, -5386, -5387, -5388, -5389, -5400, -5401, -5402, -5403, -5404, -5405, -5406, -5407, -5408, -5409, -5410, -5411, -5412, -5413, -5414, -5415, -5416, -5417, -5418, -5419, -5420, -5421, -5422, -5423, -5424, -5425, -5426, -5427, -5428, -5429, -5430, -5431, -5432, -5433, -5434, -5435, -5436, -5437, -5438, -5439, -5440, -5441, -5442, -5443, -5444, -5445, -5446, -5447, -5448, -5449, -5450, -5451, -5452, -5453, -5454, -5455, -5456, -5457, -5458, -5459, -5460, -5461, -5462, -5463, -5464, -5465, -5466, -5467, -5468, -5469, -5470, -5471, -5472, -5473, -5474, -5475, -5476, -5477, -5478, -5479, -5480, -5481, -5482, -5483, -5484, -5485, -5486, -5487, -5488, -5489, -5490, -5491, -5492, -5493, -5494, -5495, -5496, -5497, -5498, -5499, -5500, -5501, -5502, -5503, -5504, -5505, -5506, -5507, -5508, -5509, -5510, -5511, -5512, -5513, -5514, -5515, -5516, -5517, -5540, -5541, -5542, -5543, -5544, -5545, -5546, -5547, -5548, -5549, -5550, -5551, -5552, -5553, -5554, -5555, -5556, -5557, -5558, -5559, -5560, -5561, -5562, -5563, -5564, -5565, -5566, -5567, -5568, -5569, -5570, -5571, -5572, -5573, -5574, -5575, -5576, -5577, -5578, -5579, -5580, -5581, -5582, -5583, -5584, -5585, -5586, -5587, -5588, -5589, -5590, -5591, -5592, -5593, -5594, -5595, -5596, -5597, -5598, -5599, -5600, -5601, -5602, -5603, -5604, -5605, -5607, -5608, -5609, -5610, -5611, -5612, -5613, -5614, -5615, -5616, -5617, -5618, -5619, -5620, -5621, -5622, -5623, -5624, -5625, -5626, -5627, -5628, -5629, -5630, -5631, -5632, -5633, -5634, -5635, -5636, -5637, -5638, -5639, -5640, -5641, -5642, -5643, -5644, -5645, -5646, -5647, -5648, -5649, -5650, -5651, -5652, -5653, -5654, -5655, -5656, -5657, -5658, -5659, -5660, -5661, -5662, -5663, -5664, -5665, -5666, -5667, -5668, -5671, -5672, -5673, -5674, -5675, -5676, -5677, -5678, -5679, -5680, -5681, -5682, -5683, -5684, -5685, -5686, -5687, -5688, -5689, -5690, -5691, -5692, -5693, -5694, -5695, -5696, -5697, -5698, -5699, -5700, -5701, -5702, -5703, -5704, -5705, -5706, -5707, -5708, -5709, -5710, -5711, -5712, -5713, -5714, -5715, -5716, -5717, -5718, -5719, -5720, -5721, -5722, -5723, -5724, -5725, -5726, -5727, -5728, -5729, -5730, -5731, -5732, -5733, -5734, -5735, -5736, -5737, -5738, -5739, -5740, -5741, -5742, -5743, -5744, -5745, -5746, -5747, -5748, -5749, -5750, -5751, -5752, -5753, -5754, -5755, -5756, -5757, -5758, -5759, -5760, -5761, -5762, -5763, -5764, -5765, -5766, -5768, -5769, -5770, -5771, -5772, -5773, -5774, -5777, -5778, -5779, -5780, -5781, -5785, -5786, -5787, -5788, -5789, -5790, -5791, -5792, -5793, -5794, -5795, -5796, -5797, -5798, -5799, -5800, -5801, -5802, -5803, -5804, -5805, -5806, -5807, -5808, -5809, -5810, -5811, -5812, -5813, -5814, -5815, -5816, -5817, -5818, -5819, -5820, -5821, -5822, -5823, -5824, -5825, -5826, -5827, -5828, -5829, -5830, -5831, -5832, -5833, -5834, -5835, -5836, -5837, -5838, -5839, -5840, -5841, -5842, -5843, -5844, -5845, -5846, -5847, -5848, -5849, -5850, -5851, -5852, -5853, -5854, -5855, -5856, -5857, -5858, -5859, -5860, -5861, -5862, -5863, -5864, -5865, -5866, -5867, -5868, -5869, -5870, -5871, -5872, -5873, -5874, -5875, -5876, -5877, -5878, -5879, -5880, -5881, -5882, -5883, -5884, -5885, -5886, -5887, -5888, -5889, -5890, -5891, -5892, -5893, -5894, -5895, -5896, -5897, -5898, -5899, -5900, -5901, -5902, -5903, -5904, -5905, -5906, -5907, -5908, -5909, -5910, -5911, -5912, -5913, -5914, -5915, -5916, -5917, -5918, -5919, -5920, -5921, -5922, -5923, -5924, -5925, -5926, -5927, -5928, -5929, -5930, -5931, -5932, -5933, -5934, -5935, -5936, -5937, -5938, -5939, -5940, -5941, -5942, -5943, -5944, -5945, -5946, -5947, -5948, -5949, -5950, -5951, -5952, -5953, -5954, -5955, -5956, -5957, -5958, -5959, -5960, -5961, -5962, -5963, -5964, -5965, -5966, -5967, -5968, -5969, -5970, -5971, -5972, -5973, -5974, -5975, -5976, -5977, -5978, -5979, -5980, -5981, -5982, -5983, -5984, -5985, -5986, -5987, -5988, -5989, -5990, -5991, -5992, -5993, -5994, -5995, -5996, -5997, -5998, -5999, -6000, -6001, -6002, -6003, -6004, -6005, -6006, -6201, -6202, -6203, -6204, -6205, -6206, -6207, -6208, -6209, -6210, -6211, -6212, -6213, -6214, -6215, -6219, -6220, -6221, -6222, -6223, -6224, -6225, -6226, -6227, -6228, -6229, -6230, -6231, -6232, -6233, -6234, -6235, -6236, -6237, -6238, -6239, -6240, -6241, -6242, -6243, -6244, -6245, -6246, -6247, -6248, -6249, -6250, -6251, -6252, -6253, -6254, -6255, -6256, -6257, -6258, -6259, -6260, -6261, -6262, -6263, -6264, -6265, -6266, -6267, -6268, -6269, -6270, -6271, -6272, -6273, -6274, -6275, -6276, -6277, -6278, -6279, -6280, -6281, -6282, -6283, -6284, -6285, -6301, -6302, -6303, -6304, -6305, -6306, -6307, -6308, -6309, -6310, -6311, -6312, -6313, -6314, -6315, -6316, -6317, -6318, -6319, -6320, -6321, -6322, -6323, -6324, -6325, -7000, -7001, -7002, -7003, -7004, -7005, -7006, -7007, -7010, -7011, -7012, -7013, -7014, -7015, -7021, -7022, -7024, -7025, -7026, -7027, -7029, -7030, -7031, -7032, -7033, -7034, -7035, -7036, -7037, -7038, -7039, -7040, -7041, -7100, -7101, -7102, -7103, -7104, -7105, -7106, -7107, -7108, -7109, -7110, -7111, -7112, -7113, -7114, -7115, -7116, -7117, -7118, -7119, -7120, -7121, -7122, -7123, -7201, -7202, -7203, -7204, -7205, -7206, -7207, -7208, -7209, -7210, -7211, -7212, -7213, -7214, -7215, -7216, -7217, -7218, -7219, -7220, -7221, -7222, -7223, -7224, -7225, -7226, -7227, -7228, -7229, -7230, -7231, -7232, -7233, -7234, -7235, -7236, -7237, -7238, -7239, -7240, -7241, -7242, -7243, -7244, -7246, -7247, -7248, -7249, -7250, -7251, -7252, -7253, -7254, -7255, -7256, -7257, -7258, -7259, -7260, -7261, -7262, -7263, -7264, -7265, -7266, -7267, -7268, -7269, -7270, -7271, -7272, -7273, -7274, -7275, -7276, -7277, -7278, -7279, -7280, -7281, -7282, -7283, -7284, -7285, -7286, -7287, -7288, -7289, -7290, -7291, -7292, -7293, -7294, -7295, -7296, -7297, -7402, -7403, -7404, -7405, -7406, -7407, -7408, -7409, -7410, -7411, -7412, -7413, -7414, -7415, -7416, -7417, -7418, -7419, -7420, -7421, -7422, -7423, -7424, -7425, -7426, -7427, -7428, -7429, -7430, -7431, -7432, -7433, -8001, -8002, -8003, -8004, -8005, -9001, -9002, -9003, -9004, -9005, -9006, -9007, -9008, -9009, -9010, -9011, -9012, -9013, -9014, -9015, -9016, -9017, -9018, -9019, -9020, -9022, -9023, -9024, -9025, -9026, -9027, -9028, -9029, -9030, -9031, -9032, -9033, -9034, -9035, -9036, -9037, -9038, -9039, -9040, -9041, -9042, -9043, -9044, -9045, -9046, -9047, -9048, -9049, -9050, -9051, -9052, -9053, -9054, -9057, -9058, -9059, -9060, -9061, -9062, -9063, -9064, -9065, -9066, -9069, -9070, -9071, -9072, -9073, -9074, -9075, -9076, -9077, -9078, -9079, -9080, -9081, -9082, -9083, -9084, -9085, -9086, -9087, -9088, -9089, -9090, -9091, -9092, -9093, -9094, -9095, -9096, -9097, -9098, -9099, -9100, -9101, -9102, -9103, -9104, -9105, -9200, -9201, -9202, -9501, -9502, -9503, -9504, -9505, -9506, -9507, -9508, -9509, -9510, -9512, -9513, -9514, -9515, -9516, -9518, -9519, -9520, -9521, -9522, -9523, -9524, -9525, -9526, -9527, -9528, -9529, -9530, -9531, -9532, -9533, -9534, -9535, -9536, -9537, -9538, -9539, -9540, -9541, -9542, -9543, -9544, -9545, -9546, -9547, -9548, -9549, -9550, -9551, -9552, -9553, -9554, -9555, -9556, -9557, -9558, -9559, -9560, -9561, -9562, -9563, -9564, -9565, -9566, -9567, -9568, -9569, -9570, -9571, -9572, -9573, -9574, -9575, -9576, -9577, -9578, -9579, -9580, -9581, -9582, -9583, -9584, -9585, -9586, -9587, -9588, -9589, -9590, -9591, -9592, -9593, -9594, -9595, -9596, -9597, -9598, -9599, -9600, -9601, -9602, -9603, -9604, -9605, -9606, -9607, -9608, -9609, -9610, -9611, -9612, -9613, -9614, -9615, -9616, -9617, -9618, -9619, -9620, -9621, -9622, -9623, -9624, -9625, -9626, -9627, -9628, -9629, -9630, -9631, -9632, -9633, -9634, -9635, -9636, -9637, -9638, -9639, -9640, -9641, -9642, -9643, -9644, -9645, -9646, -9647, -9648, -9649, -9650, -9651, -9652, -9653, -9654, -9655, -9656, -9657, -9658, -9659, -9660, -9661, -9662, -9663, -9664, -9665, -9666, -9667, -9668, -9669, -9670, -9671, -9672, -9673, -9674, -9675, -9676, -9677, -9678, -9679, -9680, -9681, -9682, -9683, -9684, -9685, -9686, -9687, -9688, -9689, -9690, -9691, -9692, -9693, -9694, -9695, -9696, -9697, -9698, -9699, -9700, -9701, -9702, -9703, -9704, -9705, -9706, -9707, -9708, -9709, -9710, -9711, -9712, -9713, -9714, -9715, -9716, -9717, -9718, -9719, -9720, -9721, -9722, -9723, -9724, -9725, -9726, -9727, -9728, -9729, -9730, -9731, -9732, -9733, -9734, -9735, -9736, -9737, -9738, -9739, -9740, -9741, -9742, -9743, -9744, -9745, -9746, -9747, -9748, -9749, -9750, -9751, -9752, -9753, -9754, -9755, -9756, -9757, -9758, -9759, -9760, -9761, -9762, -9763, -9764, -10500, -10501, -10502, -10503, -10504, -10505, -10506, -10507, -10508, -10509, -10510, -10511, -10512, -10513, -10650, -11000, -11001, -11002, -11003, -11005, -11006, -11007, -11008, -11009, -11010, -11011, -11012, -12000, -12001, -12002, -12003, -12004, -12005, -12006, -20000, -21000, -22998, -30926, -32491, -38104, -38105}; +int g_all_ob_errnos[2210] = {0, -4000, -4001, -4002, -4003, -4004, -4005, -4006, -4007, -4008, -4009, -4010, -4011, -4012, -4013, -4014, -4015, -4016, -4017, -4018, -4019, -4020, -4021, -4022, -4023, -4024, -4025, -4026, -4027, -4028, -4029, -4030, -4031, -4032, -4033, -4034, -4035, -4036, -4037, -4038, -4039, -4041, -4042, -4043, -4044, -4045, -4046, -4047, -4048, -4049, -4050, -4051, -4052, -4053, -4054, -4055, -4057, -4058, -4060, -4061, -4062, -4063, -4064, -4065, -4066, -4067, -4068, -4070, -4071, -4072, -4073, -4074, -4075, -4076, -4077, -4078, -4080, -4081, -4084, -4085, -4090, -4097, -4098, -4099, -4100, -4101, -4102, -4103, -4104, -4105, -4106, -4107, -4108, -4109, -4110, -4111, -4112, -4113, -4114, -4115, -4116, -4117, -4118, -4119, -4120, -4121, -4122, -4123, -4124, -4125, -4126, -4127, -4128, -4133, -4138, -4139, -4142, -4143, -4144, -4146, -4147, -4149, -4150, -4151, -4152, -4153, -4154, -4155, -4156, -4157, -4158, -4159, -4160, -4161, -4162, -4163, -4164, -4165, -4166, -4167, -4168, -4169, -4170, -4171, -4172, -4173, -4174, -4175, -4176, -4177, -4178, -4179, -4180, -4181, -4182, -4183, -4184, -4185, -4186, -4187, -4188, -4189, -4190, -4191, -4192, -4200, -4201, -4204, -4205, -4206, -4207, -4208, -4209, -4210, -4211, -4212, -4213, -4214, -4215, -4216, -4217, -4218, -4219, -4220, -4221, -4222, -4223, -4224, -4225, -4226, -4227, -4228, -4229, -4230, -4231, -4232, -4233, -4234, -4235, -4236, -4237, -4238, -4239, -4240, -4241, -4242, -4243, -4244, -4245, -4246, -4247, -4248, -4249, -4250, -4251, -4252, -4253, -4254, -4255, -4256, -4257, -4258, -4260, -4261, -4262, -4263, -4264, -4265, -4266, -4267, -4268, -4269, -4270, -4271, -4273, -4274, -4275, -4276, -4277, -4278, -4279, -4280, -4281, -4282, -4283, -4284, -4285, -4286, -4287, -4288, -4289, -4290, -4291, -4292, -4293, -4294, -4295, -4296, -4297, -4298, -4299, -4300, -4301, -4302, -4303, -4304, -4305, -4306, -4307, -4308, -4309, -4310, -4311, -4312, -4313, -4314, -4315, -4316, -4317, -4318, -4319, -4320, -4321, -4322, -4323, -4324, -4325, -4326, -4327, -4328, -4329, -4330, -4331, -4332, -4333, -4334, -4335, -4336, -4337, -4338, -4339, -4340, -4341, -4342, -4343, -4344, -4345, -4346, -4347, -4348, -4349, -4350, -4351, -4352, -4353, -4354, -4355, -4356, -4357, -4358, -4359, -4360, -4361, -4362, -4363, -4364, -4365, -4366, -4367, -4368, -4369, -4370, -4371, -4372, -4373, -4374, -4375, -4376, -4377, -4378, -4379, -4380, -4381, -4382, -4383, -4385, -4386, -4387, -4388, -4389, -4390, -4391, -4392, -4393, -4394, -4395, -4396, -4397, -4398, -4399, -4400, -4401, -4402, -4403, -4505, -4507, -4510, -4512, -4515, -4517, -4518, -4519, -4523, -4524, -4525, -4526, -4527, -4528, -4529, -4530, -4531, -4532, -4533, -4537, -4538, -4539, -4540, -4541, -4542, -4543, -4544, -4545, -4546, -4547, -4548, -4549, -4550, -4551, -4552, -4553, -4554, -4600, -4601, -4602, -4603, -4604, -4605, -4606, -4607, -4608, -4609, -4610, -4611, -4613, -4614, -4615, -4620, -4621, -4622, -4623, -4624, -4625, -4626, -4628, -4629, -4630, -4631, -4632, -4633, -4634, -4636, -4637, -4638, -4639, -4640, -4641, -4642, -4643, -4644, -4645, -4646, -4647, -4648, -4649, -4650, -4651, -4652, -4653, -4654, -4655, -4656, -4657, -4658, -4659, -4660, -4661, -4662, -4663, -4664, -4665, -4666, -4667, -4668, -4669, -4670, -4671, -4672, -4673, -4674, -4675, -4676, -4677, -4678, -4679, -4680, -4681, -4682, -4683, -4684, -4685, -4686, -4687, -4688, -4689, -4690, -4691, -4692, -4693, -4694, -4695, -4696, -4697, -4698, -4699, -4700, -4701, -4702, -4703, -4704, -4705, -4706, -4707, -4708, -4709, -4710, -4711, -4712, -4713, -4714, -4715, -4716, -4717, -4718, -4719, -4720, -4721, -4722, -4723, -4724, -4725, -4726, -4727, -4728, -4729, -4730, -4731, -4732, -4733, -4734, -4735, -4736, -4737, -4738, -4739, -4740, -4741, -4742, -4743, -4744, -4745, -4746, -4747, -4748, -4749, -4750, -4751, -4752, -4753, -4754, -4755, -4756, -4757, -4758, -4759, -4760, -4761, -4762, -4763, -4764, -4765, -4766, -4767, -4768, -4769, -4770, -4771, -5000, -5001, -5002, -5003, -5006, -5007, -5008, -5010, -5011, -5012, -5014, -5015, -5016, -5017, -5018, -5019, -5020, -5022, -5023, -5024, -5025, -5026, -5027, -5028, -5029, -5030, -5031, -5032, -5034, -5035, -5036, -5037, -5038, -5039, -5040, -5041, -5042, -5043, -5044, -5046, -5047, -5050, -5051, -5052, -5053, -5054, -5055, -5056, -5057, -5058, -5059, -5061, -5063, -5064, -5065, -5066, -5067, -5068, -5069, -5070, -5071, -5072, -5073, -5074, -5080, -5081, -5083, -5084, -5085, -5086, -5087, -5088, -5089, -5090, -5091, -5092, -5093, -5094, -5095, -5096, -5097, -5098, -5099, -5100, -5101, -5102, -5103, -5104, -5105, -5106, -5107, -5108, -5109, -5110, -5111, -5112, -5113, -5114, -5115, -5116, -5117, -5118, -5119, -5120, -5121, -5122, -5123, -5124, -5125, -5130, -5131, -5133, -5134, -5135, -5136, -5137, -5138, -5139, -5140, -5142, -5143, -5144, -5145, -5146, -5147, -5148, -5149, -5150, -5151, -5153, -5154, -5155, -5156, -5157, -5158, -5159, -5160, -5161, -5162, -5163, -5164, -5165, -5166, -5167, -5168, -5169, -5170, -5171, -5172, -5173, -5174, -5175, -5176, -5177, -5178, -5179, -5180, -5181, -5182, -5183, -5184, -5185, -5187, -5188, -5189, -5190, -5191, -5192, -5193, -5194, -5195, -5196, -5197, -5198, -5199, -5200, -5201, -5202, -5203, -5204, -5205, -5206, -5207, -5208, -5209, -5210, -5211, -5212, -5213, -5214, -5215, -5216, -5217, -5218, -5219, -5220, -5221, -5222, -5223, -5224, -5225, -5226, -5227, -5228, -5229, -5230, -5231, -5233, -5234, -5235, -5236, -5237, -5238, -5239, -5240, -5241, -5242, -5243, -5244, -5245, -5246, -5247, -5248, -5249, -5250, -5251, -5252, -5253, -5254, -5255, -5256, -5257, -5258, -5259, -5260, -5261, -5262, -5263, -5264, -5265, -5266, -5267, -5268, -5269, -5270, -5271, -5272, -5273, -5274, -5275, -5276, -5277, -5278, -5279, -5280, -5281, -5282, -5283, -5284, -5285, -5286, -5287, -5288, -5289, -5290, -5291, -5292, -5293, -5294, -5295, -5296, -5297, -5298, -5299, -5300, -5301, -5302, -5303, -5304, -5305, -5306, -5307, -5308, -5309, -5310, -5311, -5312, -5313, -5314, -5315, -5316, -5317, -5318, -5319, -5320, -5321, -5322, -5323, -5324, -5325, -5326, -5327, -5328, -5329, -5330, -5331, -5332, -5333, -5334, -5335, -5336, -5337, -5338, -5339, -5340, -5341, -5342, -5343, -5344, -5345, -5346, -5347, -5348, -5349, -5350, -5351, -5352, -5353, -5354, -5355, -5356, -5357, -5358, -5359, -5360, -5361, -5362, -5363, -5364, -5365, -5366, -5367, -5368, -5369, -5370, -5371, -5372, -5373, -5374, -5375, -5376, -5377, -5378, -5379, -5380, -5381, -5382, -5383, -5384, -5385, -5386, -5387, -5388, -5389, -5400, -5401, -5402, -5403, -5404, -5405, -5406, -5407, -5408, -5409, -5410, -5411, -5412, -5413, -5414, -5415, -5416, -5417, -5418, -5419, -5420, -5421, -5422, -5423, -5424, -5425, -5426, -5427, -5428, -5429, -5430, -5431, -5432, -5433, -5434, -5435, -5436, -5437, -5438, -5439, -5440, -5441, -5442, -5443, -5444, -5445, -5446, -5447, -5448, -5449, -5450, -5451, -5452, -5453, -5454, -5455, -5456, -5457, -5458, -5459, -5460, -5461, -5462, -5463, -5464, -5465, -5466, -5467, -5468, -5469, -5470, -5471, -5472, -5473, -5474, -5475, -5476, -5477, -5478, -5479, -5480, -5481, -5482, -5483, -5484, -5485, -5486, -5487, -5488, -5489, -5490, -5491, -5492, -5493, -5494, -5495, -5496, -5497, -5498, -5499, -5500, -5501, -5502, -5503, -5504, -5505, -5506, -5507, -5508, -5509, -5510, -5511, -5512, -5513, -5514, -5515, -5516, -5517, -5540, -5541, -5542, -5543, -5544, -5545, -5546, -5547, -5548, -5549, -5550, -5551, -5552, -5553, -5554, -5555, -5556, -5557, -5558, -5559, -5560, -5561, -5562, -5563, -5564, -5565, -5566, -5567, -5568, -5569, -5570, -5571, -5572, -5573, -5574, -5575, -5576, -5577, -5578, -5579, -5580, -5581, -5582, -5583, -5584, -5585, -5586, -5587, -5588, -5589, -5590, -5591, -5592, -5593, -5594, -5595, -5596, -5597, -5598, -5599, -5600, -5601, -5602, -5603, -5604, -5605, -5607, -5608, -5609, -5610, -5611, -5612, -5613, -5614, -5615, -5616, -5617, -5618, -5619, -5620, -5621, -5622, -5623, -5624, -5625, -5626, -5627, -5628, -5629, -5630, -5631, -5632, -5633, -5634, -5635, -5636, -5637, -5638, -5639, -5640, -5641, -5642, -5643, -5644, -5645, -5646, -5647, -5648, -5649, -5650, -5651, -5652, -5653, -5654, -5655, -5656, -5657, -5658, -5659, -5660, -5661, -5662, -5663, -5664, -5665, -5666, -5667, -5668, -5671, -5672, -5673, -5674, -5675, -5676, -5677, -5678, -5679, -5680, -5681, -5682, -5683, -5684, -5685, -5686, -5687, -5688, -5689, -5690, -5691, -5692, -5693, -5694, -5695, -5696, -5697, -5698, -5699, -5700, -5701, -5702, -5703, -5704, -5705, -5706, -5707, -5708, -5709, -5710, -5711, -5712, -5713, -5714, -5715, -5716, -5717, -5718, -5719, -5720, -5721, -5722, -5723, -5724, -5725, -5726, -5727, -5728, -5729, -5730, -5731, -5732, -5733, -5734, -5735, -5736, -5737, -5738, -5739, -5740, -5741, -5742, -5743, -5744, -5745, -5746, -5747, -5748, -5749, -5750, -5751, -5752, -5753, -5754, -5755, -5756, -5757, -5758, -5759, -5760, -5761, -5762, -5763, -5764, -5765, -5766, -5768, -5769, -5770, -5771, -5772, -5773, -5774, -5777, -5778, -5779, -5780, -5781, -5785, -5786, -5787, -5788, -5789, -5790, -5791, -5792, -5793, -5794, -5795, -5796, -5797, -5798, -5799, -5800, -5801, -5802, -5803, -5804, -5805, -5806, -5807, -5808, -5809, -5810, -5811, -5812, -5813, -5814, -5815, -5816, -5817, -5818, -5819, -5820, -5821, -5822, -5823, -5824, -5825, -5826, -5827, -5828, -5829, -5830, -5831, -5832, -5833, -5834, -5835, -5836, -5837, -5838, -5839, -5840, -5841, -5842, -5843, -5844, -5845, -5846, -5847, -5848, -5849, -5850, -5851, -5852, -5853, -5854, -5855, -5856, -5857, -5858, -5859, -5860, -5861, -5862, -5863, -5864, -5865, -5866, -5867, -5868, -5869, -5870, -5871, -5872, -5873, -5874, -5875, -5876, -5877, -5878, -5879, -5880, -5881, -5882, -5883, -5884, -5885, -5886, -5887, -5888, -5889, -5890, -5891, -5892, -5893, -5894, -5895, -5896, -5897, -5898, -5899, -5900, -5901, -5902, -5903, -5904, -5905, -5906, -5907, -5908, -5909, -5910, -5911, -5912, -5913, -5914, -5915, -5916, -5917, -5918, -5919, -5920, -5921, -5922, -5923, -5924, -5925, -5926, -5927, -5928, -5929, -5930, -5931, -5932, -5933, -5934, -5935, -5936, -5937, -5938, -5939, -5940, -5941, -5942, -5943, -5944, -5945, -5946, -5947, -5948, -5949, -5950, -5951, -5952, -5953, -5954, -5955, -5956, -5957, -5958, -5959, -5960, -5961, -5962, -5963, -5964, -5965, -5966, -5967, -5968, -5969, -5970, -5971, -5972, -5973, -5974, -5975, -5976, -5977, -5978, -5979, -5980, -5981, -5982, -5983, -5984, -5985, -5986, -5987, -5988, -5989, -5990, -5991, -5992, -5993, -5994, -5995, -5996, -5997, -5998, -5999, -6000, -6001, -6002, -6003, -6004, -6005, -6006, -6201, -6202, -6203, -6204, -6205, -6206, -6207, -6208, -6209, -6210, -6211, -6212, -6213, -6214, -6215, -6219, -6220, -6221, -6222, -6223, -6224, -6225, -6226, -6227, -6228, -6229, -6230, -6231, -6232, -6233, -6234, -6235, -6236, -6237, -6238, -6239, -6240, -6241, -6242, -6243, -6244, -6245, -6246, -6247, -6248, -6249, -6250, -6251, -6252, -6253, -6254, -6255, -6256, -6257, -6258, -6259, -6260, -6261, -6262, -6263, -6264, -6265, -6266, -6267, -6268, -6269, -6270, -6271, -6272, -6273, -6274, -6275, -6276, -6277, -6278, -6279, -6280, -6281, -6282, -6283, -6284, -6285, -6301, -6302, -6303, -6304, -6305, -6306, -6307, -6308, -6309, -6310, -6311, -6312, -6313, -6314, -6315, -6316, -6317, -6318, -6319, -6320, -6321, -6322, -6323, -6324, -6325, -7000, -7001, -7002, -7003, -7004, -7005, -7006, -7007, -7010, -7011, -7012, -7013, -7014, -7015, -7021, -7022, -7024, -7025, -7026, -7027, -7029, -7030, -7031, -7032, -7033, -7034, -7035, -7036, -7037, -7038, -7039, -7040, -7041, -7100, -7101, -7102, -7103, -7104, -7105, -7106, -7107, -7108, -7109, -7110, -7111, -7112, -7113, -7114, -7115, -7116, -7117, -7118, -7119, -7120, -7121, -7122, -7123, -7201, -7202, -7203, -7204, -7205, -7206, -7207, -7208, -7209, -7210, -7211, -7212, -7213, -7214, -7215, -7216, -7217, -7218, -7219, -7220, -7221, -7222, -7223, -7224, -7225, -7226, -7227, -7228, -7229, -7230, -7231, -7232, -7233, -7234, -7235, -7236, -7237, -7238, -7239, -7240, -7241, -7242, -7243, -7244, -7246, -7247, -7248, -7249, -7250, -7251, -7252, -7253, -7254, -7255, -7256, -7257, -7258, -7259, -7260, -7261, -7262, -7263, -7264, -7265, -7266, -7267, -7268, -7269, -7270, -7271, -7272, -7273, -7274, -7275, -7276, -7277, -7278, -7279, -7280, -7281, -7282, -7283, -7284, -7285, -7286, -7287, -7288, -7289, -7290, -7291, -7292, -7293, -7294, -7295, -7296, -7297, -7402, -7403, -7404, -7405, -7406, -7407, -7408, -7409, -7410, -7411, -7412, -7413, -7414, -7415, -7416, -7417, -7418, -7419, -7420, -7421, -7422, -7423, -7424, -7425, -7426, -7427, -7428, -7429, -7430, -7431, -7432, -7433, -8001, -8002, -8003, -8004, -8005, -9001, -9002, -9003, -9004, -9005, -9006, -9007, -9008, -9009, -9010, -9011, -9012, -9013, -9014, -9015, -9016, -9017, -9018, -9019, -9020, -9022, -9023, -9024, -9025, -9026, -9027, -9028, -9029, -9030, -9031, -9032, -9033, -9034, -9035, -9036, -9037, -9038, -9039, -9040, -9041, -9042, -9043, -9044, -9045, -9046, -9047, -9048, -9049, -9050, -9051, -9052, -9053, -9054, -9057, -9058, -9059, -9060, -9061, -9062, -9063, -9064, -9065, -9066, -9069, -9070, -9071, -9072, -9073, -9074, -9075, -9076, -9077, -9078, -9079, -9080, -9081, -9082, -9083, -9084, -9085, -9086, -9087, -9088, -9089, -9090, -9091, -9092, -9093, -9094, -9095, -9096, -9097, -9098, -9099, -9100, -9101, -9102, -9103, -9104, -9105, -9200, -9201, -9202, -9501, -9502, -9503, -9504, -9505, -9506, -9507, -9508, -9509, -9510, -9512, -9513, -9514, -9515, -9516, -9518, -9519, -9520, -9521, -9522, -9523, -9524, -9525, -9526, -9527, -9528, -9529, -9530, -9531, -9532, -9533, -9534, -9535, -9536, -9537, -9538, -9539, -9540, -9541, -9542, -9543, -9544, -9545, -9546, -9547, -9548, -9549, -9550, -9551, -9552, -9553, -9554, -9555, -9556, -9557, -9558, -9559, -9560, -9561, -9562, -9563, -9564, -9565, -9566, -9567, -9568, -9569, -9570, -9571, -9572, -9573, -9574, -9575, -9576, -9577, -9578, -9579, -9580, -9581, -9582, -9583, -9584, -9585, -9586, -9587, -9588, -9589, -9590, -9591, -9592, -9593, -9594, -9595, -9596, -9597, -9598, -9599, -9600, -9601, -9602, -9603, -9604, -9605, -9606, -9607, -9608, -9609, -9610, -9611, -9612, -9613, -9614, -9615, -9616, -9617, -9618, -9619, -9620, -9621, -9622, -9623, -9624, -9625, -9626, -9627, -9628, -9629, -9630, -9631, -9632, -9633, -9634, -9635, -9636, -9637, -9638, -9639, -9640, -9641, -9642, -9643, -9644, -9645, -9646, -9647, -9648, -9649, -9650, -9651, -9652, -9653, -9654, -9655, -9656, -9657, -9658, -9659, -9660, -9661, -9662, -9663, -9664, -9665, -9666, -9667, -9668, -9669, -9670, -9671, -9672, -9673, -9674, -9675, -9676, -9677, -9678, -9679, -9680, -9681, -9682, -9683, -9684, -9685, -9686, -9687, -9688, -9689, -9690, -9691, -9692, -9693, -9694, -9695, -9696, -9697, -9698, -9699, -9700, -9701, -9702, -9703, -9704, -9705, -9706, -9707, -9708, -9709, -9710, -9711, -9712, -9713, -9714, -9715, -9716, -9717, -9718, -9719, -9720, -9721, -9722, -9723, -9724, -9725, -9726, -9727, -9728, -9729, -9730, -9731, -9732, -9733, -9734, -9735, -9736, -9737, -9738, -9739, -9740, -9741, -9742, -9743, -9744, -9745, -9746, -9747, -9748, -9749, -9750, -9751, -9752, -9753, -9754, -9755, -9756, -9757, -9758, -9759, -9760, -9761, -9762, -9763, -9764, -9765, -10500, -10501, -10502, -10503, -10504, -10505, -10506, -10507, -10508, -10509, -10510, -10511, -10512, -10513, -10650, -11000, -11001, -11002, -11003, -11005, -11006, -11007, -11008, -11009, -11010, -11011, -11012, -12000, -12001, -12002, -12003, -12004, -12005, -12006, -20000, -21000, -22998, -30926, -32491, -38104, -38105}; const char *ob_error_name(const int err) { const char *ret = "Unknown error"; diff --git a/src/share/ob_errno.def b/src/share/ob_errno.def index d1e7141466..11145ded8c 100755 --- a/src/share/ob_errno.def +++ b/src/share/ob_errno.def @@ -1628,7 +1628,7 @@ DEFINE_ERROR_EXT(OB_ERR_CHECK_OPTION_ON_NONUPDATABLE_VIEW, -5972, ER_VIEW_NONUPD DEFINE_ORACLE_ERROR(OB_ERR_NO_DESC_FOR_POS, -5973, -1, "HY000", "no descriptor for this position", 24334, "no descriptor for this position"); DEFINE_ORACLE_ERROR(OB_ERR_ILL_OBJ_FLAG, -5974, -1, "HY000", "object specified is incompatible with the flag specified", 4047, "object specified is incompatible with the flag specified"); DEFINE_ERROR_DEP(OB_ERR_DBLINK_REMOTE_ECODE, -5975, -1, "HY000", "\ndblink remote error code: %d,\nremote error msg: %.*s", "\ndblink remote error code: %d,\nremote error msg: %.*s"); -DEFINE_ERROR_DEP(OB_ERR_DBLINK_NO_LIB, -5976, -1, "HY000", "oci lib not founded"); +DEFINE_ERROR_DEP(OB_ERR_DBLINK_NO_LIB, -5976, -1, "HY000", "can not find the expected version of OCI LIB: %.*s", "can not find the expected version of OCI LIB: %.*s"); DEFINE_ORACLE_ERROR(OB_ERR_PARTITION_EXTENDED_ON_VIEW, -5977, -1, "HY000", "partition-extended object names may only be used with tables and editioning views", 14109, "partition-extended object names may only be used with tables and editioning views"); DEFINE_ORACLE_ERROR(OB_ERR_NOT_ALL_VARIABLE_BIND, -5978, -1, "HY000", "not all variables bound", 1008, "not all variables bound"); DEFINE_ORACLE_ERROR(OB_ERR_BIND_VARIABLE_NOT_EXIST, -5979, -1, "HY000", "bind variable does not exist", 1006, "bind variable does not exist"); @@ -1784,6 +1784,9 @@ DEFINE_ERROR(OB_CLOG_SLIDE_TIMEOUT, -6322, -1, "HY000", "ob clog slide timeout") DEFINE_ERROR(OB_LOG_REPLAY_ERROR, -6323, -1, "HY000", "log replay error"); DEFINE_ERROR(OB_TRY_LOCK_CONFIG_CHANGE_CONFLICT, -6324, -1, "HY000", "ob trying to lock config change conflicts"); DEFINE_ERROR(OB_CLOG_DISK_HANG, -6325, -1, "HY000", "ob clog disk hang"); +// 6326: OB_TABLE_LOCK_SPLIT_TWICE +// 6327: OB_TABLE_LOCK_IS_SPLITTING +// 6328: OB_TABLE_LOCK_SPLIT_FAIL // for freeze // DEFINE_ERROR(OB_TABLET_FREEZE_TIMEOUT, -6400, -1, "HY000", "tablet_freeze timeout"); @@ -2389,6 +2392,7 @@ DEFINE_ORACLE_ERROR_EXT(OB_ERR_MVIEW_NEVER_REFRESH, -9761, -1, "HY000", "cannot DEFINE_ERROR(OB_ERR_CLIENT_LOCAL_FILES_DISABLED, -9762, ER_CLIENT_LOCAL_FILES_DISABLED, "42000", "Loading local data is disabled; this must be enabled on both the client and server sides"); DEFINE_ORACLE_ERROR(OB_ERR_OUT_PARAM_NOT_BIND_VAR, -9763, -1, "HY000", "output parameter not a bind variable", 6577, "output parameter not a bind variable"); DEFINE_ORACLE_ERROR_EXT(OB_ERR_TIME_EARLIER_THAN_SYSDATE, -9764, -1, "HY000", "the parameter must evaluate to a time in the future", "the parameter %s must evaluate to a time in the future", 23420, "the parameter must evaluate to a time in the future", "the parameter %s must evaluate to a time in the future"); +DEFINE_PLS_ERROR_EXT(OB_ERR_NOT_FUNC_NAME, -9765, -1, "HY000", "object 'string' must be of type function or array to be used this way", "object '%.*s' must be of type function or array to be used this way", 224, "object 'string' must be of type function or array to be used this way", "object '%.*s' must be of type function or array to be used this way"); //////////////////////////////////////////////////////////////// //error code for OBKV [-10500, -10700) diff --git a/src/share/ob_errno.h b/src/share/ob_errno.h index 95d4053ee5..f461bf6230 100644 --- a/src/share/ob_errno.h +++ b/src/share/ob_errno.h @@ -1813,6 +1813,7 @@ constexpr int OB_ERR_MVIEW_NEVER_REFRESH = -9761; constexpr int OB_ERR_CLIENT_LOCAL_FILES_DISABLED = -9762; constexpr int OB_ERR_OUT_PARAM_NOT_BIND_VAR = -9763; constexpr int OB_ERR_TIME_EARLIER_THAN_SYSDATE = -9764; +constexpr int OB_ERR_NOT_FUNC_NAME = -9765; constexpr int OB_ERR_KV_GLOBAL_INDEX_ROUTE = -10500; constexpr int OB_TTL_NOT_ENABLE = -10501; constexpr int OB_TTL_COLUMN_NOT_EXIST = -10502; @@ -3326,7 +3327,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_NO_DESC_FOR_POS__USER_ERROR_MSG "no descriptor for this position" #define OB_ERR_ILL_OBJ_FLAG__USER_ERROR_MSG "object specified is incompatible with the flag specified" #define OB_ERR_DBLINK_REMOTE_ECODE__USER_ERROR_MSG "\ndblink remote error code: %d,\nremote error msg: %.*s" -#define OB_ERR_DBLINK_NO_LIB__USER_ERROR_MSG "oci lib not founded" +#define OB_ERR_DBLINK_NO_LIB__USER_ERROR_MSG "can not find the expected version of OCI LIB: %.*s" #define OB_ERR_PARTITION_EXTENDED_ON_VIEW__USER_ERROR_MSG "partition-extended object names may only be used with tables and editioning views" #define OB_ERR_NOT_ALL_VARIABLE_BIND__USER_ERROR_MSG "not all variables bound" #define OB_ERR_BIND_VARIABLE_NOT_EXIST__USER_ERROR_MSG "bind variable does not exist" @@ -4019,6 +4020,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_CLIENT_LOCAL_FILES_DISABLED__USER_ERROR_MSG "Loading local data is disabled; this must be enabled on both the client and server sides" #define OB_ERR_OUT_PARAM_NOT_BIND_VAR__USER_ERROR_MSG "output parameter not a bind variable" #define OB_ERR_TIME_EARLIER_THAN_SYSDATE__USER_ERROR_MSG "the parameter %s must evaluate to a time in the future" +#define OB_ERR_NOT_FUNC_NAME__USER_ERROR_MSG "object '%.*s' must be of type function or array to be used this way" #define OB_ERR_KV_GLOBAL_INDEX_ROUTE__USER_ERROR_MSG "incorrect route for obkv global index, client router should refresh." #define OB_TTL_NOT_ENABLE__USER_ERROR_MSG "TTL feature is not enabled" #define OB_TTL_COLUMN_NOT_EXIST__USER_ERROR_MSG "TTL column '%.*s' not exists" @@ -5539,7 +5541,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_NO_DESC_FOR_POS__ORA_USER_ERROR_MSG "ORA-24334: no descriptor for this position" #define OB_ERR_ILL_OBJ_FLAG__ORA_USER_ERROR_MSG "ORA-04047: object specified is incompatible with the flag specified" #define OB_ERR_DBLINK_REMOTE_ECODE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -5975, \ndblink remote error code: %d,\nremote error msg: %.*s" -#define OB_ERR_DBLINK_NO_LIB__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -5976, oci lib not founded" +#define OB_ERR_DBLINK_NO_LIB__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -5976, can not find the expected version of OCI LIB: %.*s" #define OB_ERR_PARTITION_EXTENDED_ON_VIEW__ORA_USER_ERROR_MSG "ORA-14109: partition-extended object names may only be used with tables and editioning views" #define OB_ERR_NOT_ALL_VARIABLE_BIND__ORA_USER_ERROR_MSG "ORA-01008: not all variables bound" #define OB_ERR_BIND_VARIABLE_NOT_EXIST__ORA_USER_ERROR_MSG "ORA-01006: bind variable does not exist" @@ -6232,6 +6234,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_CLIENT_LOCAL_FILES_DISABLED__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -9762, Loading local data is disabled; this must be enabled on both the client and server sides" #define OB_ERR_OUT_PARAM_NOT_BIND_VAR__ORA_USER_ERROR_MSG "ORA-06577: output parameter not a bind variable" #define OB_ERR_TIME_EARLIER_THAN_SYSDATE__ORA_USER_ERROR_MSG "ORA-23420: the parameter %s must evaluate to a time in the future" +#define OB_ERR_NOT_FUNC_NAME__ORA_USER_ERROR_MSG "PLS-00224: object '%.*s' must be of type function or array to be used this way" #define OB_ERR_KV_GLOBAL_INDEX_ROUTE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -10500, incorrect route for obkv global index, client router should refresh." #define OB_TTL_NOT_ENABLE__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -10501, TTL feature is not enabled" #define OB_TTL_COLUMN_NOT_EXIST__ORA_USER_ERROR_MSG "ORA-00600: internal error code, arguments: -10502, TTL column '%.*s' not exists" @@ -6276,7 +6279,7 @@ constexpr int OB_ERR_INVALID_DATE_MSG_FMT_V2 = -4219; #define OB_ERR_DATA_TOO_LONG_MSG_FMT_V2__ORA_USER_ERROR_MSG "ORA-12899: value too large for column %.*s (actual: %ld, maximum: %ld)" #define OB_ERR_INVALID_DATE_MSG_FMT_V2__ORA_USER_ERROR_MSG "ORA-01861: Incorrect datetime value for column '%.*s' at row %ld" -extern int g_all_ob_errnos[2209]; +extern int g_all_ob_errnos[2210]; const char *ob_error_name(const int oberr); const char* ob_error_cause(const int oberr); diff --git a/src/share/ob_rpc_struct.cpp b/src/share/ob_rpc_struct.cpp index bddf93b884..7a97f5062e 100755 --- a/src/share/ob_rpc_struct.cpp +++ b/src/share/ob_rpc_struct.cpp @@ -80,6 +80,7 @@ int ObDDLArg::assign(const ObDDLArg &other) parallelism_ = other.parallelism_; task_id_ = other.task_id_; consumer_group_id_ = other.consumer_group_id_; + is_parallel_ = other.is_parallel_; } return ret; } @@ -1265,7 +1266,8 @@ OB_SERIALIZE_MEMBER(ObDDLArg, based_schema_object_infos_, parallelism_, task_id_, - consumer_group_id_); + consumer_group_id_, + is_parallel_); ////////////////////////////////////////////// // @@ -5112,6 +5114,7 @@ int ObAdminFlushCacheArg::assign(const ObAdminFlushCacheArg &other) sql_id_ = other.sql_id_; is_fine_grained_ = other.is_fine_grained_; ns_type_ = other.ns_type_; + schema_id_ = other.schema_id_; if (OB_FAIL(tenant_ids_.assign(other.tenant_ids_))) { LOG_WARN("failed to assign tenant ids", K(ret)); } else if (OB_FAIL(db_ids_.assign(other.db_ids_))) { @@ -5120,7 +5123,7 @@ int ObAdminFlushCacheArg::assign(const ObAdminFlushCacheArg &other) return ret; } -OB_SERIALIZE_MEMBER(ObAdminFlushCacheArg, tenant_ids_, cache_type_, db_ids_, sql_id_, is_fine_grained_, ns_type_); +OB_SERIALIZE_MEMBER(ObAdminFlushCacheArg, tenant_ids_, cache_type_, db_ids_, sql_id_, is_fine_grained_, ns_type_, schema_id_); int ObFlushCacheArg::assign(const ObFlushCacheArg &other) { @@ -5131,13 +5134,14 @@ int ObFlushCacheArg::assign(const ObFlushCacheArg &other) sql_id_ = other.sql_id_; is_fine_grained_ = other.is_fine_grained_; ns_type_ = other.ns_type_; + schema_id_ = other.schema_id_; if (OB_FAIL(db_ids_.assign(other.db_ids_))) { LOG_WARN("failed to assign db ids", K(ret)); } return ret; } -OB_SERIALIZE_MEMBER(ObFlushCacheArg, is_all_tenant_, tenant_id_, cache_type_, db_ids_, sql_id_, is_fine_grained_, ns_type_); +OB_SERIALIZE_MEMBER(ObFlushCacheArg, is_all_tenant_, tenant_id_, cache_type_, db_ids_, sql_id_, is_fine_grained_, ns_type_, schema_id_); OB_SERIALIZE_MEMBER(ObGetAllSchemaArg, schema_version_, diff --git a/src/share/ob_rpc_struct.h b/src/share/ob_rpc_struct.h index 3ca8c0eabd..6f88a916aa 100755 --- a/src/share/ob_rpc_struct.h +++ b/src/share/ob_rpc_struct.h @@ -229,7 +229,8 @@ public: based_schema_object_infos_(), parallelism_(0), task_id_(0), - consumer_group_id_(0) + consumer_group_id_(0), + is_parallel_(false) { } virtual ~ObDDLArg() = default; bool is_need_check_based_schema_objects() const @@ -257,6 +258,7 @@ public: parallelism_ = 0; task_id_ = 0; consumer_group_id_ = 0; + is_parallel_ = false; } DECLARE_TO_STRING; @@ -268,6 +270,8 @@ public: int64_t parallelism_; int64_t task_id_; int64_t consumer_group_id_; + //some parallel ddl is effect before 4220, this member is valid after 4220 + bool is_parallel_; }; struct ObAlterResourceUnitArg : public ObDDLArg @@ -5914,7 +5918,8 @@ public: ObAdminFlushCacheArg() : cache_type_(CACHE_TYPE_INVALID), is_fine_grained_(false), - ns_type_(sql::ObLibCacheNameSpace::NS_INVALID) + ns_type_(sql::ObLibCacheNameSpace::NS_INVALID), + schema_id_(common::OB_INVALID_ID) { } virtual ~ObAdminFlushCacheArg() {} @@ -5933,6 +5938,7 @@ public: common::ObString sql_id_; bool is_fine_grained_; sql::ObLibCacheNameSpace ns_type_; + uint64_t schema_id_; }; struct ObAdminMigrateUnitArg @@ -6599,7 +6605,8 @@ public: tenant_id_(common::OB_INVALID_TENANT_ID), cache_type_(CACHE_TYPE_INVALID), is_fine_grained_(false), - ns_type_(sql::ObLibCacheNameSpace::NS_INVALID) + ns_type_(sql::ObLibCacheNameSpace::NS_INVALID), + schema_id_(common::OB_INVALID_ID) {} virtual ~ObFlushCacheArg() {} bool is_valid() const @@ -6623,6 +6630,7 @@ public: common::ObString sql_id_; bool is_fine_grained_; sql::ObLibCacheNameSpace ns_type_; + uint64_t schema_id_; }; struct ObGetAllSchemaArg diff --git a/src/share/object/ob_obj_cast.cpp b/src/share/object/ob_obj_cast.cpp index 423364c76a..939e57062a 100644 --- a/src/share/object/ob_obj_cast.cpp +++ b/src/share/object/ob_obj_cast.cpp @@ -14343,30 +14343,14 @@ int ObObjCaster::to_type(const ObExpectType &expect_type, const char OB_JSON_NULL[2] = {'\0', '\0'}; // binary json null -int ObObjCaster::get_zero_value(const ObObjType expect_type, - ObCollationType expect_cs_type, - int64_t data_len, - ObIAllocator &alloc, - ObObj &zero_obj) +int ObObjCaster::get_zero_value(const ObObjType expect_type, ObCollationType expect_cs_type, ObObj &zero_obj) { int ret = OB_SUCCESS; ObObjCastParams params; //构造一个空的cast_param对象，适配SET_RES_XXX宏定义 ObCastMode cast_mode = CM_WARN_ON_FAIL; params.warning_ = 1; //将warning code设置为1，避免SET_RES_XXX宏将其当做真实的warning处理 - if (ob_is_string_tc(expect_type)) { - //zero_obj.set_string(expect_type, ""); - ObString padding_res; - if (ObVarcharType == expect_type) { - zero_obj.set_string(expect_type, ""); - } else if (0 > data_len) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("The length of fix-len string is less than zero", K(ret), K(data_len)); - } else if (OB_FAIL(sql::padding_char_for_cast(data_len, expect_cs_type, alloc, - padding_res))) { - LOG_WARN("padding char failed", K(ret), K(data_len), K(expect_cs_type)); - } else { - zero_obj.set_string(expect_type, padding_res); - } + if (ob_is_string_tc(expect_type) || ob_is_text_tc(expect_type)) { + zero_obj.set_string(expect_type, ""); } else if (ob_is_text_tc(expect_type)) { zero_obj.set_lob_value(expect_type, static_cast(NULL), 0); } else if (ob_is_int_tc(expect_type)) { diff --git a/src/share/object/ob_obj_cast.h b/src/share/object/ob_obj_cast.h index 3d628b6ce9..49c3ecd5b3 100644 --- a/src/share/object/ob_obj_cast.h +++ b/src/share/object/ob_obj_cast.h @@ -400,11 +400,7 @@ public: const ObObj &in_obj, ObObj &out_obj); static int to_type(const ObObjType expect_type, ObCollationType expect_cs_type, ObCastCtx &cast_ctx, const ObObj &in_obj, ObObj &out_obj); - static int get_zero_value(const ObObjType expect_type, - ObCollationType expect_cs_type, - int64_t data_len, - ObIAllocator &alloc, - ObObj &zero_obj); + static int get_zero_value(const ObObjType expect_type, ObCollationType expect_cs_type, ObObj &zero_obj); static int enumset_to_inner(const ObObjMeta &expect_meta, const ObObj &in_obj, ObObj &out_obj, common::ObIAllocator &allocator, diff --git a/src/share/parameter/ob_parameter_seed.ipp b/src/share/parameter/ob_parameter_seed.ipp index 12d853c8a9..12d85bdf91 100644 --- a/src/share/parameter/ob_parameter_seed.ipp +++ b/src/share/parameter/ob_parameter_seed.ipp @@ -311,9 +311,9 @@ DEF_DBL(large_query_worker_percentage, OB_CLUSTER_PARAMETER, "30", "[0,100]", "the percentage of the workers reserved to serve large query request. " "Range: [0, 100] in percentage", ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)) -DEF_TIME(large_query_threshold, OB_CLUSTER_PARAMETER, "5s", "[1ms,)", +DEF_TIME(large_query_threshold, OB_CLUSTER_PARAMETER, "5s", "[0ms,)", "threshold for execution time beyond " - "which a request may be paused and rescheduled as a \\'large request\\'. Range: [1ms, +∞)", + "which a request may be paused and rescheduled as a \\'large request\\', 0ms means disable \\'large request\\'. Range: [0ms, +∞)", ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); DEF_INT(_ob_max_thread_num, OB_CLUSTER_PARAMETER, "0", "[0,10000)", "ob max thread number " @@ -1820,3 +1820,8 @@ DEF_BOOL(enable_rpc_authentication_bypass, OB_CLUSTER_PARAMETER, "True", "specifies whether allow arbitration service, standby cluster, OMS service to connect " "cluster and provide service when rpc authentication is turned on.", ObParameterAttr(Section::OBSERVER, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); + +// take up space on the master for checkpoint diagnose in 42x +// DEF_INT(_checkpoint_diagnose_preservation_count, OB_TENANT_PARAMETER, "100", "[0,1000]", +// "the count of checkpoint diagnose info preservation", +// ObParameterAttr(Section::TENANT, Source::DEFAULT, EditLevel::DYNAMIC_EFFECTIVE)); diff --git a/src/share/rc/ob_tenant_base.h b/src/share/rc/ob_tenant_base.h index 47f737f3ee..f0eeadbf43 100755 --- a/src/share/rc/ob_tenant_base.h +++ b/src/share/rc/ob_tenant_base.h @@ -389,8 +389,6 @@ using ObTableScanIteratorObjPool = common::ObServerObjectPoolunit_memory_size() // 注意MTL_BIND调用需要在租户创建之前，否则会导致租户创建时无法调用到绑定的函数。 -#define MTL_BIND(INIT, DESTROY) \ - share::ObTenantBase::mtl_bind_func(nullptr, INIT, nullptr, nullptr, nullptr, DESTROY); #define MTL_BIND2(NEW, INIT, START, STOP, WAIT, DESTROY) \ share::ObTenantBase::mtl_bind_func(NEW, INIT, START, STOP, WAIT, DESTROY); diff --git a/src/share/scheduler/ob_tenant_dag_scheduler.cpp b/src/share/scheduler/ob_tenant_dag_scheduler.cpp index d031e753b8..f2afb9360e 100644 --- a/src/share/scheduler/ob_tenant_dag_scheduler.cpp +++ b/src/share/scheduler/ob_tenant_dag_scheduler.cpp @@ -1698,6 +1698,16 @@ bool ObTenantDagWorker::get_force_cancel_flag() return flag; } +bool ObTenantDagWorker::hold_by_compaction_dag() +{ + bool bret = false; + ObIDag *dag = nullptr; + if (OB_NOT_NULL(task_) && OB_NOT_NULL(dag = task_->get_dag())) { + bret = is_compaction_dag(dag->get_type()); + } + return bret; +} + void ObTenantDagWorker::run1() { self_ = this; diff --git a/src/share/scheduler/ob_tenant_dag_scheduler.h b/src/share/scheduler/ob_tenant_dag_scheduler.h index 7986c35b3c..84c3f9748f 100644 --- a/src/share/scheduler/ob_tenant_dag_scheduler.h +++ b/src/share/scheduler/ob_tenant_dag_scheduler.h @@ -720,6 +720,7 @@ public: static void set_mem_ctx(compaction::ObCompactionMemoryContext *mem_ctx) { if (nullptr == mem_ctx_) { mem_ctx_ = mem_ctx; } } uint64_t get_group_id() { return group_id_; } bool get_force_cancel_flag(); + bool hold_by_compaction_dag(); private: void notify(DagWorkerStatus status); void reset_compaction_thread_locals() { is_reserve_mode_ = false; mem_ctx_ = nullptr; } @@ -1419,7 +1420,14 @@ inline bool is_reserve_mode() #define SET_MEM_CTX(mem_ctx) \ ({ \ share::ObTenantDagWorker *worker = share::ObTenantDagWorker::self(); \ - worker->set_mem_ctx(&mem_ctx); \ + if (NULL != worker) { \ + if (worker->hold_by_compaction_dag()) { \ + worker->set_mem_ctx(&mem_ctx); \ + } else { \ + COMMON_LOG_RET(WARN, OB_ERR_UNEXPECTED, \ + "only compaction dag can set memctx", KPC(worker)); \ + } \ + } \ }) #define CURRENT_MEM_CTX() \ @@ -1427,7 +1435,12 @@ inline bool is_reserve_mode() compaction::ObCompactionMemoryContext *mem_ctx = nullptr; \ share::ObTenantDagWorker *worker = share::ObTenantDagWorker::self(); \ if (NULL != worker) { \ - mem_ctx = worker->get_mem_ctx(); \ + if (worker->hold_by_compaction_dag()) { \ + mem_ctx = worker->get_mem_ctx(); \ + } else { \ + COMMON_LOG_RET(WARN, OB_ERR_UNEXPECTED, \ + "memctx only provided for compaction dag", KPC(worker)); \ + } \ } \ mem_ctx; \ }) diff --git a/src/share/schema/ob_schema_printer.cpp b/src/share/schema/ob_schema_printer.cpp index b28ac7317c..681df67ccc 100644 --- a/src/share/schema/ob_schema_printer.cpp +++ b/src/share/schema/ob_schema_printer.cpp @@ -348,11 +348,17 @@ int ObSchemaPrinter::print_table_definition_columns(const ObTableSchema &table_s SHARE_SCHEMA_LOG(WARN, "fail to print sql literal", KPC(col), K(buf), K(buf_len), K(pos), K(ret)); } } else if (ob_is_string_tc(default_value.get_type())) { - ObCollationType collation_type = ObCharset::get_default_collation(charset_type); ObString out_str = default_value.get_string(); - if (OB_FAIL(ObCharset::charset_convert(allocator, default_value.get_string(), default_value.get_collation_type(), collation_type, out_str))) { - SHARE_SCHEMA_LOG(WARN, "fail to convert charset", K(ret)); - } else if (OB_FAIL(databuff_printf(buf, buf_len, pos, "'%s'", to_cstring(ObHexEscapeSqlStr(out_str))))) { + if (oceanbase::common::ObCharsetType::CHARSET_INVALID == charset_type || + oceanbase::common::ObCharsetType::CHARSET_BINARY == charset_type) { + // observer perform no conversion of result sets or error messages, you can see more detail the official website of MySQL + } else { + ObCollationType collation_type = ObCharset::get_default_collation(charset_type); + if (OB_FAIL(ObCharset::charset_convert(allocator, default_value.get_string(), default_value.get_collation_type(), collation_type, out_str))) { + SHARE_SCHEMA_LOG(WARN, "fail to convert charset", K(ret)); + } + } + if (OB_SUCC(ret) && OB_FAIL(databuff_printf(buf, buf_len, pos, "'%s'", to_cstring(ObHexEscapeSqlStr(out_str))))) { SHARE_SCHEMA_LOG(WARN, "fail to print default value of string tc", K(ret)); } } else if (OB_FAIL(default_value.print_varchar_literal(buf, buf_len, pos, tz_info))) { diff --git a/src/share/schema/ob_table_schema.cpp b/src/share/schema/ob_table_schema.cpp index e98982dbbe..42f87fbdfe 100644 --- a/src/share/schema/ob_table_schema.cpp +++ b/src/share/schema/ob_table_schema.cpp @@ -4486,6 +4486,10 @@ int ObTableSchema::check_alter_column_type(const ObColumnSchemaV2 &src_column, is_offline = true; } } + if ((src_meta.is_signed_integer() && dst_meta.is_unsigned_integer()) + || (src_meta.is_unsigned_integer() && dst_meta.is_signed_integer())) { + is_offline = true; + } if (!src_meta.is_lob_storage() && dst_meta.is_lob_storage()) { is_offline = true; } diff --git a/src/share/system_variable/ob_system_variable_init.cpp b/src/share/system_variable/ob_system_variable_init.cpp index 6e4a231a65..02b6c72b50 100644 --- a/src/share/system_variable/ob_system_variable_init.cpp +++ b/src/share/system_variable/ob_system_variable_init.cpp @@ -2367,7 +2367,7 @@ static struct VarsInit{ }(); [&] (){ - ObSysVars[166].default_value_ = "1" ; + ObSysVars[166].default_value_ = "0" ; ObSysVars[166].info_ = "enable batching of the RHS IO in NLJ" ; ObSysVars[166].name_ = "_nlj_batching_enabled" ; ObSysVars[166].data_type_ = ObIntType ; diff --git a/src/share/system_variable/ob_system_variable_init.json b/src/share/system_variable/ob_system_variable_init.json index f18e94583f..864b21112f 100644 --- a/src/share/system_variable/ob_system_variable_init.json +++ b/src/share/system_variable/ob_system_variable_init.json @@ -2379,7 +2379,7 @@ "_nlj_batching_enabled": { "id": 10080, "name": "_nlj_batching_enabled", - "default_value": "1", + "default_value": "0", "base_value": "1", "data_type": "bool", "info": "enable batching of the RHS IO in NLJ", diff --git a/src/share/table/ob_table.cpp b/src/share/table/ob_table.cpp index 60bba37613..20704ec525 100644 --- a/src/share/table/ob_table.cpp +++ b/src/share/table/ob_table.cpp @@ -1313,7 +1313,8 @@ OB_SERIALIZE_MEMBER_IF(ObHTableFilter, //////////////////////////////////////////////////////////////// ObTableQueryResult::ObTableQueryResult() :row_count_(0), - allocator_(ObModIds::TABLE_PROC), + allocator_(ObModIds::TABLE_PROC, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), + prop_name_allocator_(ObModIds::TABLE_PROC, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()), fixed_result_size_(0), curr_idx_(0) { @@ -1333,6 +1334,7 @@ void ObTableQueryResult::reset() { properties_names_.reset(); reset_except_property(); + prop_name_allocator_.reset(); } void ObTableQueryResult::rewind() @@ -1415,7 +1417,7 @@ int ObTableQueryResult::deep_copy_property_names(const ObIArray &other } for (int64_t i = 0; OB_SUCC(ret) && i < other.count(); i++) { - if (OB_FAIL(ob_write_string(allocator_, other.at(i), properties_names_.at(i)))) { + if (OB_FAIL(ob_write_string(prop_name_allocator_, other.at(i), properties_names_.at(i)))) { LOG_WARN("failed to write string", K(ret), K(other.at(i))); } } diff --git a/src/share/table/ob_table.h b/src/share/table/ob_table.h index 5226354250..296926f320 100644 --- a/src/share/table/ob_table.h +++ b/src/share/table/ob_table.h @@ -873,6 +873,7 @@ private: int64_t row_count_; // serialize common::ObDataBuffer buf_; // serialize common::ObArenaAllocator allocator_; + common::ObArenaAllocator prop_name_allocator_; int64_t fixed_result_size_; // for deserialize and read int64_t curr_idx_; diff --git a/src/share/throttle/ob_share_throttle_define.cpp b/src/share/throttle/ob_share_throttle_define.cpp index 6cbacfdbc4..10c177b333 100644 --- a/src/share/throttle/ob_share_throttle_define.cpp +++ b/src/share/throttle/ob_share_throttle_define.cpp @@ -63,7 +63,8 @@ void FakeAllocatorForTxShare::init_throttle_config(int64_t &resource_limit, * @param[out] last_update_limit_ts last update ts (for performance optimization) * @param[out] is_updated to decide if update_decay_factor() is needed */ -void FakeAllocatorForTxShare::adaptive_update_limit(const int64_t holding_size, +void FakeAllocatorForTxShare::adaptive_update_limit(const int64_t tenant_id, + const int64_t holding_size, const int64_t config_specify_resource_limit, int64_t &resource_limit, int64_t &last_update_limit_ts, @@ -76,7 +77,7 @@ void FakeAllocatorForTxShare::adaptive_update_limit(const int64_t holding_size, int64_t cur_ts = ObClockGenerator::getClock(); int64_t old_ts = last_update_limit_ts; if ((cur_ts - old_ts > UPDATE_LIMIT_INTERVAL) && ATOMIC_BCAS(&last_update_limit_ts, old_ts, cur_ts)) { - int64_t remain_memory = lib::get_tenant_memory_remain(MTL_ID()); + int64_t remain_memory = lib::get_tenant_memory_remain(tenant_id); int64_t usable_remain_memory = remain_memory / 100 * USABLE_REMAIN_MEMORY_PERCETAGE; if (remain_memory > MAX_UNUSABLE_MEMORY) { usable_remain_memory = std::max(usable_remain_memory, remain_memory - MAX_UNUSABLE_MEMORY); @@ -96,6 +97,7 @@ void FakeAllocatorForTxShare::adaptive_update_limit(const int64_t holding_size, if (is_updated && REACH_TIME_INTERVAL(10LL * 1000LL * 1000LL)) { SHARE_LOG(INFO, "adaptive update", + "Tenant ID", tenant_id, "Config Specify Resource Limit(MB)", config_specify_resource_limit / 1024 / 1024, "TxShare Current Memory Limit(MB)", resource_limit / 1024 / 1024, "Holding Memory(MB)", holding_size / 1024 / 1024, diff --git a/src/share/throttle/ob_share_throttle_define.h b/src/share/throttle/ob_share_throttle_define.h index a05fd1c41f..9353cf66df 100644 --- a/src/share/throttle/ob_share_throttle_define.h +++ b/src/share/throttle/ob_share_throttle_define.h @@ -28,7 +28,8 @@ } \ static int64_t resource_unit_size(); \ static void init_throttle_config(int64_t &resource_limit, int64_t &trigger_percentage, int64_t &max_duration); \ - static void adaptive_update_limit(const int64_t holding_size, \ + static void adaptive_update_limit(const int64_t tenant_id, \ + const int64_t holding_size, \ const int64_t config_specify_resource_limit, \ int64_t &resource_limit, \ int64_t &last_update_limit_ts, \ @@ -38,6 +39,7 @@ \ struct FakeAllocatorFor##ThrottleName { \ DEFINE_CUSTOM_FUNC_FOR_THROTTLE(ThrottleName); \ + int64_t tenant_id_; \ }; \ \ LST_DEFINE(__VA_ARGS__); \ diff --git a/src/share/throttle/ob_throttle_unit.h b/src/share/throttle/ob_throttle_unit.h index 9347ee59fe..c4f0ba11a5 100644 --- a/src/share/throttle/ob_throttle_unit.h +++ b/src/share/throttle/ob_throttle_unit.h @@ -184,6 +184,7 @@ private: int64_t last_advance_clock_ts_us_; int64_t last_print_throttle_info_ts_; int64_t last_update_limit_ts_; + int64_t tenant_id_; double decay_factor_; diff --git a/src/share/throttle/ob_throttle_unit.ipp b/src/share/throttle/ob_throttle_unit.ipp index bd62794491..acaa8c1eb5 100644 --- a/src/share/throttle/ob_throttle_unit.ipp +++ b/src/share/throttle/ob_throttle_unit.ipp @@ -24,11 +24,12 @@ namespace oceanbase { namespace share { -#define THROTTLE_UNIT_INFO \ - "Unit Name", unit_name_, "Config Specify Resource Limit(MB)", config_specify_resource_limit_ / 1024 / 1024, \ - "Resource Limit(MB)", resource_limit_ / 1024 / 1024, "Throttle Trigger(MB)", \ - resource_limit_ *throttle_trigger_percentage_ / 100 / 1024 / 1024, "Throttle Percentage", \ - throttle_trigger_percentage_, "Max Duration(us)", throttle_max_duration_, "Decay Factor", decay_factor_ +#define THROTTLE_UNIT_INFO \ + KP(this), K(enable_adaptive_limit_), "Unit Name", unit_name_, "Config Specify Resource Limit(MB)", \ + config_specify_resource_limit_ / 1024 / 1024, "Resource Limit(MB)", resource_limit_ / 1024 / 1024, \ + "Throttle Trigger(MB)", resource_limit_ *throttle_trigger_percentage_ / 100 / 1024 / 1024, \ + "Throttle Percentage", throttle_trigger_percentage_, "Max Duration(us)", throttle_max_duration_, "Decay Factor", \ + decay_factor_ template int ObThrottleUnit::init() @@ -48,9 +49,11 @@ int ObThrottleUnit::init() (void)update_decay_factor_(); config_specify_resource_limit_ = resource_limit_; enable_adaptive_limit_ = false; + tenant_id_ = MTL_ID(); is_inited_ = true; SHARE_LOG(INFO, "[Throttle]Init throttle config finish", + K(tenant_id_), K(unit_name_), K(resource_limit_), K(config_specify_resource_limit_), @@ -73,11 +76,8 @@ int ObThrottleUnit::alloc_resource(const int64_t holding_size, // do adaptive update resource limit if needed if (enable_adaptive_limit_) { bool is_updated = false; - ALLOCATOR::adaptive_update_limit(holding_size, - config_specify_resource_limit_, - resource_limit_, - last_update_limit_ts_, - is_updated); + ALLOCATOR::adaptive_update_limit( + tenant_id_, holding_size, config_specify_resource_limit_, resource_limit_, last_update_limit_ts_, is_updated); if (is_updated) { (void)update_decay_factor_(true /* is_adaptive_update */); } @@ -165,6 +165,7 @@ void ObThrottleUnit::print_throttle_info_(const int64_t holding_size, SHARE_LOG(INFO, "[Throttling] (report write throttle info) Size Info", + "tenant_id", tenant_id_, "Throttle Unit Name", unit_name_, "Allocating Resource Size", alloc_size, "Holding Resource Size", holding_size, @@ -172,6 +173,7 @@ void ObThrottleUnit::print_throttle_info_(const int64_t holding_size, "Released Sequence", cur_clock, "Release Speed", release_speed, "Total Resource Limit", resource_limit_, + "Config Specify Limit", config_specify_resource_limit_, "Throttle Trigger Threshold", throttle_trigger); } } @@ -271,6 +273,15 @@ void ObThrottleUnit::advance_clock(const int64_t holding_size) int64_t old_ts = last_advance_clock_ts_us_; const int64_t advance_us = cur_ts - old_ts; if ((advance_us > ADVANCE_CLOCK_INTERVAL) && ATOMIC_BCAS(&last_advance_clock_ts_us_, old_ts, cur_ts)) { + if (enable_adaptive_limit_) { + bool is_updated = false; + ALLOCATOR::adaptive_update_limit( + tenant_id_, holding_size, config_specify_resource_limit_, resource_limit_, last_update_limit_ts_, is_updated); + if (is_updated) { + (void)update_decay_factor_(true /* is_adaptive_update */); + } + } + bool unused = false; const int64_t throttle_trigger = resource_limit_ * throttle_trigger_percentage_ / 100; const int64_t avaliable_resource = avaliable_resource_after_dt_(holding_size, throttle_trigger, advance_us); @@ -490,6 +501,12 @@ void ObThrottleUnit::update_throttle_config(const int64_t resource_li throttle_trigger_percentage_ != throttle_trigger_percentage || throttle_max_duration_ != throttle_max_duration) { config_changed = true; + SHARE_LOG(INFO, + "[Throttle] Update Config", + THROTTLE_UNIT_INFO, + "New Resource Limit(MB)", resource_limit / 1024 / 1024, + "New trigger percentage", throttle_trigger_percentage, + "New Throttle Duration", throttle_max_duration); throttle_trigger_percentage_ = throttle_trigger_percentage; throttle_max_duration_ = throttle_max_duration; (void)inner_set_resource_limit_(resource_limit); diff --git a/src/share/vector/expr_cmp_func.cpp b/src/share/vector/expr_cmp_func.cpp index 012f512ed7..d810cbe778 100644 --- a/src/share/vector/expr_cmp_func.cpp +++ b/src/share/vector/expr_cmp_func.cpp @@ -88,6 +88,13 @@ using nullsafe_cmp_initer = InitCmpSet< static bool g_init_cmp_set = Ob2DArrayConstIniter::init(); +static bool init_row_cmp_double_func() { + ROW_CMP_FUNCS[VEC_TC_DOUBLE][VEC_TC_FIXED_DOUBLE] = VecTCCmpCalc::cmp; + ROW_CMP_FUNCS[VEC_TC_FIXED_DOUBLE][VEC_TC_DOUBLE] = VecTCCmpCalc::cmp; + return true; +} +static bool g_init_row_cmp_double_func = init_row_cmp_double_func(); + void VectorCmpExprFuncsHelper::get_cmp_set(const sql::ObDatumMeta &l_meta, const sql::ObDatumMeta &r_meta, sql::NullSafeRowCmpFunc &null_first_cmp, @@ -239,94 +246,6 @@ template <> int get_cmp_ret (const int ret) { return ret; } } \ } while (false) -#define DO_VECTOR_BETWEEN_CMP(LVec, RVec, ResVec) \ - do { \ - LVec *l_vector = static_cast(left.get_vector(ctx)); \ - RVec *r_vector = static_cast(right.get_vector(ctx)); \ - ResVec *res_vec = static_cast(expr.get_vector(ctx)); \ - ObBitVector &eval_flags = expr.get_evaluated_flags(ctx); \ - const char *l_payload = nullptr, *r_payload = nullptr; \ - ObLength l_len = 0, r_len = 0; \ - int cmp_ret = 0; \ - if (!l_vector->has_null() && !r_vector->has_null()) { \ - if (OB_LIKELY(bound.get_all_rows_active() \ - && eval_flags.accumulate_bit_cnt(bound) == 0)) { \ - for (int i = bound.start(); OB_SUCC(ret) && i < bound.end(); i++) { \ - l_vector->get_payload(i, l_payload, l_len); \ - r_vector->get_payload(i, r_payload, r_len); \ - ret = VecTCCmpCalc::cmp(left.obj_meta_, right.obj_meta_, \ - (const void *)l_payload, l_len, \ - (const void *)r_payload, r_len, cmp_ret); \ - /* Result priority: false > null > true */ \ - if (OB_FAIL(ret)) { \ - } else if (Stage == ObExprBetween::BETWEEN_LEFT) { \ - /* If the current calculation is left<=val, any result is directly filled in. \ - If the result is false, the subsequent calculation results are meaningless, \ - and skip is set to true. */ \ - res_vec->set_int(i, (cmp_ret <= 0)); \ - if (cmp_ret > 0) { \ - skip.set(i); \ - } \ - } else if (cmp_ret > 0) { /*BETWEEN_RIGHT*/ \ - /* If currently calculating val<=right, \ - only when the result is false will it be filled in. \ - Note that set_null may have been called before, \ - so unset_null should be called here. */ \ - res_vec->unset_null(i); \ - res_vec->set_int(i, 0); \ - } \ - } \ - } else { \ - for (int i = bound.start(); OB_SUCC(ret) && i < bound.end(); i++) { \ - if (skip.at(i) || eval_flags.at(i)) { continue; } \ - l_vector->get_payload(i, l_payload, l_len); \ - r_vector->get_payload(i, r_payload, r_len); \ - ret = VecTCCmpCalc::cmp(left.obj_meta_, right.obj_meta_, \ - (const void *)l_payload, l_len, \ - (const void *)r_payload, r_len, cmp_ret); \ - if (OB_FAIL(ret)) { \ - } else if (Stage == ObExprBetween::BETWEEN_LEFT) { \ - res_vec->set_int(i, (cmp_ret <= 0)); \ - if (cmp_ret > 0) { \ - skip.set(i); \ - } \ - } else if (cmp_ret > 0) { /*BETWEEN_RIGHT*/ \ - res_vec->unset_null(i); \ - res_vec->set_int(i, 0); \ - } \ - } \ - } \ - } else { \ - for (int i = bound.start(); OB_SUCC(ret) && i < bound.end(); i++) { \ - if (skip.at(i) || eval_flags.at(i)) { continue; } \ - if (l_vector->is_null(i) || r_vector->is_null(i)) { \ - res_vec->set_null(i); \ - /* Cannot set skip here. \ - Because the priority of the "between" results \ - is consistent with the "and" expression: false > null > true. \ - If the result of the right branch is false, \ - it should override the null in the left branch. */ \ - } else { \ - l_vector->get_payload(i, l_payload, l_len); \ - r_vector->get_payload(i, r_payload, r_len); \ - ret = VecTCCmpCalc::cmp(left.obj_meta_, right.obj_meta_, \ - (const void *)l_payload, l_len, \ - (const void *)r_payload, r_len, cmp_ret); \ - if (OB_FAIL(ret)) { \ - } else if (Stage == ObExprBetween::BETWEEN_LEFT) { \ - res_vec->set_int(i, (cmp_ret <= 0)); \ - if (cmp_ret > 0) { \ - skip.set(i); \ - } \ - } else if (cmp_ret > 0) { /*BETWEEN_RIGHT*/ \ - res_vec->unset_null(i); \ - res_vec->set_int(i, 0); \ - } \ - } \ - } \ - } \ - } while (false) - #define CALC_FORMAT(l, r, res) \ ((int32_t)l + (((int32_t)r) << VEC_MAX_FORMAT) + (((int32_t)res) << (VEC_MAX_FORMAT * 2))) template @@ -415,6 +334,7 @@ struct EvalVectorCmpWithNull ObBitmapNullVectorBase &res_nulls = *static_cast(expr.get_vector(ctx)); if (OB_LIKELY(bound.get_all_rows_active() && eval_flags.accumulate_bit_cnt(bound) == 0)) { res_nulls.get_nulls()->set_all(bound.start(), bound.end()); + res_nulls.set_has_null(); eval_flags.set_all(bound.start(), bound.end()); } else { for (int i = bound.start(); i < bound.end(); i++) { @@ -451,57 +371,9 @@ struct EvalVectorCmp: public EvalVectorCmpWithNull {} template struct EvalVectorCmp: public EvalVectorCmpWithNull {}; -template -struct EvalVectorBetweenCmp -{ - #define VECTOR_BETWEEN_CMP_CASE(l_fmt, r_fmt, res_fmt) \ - case CALC_FORMAT(l_fmt, r_fmt, res_fmt): { \ - DO_VECTOR_BETWEEN_CMP(L_##l_fmt##_FMT, R_##r_fmt##_FMT, RES_##res_fmt##_FMT); \ - } break - static int eval_between_vector(const ObExpr &expr, const ObExpr &left, const ObExpr &right, - ObEvalCtx &ctx, ObBitVector &skip, const EvalBound &bound) - { - using L_VEC_FIXED_FMT = ObFixedLengthFormat>; - using R_VEC_FIXED_FMT = ObFixedLengthFormat>; - using RES_VEC_FIXED_FMT = ObFixedLengthFormat; - using L_VEC_DISCRETE_FMT = ObDiscreteFormat; - using R_VEC_DISCRETE_FMT = ObDiscreteFormat; - using L_VEC_UNIFORM_FMT = ObUniformFormat; - using R_VEC_UNIFORM_FMT = ObUniformFormat; - - int ret = OB_SUCCESS; - VectorFormat left_format = left.get_format(ctx); - VectorFormat right_format = right.get_format(ctx); - VectorFormat res_format = expr.get_format(ctx); - LOG_DEBUG("eval vector cmp", K(expr), K(l_tc), K(r_tc), K(bound), - K(left_format), K(right_format), K(res_format)); - if (is_valid_format(left_format) && is_valid_format(right_format) - && is_valid_format(res_format)) { - switch (CALC_FORMAT(left_format, right_format, res_format)) { - VECTOR_BETWEEN_CMP_CASE(VEC_FIXED, VEC_FIXED, VEC_FIXED); - VECTOR_BETWEEN_CMP_CASE(VEC_FIXED, VEC_UNIFORM, VEC_FIXED); - VECTOR_BETWEEN_CMP_CASE(VEC_DISCRETE, VEC_DISCRETE, VEC_FIXED); - VECTOR_BETWEEN_CMP_CASE(VEC_DISCRETE, VEC_UNIFORM, VEC_FIXED); - VECTOR_BETWEEN_CMP_CASE(VEC_UNIFORM, VEC_FIXED, VEC_FIXED); - VECTOR_BETWEEN_CMP_CASE(VEC_UNIFORM, VEC_DISCRETE, VEC_FIXED); - VECTOR_BETWEEN_CMP_CASE(VEC_UNIFORM, VEC_UNIFORM, VEC_FIXED); - default: { - DO_VECTOR_BETWEEN_CMP(ObVectorBase, ObVectorBase, ObVectorBase); - break; - } - } - } else { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid format", K(left_format), K(right_format), K(res_format)); - } - return ret; - } -#undef VECTOR_BETWEEN_CMP_CASE -}; #undef CALC_FORMAT static sql::ObExpr::EvalVectorFunc EVAL_VECTOR_EXPR_CMP_FUNCS[MAX_VEC_TC][MAX_VEC_TC][CO_MAX]; -static sql::ObExprBetween::EvalVectorBetweenFunc EVAL_VECTOR_BETWEEN_EXPR_CMP_FUNCS[MAX_VEC_TC][MAX_VEC_TC][ObExprBetween::EvalBetweenStage::BETWEEN_MAX]; template struct VectorExprCmpFuncIniter @@ -518,9 +390,6 @@ struct VectorExprCmpFuncIniter template using EvalFunc = EvalVectorCmp(X), static_cast(Y), cmp_op>; - template - using EvalBetweenFunc = - EvalVectorBetweenCmp(X), static_cast(Y), stage>; static void init_array() { auto &funcs = EVAL_VECTOR_EXPR_CMP_FUNCS; @@ -531,12 +400,6 @@ struct VectorExprCmpFuncIniter funcs[X][Y][CO_NE] = &EvalFunc::eval_vector; funcs[X][Y][CO_EQ] = &EvalFunc::eval_vector; funcs[X][Y][CO_CMP] = &EvalFunc::eval_vector; - - auto &between_funcs = EVAL_VECTOR_BETWEEN_EXPR_CMP_FUNCS; - between_funcs[X][Y][ObExprBetween::EvalBetweenStage::BETWEEN_LEFT] = - &EvalBetweenFunc::eval_between_vector; - between_funcs[X][Y][ObExprBetween::EvalBetweenStage::BETWEEN_RIGHT] = - &EvalBetweenFunc::eval_between_vector; } }; @@ -557,48 +420,35 @@ sql::ObExpr::EvalVectorFunc VectorCmpExprFuncsHelper::get_eval_vector_expr_cmp_f return EVAL_VECTOR_EXPR_CMP_FUNCS[l_tc][r_tc][cmp_op]; } -sql::ObExprBetween::EvalVectorBetweenFunc VectorCmpExprFuncsHelper::get_eval_vector_between_expr_cmp_func( - const sql::ObDatumMeta &l_meta, const sql::ObDatumMeta &r_meta, - sql::ObExprBetween::EvalBetweenStage stage) -{ - LOG_DEBUG("eval vector between_expr_cmp_func", K(l_meta), K(r_meta), K(stage)); - VecValueTypeClass l_tc = get_vec_value_tc(l_meta.type_, l_meta.scale_, l_meta.precision_); - VecValueTypeClass r_tc = get_vec_value_tc(r_meta.type_, r_meta.scale_, r_meta.precision_); - return EVAL_VECTOR_BETWEEN_EXPR_CMP_FUNCS[l_tc][r_tc][stage]; -} - } // end namespace common namespace sql { void *g_ser_eval_vector_expr_cmp_funcs[MAX_VEC_TC * MAX_VEC_TC * 7]; -void *g_ser_eval_vector_between_expr_cmp_funcs[MAX_VEC_TC * MAX_VEC_TC * 2]; void *g_ser_nullsafe_rowcmp_funcs[MAX_VEC_TC * MAX_VEC_TC * 2]; +void *g_ser_rowcmp_funcs[MAX_VEC_TC * MAX_VEC_TC]; static_assert(sizeof(g_ser_eval_vector_expr_cmp_funcs) == sizeof(EVAL_VECTOR_EXPR_CMP_FUNCS), "unexpected size"); -static_assert(sizeof(g_ser_eval_vector_between_expr_cmp_funcs) == sizeof(EVAL_VECTOR_BETWEEN_EXPR_CMP_FUNCS), - "unexpected size"); static_assert(sizeof(g_ser_nullsafe_rowcmp_funcs) == sizeof(NULLSAFE_ROW_CMP_FUNCS), "unexpected size"); +static_assert(sizeof(g_ser_rowcmp_funcs) == sizeof(ROW_CMP_FUNCS), + "unexpected size"); bool g_ser_eval_vector_expr_cmp_funcs_init = ObFuncSerialization::convert_NxN_array( g_ser_eval_vector_expr_cmp_funcs, reinterpret_cast(EVAL_VECTOR_EXPR_CMP_FUNCS), MAX_VEC_TC, 7, 0, 7); -bool g_ser_eval_vector_between_expr_cmp_funcs_init = ObFuncSerialization::convert_NxN_array( - g_ser_eval_vector_between_expr_cmp_funcs, reinterpret_cast(EVAL_VECTOR_BETWEEN_EXPR_CMP_FUNCS), - MAX_VEC_TC, 2, 0, 2); - bool g_ser_nullsafe_rowcmp_funcs_init = ObFuncSerialization::convert_NxN_array( g_ser_nullsafe_rowcmp_funcs, reinterpret_cast(NULLSAFE_ROW_CMP_FUNCS), MAX_VEC_TC, 2, 0, 2); - +bool g_ser_rowcmp_funcs_init = ObFuncSerialization::convert_NxN_array( + g_ser_rowcmp_funcs, reinterpret_cast(ROW_CMP_FUNCS), + MAX_VEC_TC, 1, 0, 1); REG_SER_FUNC_ARRAY(OB_SFA_CMP_EXPR_EVAL_VECTOR, g_ser_eval_vector_expr_cmp_funcs, sizeof(g_ser_eval_vector_expr_cmp_funcs) / sizeof(void *)); -REG_SER_FUNC_ARRAY(OB_SFA_CMP_BETWEEN_EXPR_EVAL_VECTOR, g_ser_eval_vector_between_expr_cmp_funcs, - sizeof(g_ser_eval_vector_between_expr_cmp_funcs) / sizeof(void *)); - REG_SER_FUNC_ARRAY(OB_SFA_VECTOR_NULLSAFE_CMP, g_ser_nullsafe_rowcmp_funcs, sizeof(g_ser_nullsafe_rowcmp_funcs) / sizeof(void *)); +REG_SER_FUNC_ARRAY(OB_SFA_VECTOR_CMP, g_ser_rowcmp_funcs, + sizeof(g_ser_rowcmp_funcs) / sizeof(void *)); } // end namespace sql } // end namespace oceanabse diff --git a/src/share/vector/expr_cmp_func.h b/src/share/vector/expr_cmp_func.h index 7cb07f21b8..e5abbb1872 100644 --- a/src/share/vector/expr_cmp_func.h +++ b/src/share/vector/expr_cmp_func.h @@ -39,15 +39,6 @@ struct VectorCmpExprFuncsHelper static sql::ObExpr::EvalVectorFunc get_eval_vector_expr_cmp_func(const sql::ObDatumMeta &l_meta, const sql::ObDatumMeta &r_meta, const common::ObCmpOp cmp_op); - // Compared to get_eval_vector_expr_cmp_func, - // the obtained function has two additional expr arguments: left and right. - // Among the EvalVectorFunc functions, - // they directly use the first two arguments of the expr as left and right. - // However, in some scenarios, this may not be the case, - // so the following functions have been added. - static sql::ObExprBetween::EvalVectorBetweenFunc get_eval_vector_between_expr_cmp_func(const sql::ObDatumMeta &l_meta, - const sql::ObDatumMeta &r_meta, - sql::ObExprBetween::EvalBetweenStage stage); }; } // end namespace common diff --git a/src/sql/code_generator/ob_dml_cg_service.cpp b/src/sql/code_generator/ob_dml_cg_service.cpp index a9746545dd..e5aaacddcf 100644 --- a/src/sql/code_generator/ob_dml_cg_service.cpp +++ b/src/sql/code_generator/ob_dml_cg_service.cpp @@ -1303,6 +1303,32 @@ int ObDmlCgService::heap_table_has_not_null_uk(ObSchemaGetterGuard *schema_guard return ret; } +int ObDmlCgService::append_lob_type_column_id(const ObTableSchema *table_schema, + ObIArray &minimal_column_ids) +{ + int ret = OB_SUCCESS; + ObTableSchema::const_column_iterator iter = table_schema->column_begin(); + for (; OB_SUCC(ret) && iter != table_schema->column_end(); ++iter) { + const ObColumnSchemaV2 *column = *iter; + if (OB_ISNULL(column)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid column schema", K(column)); + } else if (is_lob_storage(column->get_meta_type().get_type())) { + // The hidden column of xml type is of lob type, so when adding the lob column here, + // the hidden column of xml type will be added naturally. + if (OB_FAIL(add_var_to_array_no_dup(minimal_column_ids, column->get_column_id()))) { + LOG_WARN("add time type column_id failed", K(ret), K(column->get_column_id())); + } + } else if (column->get_meta_type().is_user_defined_sql_type()) { + // append xml column + if (OB_FAIL(add_var_to_array_no_dup(minimal_column_ids, column->get_column_id()))) { + LOG_WARN("add time type column_id failed", K(ret), K(column->get_column_id())); + } + } + } + return ret; +} + int ObDmlCgService::append_time_type_column_id(const ObTableSchema *table_schema, ObIArray &minimal_column_ids) { @@ -1546,6 +1572,10 @@ int ObDmlCgService::append_upd_old_row_cid(ObLogicalOperator &op, } } else if (OB_FAIL(is_table_has_unique_key(schema_guard, table_schema, has_uk))) { LOG_WARN("fail to check table has UK", K(ret)); + } else if (has_uk && + OB_FAIL(append_all_uk_column_id(schema_guard, table_schema, minimal_column_ids))) { + // append UK + LOG_WARN("fail to append all uk column_id", K(ret)); } else if (OB_FAIL(append_udt_hidden_col_id(op, table_schema, index_dml_info, minimal_column_ids))) { // append UDT hidden column LOG_WARN("fail to append upd assignment column_id", K(ret), K(index_dml_info)); @@ -1554,10 +1584,6 @@ int ObDmlCgService::append_upd_old_row_cid(ObLogicalOperator &op, minimal_column_ids))) { // append update column LOG_WARN("fail to append upd assignment column_id", K(ret), K(index_dml_info)); - } else if (has_uk && - OB_FAIL(append_all_uk_column_id(schema_guard, table_schema, minimal_column_ids))) { - // append UK - LOG_WARN("fail to append all uk column_id", K(ret)); } else if (OB_FAIL(append_time_type_column_id(table_schema, minimal_column_ids))) { // append time_type column LOG_WARN("fail to append time type column_id", K(ret)); @@ -1678,6 +1704,8 @@ int ObDmlCgService::generate_minimal_delete_old_row_cid(ObTableID index_tid, } else if (OB_FAIL(append_all_pk_column_id(schema_guard, table_schema, minimal_column_ids))) { // append PK LOG_WARN("fail to append all pk to column_id", K(ret), K(index_tid)); + } else if (OB_FAIL(append_lob_type_column_id(table_schema, minimal_column_ids))) { + LOG_WARN("fail to append all lob_storage column_id", K(ret), K(index_tid)); } else if (!is_primary_index) { // index_table record PK and the dependent columns of shadow_pk // diff --git a/src/sql/code_generator/ob_dml_cg_service.h b/src/sql/code_generator/ob_dml_cg_service.h index a4407ef021..3c0f1e480d 100644 --- a/src/sql/code_generator/ob_dml_cg_service.h +++ b/src/sql/code_generator/ob_dml_cg_service.h @@ -176,6 +176,9 @@ private: int append_time_type_column_id(const ObTableSchema *table_schema, ObIArray &minimal_column_ids); + int append_lob_type_column_id(const ObTableSchema *table_schema, + ObIArray &minimal_column_ids); + int heap_table_has_not_null_uk(ObSchemaGetterGuard *schema_guard, const ObTableSchema *table_schema, bool &need_all_columns); diff --git a/src/sql/code_generator/ob_static_engine_cg.cpp b/src/sql/code_generator/ob_static_engine_cg.cpp index f97412e815..582db2d64a 100644 --- a/src/sql/code_generator/ob_static_engine_cg.cpp +++ b/src/sql/code_generator/ob_static_engine_cg.cpp @@ -635,13 +635,18 @@ int ObStaticEngineCG::check_vectorize_supported(bool &support, // Expr rownum() shows up in both operator 0 and 2, which leads circular // dependency and breaks rownum's defination. // - bool has_rownum_expr = false; - for (int64_t i = 0; !has_rownum_expr && OB_SUCC(ret) && i < op->get_num_of_child(); i++) { - OZ(op->get_child(i)->find_rownum_expr(has_rownum_expr)); + const ObRawExpr *rownum_expr = NULL; + for (int64_t i = 0; rownum_expr == NULL && OB_SUCC(ret) && i < op->get_num_of_child(); i++) { + ObLogicalOperator *child = op->get_child(i); + if (OB_ISNULL(child)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("op child is null", K(ret)); + } else if (OB_FAIL(child->find_rownum_expr(rownum_expr))) { + LOG_WARN("find rownum expr error", K(ret)); + } } - if (has_rownum_expr) { - LOG_DEBUG("rownum expr is in count operator's subplan tree. Stop vectorization execution", - K(has_rownum_expr)); + if (NULL != rownum_expr) { + LOG_DEBUG("rownum expr is in count operator's subplan tree. Stop vectorization exec"); disable_vectorize = true; } } else if (log_op_def::LOG_JOIN == op->get_type() && diff --git a/src/sql/das/ob_das_extra_data.cpp b/src/sql/das/ob_das_extra_data.cpp index ca084474f0..8f864830c2 100644 --- a/src/sql/das/ob_das_extra_data.cpp +++ b/src/sql/das/ob_das_extra_data.cpp @@ -143,7 +143,8 @@ int ObDASExtraData::get_next_rows(int64_t &count, int64_t capacity) } } else { got_row = true; - PRINT_VECTORIZED_ROWS(SQL, DEBUG, *eval_ctx_, *output_exprs_, count, KR(ret)); + const ObBitVector *skip = NULL; + PRINT_VECTORIZED_ROWS(SQL, DEBUG, *eval_ctx_, *output_exprs_, count, skip, KR(ret)); } } return ret; diff --git a/src/sql/das/ob_das_group_scan_op.cpp b/src/sql/das/ob_das_group_scan_op.cpp index 62660ee8dc..3f3fceb2df 100644 --- a/src/sql/das/ob_das_group_scan_op.cpp +++ b/src/sql/das/ob_das_group_scan_op.cpp @@ -98,6 +98,7 @@ int ObDASGroupScanOp::open_op() iter_.init_group_range(cur_group_idx_, group_size_); if (OB_FAIL(iter_.init_row_store(scan_ctdef_->result_output_, *scan_rtdef_->eval_ctx_, + scan_ctdef_->pd_expr_spec_.access_exprs_, scan_rtdef_->stmt_allocator_, max_size, scan_ctdef_->group_id_expr_, @@ -241,6 +242,7 @@ int ObDASGroupScanOp::decode_task_result(ObIDASTaskResult *task_result) iter_.init_group_range(cur_group_idx_, group_size_); if (OB_FAIL(iter_.init_row_store(scan_ctdef_->result_output_, *scan_rtdef_->eval_ctx_, + scan_ctdef_->pd_expr_spec_.access_exprs_, scan_rtdef_->stmt_allocator_, max_size, scan_ctdef_->group_id_expr_, @@ -296,6 +298,7 @@ int ObGroupLookupOp::init_group_scan_iter(int64_t cur_group_idx, group_iter_.init_group_range(cur_group_idx, group_size); OZ(group_iter_.init_row_store(lookup_ctdef_->result_output_, *lookup_rtdef_->eval_ctx_, + lookup_ctdef_->pd_expr_spec_.access_exprs_, lookup_rtdef_->stmt_allocator_, max_row_store_size, group_id_expr, diff --git a/src/sql/das/ob_das_scan_op.cpp b/src/sql/das/ob_das_scan_op.cpp index 7ec851260f..8c60ba4883 100644 --- a/src/sql/das/ob_das_scan_op.cpp +++ b/src/sql/das/ob_das_scan_op.cpp @@ -627,7 +627,8 @@ int ObDASScanOp::fill_task_result(ObIDASTaskResult &task_result, bool &has_more, } } if (OB_SUCC(ret) && has_more) { - PRINT_VECTORIZED_ROWS(SQL, DEBUG, eval_ctx, result_output, remain_row_cnt_, + const ObBitVector *skip = NULL; + PRINT_VECTORIZED_ROWS(SQL, DEBUG, eval_ctx, result_output, remain_row_cnt_, skip, K(simulate_row_cnt), K(datum_store.get_row_cnt()), K(has_more)); } @@ -806,7 +807,8 @@ int ObDASScanResult::get_next_rows(int64_t &count, int64_t capacity) } } } else { - PRINT_VECTORIZED_ROWS(SQL, DEBUG, *eval_ctx_, *output_exprs_, count); + const ObBitVector *skip = NULL; + PRINT_VECTORIZED_ROWS(SQL, DEBUG, *eval_ctx_, *output_exprs_, count, skip); } LOG_DEBUG("das result next rows", K(enable_rich_format_), K(count), K(capacity), K(ret)); return ret; diff --git a/src/sql/das/ob_das_utils.cpp b/src/sql/das/ob_das_utils.cpp index a69271c130..2da0921197 100644 --- a/src/sql/das/ob_das_utils.cpp +++ b/src/sql/das/ob_das_utils.cpp @@ -257,10 +257,23 @@ int ObDASUtils::reshape_storage_value(const ObObjMeta &col_type, const ObAccuracy &col_accuracy, ObIAllocator &allocator, ObObj &value) +{ + int ret = OB_SUCCESS; + if (lib::is_oracle_mode() && value.is_character_type() && value.get_string_len() == 0) { + // Oracle compatibility mode: '' as null + LOG_DEBUG("reshape empty string to null", K(value)); + value.set_null(); + } else if (OB_FAIL(padding_fixed_string_value(col_accuracy.get_length(), allocator, value))) { + LOG_WARN("padding char value failed", K(ret), K(col_accuracy), K(value)); + } + return ret; +} + +int ObDASUtils::padding_fixed_string_value(int64_t max_len, ObIAllocator &allocator, ObObj &value) { int ret = OB_SUCCESS; if (value.is_binary()) { - int32_t binary_len = col_accuracy.get_length(); + int32_t binary_len = max_len; int32_t len = value.get_string_len(); if (binary_len > len) { char *dest_str = NULL; @@ -275,10 +288,6 @@ int ObDASUtils::reshape_storage_value(const ObObjMeta &col_type, value.set_binary(ObString(binary_len, dest_str)); } } - } else if (lib::is_oracle_mode() && value.is_character_type() && value.get_string_len() == 0) { - // Oracle compatibility mode: '' as null - LOG_DEBUG("reshape empty string to null", K(value)); - value.set_null(); } else if (value.is_fixed_len_char_type()) { const char *str = value.get_string_ptr(); int32_t len = value.get_string_len(); diff --git a/src/sql/das/ob_das_utils.h b/src/sql/das/ob_das_utils.h index 540fe40344..16d770be78 100644 --- a/src/sql/das/ob_das_utils.h +++ b/src/sql/das/ob_das_utils.h @@ -62,6 +62,7 @@ public: const bool enable_oracle_empty_char_reshape_to_null, ObIAllocator &allocator, blocksstable::ObStorageDatum &datum_value); + static int padding_fixed_string_value(int64_t max_len, ObIAllocator &alloc, ObObj &value); static int generate_spatial_index_rows(ObIAllocator &allocator, const ObDASDMLBaseCtDef &das_ctdef, const ObString &wkb_str, diff --git a/src/sql/das/ob_group_scan_iter.cpp b/src/sql/das/ob_group_scan_iter.cpp index e6ff704cd5..fad56d7756 100644 --- a/src/sql/das/ob_group_scan_iter.cpp +++ b/src/sql/das/ob_group_scan_iter.cpp @@ -21,6 +21,7 @@ namespace sql { int ObGroupResultRows::init(const common::ObIArray &exprs, ObEvalCtx &eval_ctx, + const ExprFixedArray &access_exprs, ObIAllocator &das_op_allocator, int64_t max_size, ObExpr *group_id_expr, @@ -50,6 +51,7 @@ int ObGroupResultRows::init(const common::ObIArray &exprs, } inited_ = true; exprs_ = &exprs; + access_exprs_ = &access_exprs; eval_ctx_ = &eval_ctx; max_size_ = max_size; group_id_expr_pos_ = OB_INVALID_INDEX; @@ -283,7 +285,8 @@ int ObGroupScanIter::get_next_rows(int64_t &count, int64_t capacity) } } if (OB_SUCC(ret)) { - PRINT_VECTORIZED_ROWS(SQL, DEBUG, *row_store_.eval_ctx_, *row_store_.exprs_, storage_count); + const ObBitVector *skip = NULL; + PRINT_VECTORIZED_ROWS(SQL, DEBUG, *row_store_.eval_ctx_, *row_store_.exprs_, storage_count, skip); ObDatum *group_idx_batch = group_id_expr_->locate_batch_datums(*row_store_.eval_ctx_); int64_t i = 0; for (i = 0; i < storage_count; i++) { @@ -349,10 +352,12 @@ int ObGroupScanIter::get_next_rows(int64_t &count, int64_t capacity) void ObGroupScanIter::reset_expr_datum_ptr() { - FOREACH_CNT(e, *row_store_.exprs_) { - (*e)->locate_datums_for_update(*row_store_.eval_ctx_, row_store_.max_size_); - ObEvalInfo &info = (*e)->get_eval_info(*row_store_.eval_ctx_); - info.point_to_frame_ = true; + if (OB_NOT_NULL(row_store_.access_exprs_)) { + FOREACH_CNT(e, *row_store_.access_exprs_) { + (*e)->locate_datums_for_update(*row_store_.eval_ctx_, row_store_.max_size_); + ObEvalInfo &info = (*e)->get_eval_info(*row_store_.eval_ctx_); + info.point_to_frame_ = true; + } } } diff --git a/src/sql/das/ob_group_scan_iter.h b/src/sql/das/ob_group_scan_iter.h index a2fdc014d4..96727c7623 100644 --- a/src/sql/das/ob_group_scan_iter.h +++ b/src/sql/das/ob_group_scan_iter.h @@ -23,7 +23,7 @@ class ObGroupResultRows public: ObGroupResultRows() : inited_(false), exprs_(NULL), eval_ctx_(NULL), saved_size_(0), max_size_(1), start_pos_(0), group_id_expr_pos_(0), - rows_(NULL), need_check_output_datum_(false),reuse_alloc_(nullptr) + rows_(NULL), need_check_output_datum_(false),reuse_alloc_(nullptr),access_exprs_(nullptr) { } @@ -31,6 +31,7 @@ public: int init(const common::ObIArray &exprs, ObEvalCtx &eval_ctx, + const ExprFixedArray &access_exprs, common::ObIAllocator &das_op_allocator, int64_t max_size, ObExpr *group_id_expr, @@ -57,6 +58,7 @@ public: reuse_alloc_->~ObArenaAllocator(); reuse_alloc_ = nullptr; } + access_exprs_ = nullptr; } TO_STRING_KV(K_(saved_size), K_(start_pos), @@ -86,6 +88,7 @@ public: //be move from das into Table scan op, every thing will be easy. common::ObArenaAllocator *reuse_alloc_; char reuse_alloc_buf_[sizeof(common::ObArenaAllocator)]; + const ExprFixedArray *access_exprs_; }; class ObGroupScanIter : public ObNewRowIterator @@ -116,6 +119,7 @@ public: ObExpr *get_group_id_expr() { return group_id_expr_; } int init_row_store(const common::ObIArray &exprs, ObEvalCtx &eval_ctx, + const ExprFixedArray &access_exprs, common::ObIAllocator &das_op_allocator, int64_t max_size, ObExpr *group_id_expr, @@ -127,6 +131,7 @@ public: iter_ = iter; return row_store_.init(exprs, eval_ctx, + access_exprs, das_op_allocator, max_size, group_id_expr, diff --git a/src/sql/dtl/ob_dtl_fc_server.cpp b/src/sql/dtl/ob_dtl_fc_server.cpp index 8369ff7ba4..57f4709a3b 100644 --- a/src/sql/dtl/ob_dtl_fc_server.cpp +++ b/src/sql/dtl/ob_dtl_fc_server.cpp @@ -38,7 +38,7 @@ ObTenantDfc::ObTenantDfc(uint64_t tenant_id) ObTenantDfc::~ObTenantDfc() {} -int ObTenantDfc::mtl_init(ObTenantDfc *&tenant_dfc) +int ObTenantDfc::mtl_new(ObTenantDfc *&tenant_dfc) { int ret = OB_SUCCESS; uint64_t tenant_id = MTL_ID(); @@ -47,7 +47,16 @@ int ObTenantDfc::mtl_init(ObTenantDfc *&tenant_dfc) ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to alloc tenant dfc", K(ret)); } else if (FALSE_IT(new (tenant_dfc) ObTenantDfc(tenant_id))) { - } else { + } + return ret; +} + + +int ObTenantDfc::mtl_init(ObTenantDfc *&tenant_dfc) +{ + int ret = OB_SUCCESS; + uint64_t tenant_id = MTL_ID(); + if (OB_SUCC(ret)) { tenant_dfc->channel_total_cnt_ = 0; tenant_dfc->blocked_dfc_cnt_ = 0; tenant_dfc->max_parallel_cnt_ = 0; @@ -60,14 +69,6 @@ int ObTenantDfc::mtl_init(ObTenantDfc *&tenant_dfc) // tenant_dfc->calc_max_buffer(10); LOG_INFO("init tenant dfc", K(ret), K(tenant_dfc->tenant_id_)); } - if (OB_FAIL(ret)) { - if (nullptr != tenant_dfc) { - tenant_dfc->tenant_mem_mgr_.destroy(); - common::ob_delete(tenant_dfc); - tenant_dfc = nullptr; - } - LOG_WARN("failed to mtl init", K(ret)); - } return ret; } diff --git a/src/sql/dtl/ob_dtl_fc_server.h b/src/sql/dtl/ob_dtl_fc_server.h index e1d88b8d87..60fe30208f 100644 --- a/src/sql/dtl/ob_dtl_fc_server.h +++ b/src/sql/dtl/ob_dtl_fc_server.h @@ -36,6 +36,7 @@ public: ObTenantDfc(uint64_t tenant_id); virtual ~ObTenantDfc(); public: + static int mtl_new(ObTenantDfc *&tenant_dfc); static int mtl_init(ObTenantDfc *&tenant_dfc); static void mtl_destroy(ObTenantDfc *&tenant_dfc); diff --git a/src/sql/dtl/ob_dtl_interm_result_manager.cpp b/src/sql/dtl/ob_dtl_interm_result_manager.cpp index 417a30ecb4..349072daba 100644 --- a/src/sql/dtl/ob_dtl_interm_result_manager.cpp +++ b/src/sql/dtl/ob_dtl_interm_result_manager.cpp @@ -389,21 +389,21 @@ int ObDTLIntermResultManager::erase_tenant_interm_result_info() int ret = OB_SUCCESS; MAP::bucket_iterator bucket_it = map_.bucket_begin(); while (bucket_it != map_.bucket_end()) { - MAP::hashtable::bucket_lock_cond blc(*bucket_it); - MAP::hashtable::writelocker locker(blc.lock()); - MAP::hashtable::hashbucket::const_iterator node_it = bucket_it->node_begin(); - while (node_it != bucket_it->node_end()) { - int tmp_ret = OB_SUCCESS; - const ObDTLIntermResultKey &key = node_it->first; - node_it++; - if (OB_SUCCESS != (tmp_ret = erase_interm_result_info(key))) { - if (OB_HASH_NOT_EXIST != tmp_ret) { - LOG_WARN("fail to erase result info", K(key), K(tmp_ret)); - ret = tmp_ret; + while (true) { + ObDTLIntermResultKey key; + { + MAP::hashtable::bucket_lock_cond blc(*bucket_it); + MAP::hashtable::readlocker locker(blc.lock()); + MAP::hashtable::hashbucket::const_iterator node_it = bucket_it->node_begin(); + if (node_it == bucket_it->node_end()) { + break; + } else { + key = node_it->first; } } + erase_interm_result_info(key); } - bucket_it++; + ++bucket_it; } if (OB_SUCC(ret)) { LOG_INFO("erase_tenant_interm_result_info", K(MTL_ID()), K(map_.size())); diff --git a/src/sql/engine/basic/ob_group_join_buffer.cpp b/src/sql/engine/basic/ob_group_join_buffer.cpp index 11424c53b8..91f2d17149 100644 --- a/src/sql/engine/basic/ob_group_join_buffer.cpp +++ b/src/sql/engine/basic/ob_group_join_buffer.cpp @@ -197,9 +197,13 @@ int ObGroupJoinBufffer::has_next_left_row(bool &has_next) LOG_WARN("left row read and group idx do not match", KR(ret), K(left_store_read_), K(left_store_group_idx_.count())); } else if (above_group_idx_for_read_ == left_store_group_idx_.at(left_store_read_)) { + // above_group_idx_for_read_ < left_store_group_idx_.at(left_store_read_) means the row of above_group_idx_ is end, + // need to return iter_end, and rescan current NLJ operator // we are still reading results for the current rescan param, need to rescan right child has_next = true; } + } else { + LOG_TRACE("Left child operator has no left rows for read,and the join buffer has no left row, needs to return iter_end_"); } return ret; } @@ -345,6 +349,7 @@ int ObGroupJoinBufffer::drain_left() const ObChunkDatumStore::StoredRow *row = NULL; // drain old rows from left store for (int64_t i = left_store_read_; OB_SUCC(ret) && need_drain && i < left_store_group_idx_.count(); ++i) { + // In addition to the lines of the current group (above_group_idx_for_read_), the buffer also caches rows of subsequent groups if (above_group_idx_for_read_ != left_store_group_idx_.at(i)) { need_drain = false; } else if (OB_FAIL(left_store_iter_.get_next_row(row))) { @@ -354,6 +359,7 @@ int ObGroupJoinBufffer::drain_left() ++left_store_read_; } } + // Only rows of the current group are left in the buffer, and there are rows of the current group that have not been added to the buffer // discard unread rows from left op if (OB_SUCC(ret) && need_drain && !is_left_end_) { if (!spec_->is_vectorized()) { @@ -584,6 +590,8 @@ int ObGroupJoinBufffer::batch_fill_group_buffer(const int64_t max_row_cnt, } else if (OB_FAIL(backup_above_params(left_params_backup, right_params_backup))) { LOG_WARN("backup above params failed", KR(ret)); } + + // fill group join buffer of current op untill join buffer is full if (OB_SUCC(ret)) { ObEvalCtx::BatchInfoScopeGuard batch_info_guard(*eval_ctx_); if (save_last_batch_) { @@ -615,6 +623,7 @@ int ObGroupJoinBufffer::batch_fill_group_buffer(const int64_t max_row_cnt, } } } + // rescan left op, switch to next iter of left child op if (OB_SUCC(ret) && batch_rows->end_) { is_left_end_ = true; if (is_multi_level_) { @@ -628,6 +637,7 @@ int ObGroupJoinBufffer::batch_fill_group_buffer(const int64_t max_row_cnt, } } } + if (OB_SUCC(ret)) { if (!rescan_params_->empty()) { op_->set_pushdown_param_null(*rescan_params_); @@ -635,9 +645,11 @@ int ObGroupJoinBufffer::batch_fill_group_buffer(const int64_t max_row_cnt, if (batch_rows->size_ == 0 && batch_rows->end_) { // do nothing } else { + // if buffer is full ,but the last batch rows of left op is not added, save them to last batch last_batch_.from_exprs(*eval_ctx_, batch_rows->skip_, spec_->max_batch_size_); save_last_batch_ = true; } + op_->clear_evaluated_flag(); if (left_store_.get_row_cnt() <= 0) { // this could happen if we have skipped all rows diff --git a/src/sql/engine/basic/ob_hp_infras_vec_op.cpp b/src/sql/engine/basic/ob_hp_infras_vec_op.cpp index 91769fd777..fc8bdf2dbf 100644 --- a/src/sql/engine/basic/ob_hp_infras_vec_op.cpp +++ b/src/sql/engine/basic/ob_hp_infras_vec_op.cpp @@ -446,7 +446,7 @@ insert_batch_on_partitions(const common::ObIArray &exprs, } is_inited_vec_ = true; } - ObBitVector::flip_foreach(skip, batch_size, + ret = ObBitVector::flip_foreach(skip, batch_size, [&](int64_t idx) __attribute__((always_inline)) { int ret = OB_SUCCESS; uint16_t batch_idx = idx; diff --git a/src/sql/engine/basic/ob_json_table_op.cpp b/src/sql/engine/basic/ob_json_table_op.cpp index 3850d4eb9d..e344607546 100644 --- a/src/sql/engine/basic/ob_json_table_op.cpp +++ b/src/sql/engine/basic/ob_json_table_op.cpp @@ -29,53 +29,6 @@ using namespace common; namespace sql { -/* json table empty or error */ -const static int32_t JSN_TABLE_ERROR = 0; -const static int32_t JSN_TABLE_NULL = 1; -const static int32_t JSN_TABLE_DEFAULT = 2; -const static int32_t JSN_TABLE_IMPLICIT = 3; - -/* json query empty or error */ -const static int32_t JSN_QUERY_ERROR = 0; -const static int32_t JSN_QUERY_NULL = 1; -const static int32_t JSN_QUERY_EMPTY = 2; -const static int32_t JSN_QUERY_EMPTY_ARRAY = 3; -const static int32_t JSN_QUERY_EMPTY_OBJECT = 4; -const static int32_t JSN_QUERY_IMPLICIT = 5; - -/* json query on mismatch { error : 0, null : 1, implicit : 2 }*/ -const static int32_t JSN_QUERY_MISMATCH_ERROR = 0; -const static int32_t JSN_QUERY_MISMATCH_NULL = 1; -const static int32_t JSN_QUERY_MISMATCH_IMPLICIT = 2; - -/* json query wrapper type */ -const static int32_t JSN_QUERY_WITHOUT_WRAPPER = 0; -const static int32_t JSN_QUERY_WITHOUT_ARRAY_WRAPPER = 1; -const static int32_t JSN_QUERY_WITH_WRAPPER = 2; -const static int32_t JSN_QUERY_WITH_ARRAY_WRAPPER = 3; -const static int32_t JSN_QUERY_WITH_UNCONDITIONAL_WRAPPER = 4; -const static int32_t JSN_QUERY_WITH_CONDITIONAL_WRAPPER = 5; -const static int32_t JSN_QUERY_WITH_UNCONDITIONAL_ARRAY_WRAPPER = 6; -const static int32_t JSN_QUERY_WITH_CONDITIONAL_ARRAY_WRAPPER = 7; -const static int32_t JSN_QUERY_WRAPPER_IMPLICIT = 8; - -/* json query on scalars { allow : 0, disallow : 1, implicit : 2 }*/ -const static int32_t JSN_QUERY_SCALARS_ALLOW = 0; -const static int32_t JSN_QUERY_SCALARS_DISALLOW = 1; -const static int32_t JSN_QUERY_SCALARS_IMPLICIT = 2; - -/* json value empty or error */ -const static int32_t JSN_VALUE_ERROR = 0; -const static int32_t JSN_VALUE_NULL = 1; -const static int32_t JSN_VALUE_DEFAULT = 2; -const static int32_t JSN_VALUE_IMPLICIT = 3; - -/* json value on mismatch { error : 0, null : 1, ignore : 2 }*/ -const static int32_t JSN_VALUE_MISMATCH_ERROR = 0; -const static int32_t JSN_VALUE_MISMATCH_NULL = 1; -const static int32_t JSN_VALUE_MISMATCH_IGNORE = 2; -const static int32_t JSN_VALUE_MISMATCH_IMPLICIT = 3; - /* json value mismatch type { MISSING : 0, EXTRA : 1, TYPE : 2, EMPTY : 3} */ const static int32_t JSN_VALUE_TYPE_MISSING_DATA = 0; const static int32_t JSN_VALUE_TYPE_EXTRA_DATA = 1; @@ -3348,7 +3301,7 @@ int ObJsonTableOp::inner_get_next_row() ret = OB_ERR_JSON_SYNTAX_ERROR; SET_COVER_ERROR(&jt_ctx_, ret); jt_ctx_.is_need_end_ = 1; - if (lib::is_oracle_mode() && jt_root_->col_info_.on_error_ != JSN_TABLE_ERROR) { + if (lib::is_oracle_mode() && jt_root_->col_info_.on_error_ != JSN_VALUE_ERROR) { ret = OB_SUCCESS; } } else { diff --git a/src/sql/engine/basic/ob_json_table_op.h b/src/sql/engine/basic/ob_json_table_op.h index 850129384d..defcca2c94 100644 --- a/src/sql/engine/basic/ob_json_table_op.h +++ b/src/sql/engine/basic/ob_json_table_op.h @@ -23,6 +23,7 @@ #include "sql/resolver/dml/ob_dml_stmt.h" #include "sql/engine/ob_exec_context.h" #include "sql/engine/expr/ob_expr.h" +#include "sql/engine/expr/ob_json_param_type.h" namespace oceanbase { diff --git a/src/sql/engine/basic/ob_limit_vec_op.cpp b/src/sql/engine/basic/ob_limit_vec_op.cpp index 3c95eea848..d00022ab66 100644 --- a/src/sql/engine/basic/ob_limit_vec_op.cpp +++ b/src/sql/engine/basic/ob_limit_vec_op.cpp @@ -316,7 +316,7 @@ int ObLimitVecOp::compare_value_in_batch(bool &keep_iterating, ObExpr *expr = MY_SPEC.sort_columns_.at(col_idx); int cmp_ret = 0; if (OB_FAIL(compare_vectors_[col_idx]->null_first_cmp(*expr, row_idx, - current_waiting_compare_row->is_null(row_idx), + current_waiting_compare_row->is_null(col_idx), current_waiting_compare_row->get_cell_payload(pre_sort_columns_.row_meta_, col_idx), current_waiting_compare_row->get_length(pre_sort_columns_.row_meta_, col_idx), cmp_ret))) { diff --git a/src/sql/engine/basic/ob_vector_result_holder.cpp b/src/sql/engine/basic/ob_vector_result_holder.cpp index 182aedbd13..93c092b504 100644 --- a/src/sql/engine/basic/ob_vector_result_holder.cpp +++ b/src/sql/engine/basic/ob_vector_result_holder.cpp @@ -147,7 +147,8 @@ int ObVectorsResultHolder::ObColResultHolder::copy_uniform_base( const int64_t batch_size) { int ret = OB_SUCCESS; int64_t copy_size = is_const ? 1 : max_row_cnt_; - bool is_fixed_length_data = expr->is_fixed_length_data_; + bool need_copy_rev_buf = expr->is_fixed_length_data_ + || ObNumberTC == ob_obj_type_class(expr->datum_meta_.get_type()); if (OB_FAIL(copy_vector_base(vec))) { LOG_WARN("failed to copy vector base", K(ret)); } else { @@ -159,7 +160,7 @@ int ObVectorsResultHolder::ObColResultHolder::copy_uniform_base( K(max_row_cnt_)); } } - if (OB_SUCC(ret) && is_fixed_length_data && nullptr == frame_data_) { + if (OB_SUCC(ret) && need_copy_rev_buf && nullptr == frame_data_) { len_ = expr->res_buf_len_; if (OB_ISNULL( frame_data_ = static_cast(alloc.alloc(len_ * copy_size)))) { @@ -169,7 +170,7 @@ int ObVectorsResultHolder::ObColResultHolder::copy_uniform_base( } } if (OB_SUCC(ret)) { - if (is_fixed_length_data) { + if (need_copy_rev_buf) { MEMCPY(frame_data_, expr->get_rev_buf(eval_ctx), len_ * copy_size); } datums_ = const_cast (vec.get_datums()); @@ -233,11 +234,13 @@ void ObVectorsResultHolder::ObColResultHolder::restore_uniform_base( ObEvalCtx &eval_ctx, const int64_t batch_size) const { int ret = OB_SUCCESS; + bool need_copy_rev_buf = expr->is_fixed_length_data_ + || ObNumberTC == ob_obj_type_class(expr->datum_meta_.get_type()); int64_t copy_size = is_const ? 1 : max_row_cnt_; restore_vector_base(vec); vec.set_datums(datums_); MEMCPY(expr->locate_batch_datums(eval_ctx), frame_datums_, sizeof(ObDatum) * copy_size); - if (OB_NOT_NULL(expr_) && expr_->is_fixed_length_data_) { + if (need_copy_rev_buf) { MEMCPY(expr->get_rev_buf(eval_ctx), frame_data_, len_ * copy_size); } } diff --git a/src/sql/engine/cmd/ob_alter_system_executor.cpp b/src/sql/engine/cmd/ob_alter_system_executor.cpp index 27d7ad85a9..8b0421557b 100644 --- a/src/sql/engine/cmd/ob_alter_system_executor.cpp +++ b/src/sql/engine/cmd/ob_alter_system_executor.cpp @@ -44,9 +44,11 @@ #include "share/ob_primary_standby_service.h" // ObPrimaryStandbyService #include "rpc/obmysql/ob_sql_sock_session.h" #include "sql/plan_cache/ob_plan_cache.h" +#include "pl/pl_cache/ob_pl_cache_mgr.h" #include "sql/plan_cache/ob_ps_cache.h" #include "share/table/ob_ttl_util.h" #include "rootserver/restore/ob_tenant_clone_util.h" + namespace oceanbase { using namespace common; @@ -379,7 +381,38 @@ int ObFlushCacheExecutor::execute(ObExecContext &ctx, ObFlushCacheStmt &stmt) break; } case CACHE_TYPE_PL_OBJ: { - if (0 == tenant_num) { + if (stmt.flush_cache_arg_.is_fine_grained_) { + // purge in sql_id level, aka. fine-grained plan evict + // we assume tenant_list must not be empty and this will be checked in resolve phase + if (0 == tenant_num) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected tenant_list in fine-grained plan evict", K(tenant_num)); + } else { + bool is_evict_by_schema_id = common::OB_INVALID_ID != stmt.flush_cache_arg_.schema_id_; + for (int64_t i = 0; i < tenant_num; i++) { // ignore ret + int64_t t_id = stmt.flush_cache_arg_.tenant_ids_.at(i); + MTL_SWITCH(t_id) { + ObPlanCache* plan_cache = MTL(ObPlanCache*); + // not specified db_name, evict all dbs + if (db_num == 0) { + if (is_evict_by_schema_id) { + ret = plan_cache->flush_pl_cache_single_cache_obj(OB_INVALID_ID, stmt.flush_cache_arg_.schema_id_); + } else { + ret = plan_cache->flush_pl_cache_single_cache_obj(OB_INVALID_ID, sql_id); + } + } else { // evict db by db + for(int64_t j = 0; j < db_num; j++) { // ignore ret + if (is_evict_by_schema_id) { + ret = plan_cache->flush_pl_cache_single_cache_obj(stmt.flush_cache_arg_.db_ids_.at(j), stmt.flush_cache_arg_.schema_id_); + } else { + ret = plan_cache->flush_pl_cache_single_cache_obj(stmt.flush_cache_arg_.db_ids_.at(j), sql_id); + } + } + } + } + } + } + } else if (0 == tenant_num) { common::ObArray tenant_ids; if (OB_ISNULL(GCTX.omt_)) { ret = OB_ERR_UNEXPECTED; diff --git a/src/sql/engine/cmd/ob_load_data_direct_impl.cpp b/src/sql/engine/cmd/ob_load_data_direct_impl.cpp index 3ea8d493b8..cfef719c45 100644 --- a/src/sql/engine/cmd/ob_load_data_direct_impl.cpp +++ b/src/sql/engine/cmd/ob_load_data_direct_impl.cpp @@ -438,7 +438,7 @@ ObLoadDataDirectImpl::DataReader::DataReader() : allocator_("TLD_DataReader"), execute_ctx_(nullptr), file_reader_(nullptr), - end_offset_(0), + end_offset_(-1), read_raw_(false), is_iter_end_(false), is_inited_(false) @@ -632,7 +632,7 @@ bool ObLoadDataDirectImpl::DataReader::is_end_file() const bool ret = false; if (file_reader_->eof()) { ret = true; - } else if (end_offset_ > 0) { + } else if (end_offset_ >= 0) { ret = file_reader_->get_offset() >= end_offset_; } return ret; diff --git a/src/sql/engine/cmd/ob_table_executor.cpp b/src/sql/engine/cmd/ob_table_executor.cpp index bfe78516f6..7da1f0043d 100644 --- a/src/sql/engine/cmd/ob_table_executor.cpp +++ b/src/sql/engine/cmd/ob_table_executor.cpp @@ -578,6 +578,7 @@ int ObCreateTableExecutor::execute(ObExecContext &ctx, ObCreateTableStmt &stmt) DEBUG_SYNC(BEFORE_SEND_PARALLEL_CREATE_TABLE); int64_t start_time = ObTimeUtility::current_time(); ObTimeoutCtx ctx; + create_table_arg.is_parallel_ = true; if (OB_FAIL(ctx.set_timeout(common_rpc_proxy->get_timeout()))) { LOG_WARN("fail to set timeout ctx", K(ret)); } else if (OB_FAIL(common_rpc_proxy->parallel_create_table(create_table_arg, res))) { @@ -2226,6 +2227,7 @@ int ObTruncateTableExecutor::execute(ObExecContext &ctx, ObTruncateTableStmt &st } else { // new parallel truncate ObTimeoutCtx ctx; + tmp_arg.is_parallel_ = true; if (OB_FAIL(ctx.set_timeout(common_rpc_proxy->get_timeout()))) { LOG_WARN("fail to set timeout ctx", K(ret)); } else { diff --git a/src/sql/engine/connect_by/ob_nl_cnnt_by_with_index_op.cpp b/src/sql/engine/connect_by/ob_nl_cnnt_by_with_index_op.cpp index a431d05b55..dd0303319c 100644 --- a/src/sql/engine/connect_by/ob_nl_cnnt_by_with_index_op.cpp +++ b/src/sql/engine/connect_by/ob_nl_cnnt_by_with_index_op.cpp @@ -371,8 +371,10 @@ int ObNLConnectByWithIndexOp::read_pump_func_going() state_ = CNTB_STATE_READ_RIGHT; } } - LOG_DEBUG("debug pump going", K(ObToStringExprRow(eval_ctx_, MY_SPEC.right_prior_exprs_)), - K(ObToStringExprRow(eval_ctx_, MY_SPEC.cur_row_exprs_))); + if (OB_SUCC(ret)) { + LOG_DEBUG("debug pump going", K(ObToStringExprRow(eval_ctx_, MY_SPEC.right_prior_exprs_)), + K(ObToStringExprRow(eval_ctx_, MY_SPEC.cur_row_exprs_))); + } return ret; } diff --git a/src/sql/engine/dml/ob_dml_service.cpp b/src/sql/engine/dml/ob_dml_service.cpp index e043050b9d..51b5725d7d 100644 --- a/src/sql/engine/dml/ob_dml_service.cpp +++ b/src/sql/engine/dml/ob_dml_service.cpp @@ -39,15 +39,14 @@ using namespace transaction; namespace sql { int ObDMLService::check_row_null(const ObExprPtrIArray &row, - ObEvalCtx &eval_ctx, - int64_t row_num, - const ColContentIArray &column_infos, - const ObDASDMLBaseCtDef &das_ctdef, - bool is_single_value, - ObTableModifyOp &dml_op) + ObEvalCtx &eval_ctx, + int64_t row_num, + const ColContentIArray &column_infos, + bool is_ignore, + bool is_single_value, + ObTableModifyOp &dml_op) { int ret = OB_SUCCESS; - const bool is_ignore = das_ctdef.is_ignore_; ObSQLSessionInfo *session = NULL; CK(row.count() >= column_infos.count()); if (OB_ISNULL(session = dml_op.get_exec_ctx().get_my_session())) { @@ -65,6 +64,7 @@ int ObDMLService::check_row_null(const ObExprPtrIArray &row, if (is_ignore || (lib::is_mysql_mode() && !is_single_value && !is_strict_mode(session->get_sql_mode()))) { ObObj zero_obj; + ObExprStrResAlloc res_alloc(*row.at(col_idx), eval_ctx); ObDatum &row_datum = row.at(col_idx)->locate_datum_for_write(eval_ctx); bool is_decimal_int = ob_is_decimal_int(row.at(col_idx)->datum_meta_.type_); if (is_oracle_mode()) { @@ -77,8 +77,6 @@ int ObDMLService::check_row_null(const ObExprPtrIArray &row, } else if (OB_FAIL(ObObjCaster::get_zero_value( row.at(col_idx)->obj_meta_.get_type(), row.at(col_idx)->obj_meta_.get_collation_type(), - das_ctdef.column_accuracys_.at(col_idx).get_length(), - eval_ctx.exec_ctx_.get_allocator(), zero_obj))) { LOG_WARN("get column default zero value failed", K(ret), K(column_infos.at(i)), K(row.at(col_idx)->max_length_)); } else if (is_decimal_int) { @@ -88,6 +86,11 @@ int ObDMLService::check_row_null(const ObExprPtrIArray &row, row_datum.set_decimal_int(dec_val.get_decimal_int(), dec_val.get_int_bytes()); } if (OB_FAIL(ret)) { + LOG_WARN("get column default zero value failed", K(ret), K(column_infos.at(i))); + } else if (OB_FAIL(ObDASUtils::padding_fixed_string_value(row.at(col_idx)->max_length_, + res_alloc, + zero_obj))) { + LOG_WARN("padding fixed string value failed", K(ret)); } else if (OB_FAIL(ObTextStringResult::ob_convert_obj_temporay_lob(zero_obj, eval_ctx.exec_ctx_.get_allocator()))) { LOG_WARN("convert lob types zero obj failed", K(ret), K(zero_obj)); } else if (OB_FAIL(!is_decimal_int && row_datum.from_obj(zero_obj))) { @@ -613,7 +616,7 @@ int ObDMLService::process_insert_row(const ObInsCtDef &ins_ctdef, dml_op.get_eval_ctx(), ins_rtdef.cur_row_num_, ins_ctdef.column_infos_, - ins_ctdef.das_ctdef_, + ins_ctdef.das_ctdef_.is_ignore_, ins_ctdef.is_single_value_, dml_op))) { LOG_WARN("check row null failed", K(ret)); @@ -829,7 +832,7 @@ int ObDMLService::process_update_row(const ObUpdCtDef &upd_ctdef, dml_op.get_eval_ctx(), upd_rtdef.cur_row_num_, upd_ctdef.assign_columns_, - upd_ctdef.dupd_ctdef_, + upd_ctdef.dupd_ctdef_.is_ignore_, false, dml_op))) { LOG_WARN("check row null failed", K(ret), K(upd_ctdef), K(upd_rtdef)); diff --git a/src/sql/engine/dml/ob_dml_service.h b/src/sql/engine/dml/ob_dml_service.h index fa2805debe..6349057d81 100644 --- a/src/sql/engine/dml/ob_dml_service.h +++ b/src/sql/engine/dml/ob_dml_service.h @@ -28,7 +28,7 @@ public: ObEvalCtx &eval_ctx, int64_t row_num, const ColContentIArray &column_infos, - const ObDASDMLBaseCtDef &das_ctdef, + bool is_ignore, bool is_single_value, ObTableModifyOp &dml_op); static int check_column_type(const ExprFixedArray &dml_row, diff --git a/src/sql/engine/dml/ob_table_lock_op.cpp b/src/sql/engine/dml/ob_table_lock_op.cpp index ade08630c5..82952143d4 100644 --- a/src/sql/engine/dml/ob_table_lock_op.cpp +++ b/src/sql/engine/dml/ob_table_lock_op.cpp @@ -450,7 +450,8 @@ OB_INLINE int ObTableLockOp::get_next_batch_from_child(const int64_t max_row_cnt if (OB_FAIL(child_->get_next_batch(max_row_cnt, child_brs))) { LOG_WARN("fail to get next batch", K(ret)); } else if (OB_LIKELY(!child_brs->end_ && child_brs->size_ > 0)) { - LOG_TRACE("child output row", "row", ROWEXPR2STR(eval_ctx_, child_->get_spec().output_)); + PRINT_VECTORIZED_ROWS(SQL, TRACE, eval_ctx_, child_->get_spec().output_, child_brs->size_, + child_brs->skip_); } return ret; } diff --git a/src/sql/engine/expr/ob_expr.h b/src/sql/engine/expr/ob_expr.h index edd8a3f5ff..c12cc68c8e 100644 --- a/src/sql/engine/expr/ob_expr.h +++ b/src/sql/engine/expr/ob_expr.h @@ -1410,7 +1410,7 @@ inline const char *get_vectorized_row_str(ObEvalCtx &eval_ctx, return buffer; } -#define PRINT_VECTORIZED_ROWS(parMod, level, eval_ctx, exprs, batch_size, args...) \ +#define PRINT_VECTORIZED_ROWS(parMod, level, eval_ctx, exprs, batch_size, skip, args...) \ do { if (IS_LOG_ENABLED(level)) { \ [&](const char *_fun_name_) __attribute__((GET_LOG_FUNC_ATTR(level))) { \ if (OB_UNLIKELY(OB_LOGGER.need_to_print(::oceanbase::common::OB_LOG_ROOT::M_##parMod, \ @@ -1420,6 +1420,9 @@ inline const char *get_vectorized_row_str(ObEvalCtx &eval_ctx, ObEvalCtx::BatchInfoScopeGuard _batch_info_guard(eval_ctx); \ _batch_info_guard.set_batch_size(_batch_size); \ for (int64_t i = 0; i < _batch_size; ++i) { \ + if (NULL != skip && skip->at(i)) { \ + continue; \ + } \ _batch_info_guard.set_batch_idx(i); \ ::oceanbase::common::OB_PRINT("["#parMod"] ", OB_LOG_LEVEL(level), \ get_vectorized_row_str(eval_ctx, exprs, i), \ diff --git a/src/sql/engine/expr/ob_expr_between.cpp b/src/sql/engine/expr/ob_expr_between.cpp index 5f0ca969ed..a407ab9744 100644 --- a/src/sql/engine/expr/ob_expr_between.cpp +++ b/src/sql/engine/expr/ob_expr_between.cpp @@ -78,92 +78,99 @@ int calc_between_expr(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res_datum) return ret; } -// eval_left_and_right_operand -template -static int eval_lr_operand(const ObExpr &expr, ObEvalCtx &ctx, - ObBitVector &skip, const EvalBound &bound) -{ - int ret = OB_SUCCESS; - const ObExpr &val_expr = *expr.args_[0]; - const ObExpr &left_expr = *expr.args_[1]; - const ObExpr &right_expr = *expr.args_[2]; - ValVec *val_vec = static_cast(val_expr.get_vector(ctx)); - ResVec *res_vec = static_cast(expr.get_vector(ctx)); - if (val_vec->has_null()) { - for (int i = bound.start(); i < bound.end(); ++i) { - if (!skip.at(i) && val_vec->is_null(i)) { - res_vec->set_null(i); - skip.set(i); - } - } - } - // if skip is all true, `eval_vector` still needs to be called, for that expr format may not be inited. - if (OB_FAIL(left_expr.eval_vector(ctx, skip, bound))) { - LOG_WARN("eval left operand failed", K(ret)); - } else if (OB_FAIL(right_expr.eval_vector(ctx, skip, bound))) { - LOG_WARN("eval right operand failed", K(ret)); - } - return ret; +#define BETWEEN_DISPATCH_VECTOR_IN_LEFT_ARG_FORMAT( \ + func_name, stage, l_format, right_vec, res_vec) \ +switch (l_format) { \ + case VEC_FIXED: { \ + ret = func_name( \ + expr, ctx, my_skip, bound); \ + break; \ + } \ + case VEC_DISCRETE: { \ + ret = func_name( \ + expr, ctx, my_skip, bound); \ + break; \ + } \ + case VEC_CONTINUOUS: { \ + ret = func_name( \ + expr, ctx, my_skip, bound); \ + break; \ + } \ + case VEC_UNIFORM: { \ + ret = func_name, right_vec, res_vec, stage>( \ + expr, ctx, my_skip, bound); \ + break; \ + } \ + case VEC_UNIFORM_CONST: { \ + ret = func_name, right_vec, res_vec, stage>( \ + expr, ctx, my_skip, bound); \ + break; \ + } \ + default: { \ + ret = func_name( \ + expr, ctx, my_skip, bound); \ + } \ } -template -static int dispatch_eval_between_operands(const ObExpr &expr, ObEvalCtx &ctx, - ObBitVector &skip, const EvalBound &bound) -{ - int ret = OB_SUCCESS; - VectorFormat res_format = expr.get_format(ctx); - switch (res_format) { - case VEC_FIXED: { - ret = eval_lr_operand(expr, ctx, skip, bound); - break; - } - case VEC_UNIFORM: { - ret = eval_lr_operand>(expr, ctx, skip, bound); - break; - } - case VEC_UNIFORM_CONST: { - ret = eval_lr_operand>(expr, ctx, skip, bound); - break; - } - default: { - ret = eval_lr_operand(expr, ctx, skip, bound); - } - } - return ret; +#define BETWEEN_DISPATCH_VECTOR_IN_RIGHT_ARG_FORMAT( \ + func_name, stage, l_format, r_format, res_vec) \ +switch (r_format) { \ + case VEC_FIXED: { \ + BETWEEN_DISPATCH_VECTOR_IN_LEFT_ARG_FORMAT( \ + func_name, stage, l_format, ObFixedLengthBase, res_vec); \ + break; \ + } \ + case VEC_DISCRETE: { \ + BETWEEN_DISPATCH_VECTOR_IN_LEFT_ARG_FORMAT( \ + func_name, stage, l_format, ObDiscreteFormat, res_vec); \ + break; \ + } \ + case VEC_CONTINUOUS: { \ + BETWEEN_DISPATCH_VECTOR_IN_LEFT_ARG_FORMAT( \ + func_name, stage, l_format, ObContinuousFormat, res_vec); \ + break; \ + } \ + case VEC_UNIFORM: { \ + BETWEEN_DISPATCH_VECTOR_IN_LEFT_ARG_FORMAT( \ + func_name, stage, l_format, ObUniformFormat, res_vec); \ + break; \ + } \ + case VEC_UNIFORM_CONST: { \ + BETWEEN_DISPATCH_VECTOR_IN_LEFT_ARG_FORMAT( \ + func_name, stage, l_format, ObUniformFormat, res_vec); \ + break; \ + } \ + default: { \ + BETWEEN_DISPATCH_VECTOR_IN_LEFT_ARG_FORMAT( \ + func_name, stage, l_format, ObVectorBase, res_vec); \ + } \ } -static int eval_between_operands(const ObExpr &expr, ObEvalCtx &ctx, - ObBitVector &skip, const EvalBound &bound) -{ - int ret = OB_SUCCESS; - const ObExpr &val_expr = *expr.args_[0]; - if (OB_FAIL(val_expr.eval_vector(ctx, skip, bound))) { - LOG_WARN("eval left operand failed", K(ret)); - } else { - VectorFormat val_format = val_expr.get_format(ctx); - switch (val_format) { - case VEC_DISCRETE: - case VEC_CONTINUOUS: - case VEC_FIXED: { - ret = dispatch_eval_between_operands(expr, ctx, skip, bound); - break; - } - case VEC_UNIFORM: { - ret = dispatch_eval_between_operands>(expr, ctx, skip, bound); - break; - } - case VEC_UNIFORM_CONST: { - ret = dispatch_eval_between_operands>(expr, ctx, skip, bound); - break; - } - default: { - ret = dispatch_eval_between_operands(expr, ctx, skip, bound); - } - } - } - return ret; +#define BETWEEN_DISPATCH_VECTOR_IN_RES_ARG_FORMAT( \ + func_name, stage, l_format, r_format) \ +switch (res_format) { \ + case VEC_FIXED: { \ + BETWEEN_DISPATCH_VECTOR_IN_RIGHT_ARG_FORMAT( \ + func_name, stage, l_format, r_format, IntegerFixedVec); \ + break; \ + } \ + case VEC_UNIFORM: { \ + BETWEEN_DISPATCH_VECTOR_IN_RIGHT_ARG_FORMAT( \ + func_name, stage, l_format, r_format, IntegerUniVec); \ + break; \ + } \ + case VEC_UNIFORM_CONST: { \ + BETWEEN_DISPATCH_VECTOR_IN_RIGHT_ARG_FORMAT( \ + func_name, stage, l_format, r_format, IntegerUniCVec); \ + break; \ + } \ + default: { \ + BETWEEN_DISPATCH_VECTOR_IN_RIGHT_ARG_FORMAT( \ + func_name, stage, l_format, r_format, ObVectorBase); \ + } \ } + int ObExprBetween::eval_between_vector(const ObExpr &expr, ObEvalCtx &ctx, const ObBitVector &skip, @@ -176,17 +183,33 @@ int ObExprBetween::eval_between_vector(const ObExpr &expr, const ObExpr &right_expr = *expr.args_[2]; ObBitVector &my_skip = expr.get_pvt_skip(ctx); my_skip.deep_copy(skip, bound.start(), bound.end()); - if (OB_FAIL(eval_between_operands(expr, ctx, my_skip, bound))) { - LOG_WARN("eval between operands failed", K(ret)); - } else if (OB_FAIL((reinterpret_cast(expr.inner_functions_[2]))( - expr, left_expr, val_expr, ctx, my_skip, bound))) { // eval left <= val - LOG_WARN("compare left and val failed", K(ret)); - } else if (OB_FAIL((reinterpret_cast(expr.inner_functions_[3]))( - expr, val_expr, right_expr, ctx, my_skip, bound))) { // eval val <= right - LOG_WARN("compare val and right failed", K(ret)); + if (OB_FAIL(val_expr.eval_vector(ctx, my_skip, bound))) { + LOG_WARN("eval left operand failed", K(ret)); + } else if (OB_FAIL(left_expr.eval_vector(ctx, my_skip, bound))) { + LOG_WARN("eval left operand failed", K(ret)); + } else if (OB_FAIL(right_expr.eval_vector(ctx, my_skip, bound))) { + LOG_WARN("eval left operand failed", K(ret)); } else { - ObBitVector &eval_flags = expr.get_evaluated_flags(ctx); - eval_flags.bit_not(skip, bound); + VectorFormat val_format = val_expr.get_format(ctx); + VectorFormat left_format = left_expr.get_format(ctx); + VectorFormat right_format = right_expr.get_format(ctx); + VectorFormat res_format = expr.get_format(ctx); + BETWEEN_DISPATCH_VECTOR_IN_RES_ARG_FORMAT( + inner_eval_between_vector, BETWEEN_LEFT, + left_format, val_format); + if (OB_FAIL(ret)) { + LOG_WARN("compare left and val failed", K(ret)); + } else { + BETWEEN_DISPATCH_VECTOR_IN_RES_ARG_FORMAT( + inner_eval_between_vector, BETWEEN_RIGHT, + val_format, right_format); + if (OB_FAIL(ret)) { + LOG_WARN("compare val and right failed", K(ret)); + } else { + ObBitVector &eval_flags = expr.get_evaluated_flags(ctx); + eval_flags.bit_not(skip, bound); + } + } } return ret; } @@ -206,8 +229,8 @@ int ObExprBetween::cg_expr(ObExprCGCtx &expr_cg_ctx, } else { DatumCmpFunc cmp_func_1 = NULL; // left <= val DatumCmpFunc cmp_func_2 = NULL; // val <= right - EvalVectorBetweenFunc vec_cmp_func_1 = NULL; // left <= val - EvalVectorBetweenFunc vec_cmp_func_2 = NULL; // val <= right + RowCmpFunc vec_cmp_func_1 = NULL; // left <= val + RowCmpFunc vec_cmp_func_2 = NULL; // val <= right const ObDatumMeta &val_meta = rt_expr.args_[0]->datum_meta_; const ObDatumMeta &left_meta = rt_expr.args_[1]->datum_meta_; const ObDatumMeta &right_meta = rt_expr.args_[2]->datum_meta_; @@ -237,23 +260,22 @@ int ObExprBetween::cg_expr(ObExprCGCtx &expr_cg_ctx, ret = OB_ERR_UNEXPECTED; LOG_WARN("get_datum_expr_cmp_func failed", K(ret), K(val_meta), K(right_meta), K(is_oracle_mode()), K(rt_expr)); - } else if (OB_ISNULL(vec_cmp_func_1 = VectorCmpExprFuncsHelper::get_eval_vector_between_expr_cmp_func( - left_meta, val_meta, EvalBetweenStage::BETWEEN_LEFT))) { + } else if (OB_ISNULL(vec_cmp_func_1 = VectorCmpExprFuncsHelper::get_row_cmp_func( + left_meta, val_meta))) { ret = OB_ERR_UNEXPECTED; VecValueTypeClass value_tc = get_vec_value_tc(val_meta.type_, val_meta.scale_, val_meta.precision_); VecValueTypeClass left_tc = get_vec_value_tc(left_meta.type_, left_meta.scale_, left_meta.precision_); LOG_WARN("The result of get_eval_vector_between_expr_cmp_func(left) is null.", K(ret), K(left_meta), K(val_meta), K(right_meta), K(value_tc), K(left_tc), K(rt_expr)); - } else if (OB_ISNULL(vec_cmp_func_2 = VectorCmpExprFuncsHelper::get_eval_vector_between_expr_cmp_func( - val_meta, right_meta, EvalBetweenStage::BETWEEN_RIGHT))) { + } else if (OB_ISNULL(vec_cmp_func_2 = VectorCmpExprFuncsHelper::get_row_cmp_func( + val_meta, right_meta))) { ret = OB_ERR_UNEXPECTED; VecValueTypeClass value_tc = get_vec_value_tc(val_meta.type_, val_meta.scale_, val_meta.precision_); VecValueTypeClass right_tc = get_vec_value_tc(right_meta.type_, right_meta.scale_, right_meta.precision_); LOG_WARN("The result of get_eval_vector_between_expr_cmp_func(right) is null.", K(ret), K(left_meta), K(val_meta), K(right_meta), K(value_tc), K(right_tc), K(rt_expr)); } else if (OB_ISNULL(rt_expr.inner_functions_ = reinterpret_cast( - expr_cg_ctx.allocator_->alloc(sizeof(DatumCmpFunc) * 2 + - sizeof(EvalVectorBetweenFunc) * 2)))) { + expr_cg_ctx.allocator_->alloc(sizeof(DatumCmpFunc) * 2 + sizeof(RowCmpFunc) * 2)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc memory for inner_functions_ failed", K(ret)); } else { @@ -269,5 +291,147 @@ int ObExprBetween::cg_expr(ObExprCGCtx &expr_cg_ctx, return ret; } +#define DO_VECTOR_BETWEEN_ROW_CMP() \ + if (std::is_same::value) { \ + l_payload = fixed_base_l_payload + l_len * i; \ + } else if (!std::is_same>::value){ \ + l_vector->get_payload(i, l_payload, l_len); \ + } \ + if (std::is_same::value) { \ + r_payload = fixed_base_r_payload + r_len * i; \ + } else if (!std::is_same>::value){ \ + r_vector->get_payload(i, r_payload, r_len); \ + } \ + if (Stage == EvalBetweenStage::BETWEEN_LEFT) { \ + ret = (reinterpret_cast(expr.inner_functions_[2])) \ + (left->obj_meta_, right->obj_meta_, \ + (const void *)l_payload, l_len, \ + (const void *)r_payload, r_len, cmp_ret); \ + } else { /*BETWEEN_RIGHT*/ \ + ret = (reinterpret_cast(expr.inner_functions_[3])) \ + (left->obj_meta_, right->obj_meta_, \ + (const void *)l_payload, l_len, \ + (const void *)r_payload, r_len, cmp_ret); \ + } + +#define DO_VECTOR_BETWEEN_SET_RES() \ + /* Result priority: false > null > true */ \ + if (OB_FAIL(ret)) { \ + } else if (Stage == EvalBetweenStage::BETWEEN_LEFT) { \ + /* If the current calculation is left<=val, any result is directly filled in. \ + If the result is false, the subsequent calculation results are meaningless, \ + and skip is set to true. */ \ + res_vec->set_int(i, (cmp_ret <= 0)); \ + if (cmp_ret > 0) { \ + skip.set(i); \ + } \ + } else if (cmp_ret > 0) { /*BETWEEN_RIGHT*/ \ + /* If currently calculating val<=right, \ + only when the result is false will it be filled in. */ \ + res_vec->set_int(i, 0); \ + } + + +template +int ObExprBetween::inner_eval_between_vector(const ObExpr &expr, + ObEvalCtx &ctx, + ObBitVector &skip, + const EvalBound &bound) +{ + int ret = OB_SUCCESS; + ObExpr *left = nullptr; + ObExpr *right = nullptr; + if (Stage == EvalBetweenStage::BETWEEN_LEFT) { + left = expr.args_[1]; + right = expr.args_[0]; + } else { + left = expr.args_[0]; + right = expr.args_[2]; + } + LVec *l_vector = static_cast(left->get_vector(ctx)); + RVec *r_vector = static_cast(right->get_vector(ctx)); + ResVec *res_vec = static_cast(expr.get_vector(ctx)); + ObBitVector &eval_flags = expr.get_evaluated_flags(ctx); + const char *l_payload = nullptr, *r_payload = nullptr; + const char *fixed_base_l_payload = nullptr, *fixed_base_r_payload = nullptr; + ObLength l_len = 0, r_len = 0; + int cmp_ret = 0; + bool l_has_null = l_vector->has_null(); + bool r_has_null = r_vector->has_null(); + // If a constant value exists and that constant value is null, + // then set the entire res_vec to null. + if (std::is_same>::value && l_has_null) { + // If at this point the computation is val < right, and val is null, + // then the result must have already been set to null previously, + // and can be skipped directly. + if (Stage == EvalBetweenStage::BETWEEN_LEFT) { + for (int i = bound.start(); i < bound.end(); ++i) { + if (skip.at(i) || eval_flags.at(i)) { continue; } + res_vec->set_null(i); + } + } + } else if (std::is_same>::value && r_has_null) { + for (int i = bound.start(); i < bound.end(); ++i) { + if (skip.at(i) || eval_flags.at(i)) { continue; } + res_vec->set_null(i); + } + // For the case where both sides are constants, calculate only once, + // then fill the values in a loop; + // there is no need to consider the null situation, + // as it has already been assessed previously. + } else if (std::is_same>::value && + std::is_same>::value) { + l_vector->get_payload(0, l_payload, l_len); + r_vector->get_payload(0, r_payload, r_len); + if (Stage == EvalBetweenStage::BETWEEN_LEFT) { + ret = (reinterpret_cast(expr.inner_functions_[2])) + (left->obj_meta_, right->obj_meta_, + (const void *)l_payload, l_len, + (const void *)r_payload, r_len, cmp_ret); + } else { /*BETWEEN_RIGHT*/ + ret = (reinterpret_cast(expr.inner_functions_[3])) + (left->obj_meta_, right->obj_meta_, + (const void *)l_payload, l_len, + (const void *)r_payload, r_len, cmp_ret); + } + for (int i = bound.start(); OB_SUCC(ret) && i < bound.end(); ++i) { + if (skip.at(i) || eval_flags.at(i)) { continue; } + DO_VECTOR_BETWEEN_SET_RES(); + } + } else { + if (std::is_same::value) { + fixed_base_l_payload = (reinterpret_cast(l_vector))->get_data(); + l_len = (reinterpret_cast(l_vector))->get_length(); + } else if (std::is_same>::value) { + l_vector->get_payload(0, l_payload, l_len); + } + if (std::is_same::value) { + fixed_base_r_payload = (reinterpret_cast(r_vector))->get_data(); + r_len = (reinterpret_cast(r_vector))->get_length(); + } else if (std::is_same>::value) { + r_vector->get_payload(0, r_payload, r_len); + } + if (!(l_has_null || r_has_null)) { + for (int i = bound.start(); OB_SUCC(ret) && i < bound.end(); ++i) { + if (skip.at(i) || eval_flags.at(i)) { continue; } + DO_VECTOR_BETWEEN_ROW_CMP(); + DO_VECTOR_BETWEEN_SET_RES(); + } + } else { + for (int i = bound.start(); OB_SUCC(ret) && i < bound.end(); ++i) { + if (skip.at(i) || eval_flags.at(i)) { continue; } + if (l_vector->is_null(i) || r_vector->is_null(i)) { + res_vec->set_null(i); + } else { + DO_VECTOR_BETWEEN_ROW_CMP(); + DO_VECTOR_BETWEEN_SET_RES(); + } + } + } + } + return ret; +} + } } diff --git a/src/sql/engine/expr/ob_expr_between.h b/src/sql/engine/expr/ob_expr_between.h index ca1d95fa4c..1db8eb0b9e 100644 --- a/src/sql/engine/expr/ob_expr_between.h +++ b/src/sql/engine/expr/ob_expr_between.h @@ -30,6 +30,12 @@ public: { } + enum EvalBetweenStage { + BETWEEN_LEFT, + BETWEEN_RIGHT, + BETWEEN_MAX + }; + virtual int cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) const override; @@ -37,20 +43,13 @@ public: ObEvalCtx &ctx, const ObBitVector &skip, const EvalBound &bound); - enum EvalBetweenStage { - BETWEEN_LEFT, - BETWEEN_RIGHT, - BETWEEN_MAX - }; - struct EvalVectorBetweenCmp; - - typedef int (*EvalVectorBetweenFunc) (const ObExpr &expr, - const ObExpr &left, - const ObExpr &right, - ObEvalCtx &ctx, - ObBitVector &skip, - const EvalBound &bound); + template + static int inner_eval_between_vector(const ObExpr &expr, + ObEvalCtx &ctx, + ObBitVector &skip, + const EvalBound &bound); private: // types and constants private: diff --git a/src/sql/engine/expr/ob_expr_eval_functions.cpp b/src/sql/engine/expr/ob_expr_eval_functions.cpp index 02340a65b1..87231f1ddb 100644 --- a/src/sql/engine/expr/ob_expr_eval_functions.cpp +++ b/src/sql/engine/expr/ob_expr_eval_functions.cpp @@ -1102,6 +1102,9 @@ static ObExpr::EvalFunc g_expr_eval_functions[] = { eval_questionmark_decint2decint_eqcast, /* 660 */ eval_questionmark_decint2decint_normalcast, /* 661 */ ObExprExtractExpiredTime::eval_extract_cert_expired_time, /* 662 */ + NULL, //ObExprXmlConcat::eval_xml_concat, /* 663 */ + NULL, //ObExprXmlForest::eval_xml_forest, /* 664 */ + NULL, //ObExprExistsNodeXml::eval_existsnode_xml, /* 665 */ }; static ObExpr::EvalBatchFunc g_expr_eval_batch_functions[] = { diff --git a/src/sql/engine/expr/ob_expr_func_round.cpp b/src/sql/engine/expr/ob_expr_func_round.cpp index 4fa8e5faa8..501cce07dc 100644 --- a/src/sql/engine/expr/ob_expr_func_round.cpp +++ b/src/sql/engine/expr/ob_expr_func_round.cpp @@ -751,7 +751,7 @@ int ObExprFuncRound::calc_round_expr_numeric1_batch(const ObExpr &expr, return ret; } -#define ROUND_DISPATCH_VECTOR_IN_LEFT_ARG_FORMAT(func_name, res_vec) \ +#define ROUND_DISPATCH_VECTOR_IN_LEFT_ARG_FORMAT(func_name, res_vec) \ switch (left_format) { \ case VEC_FIXED: { \ ret = func_name(expr, ctx, skip, bound); \ @@ -778,31 +778,31 @@ switch (left_format) { } \ } -#define ROUND_DISPATCH_VECTOR_IN_RES_ARG_FORMAT(func_name) \ -switch (res_format) { \ - case VEC_FIXED: { \ - ROUND_DISPATCH_VECTOR_IN_LEFT_ARG_FORMAT(func_name, ObFixedLengthBase); \ - break; \ - } \ - case VEC_DISCRETE: { \ - ROUND_DISPATCH_VECTOR_IN_LEFT_ARG_FORMAT(func_name, ObDiscreteFormat); \ - break; \ - } \ - case VEC_CONTINUOUS: { \ - ROUND_DISPATCH_VECTOR_IN_LEFT_ARG_FORMAT(func_name, ObContinuousFormat); \ - break; \ - } \ - case VEC_UNIFORM: { \ - ROUND_DISPATCH_VECTOR_IN_LEFT_ARG_FORMAT(func_name, ObUniformFormat); \ - break; \ - } \ - case VEC_UNIFORM_CONST: { \ - ROUND_DISPATCH_VECTOR_IN_LEFT_ARG_FORMAT(func_name, ObUniformFormat); \ - break; \ - } \ - default: { \ - ROUND_DISPATCH_VECTOR_IN_LEFT_ARG_FORMAT(func_name, ObVectorBase); \ - } \ +#define ROUND_DISPATCH_VECTOR_IN_RES_ARG_FORMAT(func_name) \ +switch (res_format) { \ + case VEC_FIXED: { \ + ROUND_DISPATCH_VECTOR_IN_LEFT_ARG_FORMAT(func_name, ObFixedLengthBase); \ + break; \ + } \ + case VEC_DISCRETE: { \ + ROUND_DISPATCH_VECTOR_IN_LEFT_ARG_FORMAT(func_name, ObDiscreteFormat); \ + break; \ + } \ + case VEC_CONTINUOUS: { \ + ROUND_DISPATCH_VECTOR_IN_LEFT_ARG_FORMAT(func_name, ObContinuousFormat); \ + break; \ + } \ + case VEC_UNIFORM: { \ + ROUND_DISPATCH_VECTOR_IN_LEFT_ARG_FORMAT(func_name, ObUniformFormat); \ + break; \ + } \ + case VEC_UNIFORM_CONST: { \ + ROUND_DISPATCH_VECTOR_IN_LEFT_ARG_FORMAT(func_name, ObUniformFormat); \ + break; \ + } \ + default: { \ + ROUND_DISPATCH_VECTOR_IN_LEFT_ARG_FORMAT(func_name, ObVectorBase); \ + } \ } int ObExprFuncRound::calc_round_expr_numeric1_vector(const ObExpr &expr, diff --git a/src/sql/engine/expr/ob_expr_in.cpp b/src/sql/engine/expr/ob_expr_in.cpp index 12d3dd5411..2c26879d50 100644 --- a/src/sql/engine/expr/ob_expr_in.cpp +++ b/src/sql/engine/expr/ob_expr_in.cpp @@ -1681,13 +1681,9 @@ int ObExprInOrNotIn::inner_eval_vector_in_without_row(const ObExpr &expr, for (int i = 0; i < right_param_num; i++) { in_ctx->cmp_functions_[i] = (void *)func_ptr; } - if (std::is_same::value) { - fixed_base_l_payload = (reinterpret_cast(input_left_vec))->get_data(); - left_datum.len_ = (reinterpret_cast(input_left_vec))->get_length(); - } if (0 == in_ctx->get_static_engine_hashset_size()) { // Scenarios where in_list contains only null. - if (right_has_null) { + if (in_ctx->ctx_hash_null_) { for (int64_t left_idx = bound.start(); left_idx < bound.end(); ++left_idx) { if (skip.at(left_idx) || eval_flags.at(left_idx)) { continue; } res_vec->set_null(left_idx); @@ -1699,6 +1695,9 @@ int ObExprInOrNotIn::inner_eval_vector_in_without_row(const ObExpr &expr, LOG_WARN("static_engine_hashset_size unexpected", K(ret), K(right_has_null), K(in_ctx->get_static_engine_hashset_size())); } + } else if (std::is_same::value) { + fixed_base_l_payload = (reinterpret_cast(input_left_vec))->get_data(); + left_datum.len_ = (reinterpret_cast(input_left_vec))->get_length(); } } } diff --git a/src/sql/engine/expr/ob_expr_json_equal.cpp b/src/sql/engine/expr/ob_expr_json_equal.cpp index d5da1074ad..39a9689e8b 100644 --- a/src/sql/engine/expr/ob_expr_json_equal.cpp +++ b/src/sql/engine/expr/ob_expr_json_equal.cpp @@ -92,14 +92,6 @@ int ObExprJsonEqual::eval_json_equal(const ObExpr &expr, ObEvalCtx &ctx, ObDatum uint8_t option_on_error = 0; // json数据解析出错或非标量，此时根据on error参数返回结果 bool is_cover_by_error = false; - bool both_json = false; - - // 检查是否均为 json 类型，如果均为json类型，标量相比不报错 - // 只要有一个参数不是json类型，标量相比就会报错 - if (expr.args_[0]->datum_meta_.type_ == ObJsonType - || expr.args_[1]->datum_meta_.type_ == ObJsonType) { - both_json = true; - } ObExpr *json_arg_l = expr.args_[0]; ObObjType val_type_l = json_arg_l->datum_meta_.type_; @@ -116,8 +108,9 @@ int ObExprJsonEqual::eval_json_equal(const ObExpr &expr, ObEvalCtx &ctx, ObDatum json_candidate, is_null_result))) { if (ret == OB_ERR_JSON_SYNTAX_ERROR) is_cover_by_error = true; LOG_WARN("get_json_doc failed", K(ret)); - } else if(!is_null_result && !both_json && (is_json_scalar(json_target) - || is_json_scalar(json_candidate))) { + // if is scalar, must be json type + } else if(!is_null_result && ((is_json_scalar(json_target) && expr.args_[0]->datum_meta_.type_ != ObJsonType) + || (is_json_scalar(json_candidate) && expr.args_[1]->datum_meta_.type_ != ObJsonType))) { ret = OB_ERR_JSON_SYNTAX_ERROR; is_cover_by_error = true; LOG_USER_ERROR(OB_ERR_JSON_SYNTAX_ERROR); diff --git a/src/sql/engine/expr/ob_expr_json_query.cpp b/src/sql/engine/expr/ob_expr_json_query.cpp index d64db7dc83..0014ad5c7b 100644 --- a/src/sql/engine/expr/ob_expr_json_query.cpp +++ b/src/sql/engine/expr/ob_expr_json_query.cpp @@ -178,7 +178,7 @@ int ObExprJsonQuery::eval_json_query(const ObExpr &expr, ObEvalCtx &ctx, ObDatum // parse pretty ascii scalars uint8_t pretty_type = OB_JSON_PRE_ASC_EMPTY; uint8_t ascii_type = OB_JSON_PRE_ASC_EMPTY; - uint8_t scalars_type = OB_JSON_SCALARS_IMPLICIT; + uint8_t scalars_type = JSN_QUERY_SCALARS_IMPLICIT; if (OB_SUCC(ret) && !is_null_result) { ret = get_clause_pre_asc_sca_opt(expr, ctx, is_cover_by_error, pretty_type, ascii_type, scalars_type); } @@ -239,12 +239,12 @@ int ObExprJsonQuery::eval_json_query(const ObExpr &expr, ObEvalCtx &ctx, ObDatum } // parse error option - uint8_t error_type = OB_JSON_ON_RESPONSE_IMPLICIT; + uint8_t error_type = JSN_QUERY_IMPLICIT; ObDatum *error_val = NULL; if (OB_SUCC(ret) && !is_null_result) { - ret = get_clause_opt(expr, ctx, 8, is_cover_by_error, error_type, OB_JSON_ON_RESPONSE_COUNT); + ret = get_clause_opt(expr, ctx, 8, is_cover_by_error, error_type, JSN_QUERY_RESPONSE_COUNT); } else if (is_cover_by_error) { // always get error option on error - int temp_ret = get_clause_opt(expr, ctx, 8, is_cover_by_error, error_type, OB_JSON_ON_RESPONSE_COUNT); + int temp_ret = get_clause_opt(expr, ctx, 8, is_cover_by_error, error_type, JSN_QUERY_RESPONSE_COUNT); if (temp_ret != OB_SUCCESS) { ret = temp_ret; LOG_WARN("failed to get error option.", K(temp_ret)); @@ -252,9 +252,9 @@ int ObExprJsonQuery::eval_json_query(const ObExpr &expr, ObEvalCtx &ctx, ObDatum } // parse wrapper - uint8_t wrapper_type = OB_WRAPPER_IMPLICIT; + uint8_t wrapper_type = JSN_QUERY_WRAPPER_IMPLICIT; if (OB_SUCC(ret)) { - ret = get_clause_opt(expr, ctx, 7, is_cover_by_error, wrapper_type, OB_WRAPPER_COUNT); + ret = get_clause_opt(expr, ctx, 7, is_cover_by_error, wrapper_type, JSN_QUERY_WRAPPER_COUNT); } if (OB_SUCC(ret) && j_path->get_last_node_type() > JPN_BEGIN_FUNC_FLAG @@ -264,18 +264,18 @@ int ObExprJsonQuery::eval_json_query(const ObExpr &expr, ObEvalCtx &ctx, ObDatum || j_path->get_last_node_type() == JPN_LENGTH || j_path->get_last_node_type() == JPN_TYPE || j_path->get_last_node_type() == JPN_SIZE ) - && (wrapper_type == OB_WITHOUT_WRAPPER || wrapper_type == OB_WITHOUT_ARRAY_WRAPPER - || wrapper_type == OB_WRAPPER_IMPLICIT)) { + && (wrapper_type == JSN_QUERY_WITHOUT_WRAPPER || wrapper_type == JSN_QUERY_WITHOUT_ARRAY_WRAPPER + || wrapper_type == JSN_QUERY_WRAPPER_IMPLICIT)) { is_cover_by_error = false; ret = OB_ERR_WITHOUT_ARR_WRAPPER; // result cannot be returned without array wrapper LOG_WARN("result cannot be returned without array wrapper.", K(ret), K(j_path->get_last_node_type()), K(wrapper_type)); } // mismatch // if mismatch_type == 3 from dot notation - uint8_t mismatch_type = OB_JSON_ON_MISMATCH_IMPLICIT; + uint8_t mismatch_type = JSN_QUERY_MISMATCH_IMPLICIT; uint8_t mismatch_val = 7; if (OB_SUCC(ret) && !is_null_result) { - if (OB_FAIL(get_clause_opt(expr, ctx, 10, is_cover_by_error, mismatch_type, OB_JSON_ON_MISMATCH_COUNT))) { + if (OB_FAIL(get_clause_opt(expr, ctx, 10, is_cover_by_error, mismatch_type, JSN_QUERY_MISMATCH_COUNT))) { LOG_WARN("failed to get mismatch option.", K(ret), K(mismatch_type)); } } @@ -304,7 +304,7 @@ int ObExprJsonQuery::eval_json_query(const ObExpr &expr, ObEvalCtx &ctx, ObDatum } LOG_WARN("json seek failed", K(json_datum->get_string()), K(ret)); } else if (hits.size() == 1) { - if (mismatch_type == OB_JSON_ON_MISMATCH_DOT) { + if (mismatch_type == JSN_QUERY_MISMATCH_DOT) { if (hits[0]->json_type() == ObJsonNodeType::J_NULL && hits[0]->is_real_json_null(hits[0]) && dst_type != ObJsonType) { is_null_result = true; } @@ -346,9 +346,9 @@ int ObExprJsonQuery::eval_json_query(const ObExpr &expr, ObEvalCtx &ctx, ObDatum } } else if (hits.size() == 0) { // parse empty option - uint8_t empty_type = OB_JSON_ON_RESPONSE_IMPLICIT; + uint8_t empty_type = JSN_QUERY_IMPLICIT; if (OB_SUCC(ret) && !is_null_result) { - ret = get_clause_opt(expr, ctx, 9, is_cover_by_error, empty_type, OB_JSON_ON_RESPONSE_COUNT); + ret = get_clause_opt(expr, ctx, 9, is_cover_by_error, empty_type, JSN_QUERY_RESPONSE_COUNT); } if (OB_SUCC(ret) && OB_FAIL(get_empty_option(hits, is_cover_by_error, empty_type, is_null_result, is_null_json_obj, is_null_json_array))) { LOG_WARN("get empty type", K(ret)); @@ -372,12 +372,12 @@ int ObExprJsonQuery::eval_json_query(const ObExpr &expr, ObEvalCtx &ctx, ObDatum LOG_WARN("json_query failed", K(ret)); } else if (is_null_result) { res.set_null(); - } else if (mismatch_type == OB_JSON_ON_MISMATCH_DOT && hits.size() == 1 && dst_type != ObJsonType) { + } else if (mismatch_type == JSN_QUERY_MISMATCH_DOT && hits.size() == 1 && dst_type != ObJsonType) { ObVector mismatch_val_tmp; ObVector mismatch_type_tmp; //OB_JSON_TYPE_IMPLICIT ObCollationType in_coll_type = expr.args_[0]->datum_meta_.cs_type_; ObCollationType dst_coll_type = expr.datum_meta_.cs_type_; - ret = ObExprJsonValue::cast_to_res(&temp_allocator, expr, ctx, hits[0], OB_JSON_ON_RESPONSE_NULL, error_val, + ret = ObExprJsonValue::cast_to_res(&temp_allocator, expr, ctx, hits[0], JSN_QUERY_NULL, error_val, accuracy, dst_type, in_coll_type, dst_coll_type, res, mismatch_val_tmp, mismatch_type_tmp, is_type_cast, ascii_type, is_truncate); } else { if (is_null_json_obj) { @@ -495,7 +495,7 @@ int ObExprJsonQuery::set_result(ObObjType dst_type, } ret = OB_OPERATE_OVERFLOW; LOG_USER_ERROR(OB_OPERATE_OVERFLOW, res_ptr, "json_query"); - if (!try_set_error_val(allocator, ctx, expr, res, ret, error_type, OB_JSON_ON_MISMATCH_IMPLICIT, dst_type)) { + if (!try_set_error_val(allocator, ctx, expr, res, ret, error_type, JSN_QUERY_MISMATCH_IMPLICIT, dst_type)) { LOG_WARN("set error val fail", K(ret)); } } @@ -573,7 +573,7 @@ int ObExprJsonQuery::get_clause_pre_asc_sca_opt(const ObExpr &expr, ObEvalCtx &c } // parse scalars if (OB_SUCC(ret)) { - ret = get_clause_opt(expr, ctx, 4, is_cover_by_error, scalars_type, OB_JSON_SCALARS_COUNT); + ret = get_clause_opt(expr, ctx, 4, is_cover_by_error, scalars_type, JSN_QUERY_SCALARS_COUNT); } return ret; } @@ -661,29 +661,29 @@ int ObExprJsonQuery::get_empty_option(ObJsonBaseVector &hits, bool &is_cover_by_ { INIT_SUCC(ret); switch (empty_type) { - case OB_JSON_ON_RESPONSE_IMPLICIT: { + case JSN_QUERY_IMPLICIT: { ret = OB_ERR_JSON_VALUE_NO_VALUE; LOG_USER_ERROR(OB_ERR_JSON_VALUE_NO_VALUE); LOG_WARN("json value seek result empty.", K(hits.size())); break; } - case OB_JSON_ON_RESPONSE_ERROR: { + case JSN_QUERY_ERROR: { is_cover_by_error = false; ret = OB_ERR_JSON_VALUE_NO_VALUE; LOG_USER_ERROR(OB_ERR_JSON_VALUE_NO_VALUE); LOG_WARN("json value seek result empty.", K(hits.size())); break; } - case OB_JSON_ON_RESPONSE_EMPTY_OBJECT: { + case JSN_QUERY_EMPTY_OBJECT: { is_null_json_obj = true; break; } - case OB_JSON_ON_RESPONSE_NULL: { + case JSN_QUERY_NULL: { is_null_result = true; break; } - case OB_JSON_ON_RESPONSE_EMPTY: - case OB_JSON_ON_RESPONSE_EMPTY_ARRAY: { + case JSN_QUERY_EMPTY: + case JSN_QUERY_EMPTY_ARRAY: { is_null_json_array = true; // set_json_array break; } @@ -697,27 +697,27 @@ int ObExprJsonQuery::get_single_obj_wrapper(uint8_t wrapper_type, int &use_wrapp { INIT_SUCC(ret); switch (wrapper_type) { - case OB_WITHOUT_WRAPPER: - case OB_WITHOUT_ARRAY_WRAPPER: - case OB_WRAPPER_IMPLICIT: { + case JSN_QUERY_WITHOUT_WRAPPER: + case JSN_QUERY_WITHOUT_ARRAY_WRAPPER: + case JSN_QUERY_WRAPPER_IMPLICIT: { if ((in_type != ObJsonNodeType::J_OBJECT && in_type != ObJsonNodeType::J_ARRAY - && scalars_type == OB_JSON_SCALARS_DISALLOW)) { + && scalars_type == JSN_QUERY_SCALARS_DISALLOW)) { ret = OB_ERR_WITHOUT_ARR_WRAPPER; // result cannot be returned without array wrapper LOG_USER_ERROR(OB_ERR_WITHOUT_ARR_WRAPPER); LOG_WARN("result cannot be returned without array wrapper.", K(ret)); } break; } - case OB_WITH_WRAPPER: - case OB_WITH_ARRAY_WRAPPER: - case OB_WITH_UNCONDITIONAL_WRAPPER: - case OB_WITH_UNCONDITIONAL_ARRAY_WRAPPER: { + case JSN_QUERY_WITH_WRAPPER: + case JSN_QUERY_WITH_ARRAY_WRAPPER: + case JSN_QUERY_WITH_UNCONDITIONAL_WRAPPER: + case JSN_QUERY_WITH_UNCONDITIONAL_ARRAY_WRAPPER: { use_wrapper = 1; break; } - case OB_WITH_CONDITIONAL_WRAPPER: - case OB_WITH_CONDITIONAL_ARRAY_WRAPPER: { - if (in_type != ObJsonNodeType::J_OBJECT && in_type != ObJsonNodeType::J_ARRAY && scalars_type == OB_JSON_SCALARS_DISALLOW ) { + case JSN_QUERY_WITH_CONDITIONAL_WRAPPER: + case JSN_QUERY_WITH_CONDITIONAL_ARRAY_WRAPPER: { + if (in_type != ObJsonNodeType::J_OBJECT && in_type != ObJsonNodeType::J_ARRAY && scalars_type == JSN_QUERY_SCALARS_DISALLOW ) { use_wrapper = 1; } break; @@ -732,23 +732,23 @@ int ObExprJsonQuery::get_multi_scalars_wrapper_type(uint8_t wrapper_type, int &u { INIT_SUCC(ret); switch (wrapper_type) { - case OB_WITHOUT_WRAPPER: - case OB_WITHOUT_ARRAY_WRAPPER: - case OB_WRAPPER_IMPLICIT: { + case JSN_QUERY_WITHOUT_WRAPPER: + case JSN_QUERY_WITHOUT_ARRAY_WRAPPER: + case JSN_QUERY_WRAPPER_IMPLICIT: { ret = OB_ERR_WITHOUT_ARR_WRAPPER; // result cannot be returned without array wrapper LOG_USER_ERROR(OB_ERR_WITHOUT_ARR_WRAPPER); LOG_WARN("result cannot be returned without array wrapper.", K(ret), K(hits.size())); break; } - case OB_WITH_WRAPPER: - case OB_WITH_ARRAY_WRAPPER: - case OB_WITH_UNCONDITIONAL_WRAPPER: - case OB_WITH_UNCONDITIONAL_ARRAY_WRAPPER: { + case JSN_QUERY_WITH_WRAPPER: + case JSN_QUERY_WITH_ARRAY_WRAPPER: + case JSN_QUERY_WITH_UNCONDITIONAL_WRAPPER: + case JSN_QUERY_WITH_UNCONDITIONAL_ARRAY_WRAPPER: { use_wrapper = 1; break; } - case OB_WITH_CONDITIONAL_WRAPPER: - case OB_WITH_CONDITIONAL_ARRAY_WRAPPER: { + case JSN_QUERY_WITH_CONDITIONAL_WRAPPER: + case JSN_QUERY_WITH_CONDITIONAL_ARRAY_WRAPPER: { use_wrapper = 1; break; } @@ -771,7 +771,7 @@ bool ObExprJsonQuery::try_set_error_val(common::ObIAllocator *allocator, bool mismatch_error = true; if (OB_FAIL(ret)) { - if (error_type == OB_JSON_ON_RESPONSE_EMPTY_ARRAY || error_type == OB_JSON_ON_RESPONSE_EMPTY) { + if (error_type == JSN_QUERY_EMPTY_ARRAY || error_type == JSN_QUERY_EMPTY) { ret = OB_SUCCESS; ObJsonArray j_arr_res(allocator); ObIJsonBase *jb_res = NULL; @@ -779,7 +779,7 @@ bool ObExprJsonQuery::try_set_error_val(common::ObIAllocator *allocator, if (OB_FAIL(set_result(dst_type, OB_MAX_TEXT_LENGTH, jb_res, allocator, ctx, expr, res, error_type, 0, 0))) { LOG_WARN("result set fail", K(ret)); } - } else if (error_type == OB_JSON_ON_RESPONSE_EMPTY_OBJECT) { + } else if (error_type == JSN_QUERY_EMPTY_OBJECT) { ret = OB_SUCCESS; ObJsonObject j_node_null(allocator); ObIJsonBase *jb_res = NULL; @@ -787,7 +787,7 @@ bool ObExprJsonQuery::try_set_error_val(common::ObIAllocator *allocator, if (OB_FAIL(set_result(dst_type, OB_MAX_TEXT_LENGTH, jb_res, allocator, ctx, expr, res, error_type, 0, 0))) { LOG_WARN("result set fail", K(ret)); } - } else if (error_type == OB_JSON_ON_RESPONSE_NULL || error_type == OB_JSON_ON_RESPONSE_IMPLICIT) { + } else if (error_type == JSN_QUERY_NULL || error_type == JSN_QUERY_IMPLICIT) { res.set_null(); ret = OB_SUCCESS; } diff --git a/src/sql/engine/expr/ob_expr_json_query.h b/src/sql/engine/expr/ob_expr_json_query.h index dc977770e4..a8fe9987e6 100644 --- a/src/sql/engine/expr/ob_expr_json_query.h +++ b/src/sql/engine/expr/ob_expr_json_query.h @@ -17,7 +17,7 @@ #include "sql/engine/expr/ob_expr_operator.h" #include "lib/json_type/ob_json_tree.h" #include "lib/json_type/ob_json_base.h" - +#include "ob_json_param_type.h" using namespace oceanbase::common; @@ -58,45 +58,7 @@ private: common::ObIAllocator *allocator, ObEvalCtx &ctx, const ObExpr &expr, ObDatum &res, uint8_t error_type, uint8_t ascii_type, uint8_t pretty_type = 0, uint8_t is_truncate = 0); - /* process empty or error */ - const static uint8_t OB_JSON_ON_RESPONSE_COUNT = 6; - const static uint8_t OB_JSON_ON_RESPONSE_ERROR = 0; - const static uint8_t OB_JSON_ON_RESPONSE_NULL = 1; - const static uint8_t OB_JSON_ON_RESPONSE_EMPTY = 2; - const static uint8_t OB_JSON_ON_RESPONSE_EMPTY_ARRAY = 3; - const static uint8_t OB_JSON_ON_RESPONSE_EMPTY_OBJECT = 4; - const static uint8_t OB_JSON_ON_RESPONSE_IMPLICIT = 5; - /* process on mismatch { error : 0, null : 1, implicit : 2 }*/ - const static uint8_t OB_JSON_ON_MISMATCH_COUNT = 4; - const static uint8_t OB_JSON_ON_MISMATCH_ERROR = 0; - const static uint8_t OB_JSON_ON_MISMATCH_NULL = 1; - const static uint8_t OB_JSON_ON_MISMATCH_IMPLICIT = 2; - const static uint8_t OB_JSON_ON_MISMATCH_DOT = 3; - - - /* process wrapper type */ - const static uint8_t OB_WRAPPER_COUNT = 9; - const static uint8_t OB_WITHOUT_WRAPPER = 0; - const static uint8_t OB_WITHOUT_ARRAY_WRAPPER = 1; - const static uint8_t OB_WITH_WRAPPER = 2; - const static uint8_t OB_WITH_ARRAY_WRAPPER = 3; - const static uint8_t OB_WITH_UNCONDITIONAL_WRAPPER = 4; - const static uint8_t OB_WITH_CONDITIONAL_WRAPPER = 5; - const static uint8_t OB_WITH_UNCONDITIONAL_ARRAY_WRAPPER = 6; - const static uint8_t OB_WITH_CONDITIONAL_ARRAY_WRAPPER = 7; - const static uint8_t OB_WRAPPER_IMPLICIT = 8; - - /* process on scalars { allow : 0, disallow : 1, implicit : 2 }*/ - const static uint8_t OB_JSON_SCALARS_COUNT = 3; - const static uint8_t OB_JSON_SCALARS_ALLOW = 0; - const static uint8_t OB_JSON_SCALARS_DISALLOW = 1; - const static uint8_t OB_JSON_SCALARS_IMPLICIT = 2; - - /* pretty ascii 0 : null 1 : yes */ - const static uint8_t OB_JSON_PRE_ASC_COUNT = 2; - const static uint8_t OB_JSON_PRE_ASC_EMPTY = 0; - const static uint8_t OB_JSON_PRE_ASC_SET = 1; static int get_clause_opt(const ObExpr &expr, ObEvalCtx &ctx, diff --git a/src/sql/engine/expr/ob_expr_json_value.cpp b/src/sql/engine/expr/ob_expr_json_value.cpp index 5cc45ab104..989a0406bf 100644 --- a/src/sql/engine/expr/ob_expr_json_value.cpp +++ b/src/sql/engine/expr/ob_expr_json_value.cpp @@ -70,12 +70,12 @@ int ObExprJsonValue::calc_result_typeN(ObExprResType& type, bool is_oracle_mode = lib::is_oracle_mode(); //type.set_json(); // json doc : 0 - ObObjType doc_type = types_stack[json_doc_id].get_type(); - if (types_stack[json_doc_id].get_type() == ObNullType) { + ObObjType doc_type = types_stack[JSN_VAL_DOC].get_type(); + if (types_stack[JSN_VAL_DOC].get_type() == ObNullType) { } else if (!ObJsonExprHelper::is_convertible_to_json(doc_type)) { if (lib::is_oracle_mode()) { ret = OB_ERR_INVALID_TYPE_FOR_OP; - LOG_USER_ERROR(OB_ERR_INVALID_TYPE_FOR_OP, ob_obj_type_str(types_stack[json_doc_id].get_type()), "JSON"); + LOG_USER_ERROR(OB_ERR_INVALID_TYPE_FOR_OP, ob_obj_type_str(types_stack[JSN_VAL_DOC].get_type()), "JSON"); } else { ret = OB_ERR_INVALID_TYPE_FOR_JSON; LOG_USER_ERROR(OB_ERR_INVALID_TYPE_FOR_JSON, 1, "json_value"); @@ -83,50 +83,50 @@ int ObExprJsonValue::calc_result_typeN(ObExprResType& type, } } else if (ob_is_string_type(doc_type)) { if (is_oracle_mode) { - if (types_stack[json_doc_id].get_collation_type() == CS_TYPE_BINARY) { - types_stack[json_doc_id].set_calc_collation_type(CS_TYPE_BINARY); - } else if (types_stack[json_doc_id].get_charset_type() != CHARSET_UTF8MB4) { - types_stack[json_doc_id].set_calc_collation_type(CS_TYPE_UTF8MB4_BIN); + if (types_stack[JSN_VAL_DOC].get_collation_type() == CS_TYPE_BINARY) { + types_stack[JSN_VAL_DOC].set_calc_collation_type(CS_TYPE_BINARY); + } else if (types_stack[JSN_VAL_DOC].get_charset_type() != CHARSET_UTF8MB4) { + types_stack[JSN_VAL_DOC].set_calc_collation_type(CS_TYPE_UTF8MB4_BIN); } } else { - if (types_stack[json_doc_id].get_collation_type() == CS_TYPE_BINARY) { + if (types_stack[JSN_VAL_DOC].get_collation_type() == CS_TYPE_BINARY) { // unsuport string type with binary charset ret = OB_ERR_INVALID_JSON_CHARSET; LOG_WARN("Unsupport for string type with binary charset input.", K(ret), K(doc_type)); - } else if (types_stack[json_doc_id].get_charset_type() != CHARSET_UTF8MB4) { - types_stack[json_doc_id].set_calc_collation_type(CS_TYPE_UTF8MB4_BIN); + } else if (types_stack[JSN_VAL_DOC].get_charset_type() != CHARSET_UTF8MB4) { + types_stack[JSN_VAL_DOC].set_calc_collation_type(CS_TYPE_UTF8MB4_BIN); } } } else if (doc_type == ObJsonType) { // do nothing } else { - types_stack[json_doc_id].set_calc_type(ObLongTextType); - types_stack[json_doc_id].set_calc_collation_type(CS_TYPE_UTF8MB4_BIN); + types_stack[JSN_VAL_DOC].set_calc_type(ObLongTextType); + types_stack[JSN_VAL_DOC].set_calc_collation_type(CS_TYPE_UTF8MB4_BIN); } // json path : 1 if (OB_SUCC(ret)) { - if (types_stack[json_path_id].get_type() == ObNullType) { + if (types_stack[JSN_VAL_PATH].get_type() == ObNullType) { if (lib::is_oracle_mode()) { ret = OB_ERR_PATH_EXPRESSION_NOT_LITERAL; LOG_USER_ERROR(OB_ERR_PATH_EXPRESSION_NOT_LITERAL); } // do nothing - } else if (ob_is_string_type(types_stack[json_path_id].get_type())) { - if (types_stack[json_path_id].get_charset_type() != CHARSET_UTF8MB4) { - types_stack[json_path_id].set_calc_collation_type(CS_TYPE_UTF8MB4_BIN); + } else if (ob_is_string_type(types_stack[JSN_VAL_PATH].get_type())) { + if (types_stack[JSN_VAL_PATH].get_charset_type() != CHARSET_UTF8MB4) { + types_stack[JSN_VAL_PATH].set_calc_collation_type(CS_TYPE_UTF8MB4_BIN); } } else { - types_stack[json_path_id].set_calc_type(ObLongTextType); - types_stack[json_path_id].set_calc_collation_type(CS_TYPE_UTF8MB4_BIN); + types_stack[JSN_VAL_PATH].set_calc_type(ObLongTextType); + types_stack[JSN_VAL_PATH].set_calc_collation_type(CS_TYPE_UTF8MB4_BIN); } } // returning type : 2 ObExprResType dst_type; if (OB_SUCC(ret)) { - if (OB_FAIL(get_cast_type(types_stack[ret_type_id], dst_type, type_ctx))) { + if (OB_FAIL(get_cast_type(types_stack[JSN_VAL_RET], dst_type, type_ctx))) { LOG_WARN("get cast dest type failed", K(ret)); - } else if (OB_FAIL(set_dest_type(types_stack[json_doc_id], type, dst_type, type_ctx))) { + } else if (OB_FAIL(set_dest_type(types_stack[JSN_VAL_DOC], type, dst_type, type_ctx))) { LOG_WARN("set dest type failed", K(ret)); } else { type.set_calc_collation_type(type.get_collation_type()); @@ -143,63 +143,63 @@ int ObExprJsonValue::calc_result_typeN(ObExprResType& type, // empty : 4, 5， 6 if (OB_SUCC(ret)) { ObExprResType temp_type; - if (types_stack[empty_type_id].get_type() == ObNullType) { + if (types_stack[JSN_VAL_EMPTY].get_type() == ObNullType) { ret = OB_ERR_UNEXPECTED; - LOG_WARN(" param type is unexpected", K(types_stack[empty_type_id].get_type())); - } else if (types_stack[empty_type_id].get_type() != ObIntType) { - types_stack[empty_type_id].set_calc_type(ObIntType); - } else if (types_stack[empty_val_id].get_type() == ObNullType) { + LOG_WARN(" param type is unexpected", K(types_stack[JSN_VAL_EMPTY].get_type())); + } else if (types_stack[JSN_VAL_EMPTY].get_type() != ObIntType) { + types_stack[JSN_VAL_EMPTY].set_calc_type(ObIntType); + } else if (types_stack[JSN_VAL_EMPTY_DEF].get_type() == ObNullType) { // do nothing - } else if (OB_FAIL(set_dest_type(types_stack[empty_val_id], temp_type, dst_type, type_ctx))) { + } else if (OB_FAIL(set_dest_type(types_stack[JSN_VAL_EMPTY_DEF], temp_type, dst_type, type_ctx))) { LOG_WARN("set dest type failed", K(ret)); } else { - types_stack[empty_val_id].set_calc_type(temp_type.get_type()); - types_stack[empty_val_id].set_calc_collation_type(temp_type.get_collation_type()); - types_stack[empty_val_id].set_calc_collation_level(temp_type.get_collation_level()); - types_stack[empty_val_id].set_calc_accuracy(temp_type.get_accuracy()); + types_stack[JSN_VAL_EMPTY_DEF].set_calc_type(temp_type.get_type()); + types_stack[JSN_VAL_EMPTY_DEF].set_calc_collation_type(temp_type.get_collation_type()); + types_stack[JSN_VAL_EMPTY_DEF].set_calc_collation_level(temp_type.get_collation_level()); + types_stack[JSN_VAL_EMPTY_DEF].set_calc_accuracy(temp_type.get_accuracy()); } - if (types_stack[empty_val_pre_id].get_type() == ObNullType) { + if (types_stack[JSN_VAL_EMPTY_DEF_PRE].get_type() == ObNullType) { // do nothing } else { - types_stack[empty_val_pre_id].set_calc_type(types_stack[empty_val_pre_id].get_type()); - types_stack[empty_val_pre_id].set_calc_collation_type(types_stack[empty_val_pre_id].get_collation_type()); - types_stack[empty_val_pre_id].set_calc_collation_level(types_stack[empty_val_pre_id].get_collation_level()); - types_stack[empty_val_pre_id].set_calc_accuracy(types_stack[empty_val_pre_id].get_accuracy()); + types_stack[JSN_VAL_EMPTY_DEF_PRE].set_calc_type(types_stack[JSN_VAL_EMPTY_DEF_PRE].get_type()); + types_stack[JSN_VAL_EMPTY_DEF_PRE].set_calc_collation_type(types_stack[JSN_VAL_EMPTY_DEF_PRE].get_collation_type()); + types_stack[JSN_VAL_EMPTY_DEF_PRE].set_calc_collation_level(types_stack[JSN_VAL_EMPTY_DEF_PRE].get_collation_level()); + types_stack[JSN_VAL_EMPTY_DEF_PRE].set_calc_accuracy(types_stack[JSN_VAL_EMPTY_DEF_PRE].get_accuracy()); } } // error : 7, 8，9 if (OB_SUCC(ret)) { ObExprResType temp_type; - if (types_stack[error_type_id].get_type() == ObNullType) { + if (types_stack[JSN_VAL_ERROR].get_type() == ObNullType) { ret = OB_ERR_UNEXPECTED; - LOG_WARN(" param type is unexpected", K(types_stack[error_type_id].get_type())); - } else if (types_stack[error_type_id].get_type() != ObIntType) { - types_stack[error_type_id].set_calc_type(ObIntType); - } else if (types_stack[error_val_id].get_type() == ObNullType) { + LOG_WARN(" param type is unexpected", K(types_stack[JSN_VAL_ERROR].get_type())); + } else if (types_stack[JSN_VAL_ERROR].get_type() != ObIntType) { + types_stack[JSN_VAL_ERROR].set_calc_type(ObIntType); + } else if (types_stack[JSN_VAL_ERROR_DEF].get_type() == ObNullType) { // do nothing - } else if (OB_FAIL(set_dest_type(types_stack[error_val_id], temp_type, dst_type, type_ctx))) { + } else if (OB_FAIL(set_dest_type(types_stack[JSN_VAL_ERROR_DEF], temp_type, dst_type, type_ctx))) { LOG_WARN("set dest type failed", K(ret)); } else { - types_stack[error_val_id].set_calc_type(temp_type.get_type()); - types_stack[error_val_id].set_calc_collation_type(temp_type.get_collation_type()); - types_stack[error_val_id].set_calc_collation_level(temp_type.get_collation_level()); - types_stack[error_val_id].set_calc_accuracy(temp_type.get_accuracy()); + types_stack[JSN_VAL_ERROR_DEF].set_calc_type(temp_type.get_type()); + types_stack[JSN_VAL_ERROR_DEF].set_calc_collation_type(temp_type.get_collation_type()); + types_stack[JSN_VAL_ERROR_DEF].set_calc_collation_level(temp_type.get_collation_level()); + types_stack[JSN_VAL_ERROR_DEF].set_calc_accuracy(temp_type.get_accuracy()); } - if (types_stack[error_val_pre_id].get_type() == ObNullType) { + if (types_stack[JSN_VAL_ERROR_DEF_PRE].get_type() == ObNullType) { // do nothing } else { - types_stack[error_val_pre_id].set_calc_type(types_stack[error_val_pre_id].get_type()); - types_stack[error_val_pre_id].set_calc_collation_type(types_stack[error_val_pre_id].get_collation_type()); - types_stack[error_val_pre_id].set_calc_collation_level(types_stack[error_val_pre_id].get_collation_level()); - types_stack[error_val_pre_id].set_calc_accuracy(types_stack[error_val_pre_id].get_accuracy()); + types_stack[JSN_VAL_ERROR_DEF_PRE].set_calc_type(types_stack[JSN_VAL_ERROR_DEF_PRE].get_type()); + types_stack[JSN_VAL_ERROR_DEF_PRE].set_calc_collation_type(types_stack[JSN_VAL_ERROR_DEF_PRE].get_collation_type()); + types_stack[JSN_VAL_ERROR_DEF_PRE].set_calc_collation_level(types_stack[JSN_VAL_ERROR_DEF_PRE].get_collation_level()); + types_stack[JSN_VAL_ERROR_DEF_PRE].set_calc_accuracy(types_stack[JSN_VAL_ERROR_DEF_PRE].get_accuracy()); } } // mismatch : 10, if (OB_SUCC(ret)) { - for (size_t i = opt_mismatch_id; OB_SUCC(ret) && i < param_num; i++) { + for (size_t i = JSN_VAL_MISMATCH; OB_SUCC(ret) && i < param_num; i++) { if (types_stack[i].get_type() == ObNullType) { ret = OB_ERR_UNEXPECTED; LOG_WARN(" param type is unexpected", K(types_stack[i].get_type()), K(ret), K(i)); @@ -289,7 +289,7 @@ int ObExprJsonValue::eval_json_value(const ObExpr &expr, ObEvalCtx &ctx, ObDatum // parse empty option ObDatum *empty_datum = NULL; ObObjType empty_val_type; - uint8_t empty_type = OB_JSON_ON_RESPONSE_IMPLICIT; + uint8_t empty_type = JSN_VALUE_IMPLICIT; if (OB_SUCC(ret) && !is_null_result) { ret = get_on_empty_or_error(expr, ctx, 5, is_cover_by_error, accuracy, empty_type, &empty_datum, dst_type, empty_val_type); } @@ -297,7 +297,7 @@ int ObExprJsonValue::eval_json_value(const ObExpr &expr, ObEvalCtx &ctx, ObDatum // parse error option ObDatum *error_val = NULL; ObObjType error_val_type; - uint8_t error_type = OB_JSON_ON_RESPONSE_IMPLICIT; + uint8_t error_type = JSN_VALUE_IMPLICIT; if (OB_SUCC(ret) && !is_null_result) { ret = get_on_empty_or_error(expr, ctx, 8, is_cover_by_error, accuracy, error_type, &error_val, dst_type, error_val_type); } else if (is_cover_by_error) { // always get error option for return default value on error @@ -348,7 +348,7 @@ int ObExprJsonValue::eval_json_value(const ObExpr &expr, ObEvalCtx &ctx, ObDatum ObVector mismatch_val; ObVector mismatch_type; //OB_JSON_TYPE_IMPLICIT if (OB_SUCC(ret) && !is_null_result) { - ret = get_on_mismatch(expr, ctx, opt_mismatch_id, is_cover_by_error, accuracy, mismatch_val, mismatch_type); + ret = get_on_mismatch(expr, ctx, JSN_VAL_MISMATCH, is_cover_by_error, accuracy, mismatch_val, mismatch_type); if (ret != OB_SUCCESS || mismatch_type.size() == 0 || mismatch_val.size() == 0) { LOG_WARN("failed to get mismatch option.", K(ret), K(mismatch_type.size()), K(mismatch_val.size())); } @@ -382,7 +382,7 @@ int ObExprJsonValue::eval_ora_json_value(const ObExpr &expr, ObEvalCtx &ctx, ObD { INIT_SUCC(ret); ObDatum *json_datum = NULL; - ObExpr *json_arg = expr.args_[json_path_id]; + ObExpr *json_arg = expr.args_[JSN_VAL_PATH]; ObObjType type = json_arg->datum_meta_.type_; bool is_cover_by_error = true; bool is_null_result = false; @@ -435,12 +435,12 @@ int ObExprJsonValue::eval_ora_json_value(const ObExpr &expr, ObEvalCtx &ctx, ObD // parse ascii uint8_t ascii_type = OB_JSON_ON_ASCII_IMPLICIT; if (OB_SUCC(ret) && !is_null_result) { - ret = get_on_ascii(expr, ctx, opt_ascii_id, is_cover_by_error, ascii_type); + ret = get_on_ascii(expr, ctx, JSN_VAL_ASCII, is_cover_by_error, ascii_type); } uint8_t is_truncate = 0; if (OB_SUCC(ret) && !is_null_result) { - if (OB_FAIL(get_on_truncate(expr, ctx, opt_truncate_id, is_cover_by_error, is_truncate))) { + if (OB_FAIL(get_on_truncate(expr, ctx, JSN_VAL_TRUNC, is_cover_by_error, is_truncate))) { LOG_WARN("eval truncate option error", K(ret)); } } @@ -465,7 +465,7 @@ int ObExprJsonValue::eval_ora_json_value(const ObExpr &expr, ObEvalCtx &ctx, ObD } // parse json doc - json_arg = expr.args_[json_doc_id]; + json_arg = expr.args_[JSN_VAL_DOC]; type = json_arg->datum_meta_.type_; ObCollationType cs_type = json_arg->datum_meta_.cs_type_; ObJsonInType j_in_type; @@ -504,15 +504,15 @@ int ObExprJsonValue::eval_ora_json_value(const ObExpr &expr, ObEvalCtx &ctx, ObD // error default val type ObObjType empty_val_type; ObDatum *empty_datum = NULL; - uint8_t empty_type = OB_JSON_ON_RESPONSE_IMPLICIT; + uint8_t empty_type = JSN_VALUE_IMPLICIT; if (OB_SUCC(ret) && !is_null_result) { - ret = get_on_empty_or_error(expr, ctx, empty_type_id, is_cover_by_error, accuracy, empty_type, &empty_datum, dst_type, empty_val_type); + ret = get_on_empty_or_error(expr, ctx, JSN_VAL_EMPTY, is_cover_by_error, accuracy, empty_type, &empty_datum, dst_type, empty_val_type); } // parse error option ObDatum *error_val = NULL; - uint8_t error_type = OB_JSON_ON_RESPONSE_IMPLICIT; - json_arg = expr.args_[error_type_id + 2]; + uint8_t error_type = JSN_VALUE_IMPLICIT; + json_arg = expr.args_[JSN_VAL_ERROR + 2]; ObObjType val_type = json_arg->datum_meta_.type_; if ((OB_SUCC(ret) && !is_null_result) || is_cover_by_error) { int temp_ret = OB_SUCCESS; @@ -549,12 +549,12 @@ int ObExprJsonValue::eval_ora_json_value(const ObExpr &expr, ObEvalCtx &ctx, ObD ObVector mismatch_val; ObVector mismatch_type; //OB_JSON_TYPE_IMPLICIT if (OB_SUCC(ret) && !is_null_result) { - ret = get_on_mismatch(expr, ctx, opt_mismatch_id, is_cover_by_error, accuracy, mismatch_val, mismatch_type); + ret = get_on_mismatch(expr, ctx, JSN_VAL_MISMATCH, is_cover_by_error, accuracy, mismatch_val, mismatch_type); if (ret != OB_SUCCESS || mismatch_type.size() == 0 || mismatch_val.size() == 0) { LOG_WARN("failed to get mismatch option.", K(ret), K(mismatch_type.size()), K(mismatch_val.size())); } } else if (is_type_cast) { - int tmp_ret = get_on_mismatch(expr, ctx, opt_mismatch_id, is_cover_by_error, accuracy, mismatch_val, mismatch_type); + int tmp_ret = get_on_mismatch(expr, ctx, JSN_VAL_MISMATCH, is_cover_by_error, accuracy, mismatch_val, mismatch_type); if (tmp_ret != OB_SUCCESS || mismatch_type.size() == 0 || mismatch_val.size() == 0) { LOG_WARN("failed to get mismatch option.", K(ret), K(mismatch_type.size()), K(mismatch_val.size())); } @@ -747,7 +747,8 @@ int ObExprJsonValue::doc_do_seek(ObJsonBaseVector &hits, bool &is_null_result, O } else if (hits.size() == 0) { if (OB_SUCC(ret)) { switch (empty_type) { - case OB_JSON_ON_RESPONSE_ERROR: { + case JSN_VALUE_ERROR +: { is_cover_by_error = false; if (lib::is_oracle_mode()) { ret = OB_ERR_JSON_VALUE_NO_VALUE; @@ -759,15 +760,15 @@ int ObExprJsonValue::doc_do_seek(ObJsonBaseVector &hits, bool &is_null_result, O LOG_WARN("json value seek result empty.", K(hits.size())); break; } - case OB_JSON_ON_RESPONSE_DEFAULT: { + case JSN_VALUE_DEFAULT: { return_val = empty_datum; break; } - case OB_JSON_ON_RESPONSE_NULL: { + case JSN_VALUE_NULL: { is_null_result = true; break; } - case OB_JSON_ON_RESPONSE_IMPLICIT: { + case JSN_VALUE_IMPLICIT: { if (lib::is_oracle_mode()) { ret = OB_ERR_JSON_VALUE_NO_VALUE; LOG_USER_ERROR(OB_ERR_JSON_VALUE_NO_VALUE); @@ -786,18 +787,18 @@ int ObExprJsonValue::doc_do_seek(ObJsonBaseVector &hits, bool &is_null_result, O // return val decide by error option if (lib::is_mysql_mode()) { switch (error_type) { - case OB_JSON_ON_RESPONSE_ERROR: { + case JSN_VALUE_ERROR: { ret = OB_ERR_MULTIPLE_JSON_VALUES; LOG_USER_ERROR(OB_ERR_MULTIPLE_JSON_VALUES, "json_value"); LOG_WARN("json value seek result more than one.", K(hits.size())); break; } - case OB_JSON_ON_RESPONSE_DEFAULT: { + case JSN_VALUE_DEFAULT: { return_val = error_datum; break; } - case OB_JSON_ON_RESPONSE_NULL: - case OB_JSON_ON_RESPONSE_IMPLICIT: { + case JSN_VALUE_NULL: + case JSN_VALUE_IMPLICIT: { is_null_result = true; break; } @@ -1761,24 +1762,24 @@ bool ObExprJsonValue::try_set_error_val(const ObExpr &expr, if (OB_FAIL(ret)) { int temp_ret = 0; - if (lib::is_oracle_mode() && error_type == OB_JSON_ON_RESPONSE_IMPLICIT) { - temp_ret = get_on_empty_or_error(expr, ctx, error_type_id, is_cover_by_error, accuracy, error_type, &error_val, dst_type, default_val_type); + if (lib::is_oracle_mode() && error_type == JSN_VALUE_IMPLICIT) { + temp_ret = get_on_empty_or_error(expr, ctx, JSN_VAL_ERROR, is_cover_by_error, accuracy, error_type, &error_val, dst_type, default_val_type); } if (temp_ret != OB_SUCCESS && !is_cover_by_error) { ret = temp_ret; LOG_WARN("failed to get error option.", K(temp_ret)); } else { - if (error_type == OB_JSON_ON_RESPONSE_DEFAULT) { + if (error_type == JSN_VALUE_DEFAULT) { set_default_val = true; - } else if (error_type == OB_JSON_ON_RESPONSE_NULL || error_type == OB_JSON_ON_RESPONSE_IMPLICIT) { + } else if (error_type == JSN_VALUE_NULL || error_type == JSN_VALUE_IMPLICIT) { is_null_res = true; } if (lib::is_oracle_mode() && is_type_cast == 1) { for(size_t i = 0; i < mismatch_val.size(); i++) { // 目前不支持UDT，因此只考虑第一个参数中的 error 和 null。 - if (mismatch_val[i] == OB_JSON_ON_MISMATCH_ERROR) { + if (mismatch_val[i] == JSN_QUERY_MISMATCH_ERROR) { mismatch_error = false; - } else if (mismatch_val[i] == OB_JSON_ON_MISMATCH_NULL || mismatch_val[i] == OB_JSON_ON_MISMATCH_IGNORE) { + } else if (mismatch_val[i] == JSN_VALUE_MISMATCH_NULL || mismatch_val[i] == JSN_VALUE_MISMATCH_IGNORE) { is_null_res = true; } } @@ -1837,8 +1838,8 @@ int ObExprJsonValue::get_on_mismatch(const ObExpr &expr, LOG_WARN("input type error", K(val_type), K(ret)); } else { int64_t option_type = json_datum->get_int(); - if (option_type >= OB_JSON_ON_MISMATCH_ERROR && - option_type <= OB_JSON_ON_MISMATCH_IMPLICIT) { + if (option_type >= JSN_QUERY_MISMATCH_ERROR && + option_type <= JSN_VALUE_MISMATCH_IMPLICIT) { pos ++; if (OB_FAIL(val.push_back(static_cast(option_type)))) { LOG_WARN("mismtach add fail", K(ret)); @@ -1928,8 +1929,8 @@ int ObExprJsonValue::get_on_empty_or_error(const ObExpr &expr, LOG_WARN("input type error", K(val_type)); } else { int64_t option_type = json_datum->get_int(); - if (option_type < OB_JSON_ON_RESPONSE_ERROR || - option_type > OB_JSON_ON_RESPONSE_IMPLICIT) { + if (option_type < JSN_VALUE_ERROR || + option_type > JSN_VALUE_IMPLICIT) { ret = OB_ERR_UNEXPECTED; LOG_WARN("input option type error", K(option_type)); } else { @@ -1938,7 +1939,7 @@ int ObExprJsonValue::get_on_empty_or_error(const ObExpr &expr, } json_arg = expr.args_[index + 2]; val_type = json_arg->datum_meta_.type_; - if (OB_SUCC(ret) && index != error_type_id) { + if (OB_SUCC(ret) && index != JSN_VAL_ERROR) { if (lib::is_oracle_mode() && (val_type == ObCharType || val_type == ObNumberType || val_type == ObDecimalIntType)) { if (OB_FAIL(json_arg->eval(ctx, json_datum))) { @@ -1971,7 +1972,7 @@ int ObExprJsonValue::get_on_empty_or_error(const ObExpr &expr, is_cover_by_error = false; LOG_WARN("eval json arg failed", K(ret)); } else if (val_type == ObNullType || json_datum->is_null()) { - } else if ((lib::is_mysql_mode() || index == empty_type_id) && OB_FAIL(check_default_val_accuracy(accuracy, val_type, json_datum))) { + } else if ((lib::is_mysql_mode() || index == JSN_VAL_EMPTY) && OB_FAIL(check_default_val_accuracy(accuracy, val_type, json_datum))) { is_cover_by_error = false; } else { *default_value = json_datum; diff --git a/src/sql/engine/expr/ob_expr_json_value.h b/src/sql/engine/expr/ob_expr_json_value.h index 94cf761101..3368e92922 100644 --- a/src/sql/engine/expr/ob_expr_json_value.h +++ b/src/sql/engine/expr/ob_expr_json_value.h @@ -17,6 +17,7 @@ #include "sql/engine/expr/ob_expr_operator.h" #include "lib/json_type/ob_json_tree.h" #include "lib/json_type/ob_json_base.h" +#include "ob_json_param_type.h" using namespace oceanbase::common; @@ -163,19 +164,6 @@ private: const static uint8_t OB_JSON_ON_ASCII_IMPLICIT = 0; const static uint8_t OB_JSON_ON_ASCII_USE = 1; - /* process empty or error */ - const static uint8_t OB_JSON_ON_RESPONSE_ERROR = 0; - const static uint8_t OB_JSON_ON_RESPONSE_NULL = 1; - const static uint8_t OB_JSON_ON_RESPONSE_DEFAULT = 2; - const static uint8_t OB_JSON_ON_RESPONSE_IMPLICIT = 3; - - /* process on mismatch { error : 0, null : 1, ignore : 2 }*/ - const static uint8_t OB_JSON_ON_MISMATCH_ERROR = 0; - const static uint8_t OB_JSON_ON_MISMATCH_NULL = 1; - const static uint8_t OB_JSON_ON_MISMATCH_IGNORE = 2; - const static uint8_t OB_JSON_ON_MISMATCH_IMPLICIT = 3; - - /* process mismatch type { MISSING : 4 (1), EXTRA : 5 (2), TYPE : 6 (4), EMPTY : 7 (0)} make diff with mismatch type */ const static uint8_t OB_JSON_TYPE_MISSING_DATA = 4; const static uint8_t OB_JSON_TYPE_EXTRA_DATA = 5; @@ -183,19 +171,6 @@ private: const static uint8_t OB_JSON_TYPE_IMPLICIT = 7; const static uint8_t OB_JSON_TYPE_DOT = 8; - const static uint8_t json_doc_id = 0; - const static uint8_t json_path_id = 1; - const static uint8_t ret_type_id = 2; - const static uint8_t opt_truncate_id = 3; - const static uint8_t opt_ascii_id = 4; - const static uint8_t empty_type_id = 5; - const static uint8_t empty_val_id = 6; - const static uint8_t empty_val_pre_id = 7; - const static uint8_t error_type_id = 8; - const static uint8_t error_val_id = 9; - const static uint8_t error_val_pre_id = 10; - const static uint8_t opt_mismatch_id = 11; - static int get_on_empty_or_error(const ObExpr &expr, ObEvalCtx &ctx, uint8_t index, diff --git a/src/sql/engine/expr/ob_expr_merge_result_type_oracle.map b/src/sql/engine/expr/ob_expr_merge_result_type_oracle.map index a010b11052..e57eac95c1 100644 --- a/src/sql/engine/expr/ob_expr_merge_result_type_oracle.map +++ b/src/sql/engine/expr/ob_expr_merge_result_type_oracle.map @@ -1299,7 +1299,7 @@ static ObObjType MERGE_RESULT_TYPE[ObMaxType][ObMaxType] = { ObLongTextType, /* CharType=>ObLongTextType */ ObVarcharType, /* CharType=>ObBitType */ ObVarcharType, /* CharType=>ObEnumType */ - ObCharType, /* CharType=>ObSetType */ + ObVarcharType, /* CharType=>ObSetType */ ObMaxType, /* CharType=>ObEnumInnerType */ ObMaxType, /* CharType=>ObSetInnerType */ ObCharType, /*CharType=>ObTimestampTZType */ @@ -1834,7 +1834,7 @@ static ObObjType MERGE_RESULT_TYPE[ObMaxType][ObMaxType] = { ObVarcharType, /*ObSetType=>TimeType */ ObVarcharType, /*ObSetType=>YearType */ ObVarcharType, /*ObSetType=>VarcharType */ - ObCharType, /*ObSetType=>CharType */ + ObVarcharType, /*ObSetType=>CharType */ ObVarcharType, /*ObSetType=>HexStringType */ ObMaxType, /*ObSetType=>ExtendType */ ObVarcharType, /*ObSetType=>UnknownType */ diff --git a/src/sql/engine/expr/ob_expr_operator.cpp b/src/sql/engine/expr/ob_expr_operator.cpp index ad6e172b03..2d126ee868 100644 --- a/src/sql/engine/expr/ob_expr_operator.cpp +++ b/src/sql/engine/expr/ob_expr_operator.cpp @@ -1907,6 +1907,9 @@ bool ObRelationalExprOperator::can_cmp_without_cast(ObExprResType type1, if (ob_is_enum_or_set_type(type1.get_type()) && ob_is_enum_or_set_type(type2.get_type())) { need_no_cast = false; + } else if ((type1.is_null() && ObDatumFuncs::is_null_aware_hash_type(type2.get_type())) || + (type2.is_null() && ObDatumFuncs::is_null_aware_hash_type(type1.get_type()))) { + need_no_cast = false; } else { if (ObDatumFuncs::is_string_type(type1.get_type()) && ObDatumFuncs::is_string_type(type2.get_type())) { diff --git a/src/sql/engine/expr/ob_expr_regexp_context.h b/src/sql/engine/expr/ob_expr_regexp_context.h index ee3bed486f..0db93ac593 100644 --- a/src/sql/engine/expr/ob_expr_regexp_context.h +++ b/src/sql/engine/expr/ob_expr_regexp_context.h @@ -30,6 +30,7 @@ namespace sql { struct ObExprRegexpSessionVariables { + TO_STRING_KV(K_(regexp_stack_limit), K_(regexp_time_limit)); ObExprRegexpSessionVariables(): regexp_stack_limit_(0), regexp_time_limit_(0) diff --git a/src/sql/engine/expr/ob_expr_substr.cpp b/src/sql/engine/expr/ob_expr_substr.cpp index 96cdf02926..3c8b386fdc 100644 --- a/src/sql/engine/expr/ob_expr_substr.cpp +++ b/src/sql/engine/expr/ob_expr_substr.cpp @@ -111,7 +111,7 @@ int ObExprSubstr::calc_result_length(ObExprResType *types_array, mbmaxlen = 1; } if (start_pos > 0 && substr_len > 0) { - if (start_pos + substr_len <= result_len + 1) { + if (start_pos <= INT64_MAX - substr_len && start_pos + substr_len <= result_len + 1) { if (is_oracle_mode) { res_len = substr_len * mbmaxlen; } else { diff --git a/src/sql/engine/expr/ob_json_param_type.h b/src/sql/engine/expr/ob_json_param_type.h new file mode 100644 index 0000000000..9e9256fb6d --- /dev/null +++ b/src/sql/engine/expr/ob_json_param_type.h @@ -0,0 +1,136 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_SRC_SQL_ENGINE_EXPR_OB_JSON_PARAM_TYPE_H +#define OCEANBASE_SRC_SQL_ENGINE_EXPR_OB_JSON_PARAM_TYPE_H +namespace oceanbase +{ +namespace sql +{ + +// json_query +/* process empty or error */ +typedef enum JsnQueryType { + JSN_QUERY_ERROR, // 0 + JSN_QUERY_NULL, // 1 + JSN_QUERY_EMPTY, // 2 + JSN_QUERY_EMPTY_ARRAY, // 3 + JSN_QUERY_EMPTY_OBJECT, // 4 + JSN_QUERY_IMPLICIT, // 5 + JSN_QUERY_RESPONSE_COUNT, // 6 +} JsnQueryType; + +/* process on mismatch { error : 0, null : 1, implicit : 2 }*/ +typedef enum JsnQueryMisMatch { + JSN_QUERY_MISMATCH_ERROR, // 0 + JSN_QUERY_MISMATCH_NULL, // 1 + JSN_QUERY_MISMATCH_IMPLICIT, // 2 + JSN_QUERY_MISMATCH_DOT, // 3 + JSN_QUERY_MISMATCH_COUNT, // 4 +} JsnQueryMisMatch; + +/* process wrapper type */ +typedef enum JsnQueryWrapper { + JSN_QUERY_WITHOUT_WRAPPER, // 0 + JSN_QUERY_WITHOUT_ARRAY_WRAPPER, // 1 + JSN_QUERY_WITH_WRAPPER, // 2 + JSN_QUERY_WITH_ARRAY_WRAPPER, // 3 + JSN_QUERY_WITH_UNCONDITIONAL_WRAPPER, // 4 + JSN_QUERY_WITH_CONDITIONAL_WRAPPER, // 5 + JSN_QUERY_WITH_UNCONDITIONAL_ARRAY_WRAPPER, // 6 + JSN_QUERY_WITH_CONDITIONAL_ARRAY_WRAPPER, // 7 + JSN_QUERY_WRAPPER_IMPLICIT , // 8 + JSN_QUERY_WRAPPER_COUNT, // 9 +} JsnQueryWrapper; + +/* process on scalars { allow : 0, disallow : 1, implicit : 2 }*/ +typedef enum JsnQueryScalar { + JSN_QUERY_SCALARS_ALLOW, // 0 + JSN_QUERY_SCALARS_DISALLOW, // 1 + JSN_QUERY_SCALARS_IMPLICIT, // 2 + JSN_QUERY_SCALARS_COUNT // 3 +} JsnQueryScalar; + +/* pretty ascii 0 : null 1 : yes */ +typedef enum JsnQueryAsc { + OB_JSON_PRE_ASC_EMPTY, // 0 + OB_JSON_PRE_ASC_SET, // 1 + OB_JSON_PRE_ASC_COUNT // 2 +} JsnQueryAsc; + +// json query clause position +typedef enum JsnQueryClause { + JSN_QUE_DOC, // 0 + JSN_QUE_PATH, // 1 + JSN_QUE_RET, // 2 + JSN_QUE_TRUNC, // 3 + JSN_QUE_SCALAR, // 4 + JSN_QUE_PRETTY, // 5 + JSN_QUE_ASCII, // 6 + JSN_QUE_WRAPPER, // 7 + JSN_QUE_ERROR, // 8 + JSN_QUE_EMPTY, // 9 + JSN_QUE_MISMATCH, //10 +} JsnQueryClause; + +typedef enum JsnQueryOpt { + JSN_QUE_TRUNC_OPT, // 0 + JSN_QUE_SCALAR_OPT, // 1 + JSN_QUE_PRETTY_OPT, // 2 + JSN_QUE_ASCII_OPT, // 3 + JSN_QUE_WRAPPER_OPT, // 4 + JSN_QUE_ERROR_OPT, // 5 + JSN_QUE_EMPTY_OPT, // 6 + JSN_QUE_MISMATCH_OPT, // 7 +} JsnQueryOpt; + +// json_value +/* process empty or error */ +typedef enum JsnValueType { + JSN_VALUE_ERROR, // 0 + JSN_VALUE_NULL, // 1 + JSN_VALUE_DEFAULT, // 2 + JSN_VALUE_IMPLICIT, // 3 +} JsnValueType; +/* process mismatch type { MISSING : 4 (1), EXTRA : 5 (2), TYPE : 6 (4), EMPTY : 7 (0)} make diff with mismatch type */ +typedef enum JsnValueClause { + JSN_VAL_DOC, // 0 + JSN_VAL_PATH, // 1 + JSN_VAL_RET, // 2 + JSN_VAL_TRUNC, // 3 + JSN_VAL_ASCII, // 4 + JSN_VAL_EMPTY, // 5 + JSN_VAL_EMPTY_DEF, // 6 + JSN_VAL_EMPTY_DEF_PRE, // 7 + JSN_VAL_ERROR, // 8 + JSN_VAL_ERROR_DEF, // 9 + JSN_VAL_ERROR_DEF_PRE, // 10 + JSN_VAL_MISMATCH // 11 +} JsnValueClause; + +typedef enum JsnValueMisMatch { + JSN_VALUE_MISMATCH_ERROR, // 0 + JSN_VALUE_MISMATCH_NULL, // 1 + JSN_VALUE_MISMATCH_IGNORE, // 2 + JSN_VALUE_MISMATCH_IMPLICIT // 3 +} JsnValueMisMatch; + +typedef enum JsnValueOpt { + JSN_VAL_TRUNC_OPT, // 0 + JSN_VAL_ASCII_OPT, // 1 + JSN_VAL_EMPTY_OPT, // 2 + JSN_VAL_ERROR_OPT, // 3 +} JsnValueOpt; + +} // namespace sql +} // namespace oceanbase +#endif /* OCEANBASE_SRC_SQL_ENGINE_EXPR_OB_JSON_PARAM_TYPE_H */ \ No newline at end of file diff --git a/src/sql/engine/join/hash_join/ob_hash_join_vec_op.cpp b/src/sql/engine/join/hash_join/ob_hash_join_vec_op.cpp index a14232f9a6..cf92f685f5 100644 --- a/src/sql/engine/join/hash_join/ob_hash_join_vec_op.cpp +++ b/src/sql/engine/join/hash_join/ob_hash_join_vec_op.cpp @@ -647,7 +647,7 @@ int ObHashJoinVecOp::process_left(bool &need_not_read_right) // copy ObOperator::drain_exch // It's same as the base operator, but only need add sync to wait exit for shared hash join -int ObHashJoinVecOp::drain_exch() +int ObHashJoinVecOp::do_drain_exch() { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; diff --git a/src/sql/engine/join/hash_join/ob_hash_join_vec_op.h b/src/sql/engine/join/hash_join/ob_hash_join_vec_op.h index c68dcd1526..cc936289be 100644 --- a/src/sql/engine/join/hash_join/ob_hash_join_vec_op.h +++ b/src/sql/engine/join/hash_join/ob_hash_join_vec_op.h @@ -315,7 +315,7 @@ private: public: virtual int inner_open() override; virtual int inner_rescan() override; - virtual int drain_exch() override; + virtual int do_drain_exch() override; virtual int inner_get_next_batch(const int64_t max_row_cnt) override; virtual int inner_get_next_row() { return common::OB_NOT_IMPLEMENT; }; virtual void destroy() override; diff --git a/src/sql/engine/join/ob_nested_loop_join_op.cpp b/src/sql/engine/join/ob_nested_loop_join_op.cpp index d33c996fe6..f35d8e7cf6 100644 --- a/src/sql/engine/join/ob_nested_loop_join_op.cpp +++ b/src/sql/engine/join/ob_nested_loop_join_op.cpp @@ -187,6 +187,54 @@ int ObNestedLoopJoinOp::rescan() return ret; } +int ObNestedLoopJoinOp::do_drain_exch_multi_lvel_bnlj() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(try_open())) { + LOG_WARN("fail to open operator", K(ret)); + } else if (!exch_drained_) { + // the drain request is triggered by current NLJ operator, and current NLJ is a multi level Batch NLJ + // It will block rescan request for it's child operator, if the drain request is passed to it's child operator + // The child operators will be marked as iter-end_, and will not get any row if rescan is blocked + // So we block the drain request here; Only set current operator to end; + int tmp_ret = inner_drain_exch(); + exch_drained_ = true; + brs_.end_ = true; + batch_reach_end_ = true; + row_reach_end_ = true; + if (OB_SUCC(ret)) { + ret = tmp_ret; + } + } + return ret; +} + +int ObNestedLoopJoinOp::do_drain_exch() +{ + int ret = OB_SUCCESS; + if (!MY_SPEC.group_rescan_) { + if (OB_FAIL( ObOperator::do_drain_exch())) { + LOG_WARN("failed to drain NLJ operator", K(ret)); + } + } else if (!group_join_buffer_.is_multi_level()) { + if (OB_FAIL( ObOperator::do_drain_exch())) { + LOG_WARN("failed to drain NLJ operator", K(ret)); + } + } else { + if (!is_operator_end()) { + // the drain request is triggered by parent operator + // NLJ needs to pass the drain request to it's child operator + LOG_TRACE("The drain request is passed by parent operator"); + if (OB_FAIL( ObOperator::do_drain_exch())) { + LOG_WARN("failed to drain normal NLJ operator", K(ret)); + } + } else if (OB_FAIL(do_drain_exch_multi_lvel_bnlj())) { + LOG_WARN("failed to drain multi level NLJ operator", K(ret)); + } + } + return ret; +} + int ObNestedLoopJoinOp::inner_rescan() { int ret = OB_SUCCESS; diff --git a/src/sql/engine/join/ob_nested_loop_join_op.h b/src/sql/engine/join/ob_nested_loop_join_op.h index 21480bce28..6710d17075 100644 --- a/src/sql/engine/join/ob_nested_loop_join_op.h +++ b/src/sql/engine/join/ob_nested_loop_join_op.h @@ -94,6 +94,8 @@ public: virtual int switch_iterator() override; virtual int rescan() override; virtual int inner_rescan() override; + + virtual int inner_get_next_row() override; virtual void destroy() override { @@ -121,7 +123,7 @@ public: ObBatchRescanCtl &get_batch_rescan_ctl() { return batch_rescan_ctl_; } int fill_cur_row_rescan_param(); int calc_other_conds(bool &is_match); - + int do_drain_exch_multi_lvel_bnlj(); private: // state operation and transfer function type. typedef int (ObNestedLoopJoinOp::*state_operation_func_type)(); @@ -167,6 +169,8 @@ private: // for refactor vectorized end bool continue_fetching() { return !(left_brs_->end_ || is_full());} + virtual int do_drain_exch() override; + virtual int inner_drain_exch() { return OB_SUCCESS; } public: ObJoinState state_; // for bnl join diff --git a/src/sql/engine/ob_operator.h b/src/sql/engine/ob_operator.h index f6cdd96bad..aa127c9ec4 100644 --- a/src/sql/engine/ob_operator.h +++ b/src/sql/engine/ob_operator.h @@ -478,11 +478,14 @@ public: bool &filtered); ObBatchRows &get_brs() { return brs_; } // Drain exchange in data for PX, or producer DFO will be blocked. - virtual int drain_exch(); + int drain_exch(); void set_pushdown_param_null(const common::ObIArray &rescan_params); void set_feedback_node_idx(int64_t idx) { fb_node_idx_ = idx; } + + bool is_operator_end() { return batch_reach_end_ || row_reach_end_ ; } protected: + virtual int do_drain_exch(); int init_skip_vector(); // Execute filter // Calc buffer does not reset internally, you need to reset it appropriately. @@ -565,7 +568,6 @@ private: int output_expr_decint_datum_len_check(); int output_expr_decint_datum_len_check_batch(); int setup_op_feedback_info(); - int do_drain_exch(); // child can implement this interface, but can't call this directly virtual int inner_drain_exch() { return common::OB_SUCCESS; }; protected: diff --git a/src/sql/engine/ob_physical_plan_ctx.cpp b/src/sql/engine/ob_physical_plan_ctx.cpp index d8c28f792b..b99f460195 100644 --- a/src/sql/engine/ob_physical_plan_ctx.cpp +++ b/src/sql/engine/ob_physical_plan_ctx.cpp @@ -949,22 +949,6 @@ OB_DEF_DESERIALIZE(ObPhysicalPlanCtx) } } } - if (OB_SUCC(ret) && array_group_count > 0 && - datum_param_store_.count() == 0 && - datum_param_store_.count() != param_store_.count()) { - if (OB_FAIL(datum_param_store_.prepare_allocate(param_store_.count()))) { - LOG_WARN("fail to prepare allocate", K(ret), K(param_store_.count())); - } - for (int64_t i = 0; OB_SUCC(ret) && i < param_store_.count(); i++) { - ObDatumObjParam &datum_param = datum_param_store_.at(i); - if (OB_FAIL(datum_param.alloc_datum_reserved_buff( - param_store_.at(i).meta_, param_store_.at(i).get_precision(), allocator_))) { - LOG_WARN("alloc datum reserved buffer failed", K(ret)); - } else if (OB_FAIL(datum_param.from_objparam(param_store_.at(i), &allocator_))) { - LOG_WARN("fail to convert obj param", K(ret), K(param_store_.at(i))); - } - } - } OB_UNIS_DECODE(enable_rich_format_); OB_UNIS_DECODE(local_var_array_cnt); if (OB_SUCC(ret)) { @@ -984,6 +968,15 @@ OB_DEF_DESERIALIZE(ObPhysicalPlanCtx) OB_UNIS_DECODE(*local_vars); } } + + // following is not deserialize, please add deserialize ahead. + if (OB_SUCC(ret) && array_group_count > 0 && + datum_param_store_.count() == 0 && + datum_param_store_.count() != param_store_.count()) { + if (OB_FAIL(init_param_store_after_deserialize())) { + LOG_WARN("failed to deserialize param store", K(ret)); + } + } return ret; } @@ -1041,5 +1034,40 @@ int ObPhysicalPlanCtx::get_local_session_vars(int64_t local_var_array_id, const return ret; } +// white list: init param_store after deserialize which it's needed really. +int ObPhysicalPlanCtx::init_param_store_after_deserialize() +{ + int ret = OB_SUCCESS; + datum_param_store_.reuse(); + if (OB_FAIL(datum_param_store_.prepare_allocate(param_store_.count()))) { + LOG_WARN("fail to prepare allocate", K(ret), K(param_store_.count())); + } + for (int64_t i = 0; OB_SUCC(ret) && i < param_store_.count(); i++) { + ObObjParam &obj_param = param_store_.at(i); + ObDatumObjParam &datum_param = datum_param_store_.at(i); + if (obj_param.is_ext_sql_array()) { + ObSqlArrayObj *array_obj = NULL; + if (OB_FAIL(ObSqlArrayObj::do_real_deserialize(allocator_, + reinterpret_cast(obj_param.get_ext()), + obj_param.get_val_len(), + array_obj))) { + LOG_WARN("failed to alloc array_obj after decode", K(ret)); + } else { + obj_param.set_extend(reinterpret_cast(array_obj), T_EXT_SQL_ARRAY); + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(datum_param.alloc_datum_reserved_buff(obj_param.meta_, + obj_param.get_precision(), + allocator_))) { + LOG_WARN("alloc datum reserved buffer failed", K(ret)); + } else if (OB_FAIL(datum_param.from_objparam(obj_param, &allocator_))) { + LOG_WARN("fail to convert obj param", K(ret), K(obj_param)); + } + } + } + return ret; +} + } //sql } //oceanbase diff --git a/src/sql/engine/ob_physical_plan_ctx.h b/src/sql/engine/ob_physical_plan_ctx.h index eabf8c6d16..cbcc6e3d0d 100644 --- a/src/sql/engine/ob_physical_plan_ctx.h +++ b/src/sql/engine/ob_physical_plan_ctx.h @@ -466,6 +466,7 @@ public: int set_all_local_session_vars(ObIArray &all_local_session_vars); int get_local_session_vars(int64_t idx, const ObLocalSessionVar *&local_vars); private: + int init_param_store_after_deserialize(); void reset_datum_frame(char *frame, int64_t expr_cnt); int extend_param_frame(const int64_t old_size); int reserve_param_frame(const int64_t capacity); diff --git a/src/sql/engine/ob_serializable_function.h b/src/sql/engine/ob_serializable_function.h index f1a893a970..6d0cada88e 100644 --- a/src/sql/engine/ob_serializable_function.h +++ b/src/sql/engine/ob_serializable_function.h @@ -120,7 +120,7 @@ typedef void (*ser_eval_vector_function)(ObEvalVectorFuncTag &); OB_SFA_DECIMAL_INT_BASIC_PART1, \ OB_SFA_DECIMAL_INT_BASIC_PART2, \ OB_SFA_DECIMAL_INT_NULLSAFE_CMP, \ - OB_SFA_CMP_BETWEEN_EXPR_EVAL_VECTOR, \ + OB_SFA_VECTOR_CMP, \ OB_SFA_SQL_EXPR_ABS_EVAL_VEC, \ OB_SFA_VECTOR_CAST, \ OB_SFA_VECTOR_EVAL_ARG_CAST, \ diff --git a/src/sql/engine/ob_tenant_sql_memory_manager.cpp b/src/sql/engine/ob_tenant_sql_memory_manager.cpp index 2182d0b3c7..1b6c77836d 100644 --- a/src/sql/engine/ob_tenant_sql_memory_manager.cpp +++ b/src/sql/engine/ob_tenant_sql_memory_manager.cpp @@ -285,11 +285,12 @@ int ObTenantSqlMemoryManager::ObSqlWorkAreaCalcInfo::calculate_global_bound_size const bool auto_calc) { int ret = OB_SUCCESS; + int64_t error_sim = std::abs(OB_E(EventTable::EN_SQL_MEMORY_MRG_OPTION) 0); int64_t max_wa_size = wa_max_memory_size; // int64_t max_wa_size = wa_max_memory_size; // 最大占比6.25%（oracle 5%） // 这里改为按照8个并发来设置 - int64_t max_bound_size = (max_wa_size >> 3); + int64_t max_bound_size = (0 == error_sim) ? (max_wa_size >> 3) : error_sim; profile_cnt_ = profile_cnt; int64_t avg_bound_size = (0 == profile_cnt_) ? max_bound_size : max_wa_size / profile_cnt_; int64_t best_interval_idx = -1; @@ -326,12 +327,10 @@ int ObTenantSqlMemoryManager::ObSqlWorkAreaCalcInfo::calculate_global_bound_size return ret; } -//////////////////////////////////////////////////////////////////////////////////// -int ObTenantSqlMemoryManager::mtl_init(ObTenantSqlMemoryManager *&sql_mem_mgr) +int ObTenantSqlMemoryManager::mtl_new(ObTenantSqlMemoryManager *&sql_mem_mgr) { int ret = OB_SUCCESS; uint64_t tenant_id = MTL_ID(); - sql_mem_mgr = nullptr; // 系统租户不创建 if (OB_MAX_RESERVED_TENANT_ID < tenant_id) { sql_mem_mgr = OB_NEW(ObTenantSqlMemoryManager, @@ -339,7 +338,19 @@ int ObTenantSqlMemoryManager::mtl_init(ObTenantSqlMemoryManager *&sql_mem_mgr) if (nullptr == sql_mem_mgr) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to alloc tenant sql memory manager", K(ret)); - } else if (OB_FAIL(sql_mem_mgr->allocator_.init( + } + } + return ret; +} + +//////////////////////////////////////////////////////////////////////////////////// +int ObTenantSqlMemoryManager::mtl_init(ObTenantSqlMemoryManager *&sql_mem_mgr) +{ + int ret = OB_SUCCESS; + uint64_t tenant_id = MTL_ID(); + // 系统租户不init + if (OB_MAX_RESERVED_TENANT_ID < tenant_id) { + if (OB_FAIL(sql_mem_mgr->allocator_.init( lib::ObMallocAllocator::get_instance(), OB_MALLOC_NORMAL_BLOCK_SIZE, ObMemAttr(tenant_id, "SqlMemMgr")))) { diff --git a/src/sql/engine/ob_tenant_sql_memory_manager.h b/src/sql/engine/ob_tenant_sql_memory_manager.h index 250c81e84b..33c9fe56e6 100644 --- a/src/sql/engine/ob_tenant_sql_memory_manager.h +++ b/src/sql/engine/ob_tenant_sql_memory_manager.h @@ -584,6 +584,7 @@ public: {} ~ObTenantSqlMemoryManager() {} public: + static int mtl_new(ObTenantSqlMemoryManager *&sql_mem_mgr); static int mtl_init(ObTenantSqlMemoryManager *&sql_mem_mgr); static void mtl_destroy(ObTenantSqlMemoryManager *&sql_mem_mgr); diff --git a/src/sql/engine/pdml/static/ob_pdml_op_batch_row_cache.cpp b/src/sql/engine/pdml/static/ob_pdml_op_batch_row_cache.cpp index 979c630554..436aefbdc6 100644 --- a/src/sql/engine/pdml/static/ob_pdml_op_batch_row_cache.cpp +++ b/src/sql/engine/pdml/static/ob_pdml_op_batch_row_cache.cpp @@ -36,11 +36,9 @@ int ObPDMLOpRowIterator::get_next_row(const ObExprPtrIArray &row) if (OB_ISNULL(eval_ctx_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("not init the eval_ctx", K(ret)); - } else if (OB_SUCC(row_store_it_.get_next_row(row, *eval_ctx_))) { - // we should do uniq row checking after data being stored in row_store - // rather than before this. because we need to return all rows unfiltered to upper ops. - if (uniq_row_checker_) { - ret = uniq_row_checker_->check_rowkey_distinct(row, is_distinct); + } else if (OB_FAIL(row_store_it_.get_next_row(row, *eval_ctx_))) { + if (OB_ITER_END != ret) { + LOG_WARN("get next row from row store iter failed", K(ret)); } } } while (OB_SUCC(ret) && !is_distinct); diff --git a/src/sql/engine/pdml/static/ob_pdml_op_batch_row_cache.h b/src/sql/engine/pdml/static/ob_pdml_op_batch_row_cache.h index d14a18414e..44d538b9f0 100644 --- a/src/sql/engine/pdml/static/ob_pdml_op_batch_row_cache.h +++ b/src/sql/engine/pdml/static/ob_pdml_op_batch_row_cache.h @@ -29,12 +29,6 @@ class ObNewRow; namespace sql { struct ObDASTabletLoc; -class ObDMLOpUniqueRowChecker -{ -public: - virtual int check_rowkey_distinct(const ObExprPtrIArray &row, bool &is_distinct) = 0; -}; - class ObExecContext; // 单个分区的新引擎数据缓存器 @@ -42,22 +36,18 @@ class ObPDMLOpRowIterator { public: friend class ObPDMLOpBatchRowCache; - ObPDMLOpRowIterator() : eval_ctx_(nullptr), uniq_row_checker_(nullptr) {} + ObPDMLOpRowIterator() : eval_ctx_(nullptr) {} virtual ~ObPDMLOpRowIterator() = default; // 获得row_store_it_中的下一行数据 // 返回的数据是对应存储数据的exprs int get_next_row(const ObExprPtrIArray &row); - void set_uniq_row_checker(ObDMLOpUniqueRowChecker *uniq_row_checker) - { uniq_row_checker_ = uniq_row_checker; } - void close() - { row_store_it_.reset(); } + void close() { row_store_it_.reset(); } private: int init_data_source(ObChunkDatumStore &row_datum_store, ObEvalCtx *eval_ctx); private: ObChunkDatumStore::Iterator row_store_it_; ObEvalCtx *eval_ctx_; - ObDMLOpUniqueRowChecker *uniq_row_checker_; DISALLOW_COPY_AND_ASSIGN(ObPDMLOpRowIterator); }; @@ -132,7 +122,8 @@ public: // 一般来说，分区id存储在行中的一个伪列里 virtual int read_row(ObExecContext &ctx, const ObExprPtrIArray *&row, - common::ObTabletID &tablet_id) = 0; + common::ObTabletID &tablet_id, + bool &is_skipped) = 0; }; class ObDMLOpDataWriter diff --git a/src/sql/engine/pdml/static/ob_pdml_op_data_driver.cpp b/src/sql/engine/pdml/static/ob_pdml_op_data_driver.cpp index fbba1eb6c9..a513185d37 100644 --- a/src/sql/engine/pdml/static/ob_pdml_op_data_driver.cpp +++ b/src/sql/engine/pdml/static/ob_pdml_op_data_driver.cpp @@ -35,7 +35,6 @@ int ObPDMLOpDataDriver::init(const ObTableModifySpec &spec, ObDMLBaseRtDef &dml_rtdef, ObDMLOpDataReader *reader, ObDMLOpDataWriter *writer, - ObDMLOpUniqueRowChecker *uniq_checker, const bool is_heap_table_insert, const bool with_barrier/*false*/) { @@ -53,7 +52,6 @@ int ObPDMLOpDataDriver::init(const ObTableModifySpec &spec, } else { reader_ = reader; writer_ = writer; - uniq_checker_ = uniq_checker; dml_rtdef_ = &dml_rtdef; is_heap_table_insert_ = is_heap_table_insert; with_barrier_ = with_barrier; @@ -179,13 +177,16 @@ int ObPDMLOpDataDriver::fill_cache_unitl_cache_full_or_child_iter_end(ObExecCont do { const ObExprPtrIArray *row = nullptr; ObTabletID tablet_id; - if (OB_FAIL(reader_->read_row(ctx, row, tablet_id))) { + bool is_skipped = false; + if (OB_FAIL(reader_->read_row(ctx, row, tablet_id, is_skipped))) { if (OB_ITER_END == ret) { // 当前reader的数据已经读取结束 // do nothing } else { LOG_WARN("failed to read row from reader", K(ret)); } + } else if (is_skipped) { + //need to skip this row } else if (is_heap_table_insert_ && OB_FAIL(set_heap_table_hidden_pk(row, tablet_id))) { LOG_WARN("fail to set heap table hidden pk", K(ret), K(*row), K(tablet_id)); } else if (OB_FAIL(cache_.add_row(*row, tablet_id))) { @@ -247,8 +248,6 @@ int ObPDMLOpDataDriver::write_partitions(ObExecContext &ctx) LOG_WARN("fail get row iterator", K(tablet_id), K(ret)); } else if (OB_FAIL(DAS_CTX(ctx).extended_tablet_loc(*table_loc, tablet_id, tablet_loc))) { LOG_WARN("extended tablet location failed", K(ret)); - } else if (FALSE_IT(row_iter->set_uniq_row_checker(uniq_checker_))) { - // nop } else if (OB_FAIL(writer_->write_rows(ctx, tablet_loc, *row_iter))) { LOG_WARN("fail write rows", K(tablet_id), K(ret)); } @@ -392,7 +391,6 @@ int ObPDMLOpDataDriver::switch_row_iter_to_next_partition() LOG_WARN("failed to get next partition iterator", K(ret), "part_id", returning_ctx_.tablet_id_array_.at(next_idx), K(next_idx)); } else { - returning_ctx_.row_iter_->set_uniq_row_checker(nullptr); returning_ctx_.next_idx_++; } return ret; diff --git a/src/sql/engine/pdml/static/ob_pdml_op_data_driver.h b/src/sql/engine/pdml/static/ob_pdml_op_data_driver.h index dfd57f742d..1e404b66f1 100644 --- a/src/sql/engine/pdml/static/ob_pdml_op_data_driver.h +++ b/src/sql/engine/pdml/static/ob_pdml_op_data_driver.h @@ -40,7 +40,6 @@ public: cache_(eval_ctx, op_monitor_info), reader_(nullptr), writer_(nullptr), - uniq_checker_(nullptr), dml_rtdef_(nullptr), state_(FILL_CACHE), eval_ctx_(eval_ctx), @@ -61,7 +60,6 @@ public: ObDMLBaseRtDef &dml_rtdef, ObDMLOpDataReader *reader, ObDMLOpDataWriter *writer, - ObDMLOpUniqueRowChecker *uniq_checker, const bool is_heap_table_insert, const bool with_barrier = false); @@ -143,7 +141,6 @@ private: ObPDMLOpBatchRowCache cache_; // 用于缓存数据，需要在init函数中初始化，并且分配alloctor ObDMLOpDataReader *reader_; ObDMLOpDataWriter *writer_; - ObDMLOpUniqueRowChecker *uniq_checker_; ObDMLBaseRtDef *dml_rtdef_; DriverState state_; // Driver 当前状态：读写数据状态、向上返回数据状态 diff --git a/src/sql/engine/pdml/static/ob_px_multi_part_delete_op.cpp b/src/sql/engine/pdml/static/ob_px_multi_part_delete_op.cpp index e832c48377..9db86bc3fe 100644 --- a/src/sql/engine/pdml/static/ob_px_multi_part_delete_op.cpp +++ b/src/sql/engine/pdml/static/ob_px_multi_part_delete_op.cpp @@ -65,7 +65,7 @@ int ObPxMultiPartDeleteOp::inner_open() LOG_WARN("failed to inner open", K(ret)); } else if (OB_FAIL(ObDMLService::init_del_rtdef(dml_rtctx_, del_rtdef_, MY_SPEC.del_ctdef_))) { LOG_WARN("init delete rtdef failed", K(ret)); - } else if (OB_FAIL(data_driver_.init(get_spec(), ctx_.get_allocator(), del_rtdef_, this, this, this, false, MY_SPEC.with_barrier_))) { + } else if (OB_FAIL(data_driver_.init(get_spec(), ctx_.get_allocator(), del_rtdef_, this, this, false, MY_SPEC.with_barrier_))) { LOG_WARN("failed to init data driver", K(ret)); } else if (MY_SPEC.with_barrier_) { if (OB_ISNULL(input_)) { @@ -81,24 +81,6 @@ int ObPxMultiPartDeleteOp::inner_open() return ret; } -int ObPxMultiPartDeleteOp::check_rowkey_distinct(const ObExprPtrIArray &row, - bool &is_distinct) -{ - int ret = OB_SUCCESS; - if (DistinctType::T_DISTINCT_NONE != MY_SPEC.del_ctdef_.distinct_algo_) { - ret = ObDMLService::check_rowkey_whether_distinct(row, - MY_SPEC.del_ctdef_.distinct_algo_, - eval_ctx_, - ctx_, - del_rtdef_.table_rowkey_, - del_rtdef_.se_rowkey_dist_ctx_, - is_distinct); - } else { - is_distinct = true; - } - return ret; -} - int ObPxMultiPartDeleteOp::inner_get_next_row() { int ret = OB_SUCCESS; @@ -149,7 +131,8 @@ int ObPxMultiPartDeleteOp::inner_close() //////////// pdml data interface implementation: reader & writer //////////// int ObPxMultiPartDeleteOp::read_row(ObExecContext &ctx, const ObExprPtrIArray *&row, - ObTabletID &tablet_id) + ObTabletID &tablet_id, + bool &is_skipped) { int ret = OB_SUCCESS; UNUSED(ctx); @@ -164,24 +147,28 @@ int ObPxMultiPartDeleteOp::read_row(ObExecContext &ctx, } else { // 每一次从child节点获得新的数据都需要进行清除计算标记 clear_evaluated_flag(); - // 通过partition id expr获得对应行对应的分区 - const int64_t part_id_idx = MY_SPEC.row_desc_.get_part_id_index(); - // 返回的值是child的output exprs - row = &child_->get_spec().output_; - if (NO_PARTITION_ID_FLAG == part_id_idx) { - // 如果row中没有partition id expr对应的cell，默认partition id为0 - ObDASTableLoc *table_loc = del_rtdef_.das_rtdef_.table_loc_; - if (OB_ISNULL(table_loc) || table_loc->get_tablet_locs().size() != 1) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("insert table location is invalid", K(ret), KPC(table_loc)); - } else { - tablet_id = table_loc->get_first_tablet_loc()->tablet_id_; + if (OB_FAIL(ObDMLService::process_delete_row(MY_SPEC.del_ctdef_, del_rtdef_, is_skipped, *this))) { + LOG_WARN("process delete row failed", K(ret)); + } else if (!is_skipped) { + // 通过partition id expr获得对应行对应的分区 + const int64_t part_id_idx = MY_SPEC.row_desc_.get_part_id_index(); + // 返回的值是child的output exprs + row = &child_->get_spec().output_; + if (NO_PARTITION_ID_FLAG == part_id_idx) { + // 如果row中没有partition id expr对应的cell，默认partition id为0 + ObDASTableLoc *table_loc = del_rtdef_.das_rtdef_.table_loc_; + if (OB_ISNULL(table_loc) || table_loc->get_tablet_locs().size() != 1) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("insert table location is invalid", K(ret), KPC(table_loc)); + } else { + tablet_id = table_loc->get_first_tablet_loc()->tablet_id_; + } + } else if (child_->get_spec().output_.count() > part_id_idx) { + ObExpr *expr = child_->get_spec().output_.at(part_id_idx); + ObDatum &expr_datum = expr->locate_expr_datum(get_eval_ctx()); + tablet_id = expr_datum.get_int(); + LOG_DEBUG("get the part id", K(ret), K(expr_datum)); } - } else if (child_->get_spec().output_.count() > part_id_idx) { - ObExpr *expr = child_->get_spec().output_.at(part_id_idx); - ObDatum &expr_datum = expr->locate_expr_datum(get_eval_ctx()); - tablet_id = expr_datum.get_int(); - LOG_DEBUG("get the part id", K(ret), K(expr_datum)); } } return ret; diff --git a/src/sql/engine/pdml/static/ob_px_multi_part_delete_op.h b/src/sql/engine/pdml/static/ob_px_multi_part_delete_op.h index dae641309f..501a67c916 100644 --- a/src/sql/engine/pdml/static/ob_px_multi_part_delete_op.h +++ b/src/sql/engine/pdml/static/ob_px_multi_part_delete_op.h @@ -76,7 +76,6 @@ public: class ObPxMultiPartDeleteOp : public ObDMLOpDataReader, public ObDMLOpDataWriter, - public ObDMLOpUniqueRowChecker, public ObTableModifyOp { OB_UNIS_VERSION(1); @@ -96,7 +95,8 @@ public: // 同时还负责计算出这一行对应的 partition_id int read_row(ObExecContext &ctx, const ObExprPtrIArray *&row, - common::ObTabletID &tablet_id) override; + common::ObTabletID &tablet_id, + bool &is_skipped) override; // impl. ObDMLDataWriter // 将缓存的数据批量写入到存储层 int write_rows(ObExecContext &ctx, @@ -107,9 +107,6 @@ public: virtual int inner_get_next_row(); virtual int inner_open(); virtual int inner_close(); - -private: - int check_rowkey_distinct(const ObExprPtrIArray &row, bool &is_distinct) override; private: ObPDMLOpDataDriver data_driver_; ObDelRtDef del_rtdef_; diff --git a/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.cpp b/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.cpp index ef097295c4..6200051162 100644 --- a/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.cpp +++ b/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.cpp @@ -46,7 +46,7 @@ int ObPxMultiPartInsertOp::inner_open() ret = OB_ERR_UNEXPECTED; LOG_WARN("table or row desc is invalid", K(ret), K(MY_SPEC.row_desc_)); } else if (OB_FAIL(data_driver_.init(get_spec(), ctx_.get_allocator(), ins_rtdef_, this, this, - nullptr, MY_SPEC.ins_ctdef_.is_heap_table_))) { + MY_SPEC.ins_ctdef_.is_heap_table_))) { LOG_WARN("failed to init data driver", K(ret)); } if (OB_SUCC(ret)) { @@ -126,28 +126,11 @@ int ObPxMultiPartInsertOp::inner_close() return ret; } -int ObPxMultiPartInsertOp::process_row() -{ - int ret = OB_SUCCESS; - bool is_filtered = false; - OZ (ObDMLService::check_row_null(MY_SPEC.ins_ctdef_.new_row_, - eval_ctx_, - ins_rtdef_.cur_row_num_, - MY_SPEC.ins_ctdef_.column_infos_, - MY_SPEC.ins_ctdef_.das_ctdef_, - MY_SPEC.ins_ctdef_.is_single_value_, - *this)); - OZ(ObDMLService::filter_row_for_view_check(MY_SPEC.ins_ctdef_.view_check_exprs_, eval_ctx_, is_filtered)); - OV(!is_filtered, OB_ERR_CHECK_OPTION_VIOLATED); - OZ(ObDMLService::filter_row_for_check_cst(MY_SPEC.ins_ctdef_.check_cst_exprs_, eval_ctx_, is_filtered)); - OV(!is_filtered, OB_ERR_CHECK_CONSTRAINT_VIOLATED); - return ret; -} - //////////// pdml data interface implementation: reader & writer //////////// int ObPxMultiPartInsertOp::read_row(ObExecContext &ctx, const ObExprPtrIArray *&row, - common::ObTabletID &tablet_id) + common::ObTabletID &tablet_id, + bool &is_skipped) { int ret = OB_SUCCESS; UNUSED(ctx); @@ -162,32 +145,31 @@ int ObPxMultiPartInsertOp::read_row(ObExecContext &ctx, } else { // 每一次从child节点获得新的数据都需要进行清除计算标记 clear_evaluated_flag(); - // 通过partition id expr获得对应行对应的分区 - const int64_t part_id_idx = MY_SPEC.row_desc_.get_part_id_index(); - // 返回的值是child的output exprs - row = &child_->get_spec().output_; - if (NO_PARTITION_ID_FLAG == part_id_idx) { - ObDASTableLoc *table_loc = ins_rtdef_.das_rtdef_.table_loc_; - if (OB_ISNULL(table_loc) || table_loc->get_tablet_locs().size() != 1) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("insert table location is invalid", K(ret), KPC(table_loc)); - } else { - tablet_id = table_loc->get_first_tablet_loc()->tablet_id_; + if (OB_FAIL(ObDMLService::process_insert_row(MY_SPEC.ins_ctdef_, ins_rtdef_, *this, is_skipped))) { + LOG_WARN("process insert row failed", K(ret)); + } else if (!is_skipped) { + // 通过partition id expr获得对应行对应的分区 + const int64_t part_id_idx = MY_SPEC.row_desc_.get_part_id_index(); + // 返回的值是child的output exprs + row = &child_->get_spec().output_; + if (NO_PARTITION_ID_FLAG == part_id_idx) { + ObDASTableLoc *table_loc = ins_rtdef_.das_rtdef_.table_loc_; + if (OB_ISNULL(table_loc) || table_loc->get_tablet_locs().size() != 1) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("insert table location is invalid", K(ret), KPC(table_loc)); + } else { + tablet_id = table_loc->get_first_tablet_loc()->tablet_id_; + } + } else if (child_->get_spec().output_.count() > part_id_idx) { + ObExpr *expr = child_->get_spec().output_.at(part_id_idx); + ObDatum &expr_datum = expr->locate_expr_datum(get_eval_ctx()); + tablet_id = expr_datum.get_int(); + LOG_DEBUG("get the part id", K(ret), K(expr_datum)); } - } else if (child_->get_spec().output_.count() > part_id_idx) { - ObExpr *expr = child_->get_spec().output_.at(part_id_idx); - ObDatum &expr_datum = expr->locate_expr_datum(get_eval_ctx()); - tablet_id = expr_datum.get_int(); - LOG_DEBUG("get the part id", K(ret), K(expr_datum)); - } - } - if (!MY_SPEC.is_pdml_index_maintain_ && OB_SUCC(ret)) { - if (OB_FAIL(process_row())) { - LOG_WARN("fail process row", K(ret)); } } if (OB_SUCC(ret)) { - LOG_TRACE("read row from pdml cache", "read_row", ROWEXPR2STR(eval_ctx_, *row), K(tablet_id)); + LOG_TRACE("read row from pdml cache", "read_row", ROWEXPR2STR(eval_ctx_, *row), K(tablet_id), K(is_skipped)); } return ret; } diff --git a/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.h b/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.h index eac99d0ba8..6c7f4ffbd8 100644 --- a/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.h +++ b/src/sql/engine/pdml/static/ob_px_multi_part_insert_op.h @@ -93,7 +93,10 @@ public: public: virtual bool has_foreign_key() const { return false; } // 默认实现，先不考虑外键的问题 - int read_row(ObExecContext &ctx, const ObExprPtrIArray *&row, common::ObTabletID &tablet_id) override; + int read_row(ObExecContext &ctx, + const ObExprPtrIArray *&row, + common::ObTabletID &tablet_id, + bool &is_skipped) override; int write_rows(ObExecContext &ctx, const ObDASTabletLoc *tablet_loc, ObPDMLOpRowIterator &iterator) override; @@ -101,8 +104,6 @@ public: virtual int inner_get_next_row(); virtual int inner_open(); virtual int inner_close(); -private: - int process_row(); protected: ObPDMLOpDataDriver data_driver_; ObInsRtDef ins_rtdef_; diff --git a/src/sql/engine/pdml/static/ob_px_multi_part_update_op.cpp b/src/sql/engine/pdml/static/ob_px_multi_part_update_op.cpp index 47d180905e..eee919ab46 100644 --- a/src/sql/engine/pdml/static/ob_px_multi_part_update_op.cpp +++ b/src/sql/engine/pdml/static/ob_px_multi_part_update_op.cpp @@ -36,8 +36,7 @@ int ObPxMultiPartUpdateOp::inner_open() } else if (!(MY_SPEC.row_desc_.is_valid())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("table or row desc is invalid", K(ret), K(MY_SPEC.row_desc_)); - } else if (OB_FAIL(data_driver_.init(get_spec(), ctx_.get_allocator(), upd_rtdef_, this, this, - nullptr, false))) { + } else if (OB_FAIL(data_driver_.init(get_spec(), ctx_.get_allocator(), upd_rtdef_, this, this, false))) { LOG_WARN("failed to init data driver", K(ret)); } else if (OB_FAIL(ObDMLService::init_upd_rtdef(dml_rtctx_, upd_rtdef_, @@ -100,17 +99,9 @@ int ObPxMultiPartUpdateOp::inner_close() int ObPxMultiPartUpdateOp::update_row_to_das(const ObDASTabletLoc *tablet_loc) { int ret = OB_SUCCESS; - bool is_skipped = false; ObChunkDatumStore::StoredRow* stored_row = nullptr; - ++upd_rtdef_.cur_row_num_; - if (OB_FAIL(ObDMLService::process_update_row(MY_SPEC.upd_ctdef_, upd_rtdef_, is_skipped, *this))) { - LOG_WARN("process update row failed", K(ret)); - } else if (is_skipped) { - //do nothing - } else if (OB_FAIL(ObDMLService::update_row(MY_SPEC.upd_ctdef_, upd_rtdef_, tablet_loc, tablet_loc, dml_rtctx_, stored_row, stored_row, stored_row))) { + if (OB_FAIL(ObDMLService::update_row(MY_SPEC.upd_ctdef_, upd_rtdef_, tablet_loc, tablet_loc, dml_rtctx_, stored_row, stored_row, stored_row))) { LOG_WARN("insert row with das failed", K(ret)); - } else { - ++upd_rtdef_.found_rows_; } return ret; } @@ -118,12 +109,12 @@ int ObPxMultiPartUpdateOp::update_row_to_das(const ObDASTabletLoc *tablet_loc) //////////// pdml data interface implementation: reader & writer //////////// int ObPxMultiPartUpdateOp::read_row(ObExecContext &ctx, const ObExprPtrIArray *&row, - common::ObTabletID &tablet_id) + common::ObTabletID &tablet_id, + bool &is_skipped) { // 从child中读取数据，数据存储在child的output exprs中 int ret = OB_SUCCESS; ObPhysicalPlanCtx *plan_ctx = NULL; - bool is_update_timestamp = false; if (OB_ISNULL(plan_ctx = ctx.get_physical_plan_ctx())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get physical plan context failed", K(ret)); @@ -137,78 +128,34 @@ int ObPxMultiPartUpdateOp::read_row(ObExecContext &ctx, } else { // 每一次从child节点获得新的数据都需要进行清除计算标记 clear_evaluated_flag(); - // 通过partition id expr获得对应行对应的分区 - const int64_t part_id_idx = MY_SPEC.row_desc_.get_part_id_index(); - // 返回的值是child的output exprs - row = &child_->get_spec().output_; - if (NO_PARTITION_ID_FLAG == part_id_idx) { - ObDASTableLoc *table_loc = upd_rtdef_.dupd_rtdef_.table_loc_; - if (OB_ISNULL(table_loc) || table_loc->get_tablet_locs().size() != 1) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("insert table location is invalid", K(ret), KPC(table_loc)); - } else { - tablet_id = table_loc->get_first_tablet_loc()->tablet_id_; - } - } else if (child_->get_spec().output_.count() > part_id_idx) { - ObExpr *expr = child_->get_spec().output_.at(part_id_idx); - ObDatum &expr_datum = expr->locate_expr_datum(get_eval_ctx()); - tablet_id = expr_datum.get_int(); - LOG_DEBUG("get the part id", K(ret), K(expr_datum)); - } - - // 做auto_failed_timestamp的检查 - // 如果assign_column包含了ON UPDATE CURRENT_TIMESTAMP类型的时间戳列 - // 第一步先检查主表是否发生update，如果主表当前行没有发生update，那么吧old_row中的当前列拷贝到new_row中 - // pdml的索引表在检查当前行是否发生update时，不会跳过ON UPDATE CURRENT_TIMESTAMP列，会比较value - if (MY_SPEC.upd_ctdef_.is_primary_index_ - && lib::is_mysql_mode() - && is_update_auto_filled_timestamp()) { - // update the timestamp column, then check whether row is updated - if (OB_FAIL(ObDMLService::check_row_whether_changed(MY_SPEC.upd_ctdef_, upd_rtdef_, get_eval_ctx()))) { - LOG_WARN("fail to check_row_whether_changed", K(ret)); - } else if (!upd_rtdef_.is_row_changed_) { - // copy old_row timestamp column as new_row timestamp column - const ObExprPtrIArray &old_row = MY_SPEC.upd_ctdef_.old_row_; - const ObExprPtrIArray &new_row = MY_SPEC.upd_ctdef_.new_row_; - FOREACH_CNT_X(info, MY_SPEC.upd_ctdef_.assign_columns_, OB_SUCC(ret)) { - const uint64_t idx = info->projector_index_; - if (info->auto_filled_timestamp_) { - ObDatum *old_datum = NULL; - ObDatum *new_datum = NULL; - if (OB_FAIL(old_row.at(idx)->eval(get_eval_ctx(), old_datum)) - || OB_FAIL(new_row.at(idx)->eval(get_eval_ctx(), new_datum))) { - LOG_WARN("evaluate value failed", K(ret)); - } else if (OB_ISNULL(old_datum) || OB_ISNULL(new_datum)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("datum is null, unexpected", K(ret), KPC(old_datum), KPC(new_datum)); - } else { - new_row.at(idx)->locate_datum_for_write(get_eval_ctx()) = *old_datum; - new_row.at(idx)->set_evaluated_projected(get_eval_ctx()); - } - LOG_TRACE("after copy timestamp column to new_row", K(ret), K(idx), - "old_row", ROWEXPR2STR(get_eval_ctx(), old_row), - "new_row", ROWEXPR2STR(get_eval_ctx(), new_row)); - } + ++upd_rtdef_.cur_row_num_; + if (OB_FAIL(ObDMLService::process_update_row(MY_SPEC.upd_ctdef_, upd_rtdef_, is_skipped, *this))) { + LOG_WARN("process update row failed", K(ret)); + } else if (!is_skipped) { + // 通过partition id expr获得对应行对应的分区 + ++upd_rtdef_.found_rows_; + const int64_t part_id_idx = MY_SPEC.row_desc_.get_part_id_index(); + // 返回的值是child的output exprs + row = &child_->get_spec().output_; + if (NO_PARTITION_ID_FLAG == part_id_idx) { + ObDASTableLoc *table_loc = upd_rtdef_.dupd_rtdef_.table_loc_; + if (OB_ISNULL(table_loc) || table_loc->get_tablet_locs().size() != 1) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("insert table location is invalid", K(ret), KPC(table_loc)); + } else { + tablet_id = table_loc->get_first_tablet_loc()->tablet_id_; } + } else if (child_->get_spec().output_.count() > part_id_idx) { + ObExpr *expr = child_->get_spec().output_.at(part_id_idx); + ObDatum &expr_datum = expr->locate_expr_datum(get_eval_ctx()); + tablet_id = expr_datum.get_int(); + LOG_DEBUG("get the part id", K(ret), K(expr_datum)); } } - } return ret; } -bool ObPxMultiPartUpdateOp::is_update_auto_filled_timestamp() -{ - bool updated = false; - FOREACH_CNT_X(info, MY_SPEC.upd_ctdef_.assign_columns_, !updated) { - const uint64_t idx = info->projector_index_; - if (info->auto_filled_timestamp_) { - updated = true; - } - } - return updated; -} - int ObPxMultiPartUpdateOp::write_rows(ObExecContext &ctx, const ObDASTabletLoc *tablet_loc, diff --git a/src/sql/engine/pdml/static/ob_px_multi_part_update_op.h b/src/sql/engine/pdml/static/ob_px_multi_part_update_op.h index 2864fc32e9..d6cd4cbed0 100644 --- a/src/sql/engine/pdml/static/ob_px_multi_part_update_op.h +++ b/src/sql/engine/pdml/static/ob_px_multi_part_update_op.h @@ -79,14 +79,13 @@ public: public: int read_row(ObExecContext &ctx, const ObExprPtrIArray *&row, - common::ObTabletID &tablet_id) override; + common::ObTabletID &tablet_id, + bool &is_skipped) override; int write_rows(ObExecContext &ctx, const ObDASTabletLoc *tablet_loc, ObPDMLOpRowIterator &iterator) override; - bool is_update_auto_filled_timestamp(); - virtual int inner_get_next_row(); virtual int inner_open(); virtual int inner_close(); diff --git a/src/sql/engine/px/exchange/ob_px_ms_receive_vec_op.cpp b/src/sql/engine/px/exchange/ob_px_ms_receive_vec_op.cpp index 36b524e606..936009eaf5 100644 --- a/src/sql/engine/px/exchange/ob_px_ms_receive_vec_op.cpp +++ b/src/sql/engine/px/exchange/ob_px_ms_receive_vec_op.cpp @@ -675,7 +675,7 @@ int ObPxMSReceiveVecOp::inner_get_next_row() int ObPxMSReceiveVecOp::inner_get_next_batch(const int64_t max_row_cnt) { int ret = OB_SUCCESS; - do_clear_datum_eval_flag(); + clear_evaluated_flag(); clear_dynamic_const_parent_flag(); // 从channel sets 读取数据，并向上迭代 const ObPxReceiveSpec &spec = static_cast(get_spec()); diff --git a/src/sql/engine/px/p2p_datahub/ob_runtime_filter_query_range.h b/src/sql/engine/px/p2p_datahub/ob_runtime_filter_query_range.h index ba80e982d1..12542b7136 100644 --- a/src/sql/engine/px/p2p_datahub/ob_runtime_filter_query_range.h +++ b/src/sql/engine/px/p2p_datahub/ob_runtime_filter_query_range.h @@ -77,6 +77,7 @@ public: // and we should fill (min, max) for c3 ObFixedArray prefix_col_idxs_; ObFixedArray prefix_col_obj_metas_; + TO_STRING_KV(K_(table_id), K_(range_column_cnt), K_(prefix_col_idxs), K_(prefix_col_obj_metas)); }; } //end name space sql diff --git a/src/sql/engine/px/p2p_datahub/ob_runtime_filter_vec_msg.h b/src/sql/engine/px/p2p_datahub/ob_runtime_filter_vec_msg.h index cbd4d5e1d2..0aa0a26bf3 100644 --- a/src/sql/engine/px/p2p_datahub/ob_runtime_filter_vec_msg.h +++ b/src/sql/engine/px/p2p_datahub/ob_runtime_filter_vec_msg.h @@ -205,6 +205,7 @@ public: inline int add_row(ObCompactRow *new_row, int64_t row_size); int create_and_add_row(const common::ObIArray &exprs, const RowMeta &row_meta, const int64_t row_size, ObEvalCtx &ctx, ObCompactRow *&new_row, uint64_t hash_val); + TO_STRING_KV(K(get_row_cnt()), K_(row_sizes), K_(serial_rows)) private: common::ObIAllocator &allocator_; ObSArray serial_rows_; diff --git a/src/sql/engine/sort/ob_sort_vec_op_eager_filter.cpp b/src/sql/engine/sort/ob_sort_vec_op_eager_filter.cpp index 8b5493f07a..7b34fc9b3d 100644 --- a/src/sql/engine/sort/ob_sort_vec_op_eager_filter.cpp +++ b/src/sql/engine/sort/ob_sort_vec_op_eager_filter.cpp @@ -42,6 +42,8 @@ int ObSortVecOpEagerFilter::filter( if (OB_UNLIKELY(!is_inited())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("ObSortVecOpEagerFilter is not initialized", K(ret)); + } else if (is_by_pass()) { + // do nothing } else { output_brs_.copy(&input_brs); if (bucket_heap_->count() < bucket_num_) { @@ -72,10 +74,10 @@ int ObSortVecOpEagerFilter::update_filter( const Store_Row *bucket_head_row, bool &updated) { int ret = OB_SUCCESS; if (!is_inited()) { - // filter is not initialized, so the filter will not be updated - // this is not an error ret = OB_ERR_UNEXPECTED; - LOG_WARN("is not initialized", K(ret)); + LOG_WARN("ObSortVecOpEagerFilter is not initialized", K(ret)); + } else if (is_by_pass()) { + updated = false; } else { Store_Row *reuse_row = nullptr; if (bucket_heap_->count() < bucket_num_) { diff --git a/src/sql/engine/sort/ob_sort_vec_op_eager_filter.h b/src/sql/engine/sort/ob_sort_vec_op_eager_filter.h index 719e97ea06..8a7e795045 100644 --- a/src/sql/engine/sort/ob_sort_vec_op_eager_filter.h +++ b/src/sql/engine/sort/ob_sort_vec_op_eager_filter.h @@ -64,15 +64,15 @@ public: bucket_num_ = bucket_num_ < MIN_BUCKET_NUM ? 0 : bucket_num_; if (bucket_num_ != 0) { bucket_size_ = (topn_cnt + bucket_num_ - 1) / bucket_num_; - is_inited_ = true; + if (OB_FAIL(init_output_brs(max_batch_size))) { + SQL_ENG_LOG(WARN, "init ouput batch rows failed", K(ret)); + } } else { is_by_pass_ = true; SQL_ENG_LOG(INFO, "no need to use filter ", K(dumped_rows_cnt), K(topn_cnt)); } - if (OB_FAIL(init_output_brs(max_batch_size))) { - SQL_ENG_LOG(WARN, "init ouput batch rows failed", K(ret)); - } + is_inited_ = true; } return ret; } diff --git a/src/sql/engine/sort/ob_sort_vec_op_impl.ipp b/src/sql/engine/sort/ob_sort_vec_op_impl.ipp index 58a733f75c..50fbf2f0fd 100644 --- a/src/sql/engine/sort/ob_sort_vec_op_impl.ipp +++ b/src/sql/engine/sort/ob_sort_vec_op_impl.ipp @@ -356,7 +356,9 @@ int ObSortVecOpImpl::add_batch(const ObBatchRows if (OB_SUCC(ret)) { const ObBatchRows *input_brs_ptr = nullptr; if (is_topn_sort() && OB_NOT_NULL(topn_filter_) && OB_LIKELY(!topn_filter_->is_by_pass())) { - if (OB_FAIL(topn_filter_->filter(all_exprs_, *eval_ctx_, start_pos, + if (OB_FAIL(batch_eval_vector(all_exprs_, input_brs))) { + SQL_ENG_LOG(WARN, "failed to eval vector", K(ret)); + } else if (OB_FAIL(topn_filter_->filter(all_exprs_, *eval_ctx_, start_pos, input_brs))) { SQL_ENG_LOG(WARN, "failed to do topn filter", K(ret)); } else { @@ -1048,14 +1050,18 @@ template int ObSortVecOpImpl::eager_topn_filter_update( const common::ObIArray *sorted_dumped_rows) { int ret = OB_SUCCESS; - int64_t dumped_rows_count = sorted_dumped_rows->count(); - int64_t bucket_size = topn_filter_->bucket_size(); - bool updated = true; - for (int64_t i = bucket_size - 1; - OB_SUCC(ret) && updated && i < dumped_rows_count; i += bucket_size) { - if (OB_FAIL( - topn_filter_->update_filter(sorted_dumped_rows->at(i), updated))) { - SQL_ENG_LOG(WARN, "failed to eager topn filter update", K(ret)); + if (topn_filter_->is_by_pass()) { + // do nothing + } else { + int64_t dumped_rows_count = sorted_dumped_rows->count(); + int64_t bucket_size = topn_filter_->bucket_size(); + bool updated = true; + for (int64_t i = bucket_size - 1; + OB_SUCC(ret) && updated && i < dumped_rows_count; i += bucket_size) { + if (OB_FAIL( + topn_filter_->update_filter(sorted_dumped_rows->at(i), updated))) { + SQL_ENG_LOG(WARN, "failed to eager topn filter update", K(ret)); + } } } return ret; diff --git a/src/sql/engine/table/ob_index_lookup_op_impl.cpp b/src/sql/engine/table/ob_index_lookup_op_impl.cpp index 4f19276d6c..001ce282d4 100644 --- a/src/sql/engine/table/ob_index_lookup_op_impl.cpp +++ b/src/sql/engine/table/ob_index_lookup_op_impl.cpp @@ -157,7 +157,8 @@ int ObIndexLookupOpImpl::get_next_rows(int64_t &count, int64_t capacity) } else { got_next_row = true; update_state_in_output_rows_state(count); - PRINT_VECTORIZED_ROWS(SQL, DEBUG, get_eval_ctx(), get_output_expr(), count, + const ObBitVector *skip = NULL; + PRINT_VECTORIZED_ROWS(SQL, DEBUG, get_eval_ctx(), get_output_expr(), count, skip, K(ret), K(lookup_row_cnt_), K(lookup_rowkey_cnt_), K(lookup_group_cnt_), K(index_group_cnt_)); } diff --git a/src/sql/engine/table/ob_table_scan_op.cpp b/src/sql/engine/table/ob_table_scan_op.cpp index a7eab7bc9a..1007dff4ad 100644 --- a/src/sql/engine/table/ob_table_scan_op.cpp +++ b/src/sql/engine/table/ob_table_scan_op.cpp @@ -2102,7 +2102,8 @@ int ObTableScanOp::inner_get_next_batch_for_tsc(const int64_t max_row_cnt) if (OB_SUCC(ret)) { const ExprFixedArray &storage_output = MY_CTDEF.get_das_output_exprs(); if (!MY_SPEC.is_global_index_back()) { - PRINT_VECTORIZED_ROWS(SQL, DEBUG, eval_ctx_, storage_output, brs_.size_, K(MY_CTDEF.scan_ctdef_.ref_table_id_)); + PRINT_VECTORIZED_ROWS(SQL, DEBUG, eval_ctx_, storage_output, brs_.size_, brs_.skip_, + K(MY_CTDEF.scan_ctdef_.ref_table_id_)); } if (OB_FAIL(add_ddl_column_checksum_batch(brs_.size_))) { LOG_WARN("add ddl column checksum failed", K(ret)); diff --git a/src/sql/monitor/flt/ob_flt_span_mgr.cpp b/src/sql/monitor/flt/ob_flt_span_mgr.cpp index 33d4c3b9d5..9445b4da45 100644 --- a/src/sql/monitor/flt/ob_flt_span_mgr.cpp +++ b/src/sql/monitor/flt/ob_flt_span_mgr.cpp @@ -81,23 +81,12 @@ namespace sql { int ret = OB_SUCCESS; uint64_t tenant_id = lib::current_resource_owner_id(); - span_mgr = OB_NEW(ObFLTSpanMgr, ObMemAttr(tenant_id, "SqlFltSpanRec")); - if (nullptr == span_mgr) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memory for ObMySQLRequestManager", K(ret)); + int64_t mem_limit = lib::get_tenant_memory_limit(tenant_id); + int64_t queue_size = MAX_QUEUE_SIZE; + if (OB_FAIL(span_mgr->init(tenant_id, mem_limit, queue_size))) { + LOG_WARN("failed to init request manager", K(ret)); } else { - int64_t mem_limit = lib::get_tenant_memory_limit(tenant_id); - int64_t queue_size = MAX_QUEUE_SIZE; - if (OB_FAIL(span_mgr->init(tenant_id, mem_limit, queue_size))) { - LOG_WARN("failed to init request manager", K(ret)); - } else { - // do nothing - } - } - if (OB_FAIL(ret) && span_mgr != nullptr) { - // cleanup - common::ob_delete(span_mgr); - span_mgr = nullptr; + // do nothing } return ret; } diff --git a/src/sql/ob_result_set.cpp b/src/sql/ob_result_set.cpp index da276ca44d..e0284e5513 100644 --- a/src/sql/ob_result_set.cpp +++ b/src/sql/ob_result_set.cpp @@ -833,7 +833,7 @@ OB_INLINE int ObResultSet::do_close_plan(int errcode, ObExecContext &ctx) return ret; } -int ObResultSet::close(int &client_ret) +int ObResultSet::do_close(int *client_ret) { int ret = OB_SUCCESS; LinkExecCtxGuard link_guard(my_session_, get_exec_context()); @@ -948,8 +948,9 @@ int ObResultSet::close(int &client_ret) } // notify close fail to listener int err = OB_SUCCESS != do_close_plan_ret ? do_close_plan_ret : ret; - if (OB_SUCCESS != err && err != errcode_ && close_fail_cb_.is_valid()) { - close_fail_cb_(err, client_ret); + if (client_ret != NULL + && OB_SUCCESS != err && err != errcode_ && close_fail_cb_.is_valid()) { + close_fail_cb_(err, *client_ret); } //Save the current execution state to determine whether to refresh location //and perform other necessary cleanup operations when the statement exits. diff --git a/src/sql/ob_result_set.h b/src/sql/ob_result_set.h index fef87db2c7..b8f08b4cb3 100644 --- a/src/sql/ob_result_set.h +++ b/src/sql/ob_result_set.h @@ -122,9 +122,10 @@ public: /// @return OB_ITER_END when no more data available int get_next_row(const common::ObNewRow *&row); /// close the result set after get all the rows - int close() { int unused = 0; return close(unused); } + int close() { return do_close(NULL); } // close result set and rewrite the client ret - int close(int &client_ret); + int close(int &client_ret) { return do_close(&client_ret); } + /// get number of rows affected by INSERT/UPDATE/DELETE int64_t get_affected_rows() const; int64_t get_return_rows() const { return return_rows_; } @@ -346,7 +347,7 @@ private: int init_cmd_exec_context(ObExecContext &exec_ctx); int on_cmd_execute(); int auto_end_plan_trans(ObPhysicalPlan& plan, int ret, bool &async); - + int do_close(int *client_ret = NULL); void store_affected_rows(ObPhysicalPlanCtx &plan_ctx); void store_found_rows(ObPhysicalPlanCtx &plan_ctx); int store_last_insert_id(ObExecContext &ctx); diff --git a/src/sql/ob_spi.cpp b/src/sql/ob_spi.cpp index 3b83e1d133..d20bf3002d 100644 --- a/src/sql/ob_spi.cpp +++ b/src/sql/ob_spi.cpp @@ -1657,7 +1657,7 @@ int ObSPIService::spi_inner_execute(ObPLExecCtx *ctx, out_params, &retry_ctrl, is_forall))) { - LOG_WARN("failed to open", K(type), K(ret)); + LOG_WARN("failed to open", K(ret), K(sql), K(ps_sql)); } else if (OB_FAIL(inner_fetch(ctx, retry_ctrl, spi_result, @@ -3713,6 +3713,21 @@ int ObSPIService::spi_cursor_open(ObPLExecCtx *ctx, spi_cursor->fields_)); //} OZ (fill_cursor(*spi_result.get_result_set(), spi_cursor)); + if (OB_FAIL(ret) && OB_NOT_NULL(spi_result.get_result_set())) { + int cli_ret = OB_SUCCESS; + retry_ctrl.test_and_save_retry_state(GCTX, + spi_result.get_sql_ctx(), + *spi_result.get_result_set(), + ret, + cli_ret, + true, + true, + true); + LOG_WARN("failed to do fill_cursor, check if need retry", K(ret), K(cli_ret), K(retry_ctrl.need_retry()), K(sql), K(ps_sql)); + ret = cli_ret; + spi_result.get_sql_ctx().clear(); + ctx->exec_ctx_->get_my_session()->set_session_in_retry(retry_ctrl.need_retry()); + } OX (spi_cursor->row_store_.finish_add_row()) OX (cursor->open(spi_cursor)); if (OB_FAIL(ret)) { @@ -3988,7 +4003,7 @@ int ObSPIService::dbms_cursor_open(ObPLExecCtx *ctx, } } OZ (fill_cursor(*spi_result.get_result_set(), spi_cursor)); - if (OB_FAIL(ret)) { + if (OB_FAIL(ret) && OB_NOT_NULL(spi_result.get_result_set())) { int cli_ret = OB_SUCCESS; retry_ctrl.test_and_save_retry_state(GCTX, spi_result.get_sql_ctx(), @@ -5246,7 +5261,7 @@ int ObSPIService::spi_delete_collection(pl::ObPLExecCtx *ctx, if (*key1 <= *key2) { for (int64_t i = 0; OB_SUCC(ret) && i < atable->get_count(); ++i) { bool flag = key!=NULL ? (key[i] >= *key1 && key[i] <= *key2) - : (i >= key1->get_int32() && i <= key2->get_int32()); + : (i+1 >= key1->get_int32() && i+1 <= key2->get_int32()); if (flag) { OZ (atable->delete_collection_elem(i)); if (atable->get_first() - 1 == i) { @@ -6347,7 +6362,7 @@ int ObSPIService::inner_open(ObPLExecCtx *ctx, if (NULL == sql) { OZ (construct_exec_params(ctx, param_allocator, param_exprs, param_count, into_exprs, into_count, exec_params, out_params, is_forall), - K(sql), K(type), K(param_count), K(out_params), K(exec_params)); + K(sql), K(ps_sql), K(type), K(param_count), K(out_params), K(exec_params)); if (OB_SUCC(ret) && OB_NOT_NULL(ctx) && OB_NOT_NULL(ctx->exec_ctx_) @@ -8550,6 +8565,7 @@ int ObSPIService::spi_update_package_change_info( OZ (session_info->get_package_state(package_id, package_state)); CK (OB_NOT_NULL(package_state)); OZ (package_state->update_changed_vars(var_idx)); + OX (session_info->set_pl_can_retry(false)); return ret; } diff --git a/src/sql/ob_sql.cpp b/src/sql/ob_sql.cpp index 2c86add244..298dd665c5 100644 --- a/src/sql/ob_sql.cpp +++ b/src/sql/ob_sql.cpp @@ -2599,7 +2599,7 @@ OB_NOINLINE int ObSql::handle_large_query(int tmp_ret, exec_times = plan->stat_.get_execute_count(); total_process_time = plan->stat_.total_process_time_; if (exec_times > 0 - && (total_process_time / exec_times) > lqt) { + && (0 != lqt && (total_process_time / exec_times) > lqt)) { plan->inc_large_querys(); is_large_query = true; lq_from_plan = true; @@ -2609,7 +2609,7 @@ OB_NOINLINE int ObSql::handle_large_query(int tmp_ret, //实际编译时间判断是否为大请求 if (OB_SUCC(ret) && is_large_query == false) { if (OB_PC_LOCK_CONFLICT == tmp_ret - || elapsed_time > lqt) { + || (0 != lqt && elapsed_time > lqt)) { is_large_query = true; lq_from_plan = false; } diff --git a/src/sql/ob_sql_utils.cpp b/src/sql/ob_sql_utils.cpp index 1e0ba7510e..d46a09a6f7 100644 --- a/src/sql/ob_sql_utils.cpp +++ b/src/sql/ob_sql_utils.cpp @@ -5452,3 +5452,23 @@ int ObSQLUtils::print_identifier_require_quotes(ObCollationType collation_type, } return ret; } + +bool ObSQLUtils::check_json_expr(ObItemType type) +{ + bool res = false; + switch(type) { + case T_FUN_SYS_JSON_ARRAY: + case T_FUN_SYS_JSON_OBJECT: + case T_FUN_ORA_JSON_ARRAYAGG: + case T_FUN_ORA_JSON_OBJECTAGG: + case T_FUN_SYS_JSON_QUERY: + case T_FUN_SYS_JSON_MERGE_PATCH: { + res = true; + break; + } + default : { + break; + } + } + return res; +} diff --git a/src/sql/ob_sql_utils.h b/src/sql/ob_sql_utils.h index 860e555c99..00aaa36c0c 100644 --- a/src/sql/ob_sql_utils.h +++ b/src/sql/ob_sql_utils.h @@ -670,6 +670,7 @@ public: share::schema::ObObjectType &obj_type, uint64_t &schema_version); static bool check_need_disconnect_parser_err(const int ret_code); + static bool check_json_expr(ObItemType type); static int print_identifier_require_quotes(ObCollationType collation_type, const ObString &ident, diff --git a/src/sql/optimizer/ob_del_upd_log_plan.cpp b/src/sql/optimizer/ob_del_upd_log_plan.cpp index 68167752b0..8a6c3b1a7e 100644 --- a/src/sql/optimizer/ob_del_upd_log_plan.cpp +++ b/src/sql/optimizer/ob_del_upd_log_plan.cpp @@ -264,8 +264,8 @@ int ObDelUpdLogPlan::check_table_rowkey_distinct( if (OB_ISNULL(index_dml_info)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("index dml info is null", K(ret)); - } else if (!use_pdml() && !index_dml_info->is_primary_index_) { - // for PDML, primary table & index table both need unique checker. + } else if (!index_dml_info->is_primary_index_) { + // only primary table need unique checker. } else if (OB_FAIL(index_dml_info->get_rowkey_exprs(rowkey_exprs))) { LOG_WARN("failed to get rowkey exprs", K(ret)); } else if (OB_FAIL(ObOptimizerUtil::is_exprs_unique(rowkey_exprs, diff --git a/src/sql/optimizer/ob_explain_note.h b/src/sql/optimizer/ob_explain_note.h index 6cd0c8c656..9362e514dd 100644 --- a/src/sql/optimizer/ob_explain_note.h +++ b/src/sql/optimizer/ob_explain_note.h @@ -35,6 +35,7 @@ namespace sql #define PARALLEL_DISABLED_BY_PL_UDF_DAS "Degree of Parallelisim is %ld because stmt contain pl_udf which force das scan" #define DIRECT_MODE_INSERT_INTO_SELECT "Direct-mode is enabled in insert into select" #define PARALLEL_DISABLED_BY_DBLINK "Degree of Parallelisim is %ld because stmt contain dblink which force das scan" +#define PDML_DISABLED_BY_INSERT_PK_AUTO_INC "PDML disabled because the insert statement primary key has specified auto-increment column" } } diff --git a/src/sql/optimizer/ob_insert_log_plan.cpp b/src/sql/optimizer/ob_insert_log_plan.cpp index b738183536..1c13cf889b 100644 --- a/src/sql/optimizer/ob_insert_log_plan.cpp +++ b/src/sql/optimizer/ob_insert_log_plan.cpp @@ -325,6 +325,8 @@ int ObInsertLogPlan::check_need_online_stats_gather(bool &need_osg) ObObj online_sys_var_obj; const ObInsertStmt *insert_stmt = NULL; TableItem *ins_table = NULL; + bool disable_pdml = false; + bool is_pk_auto_inc = false; if (OB_ISNULL(insert_stmt = get_stmt()) || OB_ISNULL(ins_table = insert_stmt->get_table_item_by_id(insert_stmt->get_insert_table_info().table_id_))) { ret = OB_ERR_UNEXPECTED; @@ -334,6 +336,11 @@ int ObInsertLogPlan::check_need_online_stats_gather(bool &need_osg) || !insert_stmt->value_from_select() || (!get_optimizer_context().get_session_info()->is_user_session())) { need_gathering = false; + } else if (OB_FAIL(insert_stmt->check_pdml_disabled(get_optimizer_context().is_online_ddl(), + disable_pdml, is_pk_auto_inc))) { + LOG_WARN("fail to check pdml disable for insert stmt", K(ret)); + } else if (disable_pdml) { + need_gathering = false; } if (OB_FAIL(ret)) { diff --git a/src/sql/optimizer/ob_join_order.cpp b/src/sql/optimizer/ob_join_order.cpp index f6ba41f7a7..6087908e23 100644 --- a/src/sql/optimizer/ob_join_order.cpp +++ b/src/sql/optimizer/ob_join_order.cpp @@ -7636,10 +7636,6 @@ int ObJoinOrder::generate_base_table_paths(PathHelper &helper) LOG_WARN("failed to calc table location", K(ret)); } else if (OB_FAIL(estimate_size_for_base_table(helper, access_paths))) { LOG_WARN("failed to estimate_size", K(ret)); - } else if (!helper.is_inner_path_ && !is_virtual_table(ref_table_id) - && EXTERNAL_TABLE != table_meta_info_.table_type_ - && OB_FAIL(compute_one_row_info_for_table_scan(access_paths))) { - LOG_WARN("failed to compute one row info", K(ret)); } else if (OB_FAIL(pruning_unstable_access_path(helper.table_opt_info_, access_paths))) { LOG_WARN("failed to pruning unstable access path", K(ret)); } else if (OB_FAIL(get_plan()->select_location(tbl_part_infos))) { @@ -7677,6 +7673,12 @@ int ObJoinOrder::compute_base_table_property(uint64_t table_id, ref_table_id, get_restrict_infos()))) { LOG_WARN("failed to extract fd item set", K(ret)); + } else if (OB_FAIL(ObOptimizerUtil::is_exprs_unique(get_output_const_exprs(), + get_fd_item_set(), + get_output_equal_sets(), + get_output_const_exprs(), + is_at_most_one_row_))) { + LOG_WARN("failed to compute at most one row", K(ret)); } else if (OB_FAIL(compute_table_location(table_id, ref_table_id, false, @@ -13170,30 +13172,6 @@ int ObJoinOrder::compute_fd_item_set_for_subquery(const uint64_t table_id, return ret; } -int ObJoinOrder::compute_one_row_info_for_table_scan(ObIArray &access_paths) -{ - int ret = OB_SUCCESS; - AccessPath *access_path = NULL; - is_at_most_one_row_ = false; - for (int64_t i = 0; OB_SUCC(ret) && !is_at_most_one_row_ && i < access_paths.count(); i++) { - bool is_one_row = false; - if (OB_ISNULL(access_path = access_paths.at(i)) || - OB_ISNULL(access_path->pre_query_range_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret)); - } else if (access_path->is_inner_path_) { - /*do nothing*/ - } else if (OB_FAIL(access_path->pre_query_range_->is_at_most_one_row(is_one_row))) { - LOG_WARN("failed to check if is at most one row", K(ret)); - } else if (is_one_row && (1 == access_path->est_cost_info_.ranges_.count())) { - is_at_most_one_row_ = true; - } else { /*do nothing*/ } - } - LOG_TRACE("succeed to compute one row info for table scan", K(is_at_most_one_row_), - K(table_id_)); - return ret; -} - int ObJoinOrder::compute_one_row_info_for_join(const ObJoinOrder *left_tree, const ObJoinOrder *right_tree, const ObIArray &join_condition, diff --git a/src/sql/optimizer/ob_join_order.h b/src/sql/optimizer/ob_join_order.h index de32d3b03b..455d1705ff 100644 --- a/src/sql/optimizer/ob_join_order.h +++ b/src/sql/optimizer/ob_join_order.h @@ -2315,8 +2315,6 @@ struct NullAwareAntiJoinInfo { int compute_fd_item_set_for_subquery(const uint64_t table_id, ObLogicalOperator *subplan_root); - int compute_one_row_info_for_table_scan(ObIArray &access_paths); - int compute_one_row_info_for_join(const ObJoinOrder *left_tree, const ObJoinOrder *right_tree, const ObIArray &join_condition, diff --git a/src/sql/optimizer/ob_log_plan.cpp b/src/sql/optimizer/ob_log_plan.cpp index 07f8f54683..e2eab7fe61 100644 --- a/src/sql/optimizer/ob_log_plan.cpp +++ b/src/sql/optimizer/ob_log_plan.cpp @@ -7346,6 +7346,7 @@ int ObLogPlan::adjust_sort_expr_ordering(ObIArray &sort_exprs, } } if (OB_SUCC(ret) && !adjusted && stmt->get_order_item_size() > 0) { + adjusted = true; if (prefix_count > 0) { /* used part of input ordering, try adjust sort_exprs after prefix_count by order item */ if (OB_FAIL(adjust_postfix_sort_expr_ordering(stmt->get_order_items(), @@ -7368,6 +7369,11 @@ int ObLogPlan::adjust_sort_expr_ordering(ObIArray &sort_exprs, LOG_WARN("failed to adjust exprs by ordering", K(ret)); } } + if (OB_SUCC(ret) && !adjusted) { + if (OB_FAIL(ObOptimizerUtil::generate_stable_ordering(sort_exprs, sort_directions))) { + LOG_WARN("failed to generate stable ordering", K(ret)); + } + } } return ret; } @@ -9021,7 +9027,7 @@ int ObLogPlan::candi_allocate_subplan_filter(const ObIArray &subquer onetime_exprs, initplan_idxs, onetime_idxs, - NULL == filters ? NULL : &new_filters, + new_filters, for_cursor_expr, is_update_set))) { LOG_WARN("failed to allocate subplan filter", K(ret)); @@ -9036,24 +9042,83 @@ int ObLogPlan::inner_candi_allocate_subplan_filter(ObIArray &subplan ObIArray &onetime_exprs, ObBitSet<> &initplan_idxs, ObBitSet<> &onetime_idxs, - const ObIArray *filters, + const ObIArray &filters, const bool for_cursor_expr, const bool is_update_set) { int ret = OB_SUCCESS; + ObSEArray, 8> best_subplan_list; + ObSEArray, 8> best_dist_subplan_list; + ObSEArray subquery_plans; + int64_t dist_methods = DIST_INVALID_METHOD; + if (OB_FAIL(prepare_subplan_candidate_list(subplans, params, best_subplan_list, + best_dist_subplan_list))) { + LOG_WARN("failed to prepare subplan candidate list", K(ret)); + } else if (OB_FAIL(get_valid_subplan_filter_dist_method(subplans, for_cursor_expr, false, + dist_methods))) { + LOG_WARN("failed to get valid subplan filter dist method", K(ret)); + } else if (OB_FAIL(inner_candi_allocate_subplan_filter(best_subplan_list, + best_dist_subplan_list, + query_refs, + params, + onetime_exprs, + initplan_idxs, + onetime_idxs, + filters, + for_cursor_expr, + is_update_set, + dist_methods, + subquery_plans))) { + LOG_WARN("failed to allocate subplan filter", K(ret)); + } else if (!subquery_plans.empty()) { + LOG_TRACE("succeed to allocate subplan filter using hint", K(subquery_plans.count()), K(dist_methods)); + OPT_TRACE("success to generate subplan filter plan with hint"); + } else if (OB_FAIL(get_log_plan_hint().check_status())) { + LOG_WARN("failed to generate plans with hint", K(ret)); + } else if (OB_FAIL(get_valid_subplan_filter_dist_method(subplans, for_cursor_expr, true, + dist_methods))) { + LOG_WARN("failed to get valid subplan filter dist method", K(ret)); + } else if (OB_FAIL(inner_candi_allocate_subplan_filter(best_subplan_list, + best_dist_subplan_list, + query_refs, + params, + onetime_exprs, + initplan_idxs, + onetime_idxs, + filters, + for_cursor_expr, + is_update_set, + dist_methods, + subquery_plans))) { + LOG_WARN("failed to allocate subplan filter", K(ret)); + } else { + LOG_TRACE("succeed to allocate subplan filter ignore hint", K(subquery_plans.count()), K(dist_methods)); + OPT_TRACE("success to generate subplan filter plan ignore hint"); + } + + if (OB_FAIL(ret)) { + /*do nothing*/ + } else if (OB_FAIL(prune_and_keep_best_plans(subquery_plans))) { + LOG_WARN("failed to prune and keep best plans", K(ret)); + } else { /*do nothing*/ } + return ret; +} + +// get best candidate list +int ObLogPlan::prepare_subplan_candidate_list(ObIArray &subplans, + ObIArray ¶ms, + ObIArray> &best_list, + ObIArray> &dist_best_list) +{ + int ret = OB_SUCCESS; + best_list.reuse(); + dist_best_list.reuse(); ObLogPlan *log_plan = NULL; const ObDMLStmt *stmt = NULL; CandidatePlan candidate_plan; - ObSEArray move_pos; ObSEArray temp_plans; ObSEArray dist_temp_plans; - ObSEArray subquery_plans; - ObSEArray, 8> best_list; - ObSEArray, 8> dist_best_list; - ObSEArray child_ops; - ObSEArray dist_child_ops; ObExchangeInfo exch_info; - // get best candidate list for (int64_t i = 0; OB_SUCC(ret) && i < subplans.count(); i++) { temp_plans.reuse(); if (OB_ISNULL(subplans.at(i))) { @@ -9099,6 +9164,82 @@ int ObLogPlan::inner_candi_allocate_subplan_filter(ObIArray &subplan } } } + return ret; +} + +int ObLogPlan::get_valid_subplan_filter_dist_method(ObIArray &subplans, + const bool for_cursor_expr, + const bool ignore_hint, + int64_t &dist_methods) +{ + int ret = OB_SUCCESS; + dist_methods = DIST_BASIC_METHOD | DIST_PULL_TO_LOCAL + | DIST_PARTITION_WISE | DIST_PARTITION_NONE + | DIST_NONE_ALL; + const ObLogicalOperator *op = NULL; + bool contain_recursive_cte = false; + if (OB_ISNULL(get_stmt()) || OB_UNLIKELY(candidates_.candidate_plans_.empty() + || OB_UNLIKELY(subplans.empty())) + || OB_ISNULL(op = candidates_.candidate_plans_.at(0).plan_tree_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected list", K(ret), K(subplans.count()), K(op)); + } else { + contain_recursive_cte |= op->get_contains_fake_cte(); + ObSEArray sub_qb_names; + ObString qb_name; + ObLogPlan *subplan = NULL; + for (int64_t i = 0; OB_SUCC(ret) && i < subplans.count(); i++) { + if (OB_ISNULL(subplan = subplans.at(i)) || OB_ISNULL(subplan->get_stmt()) + || OB_UNLIKELY(subplan->candidates_.candidate_plans_.empty()) + || OB_ISNULL(op = subplan->candidates_.candidate_plans_.at(0).plan_tree_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected list", K(ret), K(subplan), K(op)); + } else if (OB_FAIL(subplan->get_stmt()->get_qb_name(qb_name))) { + LOG_WARN("failed to get qb name", K(ret)); + } else if (OB_FAIL(sub_qb_names.push_back(qb_name))) { + LOG_WARN("failed to push back", K(ret)); + } else { + contain_recursive_cte |= op->get_contains_fake_cte(); + } + } + + if (OB_SUCC(ret) && !ignore_hint) { + const bool implicit_hint_allowed = (subplans.count() == get_stmt()->get_subquery_expr_size()); + dist_methods &= get_log_plan_hint().get_valid_pq_subquery_dist_algo(sub_qb_names, + implicit_hint_allowed); + } + + if (OB_FAIL(ret)) { + } else if (for_cursor_expr || contain_recursive_cte) { + dist_methods &= (DIST_BASIC_METHOD | DIST_PULL_TO_LOCAL); + OPT_TRACE("SPF will use basic method"); + } else if (!get_optimizer_context().is_var_assign_only_in_root_stmt() + && get_optimizer_context().has_var_assign()) { + dist_methods &= (DIST_BASIC_METHOD | DIST_PULL_TO_LOCAL); + OPT_TRACE("SPF will use pull to local method for var assign"); + } + } + return ret; +} + +int ObLogPlan::inner_candi_allocate_subplan_filter(ObIArray> &best_list, + ObIArray> &dist_best_list, + ObIArray &query_refs, + ObIArray ¶ms, + ObIArray &onetime_exprs, + ObBitSet<> &initplan_idxs, + ObBitSet<> &onetime_idxs, + const ObIArray &filters, + const bool for_cursor_expr, + const bool is_update_set, + const int64_t dist_methods, + ObIArray &subquery_plans) +{ + int ret = OB_SUCCESS; + CandidatePlan candidate_plan; + ObSEArray move_pos; + ObSEArray child_ops; + ObSEArray dist_child_ops; // generate subplan filter for (int64_t i = 0; OB_SUCC(ret) && i < candidates_.candidate_plans_.count(); i++) { candidate_plan = candidates_.candidate_plans_.at(i); @@ -9132,17 +9273,27 @@ int ObLogPlan::inner_candi_allocate_subplan_filter(ObIArray &subplan if (OB_SUCC(ret)) { CandidatePlan curr_candidate_plan; curr_candidate_plan.plan_tree_ = candidate_plan.plan_tree_; - if (OB_FAIL(create_subplan_filter_plan(curr_candidate_plan.plan_tree_, - child_ops, - dist_child_ops, - query_refs, - params, - onetime_exprs, - initplan_idxs, - onetime_idxs, - for_cursor_expr, - filters, - is_update_set))) { + int64_t cur_dist_methods = dist_methods; + if (OB_FAIL(get_subplan_filter_distributed_method(curr_candidate_plan.plan_tree_, + child_ops, + params, + for_cursor_expr, + !onetime_idxs.is_empty(), + cur_dist_methods))) { + LOG_WARN("failed to get subplan filter distributed method", K(ret)); + } else if (0 == cur_dist_methods) { + /* do nothing */ + } else if (OB_FAIL(create_subplan_filter_plan(curr_candidate_plan.plan_tree_, + child_ops, + dist_child_ops, + query_refs, + params, + onetime_exprs, + initplan_idxs, + onetime_idxs, + cur_dist_methods, + filters, + is_update_set))) { LOG_WARN("failed to create subplan filter plan", K(ret)); } else if (OB_FAIL(subquery_plans.push_back(curr_candidate_plan))) { LOG_WARN("failed to push back subquery plans", K(ret)); @@ -9164,11 +9315,6 @@ int ObLogPlan::inner_candi_allocate_subplan_filter(ObIArray &subplan } } } - if (OB_FAIL(ret)) { - /*do nothing*/ - } else if (OB_FAIL(prune_and_keep_best_plans(subquery_plans))) { - LOG_WARN("failed to prune and keep best plans", K(ret)); - } else { /*do nothing*/ } return ret; } @@ -9310,18 +9456,12 @@ int ObLogPlan::adjust_exprs_with_onetime(ObIArray &exprs) return ret; } - -int ObLogPlan::create_subplan_filter_plan(ObLogicalOperator *&top, - const ObIArray &subquery_ops, - const ObIArray &dist_subquery_ops, - const ObIArray &query_ref_exprs, - const ObIArray ¶ms, - const ObIArray &onetime_exprs, - const ObBitSet<> &initplan_idxs, - const ObBitSet<> &onetime_idxs, - const bool for_cursor_expr, - const ObIArray *filters, - const bool is_update_set) +int ObLogPlan::get_subplan_filter_distributed_method(ObLogicalOperator *&top, + const ObIArray &subquery_ops, + const ObIArray ¶ms, + const bool for_cursor_expr, + const bool has_onetime, + int64_t &distributed_methods) { int ret = OB_SUCCESS; bool is_basic = false; @@ -9332,7 +9472,7 @@ int ObLogPlan::create_subplan_filter_plan(ObLogicalOperator *&top, bool is_all_none = false; bool is_partition_none = false; ObSEArray sf_childs; - DistAlgo dist_algo = DistAlgo::DIST_INVALID_METHOD; + if (OB_ISNULL(top)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected null", K(ret)); @@ -9344,93 +9484,105 @@ int ObLogPlan::create_subplan_filter_plan(ObLogicalOperator *&top, is_basic, is_remote))) { LOG_WARN("failed to check if match basic sharding info", K(ret)); - } else if ((!for_cursor_expr && is_basic) || - (for_cursor_expr && is_basic && !is_remote)) { - dist_algo = DistAlgo::DIST_BASIC_METHOD; - OPT_TRACE("SPF will use basic method"); - } else if (for_cursor_expr) { - dist_algo = DistAlgo::DIST_PULL_TO_LOCAL; - OPT_TRACE("SPF will use pull to local method for cursor subquery expr"); - } else if (OB_FAIL(check_contains_recursive_cte(sf_childs, - is_recursive_cte))) { - LOG_WARN("failed to check whether contains recursive cte", K(ret)); - } else if (is_recursive_cte) { - dist_algo = DistAlgo::DIST_PULL_TO_LOCAL; - OPT_TRACE("SPF will use pull to local method for recursive cte"); - } else if (!get_optimizer_context().is_var_assign_only_in_root_stmt() && - get_optimizer_context().has_var_assign()) { - dist_algo = DistAlgo::DIST_PULL_TO_LOCAL; - OPT_TRACE("SPF will use pull to local method for var assign"); - } else if (OB_FAIL(check_if_subplan_filter_match_partition_wise(top, - subquery_ops, - params, - is_partition_wise))) { - LOG_WARN("failed to check if match partition wise", K(ret)); - } else if (is_partition_wise) { - dist_algo = DistAlgo::DIST_PARTITION_WISE; - OPT_TRACE("SPF will use partition wise method"); - } else if (OB_FAIL(check_if_subplan_filter_match_repart(top, - subquery_ops, - params, - is_partition_none))) { - LOG_WARN("failed to check if match repart", K(ret)); - } else if (is_partition_none) { - dist_algo = DistAlgo::DIST_PARTITION_NONE; - OPT_TRACE("SPF will use repartition method"); - } else if (OB_FAIL(check_if_match_none_all(top, - subquery_ops, - is_none_all))) { - LOG_WARN("failed to check if match none all", K(ret)); - } else if (is_none_all && onetime_idxs.is_empty()) { - dist_algo = DistAlgo::DIST_NONE_ALL; - OPT_TRACE("SPF will use none all method"); - } else { - dist_algo = DistAlgo::DIST_PULL_TO_LOCAL; + } else if (is_basic && !is_remote) { + distributed_methods &= DistAlgo::DIST_BASIC_METHOD; + } else if (is_remote) { + distributed_methods &= (DIST_BASIC_METHOD | DIST_PULL_TO_LOCAL); + } + + if (OB_SUCC(ret) && (distributed_methods & DistAlgo::DIST_NONE_ALL)) { + if (OB_FAIL(check_if_match_none_all(top, subquery_ops, is_none_all))) { + LOG_WARN("failed to check if match repart", K(ret)); + } else if (is_none_all && !has_onetime) { + distributed_methods = DistAlgo::DIST_NONE_ALL; + OPT_TRACE("SPF will use none all method"); + } else { + distributed_methods &= ~DIST_NONE_ALL; + } + } + + if (OB_SUCC(ret) && (distributed_methods & DistAlgo::DIST_BASIC_METHOD)) { + if (is_basic && (!for_cursor_expr || !is_remote)) { + distributed_methods = DistAlgo::DIST_BASIC_METHOD; + OPT_TRACE("SPF will use basic method"); + } else { + distributed_methods &= ~DIST_BASIC_METHOD; + } + } + + if (OB_SUCC(ret) && (distributed_methods & DistAlgo::DIST_PARTITION_WISE)) { + if (OB_FAIL(check_if_subplan_filter_match_partition_wise(top, subquery_ops, params, is_partition_wise))) { + LOG_WARN("failed to check if match partition wise", K(ret)); + } else if (is_partition_wise) { + distributed_methods = DistAlgo::DIST_PARTITION_WISE; + OPT_TRACE("SPF will use partition wise method"); + } else { + distributed_methods &= ~DIST_PARTITION_WISE; + } + } + + if (OB_SUCC(ret) && (distributed_methods & DistAlgo::DIST_PARTITION_NONE)) { + if (OB_FAIL(check_if_subplan_filter_match_repart(top, subquery_ops, params, is_partition_none))) { + LOG_WARN("failed to check if match repart", K(ret)); + } else if (is_partition_none) { + distributed_methods = DistAlgo::DIST_PARTITION_NONE; + OPT_TRACE("SPF will use repartition method"); + } else { + distributed_methods &= ~DIST_PARTITION_NONE; + } + } + + if (OB_SUCC(ret) && (distributed_methods & DistAlgo::DIST_PULL_TO_LOCAL)) { + distributed_methods = DistAlgo::DIST_PULL_TO_LOCAL; OPT_TRACE("SPF will use pull to local method"); } - if (OB_SUCC(ret)) { - ObExchangeInfo exch_info; - if (DistAlgo::DIST_BASIC_METHOD == dist_algo || - DistAlgo::DIST_PARTITION_WISE == dist_algo || - DistAlgo::DIST_NONE_ALL == dist_algo) { - // is basic or is_partition_wise - if (OB_FAIL(allocate_subplan_filter_as_top(top, - subquery_ops, - query_ref_exprs, - params, - onetime_exprs, - initplan_idxs, - onetime_idxs, - filters, - dist_algo, - is_update_set))) { - LOG_WARN("failed to allocate subplan filter as top", K(ret)); - } else { /*do nothing*/ } - } else if (DistAlgo::DIST_PARTITION_NONE == dist_algo) { - if (OB_FAIL(compute_subplan_filter_repartition_distribution_info(top, - subquery_ops, - params, - exch_info))) { - LOG_WARN("failed to compute subplan filter distribution info", K(ret)); - } else if (OB_FAIL(allocate_exchange_as_top(top, exch_info))) { - LOG_WARN("failed to allocate exchange as top"); - } else if (OB_FAIL(allocate_subplan_filter_as_top(top, - subquery_ops, - query_ref_exprs, - params, - onetime_exprs, - initplan_idxs, - onetime_idxs, - filters, - dist_algo, - is_update_set))) { - LOG_WARN("failed to allocate subplan filter as top", K(ret)); - } else { /*do nothing*/ } - } else if (top->is_sharding() && - OB_FAIL(allocate_exchange_as_top(top, exch_info))) { - LOG_WARN("failed to allocate exchange as top", K(ret)); + return ret; +} + +int ObLogPlan::create_subplan_filter_plan(ObLogicalOperator *&top, + const ObIArray &subquery_ops, + const ObIArray &dist_subquery_ops, + const ObIArray &query_ref_exprs, + const ObIArray ¶ms, + const ObIArray &onetime_exprs, + const ObBitSet<> &initplan_idxs, + const ObBitSet<> &onetime_idxs, + const int64_t dist_methods, + const ObIArray &filters, + const bool is_update_set) +{ + int ret = OB_SUCCESS; + ObExchangeInfo exch_info; + const DistAlgo dist_algo = get_dist_algo(dist_methods); + if (OB_ISNULL(top)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else if (DistAlgo::DIST_BASIC_METHOD == dist_algo || + DistAlgo::DIST_PARTITION_WISE == dist_algo || + DistAlgo::DIST_NONE_ALL == dist_algo) { + // is basic or is_partition_wise + if (OB_FAIL(allocate_subplan_filter_as_top(top, + subquery_ops, + query_ref_exprs, + params, + onetime_exprs, + initplan_idxs, + onetime_idxs, + filters, + dist_algo, + is_update_set))) { + LOG_WARN("failed to allocate subplan filter as top", K(ret)); + } else { /*do nothing*/ } + } else if (DistAlgo::DIST_PARTITION_NONE == dist_algo) { + if (OB_FAIL(compute_subplan_filter_repartition_distribution_info(top, + subquery_ops, + params, + exch_info))) { + LOG_WARN("failed to compute subplan filter distribution info", K(ret)); + } else if (OB_FAIL(allocate_exchange_as_top(top, exch_info))) { + LOG_WARN("failed to allocate exchange as top"); } else if (OB_FAIL(allocate_subplan_filter_as_top(top, - dist_subquery_ops, + subquery_ops, query_ref_exprs, params, onetime_exprs, @@ -9441,23 +9593,23 @@ int ObLogPlan::create_subplan_filter_plan(ObLogicalOperator *&top, is_update_set))) { LOG_WARN("failed to allocate subplan filter as top", K(ret)); } else { /*do nothing*/ } - } - return ret; -} - -int ObLogPlan::check_contains_recursive_cte(ObIArray &child_ops, - bool &is_recursive_cte) -{ - int ret = OB_SUCCESS; - is_recursive_cte = false; - for (int64_t i = 0; OB_SUCC(ret) && !is_recursive_cte && i < child_ops.count(); i++) { - if (OB_ISNULL(child_ops.at(i))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null", K(ret)); - } else { - is_recursive_cte |= child_ops.at(i)->get_contains_fake_cte(); - } - } + } else if (OB_UNLIKELY(DistAlgo::DIST_PULL_TO_LOCAL != dist_algo)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected subplan filter distributed method", K(ret), K(dist_algo)); + } else if (top->is_sharding() && OB_FAIL(allocate_exchange_as_top(top, exch_info))) { + LOG_WARN("failed to allocate exchange as top", K(ret)); + } else if (OB_FAIL(allocate_subplan_filter_as_top(top, + dist_subquery_ops, + query_ref_exprs, + params, + onetime_exprs, + initplan_idxs, + onetime_idxs, + filters, + dist_algo, + is_update_set))) { + LOG_WARN("failed to allocate subplan filter as top", K(ret)); + } else { /*do nothing*/ } return ret; } @@ -9851,7 +10003,7 @@ int ObLogPlan::allocate_subplan_filter_as_top(ObLogicalOperator *&top, const ObIArray &onetime_exprs, const ObBitSet<> &initplan_idxs, const ObBitSet<> &onetime_idxs, - const ObIArray *filters, + const ObIArray &filters, const DistAlgo dist_algo, const bool is_update_set) { @@ -9871,7 +10023,7 @@ int ObLogPlan::allocate_subplan_filter_as_top(ObLogicalOperator *&top, } else { spf_node->set_distributed_algo(dist_algo); spf_node->set_update_set(is_update_set); - if (NULL != filters && OB_FAIL(append(spf_node->get_filter_exprs(), *filters))) { + if (OB_FAIL(append(spf_node->get_filter_exprs(), filters))) { LOG_WARN("failed to append filter exprs", K(ret)); } else if (OB_FAIL(spf_node->add_subquery_exprs(query_ref_exprs))) { LOG_WARN("failed to add subquery exprs", K(ret)); diff --git a/src/sql/optimizer/ob_log_plan.h b/src/sql/optimizer/ob_log_plan.h index 8f22353c12..9cdcb308a2 100644 --- a/src/sql/optimizer/ob_log_plan.h +++ b/src/sql/optimizer/ob_log_plan.h @@ -1017,10 +1017,32 @@ public: ObIArray &onetime_exprs, ObBitSet<> &initplan_idxs, ObBitSet<> &onetime_idxs, - const ObIArray *filters, + const ObIArray &filters, const bool or_cursor_expr, const bool is_update_set); + int inner_candi_allocate_subplan_filter(ObIArray> &best_list, + ObIArray> &dist_best_list, + ObIArray &query_refs, + ObIArray ¶ms, + ObIArray &onetime_exprs, + ObBitSet<> &initplan_idxs, + ObBitSet<> &onetime_idxs, + const ObIArray &filters, + const bool for_cursor_expr, + const bool is_update_set, + const int64_t dist_methods, + ObIArray &subquery_plans); + + int prepare_subplan_candidate_list(ObIArray &subplans, + ObIArray ¶ms, + ObIArray> &best_list, + ObIArray> &dist_best_list); + int get_valid_subplan_filter_dist_method(ObIArray &subplans, + const bool for_cursor_expr, + const bool ignore_hint, + int64_t &dist_methods); + int generate_subplan_filter_info(const ObIArray &subquery_exprs, ObIArray &subquery_ops, ObIArray &query_refs, @@ -1038,6 +1060,12 @@ public: int adjust_exprs_with_onetime(ObIArray &exprs); + int get_subplan_filter_distributed_method(ObLogicalOperator *&top, + const ObIArray &subquery_ops, + const ObIArray ¶ms, + const bool for_cursor_expr, + const bool has_onetime, + int64_t &distributed_methods); int create_subplan_filter_plan(ObLogicalOperator *&top, const ObIArray &subquery_ops, const ObIArray &dist_subquery_ops, @@ -1046,8 +1074,8 @@ public: const ObIArray &onetime_exprs, const ObBitSet<> &initplan_idxs, const ObBitSet<> &onetime_idxs, - const bool for_cursor_expr, - const ObIArray *filters, + const int64_t dist_methods, + const ObIArray &filters, const bool is_update_set); int check_contains_recursive_cte(ObIArray &child_ops, @@ -1095,7 +1123,7 @@ public: const ObIArray &onetime_exprs, const ObBitSet<> &initplan_idxs, const ObBitSet<> &onetime_idxs, - const ObIArray *filters, + const ObIArray &filters, const DistAlgo dist_algo, const bool is_update_set); int allocate_subplan_filter_as_top(ObLogicalOperator *&old_top, diff --git a/src/sql/optimizer/ob_log_subplan_filter.cpp b/src/sql/optimizer/ob_log_subplan_filter.cpp index 7d5aaa7630..4a35dca4ae 100644 --- a/src/sql/optimizer/ob_log_subplan_filter.cpp +++ b/src/sql/optimizer/ob_log_subplan_filter.cpp @@ -837,3 +837,77 @@ int ObLogSubPlanFilter::compute_op_parallel_and_server_info() } return ret; } + +int ObLogSubPlanFilter::print_outline_data(PlanText &plan_text) +{ + int ret = OB_SUCCESS; + ObPQSubqueryHint hint; + const ObDMLStmt *stmt = NULL; + ObString qb_name; + if (OB_ISNULL(get_plan()) || OB_ISNULL(stmt = get_plan()->get_stmt())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected NULl", K(ret), K(get_plan()), K(stmt)); + } else if (DistAlgo::DIST_BASIC_METHOD == get_distributed_algo()) { + /* do not print data for basic. need remove this when support split subplan filter op */ + } else if (OB_FAIL(get_sub_qb_names(hint.get_sub_qb_names()))) { + LOG_WARN("fail to get subplan qb_names", K(ret)); + } else if (OB_FAIL(stmt->get_qb_name(qb_name))) { + LOG_WARN("fail to get qb_name", K(ret), K(stmt->get_stmt_id())); + } else { + hint.set_dist_algo(get_distributed_algo()); + hint.set_qb_name(qb_name); + if (OB_FAIL(hint.print_hint(plan_text))) { + LOG_WARN("failed to print pq subquery hint", K(ret)); + } + } + return ret; +} + +int ObLogSubPlanFilter::print_used_hint(PlanText &plan_text) +{ + int ret = OB_SUCCESS; + const ObPQSubqueryHint *explicit_hint = NULL; + const ObPQSubqueryHint *implicit_hint = NULL; + const ObPQSubqueryHint *hint = NULL; + ObSEArray sub_qb_names; + if (OB_ISNULL(get_plan()) || OB_ISNULL(get_plan()->get_stmt())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected NULl", K(ret), K(get_plan())); + } else if (OB_FAIL(get_sub_qb_names(sub_qb_names))) { + LOG_WARN("fail to get subplan qb_names", K(ret)); + } else if (OB_FAIL(get_plan()->get_log_plan_hint().get_valid_pq_subquery_hint(sub_qb_names, + explicit_hint, + implicit_hint))) { + LOG_WARN("fail to get valid pq subquery hint", K(ret)); + } else if (NULL == (hint = (NULL == explicit_hint + && sub_qb_names.count() == get_plan()->get_stmt()->get_subquery_expr_size()) + ? implicit_hint : explicit_hint)) { + /* do nothing */ + } else if (get_distributed_algo() != hint->get_dist_algo()) { + /* do nothing */ + } else if (OB_FAIL(hint->print_hint(plan_text))) { + LOG_WARN("failed to print used pq pq subquery hint", K(ret)); + } + return ret; +} + +int ObLogSubPlanFilter::get_sub_qb_names(ObIArray &sub_qb_names) +{ + int ret = OB_SUCCESS; + sub_qb_names.reuse(); + ObLogicalOperator *child = NULL; + const ObDMLStmt *stmt = NULL; + ObString qb_name; + for (int64_t i = 1; OB_SUCC(ret) && i < get_num_of_child(); i++) { + if (OB_ISNULL(child = get_child(i)) || OB_ISNULL(child->get_plan()) + || OB_ISNULL(stmt = child->get_plan()->get_stmt())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(child), K(stmt)); + } else if (OB_FAIL(stmt->get_qb_name(qb_name))) { + LOG_WARN("fail to get qb_name", K(ret), K(stmt->get_stmt_id())); + } else if (OB_FAIL(sub_qb_names.push_back(qb_name))) { + LOG_WARN("failed to push back", K(ret)); + } else { /*do nothing*/ } + } + return ret; +} diff --git a/src/sql/optimizer/ob_log_subplan_filter.h b/src/sql/optimizer/ob_log_subplan_filter.h index cf49f17d4b..3f6cd1d440 100644 --- a/src/sql/optimizer/ob_log_subplan_filter.h +++ b/src/sql/optimizer/ob_log_subplan_filter.h @@ -131,11 +131,14 @@ public: ObShardingInfo *&out_sharding); virtual int compute_op_parallel_and_server_info() override; + virtual int print_outline_data(PlanText &plan_text) override; + virtual int print_used_hint(PlanText &plan_text) override; private: int extract_exist_style_subquery_exprs(ObRawExpr *expr, ObIArray &exist_style_exprs); int check_expr_contain_row_subquery(const ObRawExpr *expr, bool &contains); + int get_sub_qb_names(ObIArray& sub_qb_names); protected: DistAlgo dist_algo_; common::ObSEArray subquery_exprs_; diff --git a/src/sql/optimizer/ob_logical_operator.cpp b/src/sql/optimizer/ob_logical_operator.cpp index 94b8641dba..e4df532075 100644 --- a/src/sql/optimizer/ob_logical_operator.cpp +++ b/src/sql/optimizer/ob_logical_operator.cpp @@ -3092,11 +3092,30 @@ int ObLogicalOperator::alloc_op_pre(AllocOpContext& ctx) ctx.gen_temp_op_id_ = true; } // disable nodes in COUNT-rownum situation - if (OB_SUCC(ret) && log_op_def::LOG_COUNT == get_type()) { - ObSEArray cur_path; - for (int64_t i = 0; OB_SUCC(ret) && i < get_num_of_child(); i++) { - if (OB_FAIL(get_child(i)->disable_rownum_expr(ctx.disabled_op_set_, cur_path))) { - LOG_WARN("fail to find rownum expr", K(ret)); + if (OB_SUCC(ret) && LOG_COUNT == get_type()) { + ObRawExpr *rownum_expr = NULL; + if (OB_ISNULL(get_stmt())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("stmt is null", K(ret)); + } else if (OB_FAIL(get_stmt()->get_rownum_expr(rownum_expr))) { + LOG_WARN("get rownum expr failed", K(ret)); + } else if (OB_ISNULL(rownum_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("no rownum expr in stmt of count operator", K(ret)); + } else { + ObSysFunRawExpr *sys_rownum_expr = static_cast(rownum_expr); + sys_rownum_expr->set_op_id(op_id_); + } + } + const ObRawExpr *rownum_expr = NULL; + if (OB_SUCC(ret) && OB_SUCC(find_rownum_expr(rownum_expr)) && rownum_expr != NULL) { + const ObSysFunRawExpr *sys_rownum_expr = static_cast(rownum_expr); + uint64_t count_op_id = sys_rownum_expr->get_op_id(); + LOG_DEBUG("the coun_op_id of rownum is", K(count_op_id)); + // rownum expr may be above count + if (count_op_id != OB_INVALID_ID) { + if (OB_FAIL(disable_rownum_expr(ctx.disabled_op_set_, count_op_id))) { + LOG_WARN("fail to disable rownum", K(ret), K(count_op_id)); } } } @@ -5686,36 +5705,35 @@ int ObLogicalOperator::collect_batch_exec_param(void* ctx, return ret; } -int ObLogicalOperator::find_rownum_expr_recursively(bool &found, const ObRawExpr *expr) +int ObLogicalOperator::find_rownum_expr_recursively(const ObRawExpr *&rownum_expr, const ObRawExpr *expr) { int ret = OB_SUCCESS; if (OB_ISNULL(expr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("expr is null", K(ret)); } else if (expr->get_expr_type() == T_FUN_SYS_ROWNUM) { - found = true; + rownum_expr = expr; } else { - for (auto i = 0; OB_SUCC(ret) && !found && i < expr->get_param_count(); i++) { - if (OB_FAIL(SMART_CALL(find_rownum_expr_recursively(found, expr->get_param_expr(i))))) { + for (int64_t i = 0; OB_SUCC(ret) && rownum_expr == NULL && i < expr->get_param_count(); i++) { + if (OB_FAIL(SMART_CALL(find_rownum_expr_recursively(rownum_expr, expr->get_param_expr(i))))) { LOG_WARN("fail to find rownum expr recursively", K(ret)); } } } LOG_DEBUG("find_rownum_expr_recursively finished", K(expr->get_param_count()), - K(expr->get_expr_type()), K(found)); + K(expr->get_expr_type())); return ret; } -int ObLogicalOperator::find_rownum_expr(bool &found, const ObIArray &exprs) +int ObLogicalOperator::find_rownum_expr(const ObRawExpr *&rownum_expr, const ObIArray &exprs) { - LOG_DEBUG("find_rownum_expr begin", K(exprs.count()), K(found)); + LOG_DEBUG("find_rownum_expr begin", K(exprs.count())); int ret = OB_SUCCESS; - for (auto i = 0; OB_SUCC(ret) && !found && i < exprs.count(); i++) { + for (int64_t i = 0; OB_SUCC(ret) && rownum_expr == NULL && i < exprs.count(); i++) { ObRawExpr *expr = exprs.at(i); - ret = find_rownum_expr_recursively(found, expr); - LOG_DEBUG( - "find_rownum_expr_recursively done:", K(expr->get_expr_type()), - K(found), K(i), K(expr->get_param_count())); + ret = find_rownum_expr_recursively(rownum_expr, expr); + LOG_DEBUG("find_rownum_expr_recursively done:", K(expr->get_expr_type()), K(i), + K(expr->get_param_count())); } return ret; } @@ -5725,68 +5743,43 @@ int ObLogicalOperator::find_rownum_expr(bool &found, const ObIArray // - output expr // - join conditions: equal ("=") // - join conditions: filter (">", "<", ">=", "<=") -int ObLogicalOperator::find_rownum_expr(bool &found) +int ObLogicalOperator::find_rownum_expr(const ObRawExpr *&rownum_expr) { int ret = OB_SUCCESS; - LOG_DEBUG("find_rownum_expr debug: ", K(get_name()), K(found)); - if (OB_FAIL(find_rownum_expr(found, get_filter_exprs()))) { + LOG_DEBUG("find_rownum_expr debug: ", K(get_name())); + if (OB_FAIL(find_rownum_expr(rownum_expr, get_filter_exprs()))) { LOG_WARN("failure encountered during find rownum expr", K(ret)); - } else if (OB_FAIL(find_rownum_expr(found, get_output_exprs()))) { + } else if (OB_FAIL(find_rownum_expr(rownum_expr, get_output_exprs()))) { LOG_WARN("failure encountered during find rownum expr", K(ret)); - } else if (!found && get_type() == log_op_def::LOG_JOIN) { + } else if (rownum_expr == NULL && get_type() == log_op_def::LOG_JOIN) { ObLogJoin *join_op = dynamic_cast(this); // NO NPE check for join_op as it should NOT be nullptr if (OB_ISNULL(join_op)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("join op is null", K(ret)); - } else if (OB_FAIL(find_rownum_expr(found, join_op->get_other_join_conditions()))) { + } else if (OB_FAIL(find_rownum_expr(rownum_expr, join_op->get_other_join_conditions()))) { LOG_WARN("failure encountered during find rownum expr", K(ret)); - } else if (OB_FAIL(find_rownum_expr(found, join_op->get_equal_join_conditions()))) { + } else if (OB_FAIL(find_rownum_expr(rownum_expr, join_op->get_equal_join_conditions()))) { LOG_WARN("failure encountered during find rownum expr", K(ret)); } } - - for (auto i = 0; !found && OB_SUCC(ret) && i < get_num_of_child(); i++) { - if (OB_FAIL(SMART_CALL(get_child(i)->find_rownum_expr(found)))) { - LOG_WARN("fail to find rownum expr", K(ret)); - } - } return ret; } /** -Starting from COUNT, search downwards and add all operators on the path from COUNT to rownum() to disabled_op_set. -1. Push current op into the stack. -2. Check if the operator has rownum() first. If yes, merge cur_path into disabled_op_set. - At this point, the children nodes may still have rownum(), so cannot return and need to continue searching recursively. -3. DFS to check children nodes. -4. Pop current op out from the stack and backtrack. +After finding the rownum expression, the count operator is searched from bottom to top, and +the operators on the path are marked as not being able to add materialization. */ -int ObLogicalOperator::disable_rownum_expr(hash::ObHashSet &disabled_op_set, ObIArray &cur_path) +int ObLogicalOperator::disable_rownum_expr(hash::ObHashSet &disabled_op_set, + const uint64_t &count_op_id) { int ret = OB_SUCCESS; - bool found = false; - if (OB_FAIL(cur_path.push_back(op_id_))) { - LOG_WARN("fail to push back path", K(ret)); - } else if (OB_FAIL(find_rownum_expr(found))) { - LOG_WARN("fail to find rownum expr", K(ret)); - } else { - if (found) { - for (int64_t i = 0; OB_SUCC(ret) && i < cur_path.count(); ++i) { - ret = disabled_op_set.set_refactored(op_id_); - if (ret != OB_SUCCESS && ret != OB_HASH_EXIST) { - LOG_WARN("set_refactored fail", K(ret)); - } else { - ret = OB_SUCCESS; - } - } - } - for (int64_t i = 0; OB_SUCC(ret) && i < get_num_of_child(); i++) { - if (OB_FAIL(SMART_CALL(get_child(i)->disable_rownum_expr(disabled_op_set, cur_path)))) { - LOG_WARN("fail to disable rownum expr", K(ret)); - } - } - cur_path.pop_back(); + uint64_t op_id = op_id_; + ObLogicalOperator *parent = get_parent(); + while (OB_SUCC(ret) && op_id != count_op_id && parent != NULL) { + ret = disabled_op_set.set_refactored(op_id); + op_id = parent->get_op_id(); + parent = parent->get_parent(); } return ret; } diff --git a/src/sql/optimizer/ob_logical_operator.h b/src/sql/optimizer/ob_logical_operator.h index a84a70aff7..2611ff5547 100644 --- a/src/sql/optimizer/ob_logical_operator.h +++ b/src/sql/optimizer/ob_logical_operator.h @@ -1358,10 +1358,11 @@ public: */ int alloc_op_pre(AllocOpContext& ctx); int alloc_op_post(AllocOpContext& ctx); - int find_rownum_expr_recursively(bool &found, const ObRawExpr *expr); - int find_rownum_expr(bool &found, const ObIArray &exprs); - int find_rownum_expr(bool &found); + int find_rownum_expr_recursively(const ObRawExpr *&rownum_expr, const ObRawExpr *expr); + int find_rownum_expr(const ObRawExpr *&rownum_expr, const ObIArray &exprs); + int find_rownum_expr(const ObRawExpr *&rownum_expr); int disable_rownum_expr(hash::ObHashSet &disabled_op_set, ObIArray &cur_path); + int disable_rownum_expr(hash::ObHashSet &disabled_op_set, const uint64_t &count_op_id); int gen_temp_op_id(AllocOpContext& ctx); int recursively_disable_alloc_op_above(AllocOpContext& ctx); int alloc_nodes_above(AllocOpContext& ctx, const uint64_t &flags); diff --git a/src/sql/optimizer/ob_optimizer.cpp b/src/sql/optimizer/ob_optimizer.cpp index c631227688..445f1bfc14 100644 --- a/src/sql/optimizer/ob_optimizer.cpp +++ b/src/sql/optimizer/ob_optimizer.cpp @@ -332,6 +332,8 @@ int ObOptimizer::check_pdml_enabled(const ObDMLStmt &stmt, bool session_enable_pdml = false; bool enable_auto_dop = false; uint64_t session_pdml_dop = ObGlobalHint::UNSET_PARALLEL; + bool disable_pdml = false; + bool is_pk_auto_inc = false; if (OB_ISNULL(ctx_.get_exec_ctx()) || OB_ISNULL(query_ctx = ctx_.get_query_ctx())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret), K(ctx_.get_exec_ctx()), K(query_ctx)); @@ -348,8 +350,15 @@ int ObOptimizer::check_pdml_enabled(const ObDMLStmt &stmt, } else if (ctx_.has_var_assign() && !ctx_.is_var_assign_only_in_root_stmt()) { can_use_pdml = false; } else if (stmt::T_INSERT == stmt.get_stmt_type() && - !static_cast< const ObInsertStmt &>(stmt).value_from_select()) { + OB_FAIL(static_cast< const ObInsertStmt &>(stmt).check_pdml_disabled(ctx_.is_online_ddl(), + disable_pdml, + is_pk_auto_inc))) { + LOG_WARN("fail to check pdml disabled for insert stmt", K(ret)); + } else if (disable_pdml) { can_use_pdml = false; + if (is_pk_auto_inc) { + ctx_.add_plan_note(PDML_DISABLED_BY_INSERT_PK_AUTO_INC); + } } else if ((stmt.is_update_stmt() || stmt.is_delete_stmt()) && static_cast(stmt).dml_source_from_join() && static_cast(stmt).is_dml_table_from_join()) { diff --git a/src/sql/optimizer/ob_optimizer_util.cpp b/src/sql/optimizer/ob_optimizer_util.cpp index 57896f1484..4e7fa48d9e 100644 --- a/src/sql/optimizer/ob_optimizer_util.cpp +++ b/src/sql/optimizer/ob_optimizer_util.cpp @@ -425,39 +425,142 @@ int ObOptimizerUtil::adjust_exprs_by_ordering(ObIArray &exprs, } if (OB_SUCC(ret)) { prefix_count = adjusted_exprs.count(); - ordering_all_used = prefix_count > 0; - } - for (int64_t j = 0; OB_SUCC(ret) && j < exprs.count(); ++j) { - if (expr_idxs.has_member(j)) { - //已经和其它序的表达式对应了，不再参与比较 - } else if (OB_FAIL(adjusted_exprs.push_back(exprs.at(j)))) { - LOG_WARN("store ordered expr failed", K(ret), K(j)); - } else if (OB_FAIL(expr_map.push_back(j))) { - LOG_WARN("failed to push back expr index", K(ret)); - } else if (OB_FAIL(order_types.push_back(directions.at(j)))) { - LOG_WARN("failed to push back order type", K(ret)); - } else if (OB_FAIL(expr_idxs.add_member(j))) { - LOG_WARN("add expr idxs member failed", K(ret), K(j)); - } else { - ordering_all_used = false; - } - } - if (OB_SUCC(ret)) { - if (adjusted_exprs.count() != exprs.count() - || order_types.count() != exprs.count()) { + ordering_all_used = prefix_count > 0 && expr_idxs.num_members() == exprs.count(); + if (OB_FAIL(generate_stable_ordering(exprs, directions, expr_idxs, expr_map, + adjusted_exprs, order_types))) { + LOG_WARN("failed to generate stable ordering", K(ret)); + } else if (OB_UNLIKELY(adjusted_exprs.count() != exprs.count() + || order_types.count() != exprs.count())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("exprs don't covered completely by ordering", K(adjusted_exprs.count()), K(exprs.count()), K(order_types.count())); + } else if (OB_FAIL(exprs.assign(adjusted_exprs))) { + LOG_WARN("assign adjusted exprs failed", K(ret)); + } else if (OB_FAIL(directions.assign(order_types))) { + LOG_WARN("failed to assign order types", K(ret)); + } else if (match_map != NULL && OB_FAIL(match_map->assign(expr_map))) { + LOG_WARN("failed to assign expr indexs", K(ret)); + } + } + return ret; +} + +// when there is no input ordering or interesting ordering, generate a stable ordering use this function +int ObOptimizerUtil::generate_stable_ordering(common::ObIArray &exprs, + ObIArray &directions) +{ + int ret = OB_SUCCESS; + ObBitSet<64> used_expr_idxs; + ObSEArray expr_map; + ObSEArray adjusted_exprs; + ObSEArray adjusted_directions; + if (OB_FAIL(generate_stable_ordering(exprs, + directions, + used_expr_idxs, + expr_map, + adjusted_exprs, + adjusted_directions))) { + LOG_WARN("failed to generate stable ordering", K(ret)); + } else if (OB_FAIL(exprs.assign(adjusted_exprs))) { + LOG_WARN("assign adjusted exprs failed", K(ret)); + } else if (OB_FAIL(directions.assign(adjusted_directions))) { + LOG_WARN("failed to assign order types", K(ret)); + } + return ret; +} + +// when there is no input ordering or interesting ordering, generate a stable ordering use this function +int ObOptimizerUtil::generate_stable_ordering(common::ObIArray &exprs, + ObIArray &directions, + ObBitSet<64> &used_expr_idxs, + ObIArray &expr_map, + ObIArray &adjusted_exprs, + ObIArray &adjusted_directions) +{ + int ret = OB_SUCCESS; + ObSEArray, 8> sort_pairs; + int64_t idx = OB_INVALID_INDEX; + if (OB_UNLIKELY(exprs.count() != directions.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected array count", K(ret), K(exprs.count()), K(directions.count())); + } + for (int64_t i = 0; OB_SUCC(ret) && i < exprs.count(); ++i) { + if (!used_expr_idxs.has_member(i) && + OB_FAIL(sort_pairs.push_back(std::pair(exprs.at(i), i)))) { + LOG_WARN("failed to push back", K(ret)); + } + } + if (OB_SUCC(ret)) { + // do not generate stable ordering now, remove comment for this sort when need stable ordering + // std::sort(sort_pairs.begin(), sort_pairs.end(), stable_expr_cmp_func); + } + for (int64_t i = 0; OB_SUCC(ret) && i < sort_pairs.count(); ++i) { + idx = sort_pairs.at(i).second; + if (OB_UNLIKELY(exprs.count() <= idx)) { + LOG_WARN("unexpected idx", K(ret), K(exprs.count()), K(i), K(idx)); + } else if (OB_FAIL(adjusted_exprs.push_back(exprs.at(idx)))) { + LOG_WARN("store ordered expr failed", K(ret), K(idx)); + } else if (OB_FAIL(expr_map.push_back(idx))) { + LOG_WARN("failed to push back expr index", K(ret)); + } else if (OB_FAIL(adjusted_directions.push_back(directions.at(idx)))) { + LOG_WARN("failed to push back order type", K(ret)); + } else if (OB_FAIL(used_expr_idxs.add_member(idx))) { + LOG_WARN("add expr idxs member failed", K(ret), K(idx)); + } + } + return ret; +} + +// used to generate specific ordering exprs +bool ObOptimizerUtil::stable_expr_cmp_func(std::pair l_pair, + std::pair r_pair) +{ + int ret = OB_SUCCESS; + bool bret = false; + int64_t res = 0; + if (OB_FAIL(cmp_expr(l_pair.first, r_pair.first, res))) { + LOG_WARN("failed to compare two expr", K(ret), K(res), K(lbt()), KPC(l_pair.first), KPC(r_pair.first)); + } else { + bret = (res <= 0); + } + return bret; +} + +// used to generate specific ordering exprs +int ObOptimizerUtil::cmp_expr(ObRawExpr *l_expr, ObRawExpr *r_expr, int64_t &res) +{ + int ret = OB_SUCCESS; + res = 0; + if (OB_ISNULL(l_expr) || OB_ISNULL(r_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(l_expr), K(r_expr)); + } else if (l_expr->get_expr_type() != r_expr->get_expr_type()) { + if (l_expr->is_column_ref_expr() || r_expr->is_column_ref_expr()) { + res = l_expr->is_column_ref_expr() ? -1 : 1; } else { - exprs.reuse(); - if (OB_FAIL(exprs.assign(adjusted_exprs))) { - LOG_WARN("assign adjusted exprs failed", K(ret)); - } else if (OB_FAIL(directions.assign(order_types))) { - LOG_WARN("failed to assign order types", K(ret)); - } else if (match_map != NULL && OB_FAIL(match_map->assign(expr_map))) { - LOG_WARN("failed to assign expr indexs", K(ret)); + res = l_expr->get_expr_type() < r_expr->get_expr_type() ? -1 : 1; + } + } else if (l_expr->is_column_ref_expr()) { + ObColumnRefRawExpr *l_col = static_cast(l_expr); + ObColumnRefRawExpr *r_col = static_cast(r_expr); + if (l_col->get_table_id() != r_col->get_table_id()) { + res = l_col->get_table_id() < r_col->get_table_id() ? -1 : 1; + } else if (l_col->get_column_id() != r_col->get_column_id()) { + res = l_col->get_column_id() < r_col->get_column_id() ? -1 : 1; + } + } else if (l_expr->is_set_op_expr()) { + res = static_cast(l_expr)->get_idx() < static_cast(r_expr)->get_idx() + ? -1 : 1; + } else if (l_expr->get_param_count() != r_expr->get_param_count()) { + res = l_expr->get_param_count() < r_expr->get_param_count() ? -1 : 1; + } else if (0 != l_expr->get_param_count()) { + for (int64_t i = 0; 0 == res && OB_SUCC(ret) && i < l_expr->get_param_count(); ++i) { + if (OB_FAIL(SMART_CALL(cmp_expr(l_expr->get_param_expr(i), r_expr->get_param_expr(i), res)))) { + LOG_WARN("failed to smart call compare two expr", K(ret), K(res)); } } + } else { + /* do nothing now */ } return ret; } @@ -4615,6 +4718,8 @@ int ObOptimizerUtil::simplify_exprs(const ObFdItemSet &fd_item_set, ObRawExprSet *min_parent_exprs = NULL; bool is_contain = false; bool is_in_child = false; + ObSqlBitSet<> root_exprs_set; + int64_t expr_idx = -1; root_exprs.reset(); // 将 fd item 分为unique的和非unique的 for (int64_t i = 0; OB_SUCC(ret) && i < fd_item_set.count(); ++i) { @@ -4647,11 +4752,10 @@ int ObOptimizerUtil::simplify_exprs(const ObFdItemSet &fd_item_set, } // 找到一个unique fd item, 直接从candi exprs里找出匹配parent set的expr if (OB_SUCC(ret) && NULL != min_parent_exprs) { - int64_t expr_idx = -1; for (int64_t i = 0; OB_SUCC(ret) && i < min_parent_exprs->count(); ++i) { if (find_equal_expr(candi_exprs, min_parent_exprs->at(i), equal_sets, expr_idx)) { - if (OB_FAIL(root_exprs.push_back(candi_exprs.at(expr_idx)))) { - LOG_WARN("failed to push back root expr", K(ret)); + if (OB_FAIL(root_exprs_set.add_member(expr_idx))) { + LOG_WARN("failed to add member", K(ret)); } } else { /* not find means min_parent_exprs->at(i) is const, do nothing */ } } @@ -4671,7 +4775,6 @@ int ObOptimizerUtil::simplify_exprs(const ObFdItemSet &fd_item_set, } else if (is_contain) { ObSEArray left_domain; ObSEArray right_domain; - int64_t expr_idx = -1; // 生成新的root exprs集合 for (int64_t j = 0; OB_SUCC(ret) && j < root_exprs.count(); ++j) { if (OB_FAIL(left_domain.push_back(root_exprs.at(j)))) { @@ -4720,18 +4823,35 @@ int ObOptimizerUtil::simplify_exprs(const ObFdItemSet &fd_item_set, } } } + for (int64_t i = 0; OB_SUCC(ret) && i < root_exprs.count(); ++i) { + if(OB_UNLIKELY(!find_item(candi_exprs, root_exprs.at(i), &expr_idx))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to find item", K(ret)); + } else if (OB_FAIL(root_exprs_set.add_member(expr_idx))) { + LOG_WARN("failed to add member", K(ret)); + } + } // 所有non unique fd item都遍历过了, 但是还有其它无法被决定的expr, 直接加入到root exprs中 if (OB_SUCC(ret) && eliminate_set.num_members() < candi_exprs.count()) { for (int64_t i = 0; OB_SUCC(ret) && i < candi_exprs.count(); ++i) { if (eliminate_set.has_member(i)) { // do nothing - } else if (OB_FAIL(root_exprs.push_back(candi_exprs.at(i)))) { - LOG_WARN("failed to push back candi expr", K(ret)); + } else if (OB_FAIL(root_exprs_set.add_member(i))) { + LOG_WARN("failed to add member", K(ret)); } } } } if (OB_SUCC(ret)) { + root_exprs.reuse(); + // use bitset to keep exprs ordering + for (int64_t i = 0; OB_SUCC(ret) && i < candi_exprs.count(); ++i) { + if (!root_exprs_set.has_member(i)) { + // do nothing + } else if (OB_FAIL(root_exprs.push_back(candi_exprs.at(i)))) { + LOG_WARN("failed to push back candi expr", K(ret)); + } + } LOG_TRACE("succeed to simply exprs", K(fd_item_set), K(equal_sets), K(const_exprs), K(candi_exprs), K(root_exprs)); } diff --git a/src/sql/optimizer/ob_optimizer_util.h b/src/sql/optimizer/ob_optimizer_util.h index d665c8564b..5c11407557 100644 --- a/src/sql/optimizer/ob_optimizer_util.h +++ b/src/sql/optimizer/ob_optimizer_util.h @@ -127,6 +127,18 @@ public: bool &ordering_all_used, common::ObIArray &directions, common::ObIArray *match_map = NULL); + // when there is no input ordering or interesting ordering, generate a stable ordering use this function + static int generate_stable_ordering(common::ObIArray &exprs, + ObIArray &directions); + static int generate_stable_ordering(common::ObIArray &exprs, + ObIArray &directions, + ObBitSet<64> &used_expr_idxs, + ObIArray &expr_map, + ObIArray &adjusted_exprs, + ObIArray &adjusted_directions); + static bool stable_expr_cmp_func(std::pair l_pair, + std::pair r_pair); + static int cmp_expr(ObRawExpr *l_expr, ObRawExpr *r_expr, int64_t &res); static int adjust_exprs_by_mapping(const common::ObIArray &exprs, const common::ObIArray &match_map, diff --git a/src/sql/parser/gen_parser.sh b/src/sql/parser/gen_parser.sh index 051f0e02c6..6fe26d54f1 100755 --- a/src/sql/parser/gen_parser.sh +++ b/src/sql/parser/gen_parser.sh @@ -40,6 +40,7 @@ flex -Cfa -B -8 -o ../../../src/sql/parser/sql_parser_mysql_mode_lex.c ../../../ sed "/Setup the input buffer state to scan the given bytes/,/}/{/int i/d}" -i ../../../src/sql/parser/sql_parser_mysql_mode_lex.c sed "/Setup the input buffer state to scan the given bytes/,/}/{/for ( i = 0; i < _yybytes_len; ++i )/d}" -i ../../../src/sql/parser/sql_parser_mysql_mode_lex.c sed "/Setup the input buffer state to scan the given bytes/,/}/{s/\tbuf\[i\] = yybytes\[i\]/memcpy(buf, yybytes, _yybytes_len)/g}" -i ../../../src/sql/parser/sql_parser_mysql_mode_lex.c +sed "/obsql_mysql_yylex_init is special because it creates the scanner itself/,/Initialization is the same as for the non-reentrant scanner/{s/return 1/return errno/g}" -i ../../../src/sql/parser/sql_parser_mysql_mode_lex.c if [ -d "../../../close_modules/oracle_parser/sql/parser" ]; then @@ -82,6 +83,7 @@ flex -o ../../../src/sql/parser/sql_parser_oracle_latin1_mode_lex.c ../../../src sed "/Setup the input buffer state to scan the given bytes/,/}/{/int i/d}" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode_lex.c sed "/Setup the input buffer state to scan the given bytes/,/}/{/for ( i = 0; i < _yybytes_len; ++i )/d}" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode_lex.c sed "/Setup the input buffer state to scan the given bytes/,/}/{s/\tbuf\[i\] = yybytes\[i\]/memcpy(buf, yybytes, _yybytes_len)/g}" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode_lex.c +sed "/obsql_oracle_latin1_yylex_init is special because it creates the scanner itself/,/Initialization is the same as for the non-reentrant scanner/{s/return 1/return errno/g}" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode_lex.c cat ../../../src/sql/parser/non_reserved_keywords_oracle_mode.c > ../../../src/sql/parser/non_reserved_keywords_oracle_latin1_mode.c sed '/#include "ob_non_reserved_keywords.h"/a\#include "sql/parser/sql_parser_oracle_latin1_mode_tab.h\"' -i ../../../src/sql/parser/non_reserved_keywords_oracle_latin1_mode.c sed "s/non_reserved_keywords_oracle_mode.c is for …/non_reserved_keywords_oracle_latin1_mode.c is auto generated by gen_parser.sh/g" -i ../../../src/sql/parser/non_reserved_keywords_oracle_latin1_mode.c @@ -139,6 +141,7 @@ flex -o ../../../src/sql/parser/sql_parser_oracle_utf8_mode_lex.c ../../../src/s sed "/Setup the input buffer state to scan the given bytes/,/}/{/int i/d}" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode_lex.c sed "/Setup the input buffer state to scan the given bytes/,/}/{/for ( i = 0; i < _yybytes_len; ++i )/d}" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode_lex.c sed "/Setup the input buffer state to scan the given bytes/,/}/{s/\tbuf\[i\] = yybytes\[i\]/memcpy(buf, yybytes, _yybytes_len)/g}" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode_lex.c +sed "/obsql_oracle_utf8_yylex_init is special because it creates the scanner itself/,/Initialization is the same as for the non-reentrant scanner/{s/return 1/return errno/g}" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode_lex.c cat ../../../src/sql/parser/non_reserved_keywords_oracle_mode.c > ../../../src/sql/parser/non_reserved_keywords_oracle_utf8_mode.c sed '/#include "ob_non_reserved_keywords.h"/a\#include "sql/parser/sql_parser_oracle_utf8_mode_tab.h\"' -i ../../../src/sql/parser/non_reserved_keywords_oracle_utf8_mode.c sed "s/non_reserved_keywords_oracle_mode.c is for …/non_reserved_keywords_oracle_utf8_mode.c is auto generated by gen_parser.sh/g" -i ../../../src/sql/parser/non_reserved_keywords_oracle_utf8_mode.c @@ -194,6 +197,7 @@ flex -o ../../../src/sql/parser/sql_parser_oracle_gbk_mode_lex.c ../../../src/sq sed "/Setup the input buffer state to scan the given bytes/,/}/{/int i/d}" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode_lex.c sed "/Setup the input buffer state to scan the given bytes/,/}/{/for ( i = 0; i < _yybytes_len; ++i )/d}" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode_lex.c sed "/Setup the input buffer state to scan the given bytes/,/}/{s/\tbuf\[i\] = yybytes\[i\]/memcpy(buf, yybytes, _yybytes_len)/g}" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode_lex.c +sed "/obsql_oracle_gbk_yylex_init is special because it creates the scanner itself/,/Initialization is the same as for the non-reentrant scanner/{s/return 1/return errno/g}" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode_lex.c cat ../../../src/sql/parser/non_reserved_keywords_oracle_mode.c > ../../../src/sql/parser/non_reserved_keywords_oracle_gbk_mode.c sed '/#include "ob_non_reserved_keywords.h"/a\#include "sql/parser/sql_parser_oracle_gbk_mode_tab.h\"' -i ../../../src/sql/parser/non_reserved_keywords_oracle_gbk_mode.c sed "s/non_reserved_keywords_oracle_mode.c is for …/non_reserved_keywords_oracle_gbk_mode.c is auto generated by gen_parser.sh/g" -i ../../../src/sql/parser/non_reserved_keywords_oracle_gbk_mode.c diff --git a/src/sql/parser/non_reserved_keywords_mysql_mode.c b/src/sql/parser/non_reserved_keywords_mysql_mode.c index 9c07c9f32b..5c1dc2c4cc 100644 --- a/src/sql/parser/non_reserved_keywords_mysql_mode.c +++ b/src/sql/parser/non_reserved_keywords_mysql_mode.c @@ -809,6 +809,7 @@ static const NonReservedKeyword Mysql_none_reserved_keywords[] = {"sql_buffer_result", SQL_BUFFER_RESULT}, {"sql_cache", SQL_CACHE}, {"sql_id", SQL_ID}, + {"schema_id", SCHEMA_ID}, {"sql_no_cache", SQL_NO_CACHE}, {"sql_thread", SQL_THREAD}, {"sql_tsi_day", SQL_TSI_DAY}, diff --git a/src/sql/parser/ob_fast_parser.cpp b/src/sql/parser/ob_fast_parser.cpp index 207b9b6315..b080bb6c97 100644 --- a/src/sql/parser/ob_fast_parser.cpp +++ b/src/sql/parser/ob_fast_parser.cpp @@ -449,6 +449,8 @@ inline int64_t ObFastParserBase::is_identifier_flags(const int64_t pos) // Most of the time, if it is not an identifier character, it maybe a space, // comma, opening parenthesis, or closing parenthesis. This judgment logic is // added here to avoid the next judgment whether it is utf8 char or gbk char + } else if (!is_oracle_mode_) { + idf_pos = notascii_gb_char(pos); } else if (CHARSET_UTF8MB4 == charset_type_ || CHARSET_UTF16 == charset_type_) { idf_pos = is_utf8_char(pos); } else if (ObCharset::is_gb_charset(charset_type_)) { @@ -921,6 +923,17 @@ int ObFastParserBase::get_one_insert_row_str(ObRawSql &raw_sql, return ret; } +inline int64_t ObFastParserBase::notascii_gb_char(const int64_t pos) +{ + int64_t idf_pos = -1; + if (notascii(raw_sql_.char_at(pos))) { + idf_pos = pos + 1; + } else { + idf_pos = is_gbk_char(pos); + } + return idf_pos; +} + inline int64_t ObFastParserBase::is_latin1_char(const int64_t pos) { int64_t idf_pos = -1; @@ -1181,7 +1194,7 @@ int64_t ObFastParserBase::is_hint_begin(int64_t pos) ch = raw_sql_.char_at(pos); next_ch = raw_sql_.char_at(++pos); // check and ignore comment - while (ch != '*' && next_ch != '/' && !raw_sql_.is_search_end()) { + while (ch != '*' && next_ch != '/' && !raw_sql_.is_search_end(pos)) { ch = raw_sql_.char_at(pos); next_ch = raw_sql_.char_at(++pos); } @@ -1672,6 +1685,8 @@ inline int64_t ObFastParserBase::is_first_identifier_flags(const int64_t pos) // Most of the time, if it is not an identifier character, it maybe a space, // comma, opening parenthesis, or closing parenthesis. This judgment logic is // added here to avoid the next judgment whether it is utf8 char or gbk char + } else if (!is_oracle_mode_) { + idf_pos = notascii_gb_char(pos); } else if (CHARSET_UTF8MB4 == charset_type_ || CHARSET_UTF16 == charset_type_) { idf_pos = is_utf8_char(pos); } else if (ObCharset::is_gb_charset(charset_type_)) { @@ -2754,16 +2769,26 @@ int ObFastParserMysql::parse_next_token() } case '-': { // need to deal with sql_comment or negative sign - int64_t space_len = 0; ch = raw_sql_.scan(); - if ('-' == ch && IS_MULTI_SPACE(raw_sql_.cur_pos_ + 1, space_len)) { - // "--"{space}+{non_newline}* + if ('-' == ch && + raw_sql_.cur_pos_ + 1 < raw_sql_.raw_sql_len_ && + (raw_sql_.raw_sql_[raw_sql_.cur_pos_ + 1] == ' ' || + raw_sql_.raw_sql_[raw_sql_.cur_pos_ + 1] == '\t')) { + // "--"[ \t]+{non_newline}* cur_token_type_ = IGNORE_TOKEN; // skip the second '-' and space - raw_sql_.scan(1 + space_len); + raw_sql_.scan(1); while (!raw_sql_.is_search_end() && is_non_newline(ch)) { ch = raw_sql_.scan(); } + } else if ('-' == ch && + raw_sql_.cur_pos_ + 1 < raw_sql_.raw_sql_len_ && + (raw_sql_.raw_sql_[raw_sql_.cur_pos_ + 1] == '\n' || + raw_sql_.raw_sql_[raw_sql_.cur_pos_ + 1] == '\r')) { + // "--"[\n\r] + cur_token_type_ = IGNORE_TOKEN; + //skip the second '-' and ('\n' or \r) + raw_sql_.scan(1); } else { OZ (process_negative()); } diff --git a/src/sql/parser/ob_fast_parser.h b/src/sql/parser/ob_fast_parser.h index 5b8b4dfda0..e89a6399ab 100644 --- a/src/sql/parser/ob_fast_parser.h +++ b/src/sql/parser/ob_fast_parser.h @@ -87,6 +87,10 @@ struct ObRawSql { { return search_end_ || cur_pos_ > raw_sql_len_ - 1; } + inline bool is_search_end(const int64_t pos) + { + return search_end_ || pos > raw_sql_len_ - 1; + } inline char peek() { if (cur_pos_ >= raw_sql_len_ - 1) { @@ -377,6 +381,7 @@ protected: return is_valid_char(ch) && USER_VAR_CHAR[static_cast(ch)]; } void reset_parser_node(ParseNode *node); + int64_t notascii_gb_char(const int64_t pos); //{U} int64_t is_latin1_char(const int64_t pos); // ({U_2}{U}|{U_3}{U}{U}|{U_4}{U}{U}{U} @@ -436,6 +441,11 @@ protected: return is_valid_char(ch) && static_cast(ch) >= 0x40 && static_cast(ch) <= 0xfe; } + inline bool notascii(char ch) + { + return is_valid_char(ch) && + (static_cast(ch) >= 0x80 && static_cast(ch) <= 0xFF); + } inline bool is_latin1(char ch) { return is_valid_char(ch) && diff --git a/src/sql/parser/sql_parser_base.c b/src/sql/parser/sql_parser_base.c index c360633258..be85268dd2 100644 --- a/src/sql/parser/sql_parser_base.c +++ b/src/sql/parser/sql_parser_base.c @@ -62,7 +62,7 @@ int parse_init(ParseResult *p) static __thread char error_msg[MAX_ERROR_MSG] = {'\0'}; p->error_msg_ = error_msg; if (OB_UNLIKELY(NULL == p || NULL == p->malloc_pool_)) { - ret = -1; + ret = OB_PARSER_ERR_UNEXPECTED; if (NULL != p) { (void)snprintf(p->error_msg_, MAX_ERROR_MSG, "malloc_pool_ must be set"); } @@ -108,6 +108,15 @@ int parse_init(ParseResult *p) #ifdef OB_BUILD_ORACLE_PARSER } #endif +#define ENOMEM 12 /* Out of memory */ + //refine parser error code to OB error code + if (0 == ret) { + ret = OB_PARSER_SUCCESS; + } else if (ENOMEM == ret) { + ret = OB_PARSER_ERR_NO_MEMORY; + } else { + ret = OB_PARSER_ERR_PARSE_SQL; + } } return ret; } diff --git a/src/sql/parser/sql_parser_mysql_mode.l b/src/sql/parser/sql_parser_mysql_mode.l index 06a119459d..ae764543d5 100644 --- a/src/sql/parser/sql_parser_mysql_mode.l +++ b/src/sql/parser/sql_parser_mysql_mode.l @@ -38,14 +38,12 @@ extern void obsql_mysql_parser_fatal_error(int32_t errcoyyde, yyscan_t yyscanner /* the adq is used to process dq in ANSI_QUOTES sql_mode*/ %x adq -U [\x80-\xbf] -U_2 [\xc2-\xdf] -U_3 [\xe0-\xef] -U_4 [\xf0-\xf4] +NOTASCII [\x80-\xFF] GB_1 [\x81-\xfe] GB_2 [\x40-\xfe] GB_3 [\x30-\x39] -UTF8_GB_CHAR ({U_2}{U}|{U_3}{U}{U}|{U_4}{U}{U}{U}|{GB_1}{GB_2}|{GB_1}{GB_3}{GB_1}{GB_3}) +NOTASCII_GB_CHAR ({NOTASCII}|{GB_1}{GB_2}|{GB_1}{GB_3}{GB_1}{GB_3}) + space [ \t\n\r\f] non_newline [^\n\r] sql_comment ("--"[ \t]+{non_newline}*)|(#{non_newline}*|"--"[\n\r]) @@ -62,7 +60,7 @@ common_hint_begin (\/\*\+({space}*hint{space}+)?) c_cmt_begin \/\* c_cmt_end \*+\/ comment ({sql_comment}) -identifier (([A-Za-z0-9$_]|{UTF8_GB_CHAR})+) +identifier (([A-Za-z0-9$_]|{NOTASCII_GB_CHAR})+) system_variable (@@[A-Za-z_][A-Za-z0-9_]*)|(@@[`][`A-Za-z_][`A-Za-z_]*) user_variable (@[A-Za-z0-9_\.$]*)|(@[`'\"][`'\"A-Za-z0-9_\.$/%]*) version_num ([0-9]+\.+[0-9]*) @@ -135,6 +133,7 @@ NULL { check_value(yylval); ParseResult *p = (ParseResult *)yyextra; malloc_new_node(yylval->node, p->malloc_pool_, T_NULL, 0); + CHECK_MYSQL_COMMENT(p, yylval->node); COPY_STRING(p->input_sql_ + yylloc->first_column - 1, yyleng, yylval->node->raw_text_); yylval->node->text_len_ = yyleng; if (IS_FAST_PARAMETERIZE) { @@ -152,6 +151,7 @@ NULL { ParseResult *p = (ParseResult *)yyextra; malloc_new_node(node, p->malloc_pool_, T_INT, 0); yylval->node = node; + CHECK_MYSQL_COMMENT(p, yylval->node); int err_no = 0; COPY_NUM_STRING(p, node); // we treated '- 12' as a const node, that is to say node->str_value_ = '- 12' @@ -205,6 +205,7 @@ NULL { malloc_new_node(node, p->malloc_pool_, T_DOUBLE, 0); check_value(yylval); yylval->node = node; + CHECK_MYSQL_COMMENT(p, yylval->node); COPY_NUM_STRING(p, node); node->raw_text_ = node->str_value_; node->text_len_ = node->str_len_; @@ -224,6 +225,7 @@ NULL { malloc_new_node(node, p->malloc_pool_, T_NUMBER/* should be T_NUMBER,*/, 0); check_value(yylval); yylval->node = node; + CHECK_MYSQL_COMMENT(p, yylval->node); COPY_NUM_STRING(p, node); node->raw_text_ = node->str_value_; node->text_len_ = node->str_len_; @@ -240,6 +242,7 @@ TRUE { check_value(yylval); ParseResult *p = (ParseResult *)yyextra; malloc_new_node(yylval->node, p->malloc_pool_, T_BOOL, 0); + CHECK_MYSQL_COMMENT(p, yylval->node); yylval->node->value_ = 1; COPY_STRING(p->input_sql_ + yylloc->first_column - 1, yyleng, yylval->node->raw_text_); yylval->node->text_len_ = yyleng; @@ -265,6 +268,7 @@ FALSE { check_value(yylval); ParseResult *p = (ParseResult *)yyextra; malloc_new_node(yylval->node, p->malloc_pool_, T_BOOL, 0); + CHECK_MYSQL_COMMENT(p, yylval->node); yylval->node->value_ = 0; COPY_STRING(p->input_sql_ + yylloc->first_column - 1, yyleng, yylval->node->raw_text_); yylval->node->text_len_ = yyleng; @@ -298,6 +302,7 @@ FALSE { } check_value(yylval); malloc_new_node(yylval->node, p->malloc_pool_, T_NCHAR, 0); + CHECK_MYSQL_COMMENT(p, yylval->node); yylval->node->str_len_ = 0; if (IS_FAST_PARAMETERIZE && !IS_NEED_PARAMETERIZE) { @@ -312,6 +317,7 @@ FALSE { p->start_col_ = yylloc->first_column; check_value(yylval); malloc_new_node(yylval->node, p->malloc_pool_, T_VARCHAR, 0); + CHECK_MYSQL_COMMENT(p, yylval->node); yylval->node->str_len_ = 0; p->last_escape_check_pos_ = 0; yylval->node->str_value_ = NULL; @@ -412,6 +418,7 @@ FALSE { } malloc_new_node(yylval->node->children_[0], p->malloc_pool_, T_CONCAT_STRING, 0); + CHECK_MYSQL_COMMENT(p, yylval->node->children_[0]); (*yylval->node->children_)->str_value_ = parse_strndup(tmp_literal, yylval->node->str_len_ + 1, p->malloc_pool_); check_malloc((*yylval->node->children_)->str_value_); @@ -459,6 +466,7 @@ FALSE { check_malloc(*tmp_literal); } malloc_new_node(yylval->node, p->malloc_pool_, T_VARCHAR, 0); + CHECK_MYSQL_COMMENT(p, yylval->node); yylval->node->str_len_ = 0; p->last_escape_check_pos_ = 0; if (IS_FAST_PARAMETERIZE && !IS_NEED_PARAMETERIZE) { @@ -553,6 +561,7 @@ FALSE { } malloc_new_node(yylval->node->children_[0], p->malloc_pool_, T_CONCAT_STRING, 0); + CHECK_MYSQL_COMMENT(p, yylval->node->children_[0]); (*yylval->node->children_)->str_value_ = parse_strndup(tmp_literal, yylval->node->str_len_ + 1, p->malloc_pool_); check_malloc((*yylval->node->children_)->str_value_); @@ -730,6 +739,7 @@ X'([0-9A-F])*'|0X([0-9A-F])+ { node->str_len_ = 0; } yylval->node = node; + CHECK_MYSQL_COMMENT(p, yylval->node); COPY_STRING(p->input_sql_ + yylloc->first_column - 1, yyleng, yylval->node->raw_text_); yylval->node->text_len_ = yyleng; yylval->node->is_copy_raw_text_ = 1; @@ -773,6 +783,7 @@ B'([01])*'|0B([01])+ { node->str_len_ = 0; } yylval->node = node; + CHECK_MYSQL_COMMENT(p, yylval->node); COPY_STRING(p->input_sql_ + yylloc->first_column - 1, yyleng, yylval->node->raw_text_); yylval->node->text_len_ = yyleng; yylval->node->is_copy_raw_text_ = 1; @@ -1038,6 +1049,7 @@ Timestamp{whitespace}?\"[^\"]*\" { PQ_DISTRIBUTE_WINDOW { return PQ_DISTRIBUTE_WINDOW; } PQ_MAP { return PQ_MAP; } PQ_SET { return PQ_SET; } +PQ_SUBQUERY { return PQ_SUBQUERY; } RANDOM_LOCAL { return RANDOM_LOCAL; } RANDOM { return RANDOM; } BROADCAST { return BROADCAST; } diff --git a/src/sql/parser/sql_parser_mysql_mode.y b/src/sql/parser/sql_parser_mysql_mode.y index ec1557a8b7..ccbe3d68f7 100644 --- a/src/sql/parser/sql_parser_mysql_mode.y +++ b/src/sql/parser/sql_parser_mysql_mode.y @@ -185,7 +185,7 @@ USE_HASH_AGGREGATION NO_USE_HASH_AGGREGATION PARTITION_SORT NO_PARTITION_SORT WF_TOPN USE_LATE_MATERIALIZATION NO_USE_LATE_MATERIALIZATION PX_JOIN_FILTER NO_PX_JOIN_FILTER PX_PART_JOIN_FILTER NO_PX_PART_JOIN_FILTER -PQ_MAP PQ_DISTRIBUTE PQ_DISTRIBUTE_WINDOW PQ_SET RANDOM_LOCAL BROADCAST BC2HOST LIST +PQ_MAP PQ_DISTRIBUTE PQ_DISTRIBUTE_WINDOW PQ_SET PQ_SUBQUERY RANDOM_LOCAL BROADCAST BC2HOST LIST GBY_PUSHDOWN NO_GBY_PUSHDOWN USE_HASH_DISTINCT NO_USE_HASH_DISTINCT DISTINCT_PUSHDOWN NO_DISTINCT_PUSHDOWN @@ -340,7 +340,7 @@ END_P SET_VAR DELIMITER SERVER_IP SERVER_PORT SERVER_TYPE SERVICE SESSION SESSION_USER SET_MASTER_CLUSTER SET_SLAVE_CLUSTER SET_TP SHARE SHUTDOWN SIGNED SIMPLE SKIP_INDEX SLAVE SLOW SLOT_IDX SNAPSHOT SOCKET SOME SONAME SOUNDS SOURCE SPFILE SPLIT SQL_AFTER_GTIDS SQL_AFTER_MTS_GAPS SQL_BEFORE_GTIDS SQL_BUFFER_RESULT - SQL_CACHE SQL_NO_CACHE SQL_ID SQL_THREAD SQL_TSI_DAY SQL_TSI_HOUR SQL_TSI_MINUTE SQL_TSI_MONTH + SQL_CACHE SQL_NO_CACHE SQL_ID SCHEMA_ID SQL_THREAD SQL_TSI_DAY SQL_TSI_HOUR SQL_TSI_MINUTE SQL_TSI_MONTH SQL_TSI_QUARTER SQL_TSI_SECOND SQL_TSI_WEEK SQL_TSI_YEAR SRID STANDBY STAT START STARTS STATS_AUTO_RECALC STATS_PERSISTENT STATS_SAMPLE_PAGES STATUS STATEMENTS STATISTICS STD STDDEV STDDEV_POP STDDEV_SAMP STRONG SYNCHRONIZATION SYNCHRONOUS STOP STORAGE STORAGE_FORMAT_VERSION STORE STORING STRING @@ -475,7 +475,7 @@ END_P SET_VAR DELIMITER %type ls opt_tenant_list_or_ls_or_tablet_id ls_server_or_server_or_zone_or_tenant add_or_alter_zone_option %type opt_tenant_list_v2 %type suspend_or_resume tenant_name opt_tenant_name cache_name opt_cache_name file_id opt_file_id cancel_task_type -%type sql_id_expr opt_sql_id +%type sql_id_or_schema_id_expr opt_sql_id_or_schema_id %type namespace_expr opt_namespace %type server_action server_list opt_server_list %type zone_action upgrade_action @@ -507,7 +507,7 @@ END_P SET_VAR DELIMITER %type optimize_stmt %type dump_memory_stmt %type create_savepoint_stmt rollback_savepoint_stmt release_savepoint_stmt -%type opt_qb_name parallel_hint pq_set_hint_desc +%type opt_qb_name opt_qb_name_list_with_quotes parallel_hint pq_set_hint_desc pq_subquery_hint_desc %type create_tablespace_stmt drop_tablespace_stmt tablespace rotate_master_key_stmt %type alter_tablespace_stmt %type permanent_tablespace permanent_tablespace_options permanent_tablespace_option alter_tablespace_actions alter_tablespace_action opt_force_purge @@ -968,6 +968,7 @@ STRING_VALUE %prec LOWER_THAN_COMP $$->sql_str_off_ = @1.first_column; @$.first_column = @1.first_column; @$.last_column = @1.last_column; + $$->is_forbid_parameter_ = $1->is_forbid_parameter_; } | charset_introducer STRING_VALUE { @@ -977,6 +978,7 @@ STRING_VALUE %prec LOWER_THAN_COMP $$->raw_text_ = $2->raw_text_; $$->text_len_ = $2->text_len_; $$->sql_str_off_ = $2->sql_str_off_; + $$->is_forbid_parameter_ = $2->is_forbid_parameter_; } | charset_introducer HEX_STRING_VALUE { @@ -987,6 +989,7 @@ STRING_VALUE %prec LOWER_THAN_COMP $$->raw_text_ = $2->raw_text_; $$->text_len_ = $2->text_len_; $$->sql_str_off_ = $2->sql_str_off_; + $$->is_forbid_parameter_ = $2->is_forbid_parameter_; } | STRING_VALUE string_val_list %prec LOWER_THAN_COMP { @@ -1106,7 +1109,6 @@ literal { $$ = $1; $$->sql_str_off_ = $1->sql_str_off_; - CHECK_MYSQL_COMMENT(result, $$); } | SYSTEM_VARIABLE { $$ = $1; } | QUESTIONMARK { $$ = $1; } @@ -7890,7 +7892,7 @@ create_with_opt_hint opt_replace opt_algorithm opt_definer opt_sql_security VIEW UNUSED($2); UNUSED($3); UNUSED($4); - malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_VIEW, 11, + malloc_non_terminal_node($$, result->malloc_pool_, T_CREATE_VIEW, 12, NULL, /* opt_materialized */ $6, /* view name */ $7, /* column list */ @@ -7899,7 +7901,7 @@ create_with_opt_hint opt_replace opt_algorithm opt_definer opt_sql_security VIEW NULL, $11, /* with option */ NULL, /* force view opt */ - NULL, NULL, NULL + NULL, NULL, NULL, NULL ); dup_expr_string($10, result, @10.first_column, @10.last_column); $$->reserved_ = 1; /* is alter view */ @@ -10434,6 +10436,11 @@ INDEX_HINT '(' qb_name_option relation_factor_in_hint NAME_OB ')' { $$ = $3; } +| PQ_SUBQUERY '('qb_name_option opt_comma pq_subquery_hint_desc ')' +{ + (void)($4); /* unused */ + malloc_non_terminal_node($$, result->malloc_pool_, T_PQ_SUBQUERY, 3, $3, $5->children_[0], $5->children_[1]); +} | GBY_PUSHDOWN opt_qb_name { malloc_non_terminal_node($$, result->malloc_pool_, T_GBY_PUSHDOWN, 1, $2); @@ -10563,6 +10570,26 @@ pq_set_hint_desc: } ; +pq_subquery_hint_desc: +opt_qb_name_list_with_quotes distribute_method_list +{ + ParseNode *method_list = NULL; + merge_nodes(method_list, result, T_DISTRIBUTE_METHOD_LIST, $2); + malloc_non_terminal_node($$, result->malloc_pool_, T_INVALID, 2, $1, method_list); +} +; + +opt_qb_name_list_with_quotes: +'(' qb_name_list ')' +{ + merge_nodes($$, result, T_QB_NAME_LIST, $2); +} +| /*empty*/ +{ + $$ = NULL; +} +; + opt_qb_name: '(' qb_name_option ')' { @@ -15985,7 +16012,7 @@ ALTER SYSTEM BOOTSTRAP server_info_list malloc_non_terminal_node($$, result->malloc_pool_, T_BOOTSTRAP, 1, server_list); } | -ALTER SYSTEM FLUSH cache_type CACHE opt_namespace opt_sql_id opt_databases opt_tenant_list flush_scope +ALTER SYSTEM FLUSH cache_type CACHE opt_namespace opt_sql_id_or_schema_id opt_databases opt_tenant_list flush_scope { // system tenant use only. malloc_non_terminal_node($$, result->malloc_pool_, T_FLUSH_CACHE, 6, $4, $6, $7, $8, $9, $10); @@ -17650,16 +17677,21 @@ SUSPEND } ; -sql_id_expr: +sql_id_or_schema_id_expr: SQL_ID opt_equal_mark STRING_VALUE { (void)($2); malloc_non_terminal_node($$, result->malloc_pool_, T_SQL_ID, 1, $3); } +| SCHEMA_ID opt_equal_mark INTNUM +{ + (void)($2); + malloc_non_terminal_node($$, result->malloc_pool_, T_SCHEMA_ID, 1, $3); +} ; -opt_sql_id: -sql_id_expr +opt_sql_id_or_schema_id: +sql_id_or_schema_id_expr { $$ = $1; } @@ -19779,6 +19811,7 @@ ACCOUNT | SQL_BUFFER_RESULT | SQL_CACHE | SQL_ID +| SCHEMA_ID | SQL_NO_CACHE | SQL_THREAD | SQL_TSI_DAY diff --git a/src/sql/plan_cache/ob_plan_cache.cpp b/src/sql/plan_cache/ob_plan_cache.cpp index b260fbee93..d6467eb420 100644 --- a/src/sql/plan_cache/ob_plan_cache.cpp +++ b/src/sql/plan_cache/ob_plan_cache.cpp @@ -1342,6 +1342,8 @@ int ObPlanCache::foreach_cache_evict(CallBack &cb) } template int ObPlanCache::foreach_cache_evict(pl::ObGetPLKVEntryOp &); +template int ObPlanCache::foreach_cache_evict(pl::ObGetPLKVEntryBySchemaIdOp &); +template int ObPlanCache::foreach_cache_evict(pl::ObGetPLKVEntryBySQLIDOp &); // Remove all cache object in the lib cache int ObPlanCache::cache_evict_all_obj() @@ -2567,6 +2569,42 @@ int ObPlanCache::flush_lib_cache_by_ns(const ObLibCacheNameSpace ns) return ret; } +template +int ObPlanCache::flush_pl_cache_single_cache_obj(uint64_t db_id, EvictAttr &attr) +{ + int ret = OB_SUCCESS; + observer::ObReqTimeGuard req_timeinfo_guard; + if (OB_FAIL(ObPLCacheMgr::cache_evict_pl_cache_single(this, db_id, attr))) { + SQL_PC_LOG(ERROR, "Plan cache evict failed, please check", K(ret)); + } + ObArray deleted_objs; + int64_t safe_timestamp = INT64_MAX; + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(observer::ObGlobalReqTimeService::get_instance() + .get_global_safe_timestamp(safe_timestamp))) { + SQL_PC_LOG(ERROR, "failed to get global safe timestamp", K(ret)); + } else if (OB_FAIL(dump_deleted_objs(deleted_objs, safe_timestamp))) { + SQL_PC_LOG(WARN, "failed to get deleted sql objs", K(ret)); + } else { + int tmp_ret = OB_SUCCESS; + tmp_ret = OB_E(EventTable::EN_FLUSH_PC_NOT_CLEANUP_LEAK_MEM_ERROR) OB_SUCCESS; + if (OB_SUCCESS == tmp_ret) { + LOG_INFO("Deleted Cache Objs", K(deleted_objs)); + for (int64_t i = 0; i < deleted_objs.count(); i++) { // ignore error code and continue + if (OB_FAIL(ObCacheObjectFactory::destroy_cache_obj(true, + deleted_objs.at(i).obj_id_, + this))) { + LOG_WARN("failed to destroy cache obj", K(ret)); + } + } + } + } + return ret; +} + +template int ObPlanCache::flush_pl_cache_single_cache_obj(uint64_t db_id, uint64_t &schema_id); +template int ObPlanCache::flush_pl_cache_single_cache_obj(uint64_t db_id, common::ObString &sql_id); int ObPlanCache::flush_pl_cache() { diff --git a/src/sql/plan_cache/ob_plan_cache.h b/src/sql/plan_cache/ob_plan_cache.h index b30c83a4f1..4f45cb64d1 100644 --- a/src/sql/plan_cache/ob_plan_cache.h +++ b/src/sql/plan_cache/ob_plan_cache.h @@ -424,6 +424,8 @@ public: public: int flush_plan_cache(); int flush_plan_cache_by_sql_id(uint64_t db_id, common::ObString sql_id); + template + int flush_pl_cache_single_cache_obj(uint64_t db_id, EvictAttr &attr); int flush_lib_cache(); int flush_lib_cache_by_ns(const ObLibCacheNameSpace ns); int flush_pl_cache(); diff --git a/src/sql/plan_cache/ob_values_table_compression.cpp b/src/sql/plan_cache/ob_values_table_compression.cpp index 1b6f2a830d..33bde3d37f 100644 --- a/src/sql/plan_cache/ob_values_table_compression.cpp +++ b/src/sql/plan_cache/ob_values_table_compression.cpp @@ -189,6 +189,8 @@ int ObValuesTableCompression::rebuild_new_raw_sql(ObPlanCacheCtx &pc_ctx, const int64_t param_cnt, const int64_t delta_length, const ObString &no_param_sql, + ObIArray &no_param_pos, + ObIArray &raw_sql_offset, ObString &new_raw_sql, int64_t &no_param_sql_pos, int64_t &new_raw_pos) @@ -211,6 +213,7 @@ int ObValuesTableCompression::rebuild_new_raw_sql(ObPlanCacheCtx &pc_ctx, LOG_WARN("get unexpected NULL ptr", K(ret), KP(pc_param)); } else { int64_t param_pos = pc_param->node_->pos_ - delta_length; // get pos is in new no param sql + int64_t param_raw_offset = pc_param->node_->raw_sql_offset_; int64_t param_len = pc_param->node_->text_len_; len = param_pos - no_param_sql_pos; if (OB_UNLIKELY(len < 0) || OB_UNLIKELY(new_raw_pos + len + param_len > buff_len)) { @@ -225,10 +228,17 @@ int ObValuesTableCompression::rebuild_new_raw_sql(ObPlanCacheCtx &pc_ctx, } if (param_pos == no_param_sql_pos) { //copy raw param + param_raw_offset = new_raw_pos; MEMCPY(buff + new_raw_pos, pc_param->node_->raw_text_, param_len); new_raw_pos += param_len; no_param_sql_pos += 1; } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(no_param_pos.push_back(param_pos))) { + LOG_WARN("failed to push back", K(ret)); + } else if (OB_FAIL(raw_sql_offset.push_back(param_raw_offset))) { + LOG_WARN("failed to push back", K(ret)); + } } } } @@ -260,7 +270,8 @@ int ObValuesTableCompression::try_batch_exec_params(ObIAllocator &allocator, int64_t no_param_sql_pos = 0; int64_t new_raw_sql_pos = 0; ObString &new_raw_sql = pc_ctx.new_raw_sql_; - ObSEArray raw_pos; + ObSEArray no_param_pos; + ObSEArray raw_sql_offset; ObPhysicalPlanCtx *phy_ctx = NULL; uint64_t data_version = 0; if (pc_ctx.sql_ctx_.handle_batched_multi_stmt() || @@ -270,9 +281,8 @@ int ObValuesTableCompression::try_batch_exec_params(ObIAllocator &allocator, fp_result.values_tokens_.empty() || !GCONF._enable_values_table_folding) { /* do nothing */ - } else if (OB_FAIL(GET_MIN_DATA_VERSION(session_info.get_effective_tenant_id(), data_version))) { - LOG_WARN("get tenant data version failed", K(ret), K(session_info.get_effective_tenant_id())); - } else if (data_version < DATA_VERSION_4_2_1_0 || + /* TODO NOTE@yejingtao.yjt: remove following upgrade checking after next barrier version */ + } else if (GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_4_2_1_2 || !is_support_compress_values_table(pc_ctx.raw_sql_)) { /* do nothing */ } else if (OB_ISNULL(phy_ctx = pc_ctx.exec_ctx_.get_physical_plan_ctx())) { @@ -306,20 +316,15 @@ int ObValuesTableCompression::try_batch_exec_params(ObIAllocator &allocator, } } for (int64_t j = last_raw_param_idx; OB_SUCC(ret) && j < param_idx + param_count; j++) { - ObPCParam *pc_param = pc_ctx.fp_result_.raw_params_.at(j); - if (OB_ISNULL(pc_param) || OB_ISNULL(pc_param->node_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("pc_param is null", K(ret), KP(pc_param)); - } else if (OB_FAIL(temp_store.push_back(pc_param))) { - LOG_WARN("failed to push back", K(ret)); - } else if (OB_FAIL(raw_pos.push_back(pc_param->node_->pos_ - total_delta_len))) { + if (OB_FAIL(temp_store.push_back(pc_ctx.fp_result_.raw_params_.at(j)))) { LOG_WARN("failed to push back", K(ret)); } } if (OB_FAIL(ret)) { } else if (OB_FAIL(rebuild_new_raw_sql(pc_ctx, temp_store, new_raw_idx, temp_store.count() - new_raw_idx, total_delta_len, new_no_param_sql, - new_raw_sql, no_param_sql_pos, new_raw_sql_pos))) { + no_param_pos, raw_sql_offset, new_raw_sql, no_param_sql_pos, + new_raw_sql_pos))) { LOG_WARN("failed to rebuild new raw sql", K(ret)); } else { int64_t batch_begin_idx = new_raw_idx + param_idx - last_raw_param_idx; @@ -337,24 +342,20 @@ int ObValuesTableCompression::try_batch_exec_params(ObIAllocator &allocator, } if (OB_SUCC(ret) && can_fold_params) { for (int64_t j = last_raw_param_idx; OB_SUCC(ret) && j < pc_ctx.fp_result_.raw_params_.count(); j++) { - ObPCParam *pc_param = pc_ctx.fp_result_.raw_params_.at(j); - if (OB_ISNULL(pc_param) || OB_ISNULL(pc_param->node_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("pc_param is null", K(ret), KP(pc_param)); - } else if (OB_FAIL(temp_store.push_back(pc_param))) { - LOG_WARN("failed to push back", K(ret)); - } else if (OB_FAIL(raw_pos.push_back(pc_param->node_->pos_ - total_delta_len))) { + if (OB_FAIL(temp_store.push_back(pc_ctx.fp_result_.raw_params_.at(j)))) { LOG_WARN("failed to push back", K(ret)); } } if (OB_SUCC(ret)) { if (OB_FAIL(rebuild_new_raw_sql(pc_ctx, temp_store, new_raw_idx, temp_store.count() - new_raw_idx, total_delta_len, new_no_param_sql, - new_raw_sql, no_param_sql_pos, new_raw_sql_pos))) { + no_param_pos, raw_sql_offset, new_raw_sql, no_param_sql_pos, + new_raw_sql_pos))) { LOG_WARN("failed to rebuild new raw sql", K(ret)); - } else if (OB_UNLIKELY(raw_pos.count() != temp_store.count())) { + } else if (OB_UNLIKELY(no_param_pos.count() != temp_store.count()) || + OB_UNLIKELY(raw_sql_offset.count() != temp_store.count())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("param is invalid", K(ret)); + LOG_WARN("params is invalid", K(ret)); } else { int64_t len = new_no_param_sql.length() - no_param_sql_pos; if (OB_UNLIKELY(len < 0) || OB_UNLIKELY(new_raw_sql_pos + len > buff_len)) { @@ -385,7 +386,8 @@ int ObValuesTableCompression::try_batch_exec_params(ObIAllocator &allocator, fp_result.raw_params_.set_capacity(temp_store.count()); for (int64_t i = 0; i < temp_store.count(); i++) { // checked null before - temp_store.at(i)->node_->pos_ = raw_pos.at(i); + temp_store.at(i)->node_->pos_ = no_param_pos.at(i); + temp_store.at(i)->node_->raw_sql_offset_ = raw_sql_offset.at(i); } if (OB_FAIL(fp_result.raw_params_.assign(temp_store))) { LOG_WARN("fail to assign raw_param", K(ret)); @@ -446,12 +448,14 @@ int ObValuesTableCompression::resolve_params_for_values_clause(ObPlanCacheCtx &p obj_param, is_param, enable_decimal_int))) { LOG_WARN("failed to resolver param", K(ret), K(raw_idx)); } else if (!is_param) { - not_param_cnt++; // in value clause, which wonn't happen actually + not_param_cnt++; } else if (OB_FAIL(ab_params->push_back(obj_param))) { LOG_WARN("fail to push item to array", K(ret), K(raw_idx)); } } - + if (OB_SUCC(ret)) { + array_param_groups.at(i).start_param_idx_ -= not_param_cnt; + } // 1.2 build array_param in batch group for (int64_t j = 0; OB_SUCC(ret) && j < param_num; j++, raw_idx++, array_param_idx++) { ObArrayPCParam *raw_array_param = pc_ctx.fp_result_.array_params_.at(array_param_idx); @@ -564,7 +568,7 @@ int ObValuesTableCompression::resolve_params_for_values_clause(ObPlanCacheCtx &p bool enable_decimal_int = false; const ObIArray ¶m_charset_type = pc_ctx.param_charset_type_; if (OB_UNLIKELY(!pc_ctx.exec_ctx_.has_dynamic_values_table()) || OB_ISNULL(session) || - OB_ISNULL(phy_ctx) || OB_UNLIKELY(param_charset_type.count() != raw_param_cnt)) { + OB_ISNULL(phy_ctx) || OB_UNLIKELY(param_charset_type.count() > raw_param_cnt)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("sql should be mutil stmt", K(ret), KP(session), KP(phy_ctx), K(raw_param_cnt), K(param_charset_type.count())); @@ -573,6 +577,29 @@ int ObValuesTableCompression::resolve_params_for_values_clause(ObPlanCacheCtx &p } else { ParamStore &phy_param_store = phy_ctx->get_param_store_for_update(); ObIArray &array_param_groups = phy_ctx->get_array_param_groups(); + int64_t not_param_offset = 0; + int64_t tmp_raw_idx = 0; + for (int64_t i = 0; OB_SUCC(ret) && i < array_param_groups.count(); i++) { + int64_t array_idx = array_param_groups.at(i).start_param_idx_; + while (OB_SUCC(ret) && tmp_raw_idx < array_idx && tmp_raw_idx < raw_param_cnt) { + ObPCParam *pc_param = pc_ctx.fp_result_.raw_params_.at(tmp_raw_idx); + if (OB_ISNULL(pc_param)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("null expr", K(ret)); + } else if (pc_param->flag_ == NOT_PARAM) { + not_param_offset++; + } + tmp_raw_idx++; + } + if (OB_SUCC(ret)) { + if (OB_UNLIKELY(not_param_offset > array_idx)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected param", K(ret), K(array_idx), K(not_param_offset)); + } else { + array_param_groups.at(i).start_param_idx_ -= not_param_offset; + } + } + } for (int64_t i = 0; OB_SUCC(ret) && i < array_param_groups.count(); ++i) { int64_t param_num = array_param_groups.at(i).column_count_; int64_t batch_num = array_param_groups.at(i).row_count_; diff --git a/src/sql/plan_cache/ob_values_table_compression.h b/src/sql/plan_cache/ob_values_table_compression.h index ce5fc01461..784d43476d 100644 --- a/src/sql/plan_cache/ob_values_table_compression.h +++ b/src/sql/plan_cache/ob_values_table_compression.h @@ -74,6 +74,8 @@ private: const int64_t param_cnt, const int64_t delta_length, const common::ObString &no_param_sql, + common::ObIArray &no_param_pos, + common::ObIArray &raw_sql_offset, common::ObString &new_raw_sql, int64_t &no_param_sql_pos, int64_t &new_raw_pos); diff --git a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp index c3ff248a63..c14626db40 100644 --- a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp +++ b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp @@ -818,7 +818,8 @@ int ObFlushCacheResolver::resolve(const ParseNode &parse_tree) } else if (OB_ISNULL(sql_id_node)) { // do nothing // currently, only support plan cache's fine-grained cache evict - } else if (stmt->flush_cache_arg_.cache_type_ != CACHE_TYPE_PLAN) { + } else if (stmt->flush_cache_arg_.cache_type_ != CACHE_TYPE_PLAN && + stmt->flush_cache_arg_.cache_type_ != CACHE_TYPE_PL_OBJ) { ret = OB_NOT_SUPPORTED; LOG_WARN("only support plan cache's fine-grained cache evict", K(stmt->flush_cache_arg_.cache_type_), K(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "only support plan cache's fine-grained cache evict, other type"); @@ -827,18 +828,25 @@ int ObFlushCacheResolver::resolve(const ParseNode &parse_tree) LOG_WARN("not supported plan cache's fine-grained cache evict in oracle mode", K(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "plan cache's fine-grained cache evict in oracle mode is"); } else if (OB_ISNULL(sql_id_node->children_) - || OB_ISNULL(sql_id_node->children_[0]) - || T_SQL_ID != sql_id_node->type_) { + || OB_ISNULL(sql_id_node->children_[0])) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret)); - } else if (sql_id_node->children_[0]->str_len_ > (OB_MAX_SQL_ID_LENGTH+1)) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret)); - } else { - stmt->flush_cache_arg_.sql_id_.assign_ptr( - sql_id_node->children_[0]->str_value_, - static_cast(sql_id_node->children_[0]->str_len_)); + } else if (T_SQL_ID == sql_id_node->type_) { + if (sql_id_node->children_[0]->str_len_ > (OB_MAX_SQL_ID_LENGTH+1)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret)); + } else { + stmt->flush_cache_arg_.sql_id_.assign_ptr( + sql_id_node->children_[0]->str_value_, + static_cast(sql_id_node->children_[0]->str_len_)); + stmt->flush_cache_arg_.is_fine_grained_ = true; + } + } else if (T_SCHEMA_ID == sql_id_node->type_) { + stmt->flush_cache_arg_.schema_id_ = sql_id_node->children_[0]->value_; stmt->flush_cache_arg_.is_fine_grained_ = true; + } else { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret)); } // retrive schema guard diff --git a/src/sql/resolver/cmd/ob_call_procedure_resolver.cpp b/src/sql/resolver/cmd/ob_call_procedure_resolver.cpp index e6977b4214..008e304a52 100644 --- a/src/sql/resolver/cmd/ob_call_procedure_resolver.cpp +++ b/src/sql/resolver/cmd/ob_call_procedure_resolver.cpp @@ -240,6 +240,9 @@ int ObCallProcedureResolver::generate_pl_cache_ctx(pl::ObPLCacheCtx &pc_ctx) pc_ctx.key_.sessid_ = 0; pc_ctx.key_.key_id_ = OB_INVALID_ID; pc_ctx.key_.name_ = params_.cur_sql_; + (void)ObSQLUtils::md5(pc_ctx.raw_sql_, + pc_ctx.sql_id_, + (int32_t)sizeof(pc_ctx.sql_id_)); } return ret; } diff --git a/src/sql/resolver/ddl/ob_create_routine_resolver.cpp b/src/sql/resolver/ddl/ob_create_routine_resolver.cpp index 7b73f873eb..86488a3750 100644 --- a/src/sql/resolver/ddl/ob_create_routine_resolver.cpp +++ b/src/sql/resolver/ddl/ob_create_routine_resolver.cpp @@ -141,6 +141,8 @@ int ObCreateRoutineResolver::resolve_sp_definer(const ParseNode *parse_node, // 需要检查当前用户是否有超级权限或者set user id的权限，如果权限ok，那么host为% if (session_info_->has_user_super_privilege()) { host_name.assign_ptr("%", 1); + } else if (user_name == cur_user_name) { + host_name = cur_host_name; } else { ret = OB_ERR_NO_PRIVILEGE; LOG_WARN("no privilege", K(ret)); @@ -148,7 +150,9 @@ int ObCreateRoutineResolver::resolve_sp_definer(const ParseNode *parse_node, } else { host_name.assign_ptr(host_node->str_value_, static_cast(host_node->str_len_)); // 显式指定host为%，需要当前用户有超级权限或者set user id的权限 - if (0 == host_name.case_compare("%") && !session_info_->has_user_super_privilege()) { + if (user_name == cur_user_name && host_name == cur_host_name) { + // do nothing + } else if (0 == host_name.case_compare("%") && !session_info_->has_user_super_privilege()) { ret = OB_ERR_NO_PRIVILEGE; LOG_WARN("no privilege", K(ret)); } @@ -160,7 +164,6 @@ int ObCreateRoutineResolver::resolve_sp_definer(const ParseNode *parse_node, user_name, host_name, user_info))) { - ret = OB_ERR_UNEXPECTED; LOG_WARN("fail to get_user_info", K(ret)); } else if (OB_ISNULL(user_info)) { LOG_USER_WARN(OB_ERR_USER_NOT_EXIST); diff --git a/src/sql/resolver/ddl/ob_create_table_resolver.cpp b/src/sql/resolver/ddl/ob_create_table_resolver.cpp index cf0f373488..3d04777fa2 100644 --- a/src/sql/resolver/ddl/ob_create_table_resolver.cpp +++ b/src/sql/resolver/ddl/ob_create_table_resolver.cpp @@ -723,7 +723,7 @@ int ObCreateTableResolver::resolve(const ParseNode &parse_tree) ObTableSchema &table_schema = create_table_stmt->get_create_table_arg().schema_; if (OB_FAIL(add_hidden_tablet_seq_col())) { SQL_RESV_LOG(WARN, "failed to add hidden primary key tablet seq", K(ret)); - } else if (OB_FAIL(add_inner_index_for_heap_gtt())) { + } else if (!is_create_as_sel && OB_FAIL(add_inner_index_for_heap_gtt())) { SQL_RESV_LOG(WARN, "failed to add_inner_index_for_heap_gtt", K(ret)); } } diff --git a/src/sql/resolver/dml/ob_del_upd_resolver.cpp b/src/sql/resolver/dml/ob_del_upd_resolver.cpp index 0d18003946..80869a9672 100644 --- a/src/sql/resolver/dml/ob_del_upd_resolver.cpp +++ b/src/sql/resolver/dml/ob_del_upd_resolver.cpp @@ -2351,11 +2351,50 @@ int ObDelUpdResolver::view_pullup_part_exprs() LOG_WARN("failed to push back pullup partition expr", K(ret)); } } + + // pull up the partition expr from view stmt to root stmt + const ObTableSchema *table_schema = NULL; + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_ISNULL(table) || OB_ISNULL(schema_checker_) || OB_ISNULL(session_info_) || OB_ISNULL(table)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table item is null", K(ret)); + } else if (OB_FAIL(schema_checker_->get_table_schema(session_info_->get_effective_tenant_id(), + table->get_base_table_item().ref_id_, + table_schema))) { + LOG_WARN("fail to get table schema", K(ret), K(table->get_base_table_item().ref_id_)); + } else if (OB_NOT_NULL(table_schema)) { + const common::ObIArray &foreign_key_infos = table_schema->get_foreign_key_infos(); + for (int64_t i = 0; OB_SUCC(ret) && i < sel_stmt->get_part_exprs().count(); ++i) { + ObDMLStmt::PartExprItem pei = sel_stmt->get_part_exprs().at(i); + if (!is_fk_parent_table(foreign_key_infos, pei.index_tid_)) { + continue; + } else if (OB_FAIL(copier.copy(pei.part_expr_, pei.part_expr_))) { + LOG_WARN("failed to copy part expr", K(ret)); + } else if (OB_FAIL(copier.copy(pei.subpart_expr_, pei.subpart_expr_))) { + LOG_WARN("failed to copy subpart expr", K(ret)); + } else if (OB_FAIL(stmt->get_part_exprs().push_back(pei))) { + LOG_WARN("failed to push back pullup partition expr", K(ret)); + } + } + } } } return ret; } +bool ObDelUpdResolver::is_fk_parent_table(const common::ObIArray &foreign_key_infos, const uint64_t table_id) +{ + bool is_pk_table = false; + for (int64_t i = 0; i < foreign_key_infos.count() && !is_pk_table; i++) { + const ObForeignKeyInfo &foreign_key_info = foreign_key_infos.at(i); + const uint64_t parent_table_id = foreign_key_info.parent_table_id_; + if (table_id == parent_table_id) { + is_pk_table = true; + } + } + return is_pk_table; +} int ObDelUpdResolver::expand_record_to_columns(const ParseNode &record_node, ObIArray &value_list) { diff --git a/src/sql/resolver/dml/ob_del_upd_resolver.h b/src/sql/resolver/dml/ob_del_upd_resolver.h index b4bc2d4600..5e0c9b793a 100644 --- a/src/sql/resolver/dml/ob_del_upd_resolver.h +++ b/src/sql/resolver/dml/ob_del_upd_resolver.h @@ -155,6 +155,7 @@ protected: int view_pullup_part_exprs(); int expand_record_to_columns(const ParseNode &record_node, ObIArray &value_list); + bool is_fk_parent_table(const common::ObIArray &foreign_key_infos, const uint64_t table_id); int resolve_check_constraints(const TableItem* table_item, common::ObIArray &check_exprs); int resolve_view_check_exprs(uint64_t table_id, diff --git a/src/sql/resolver/dml/ob_dml_resolver.cpp b/src/sql/resolver/dml/ob_dml_resolver.cpp index b2b3ca0c56..b189e7d910 100755 --- a/src/sql/resolver/dml/ob_dml_resolver.cpp +++ b/src/sql/resolver/dml/ob_dml_resolver.cpp @@ -2598,7 +2598,9 @@ int ObDMLResolver::replace_pl_relative_expr_to_question_mark(ObRawExpr *&real_re || T_OP_GET_PACKAGE_VAR == real_ref_expr->get_expr_type() //package variable access, must not (system/user variable) || real_ref_expr->is_sys_func_expr() || T_FUN_PL_GET_CURSOR_ATTR == real_ref_expr->get_expr_type()) { //允许CURSOR%ROWID通过 - if (OB_FAIL(ObResolverUtils::resolve_external_param_info(params_.external_param_info_, + if (OB_FAIL(ObResolverUtils::revert_external_param_info(params_.external_param_info_, real_ref_expr))) { + LOG_WARN("failed to revert external param info", K(ret), KPC(real_ref_expr)); + } else if (OB_FAIL(ObResolverUtils::resolve_external_param_info(params_.external_param_info_, *params_.expr_factory_, params_.prepare_param_count_, real_ref_expr))) { @@ -3303,7 +3305,7 @@ int ObDMLResolver::resolve_basic_table_without_cte(const ParseNode &parse_tree, LOG_WARN("resolve table partition expr failed", K(ret), K(table_name)); } else if (OB_FAIL(resolve_generated_column_expr_temp(table_item))) { LOG_WARN("resolve generated column expr templte failed", K(ret)); - } else if (OB_FAIL(resolve_table_check_constraint_items(table_item, table_schema))) { + } else if (OB_FAIL(resolve_table_constraint_items(table_item, table_schema))) { LOG_WARN("resolve table partition expr failed", K(ret), K(table_name)); } else if (stmt->is_select_stmt() && OB_FAIL(resolve_geo_mbr_column())) { LOG_WARN("resolve geo mbr column failed", K(ret), K(table_name)); @@ -3440,6 +3442,21 @@ int ObDMLResolver::resolve_table_check_constraint_items(const TableItem *table_i return ret; } +int ObDMLResolver::resolve_table_constraint_items(const TableItem *table_item, + const ObTableSchema *table_schema) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(table_item) || OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table item or schema is null", K(ret), K(table_item), K(table_schema)); + } else if (OB_FAIL(resolve_table_check_constraint_items(table_item, table_schema))) { + LOG_WARN("failed to resolve check constraints", K(ret)); + } else if (OB_FAIL(resolve_foreign_key_constraint(table_item))) { + LOG_WARN("failed to resolve foreign key constraint", K(ret)); + } + return ret; +} + int ObDMLResolver::check_flashback_expr_validity(ObRawExpr *expr, bool &has_column) { int ret = OB_SUCCESS; @@ -4479,17 +4496,21 @@ int ObDMLResolver::resolve_generate_table_item(ObSelectStmt *ref_query, int ret = OB_SUCCESS; TableItem *item = NULL; ObDMLStmt *dml_stmt = get_stmt(); + ObString used_alias_name = alias_name; if (OB_ISNULL(dml_stmt) || OB_ISNULL(allocator_)) { ret = OB_NOT_INIT; LOG_WARN("resolver isn't init"); } else if (OB_UNLIKELY(NULL == (item = dml_stmt->create_table_item(*allocator_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_ERROR("create table item failed"); + } else if (used_alias_name.empty() + && OB_FAIL(dml_stmt->generate_anonymous_view_name(*allocator_, used_alias_name))) { + LOG_WARN("failed to generate view name", K(ret)); } else { item->ref_query_ = ref_query; item->table_id_ = generate_table_id(); - item->table_name_ = alias_name; - item->alias_name_ = alias_name; + item->table_name_ = used_alias_name; + item->alias_name_ = used_alias_name; item->type_ = TableItem::GENERATED_TABLE; item->is_view_table_ = false; if (OB_FAIL(dml_stmt->add_table_item(session_info_, item, params_.have_same_table_name_))) { @@ -5413,7 +5434,18 @@ int ObDMLResolver::resolve_fk_table_partition_expr(const TableItem &table_item, if (OB_FAIL(ret)) { } else if (OB_FAIL(dml_stmt->set_part_expr(foreign_key_info.foreign_key_id_, fk_scan_tid, parent_part_expr, parent_subpart_expr))) { - LOG_WARN("set part expr to dml stmt failed", K(ret)); + /* + create table t17(a int, b int, c int, d int, primary key (a)) partition by hash(a) partitions 3; + create table tf17(a int, b int, c int, d int, primary key (a), foreign key (a) references t17 (a)) partition by hash(a) partitions 3; + update tf17 partition(p0) as C, tf17 as P set C.d = C.d + 100 where C.a = P.a; + In follwing cases, the partition key of t17 will be resolved twice, but it needs return success for compatibility with oracle + */ + if (ret == OB_ERR_TABLE_EXIST) { + ret = OB_SUCCESS; + LOG_INFO("Duplicate foreign key", K(lbt())); + } else { + LOG_WARN("set part expr to dml stmt failed", K(ret)); + } } else { LOG_TRACE("resolve partition expr", K(table_item), KPC(parent_part_expr), K(part_str)); } @@ -9784,6 +9816,8 @@ int ObDMLResolver::resolve_generated_table_column_item(const TableItem &table_it } } else if (ob_is_geometry(select_expr->get_data_type()) && !select_expr->is_column_ref_expr()) { col_expr->set_srs_id(OB_DEFAULT_COLUMN_SRS_ID); + } else if (ObSQLUtils::check_json_expr(select_expr->get_expr_type())) { + with_is_json_constraint = true; } is_break = true; @@ -13377,6 +13411,12 @@ int ObDMLResolver::resolve_optimize_hint(const ParseNode &hint_node, } break; } + case T_PQ_SUBQUERY: { + if (OB_FAIL(resolve_pq_subquery_hint(hint_node, opt_hint))) { + LOG_WARN("failed to resolve pq subquery hint", K(ret)); + } + break; + } case T_USE_LATE_MATERIALIZATION: case T_NO_USE_LATE_MATERIALIZATION: case T_GBY_PUSHDOWN: @@ -13783,6 +13823,66 @@ int ObDMLResolver::get_valid_dist_methods(const ParseNode *dist_methods_node, return ret; } +int ObDMLResolver::resolve_pq_subquery_hint(const ParseNode &hint_node, + ObOptHint *&opt_hint) +{ + int ret = OB_SUCCESS; + opt_hint = NULL; + const ParseNode *dist_methods_node = NULL; + if (OB_UNLIKELY(3 != hint_node.num_child_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected pq_subquery hint node", K(ret), K(hint_node.num_child_)); + } else { + DistAlgo dist_algo = DistAlgo::DIST_INVALID_METHOD; + if (OB_ISNULL(dist_methods_node = hint_node.children_[2])) { + dist_algo = DistAlgo::DIST_BASIC_METHOD; + } else if (OB_UNLIKELY(T_DISTRIBUTE_METHOD_LIST != dist_methods_node->type_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected pq_subquery hint node", K(ret), K(get_type_name(dist_methods_node->type_))); + } else if (OB_UNLIKELY(2 != dist_methods_node->num_child_) + || OB_ISNULL(dist_methods_node->children_[0]) + || OB_ISNULL(dist_methods_node->children_[1])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected disdist methods node", K(ret)); + } else { + ObItemType outer = dist_methods_node->children_[0]->type_; + ObItemType inner = dist_methods_node->children_[1]->type_; + // pq_subquery(@sel$1 (sel$2 sel$3)) basic + // pq_subquery(@sel$1 (sel$2 sel$3) LOCAL LOCAL) + // pq_subquery(@sel$1 (sel$2 sel$3) NONE NONE) + // pq_subquery(@sel$1 (sel$2 sel$3) PARTITION NONE) + // pq_subquery(@sel$1 (sel$2 sel$3) NONE ALL) + if (T_DISTRIBUTE_LOCAL == outer && T_DISTRIBUTE_LOCAL == inner) { + dist_algo = DistAlgo::DIST_PULL_TO_LOCAL; + } else if (T_DISTRIBUTE_NONE == outer && T_DISTRIBUTE_NONE == inner) { + dist_algo = DistAlgo::DIST_PARTITION_WISE; + } else if (T_DISTRIBUTE_PARTITION == outer && T_DISTRIBUTE_NONE == inner) { + dist_algo = DistAlgo::DIST_PARTITION_NONE; + } else if (T_DISTRIBUTE_NONE == outer && T_DISTRIBUTE_ALL == inner) { + dist_algo = DistAlgo::DIST_NONE_ALL; + } + } + + if (DistAlgo::DIST_INVALID_METHOD != dist_algo) { + ObPQSubqueryHint *pq_subquery_hint = NULL; + ObString qb_name; + if (OB_FAIL(ObQueryHint::create_hint(allocator_, hint_node.type_, pq_subquery_hint))) { + LOG_WARN("failed to create hint", K(ret)); + } else if (OB_FAIL(resolve_qb_name_node(hint_node.children_[0], qb_name))) { + LOG_WARN("failed to resolve query block name", K(ret)); + } else if (hint_node.children_[1] != NULL && + OB_FAIL(resolve_qb_name_list(hint_node.children_[1], pq_subquery_hint->get_sub_qb_names()))) { + LOG_WARN("failed to resolve qb name list", K(ret)); + } else { + pq_subquery_hint->set_qb_name(qb_name); + pq_subquery_hint->set_dist_algo(dist_algo); + opt_hint = pq_subquery_hint; + } + } + } + return ret; +} + int ObDMLResolver::resolve_join_filter_hint(const ParseNode &hint_node, ObOptHint *&opt_hint) { @@ -13969,7 +14069,7 @@ int ObDMLResolver::resolve_count_to_exists_hint(const ParseNode &hint_node, } else { const ParseNode *qb_name_list_node = hint_node.children_[1]; if (qb_name_list_node != NULL && - OB_FAIL(resolve_qb_name_list(qb_name_list_node, count_to_exists_hint->get_qb_name_list()))) { + OB_FAIL(resolve_qb_name_list(qb_name_list_node, count_to_exists_hint->get_qb_names()))) { LOG_WARN("failed to resolve qb name list", K(ret)); } else { count_to_exists_hint->set_qb_name(qb_name); @@ -15571,16 +15671,19 @@ int ObDMLResolver::resolve_values_table_item(const ParseNode &table_node, TableI } if (OB_SUCC(ret)) { int64_t column_cnt = 0; + ObSEArray res_types; //common values table: values row(...), row(...),... if (upper_insert_resolver_ == NULL && OB_FAIL(resolve_table_values_for_select(table_node, new_table_item->table_values_, + res_types, column_cnt))) { LOG_WARN("failed to resolve table values for select", K(ret)); //insert values table: insert into ....values row(...), row(...),... } else if (upper_insert_resolver_ != NULL && OB_FAIL(resolve_table_values_for_insert(table_node, new_table_item->table_values_, + res_types, column_cnt))) { LOG_WARN("failed to resolve table values for insert", K(ret)); } else { @@ -15591,7 +15694,7 @@ int ObDMLResolver::resolve_values_table_item(const ParseNode &table_node, TableI new_table_item->is_view_table_ = false; if (OB_FAIL(dml_stmt->add_table_item(session_info_, new_table_item))) { LOG_WARN("add table item failed", K(ret)); - } else if (OB_FAIL(gen_values_table_column_items(column_cnt, *new_table_item))) { + } else if (OB_FAIL(gen_values_table_column_items(column_cnt, res_types, *new_table_item))) { LOG_WARN("failed to gen values table column items", K(ret)); } else { table_item = new_table_item; @@ -15604,6 +15707,7 @@ int ObDMLResolver::resolve_values_table_item(const ParseNode &table_node, TableI int ObDMLResolver::resolve_table_values_for_select(const ParseNode &table_node, ObIArray &table_values, + ObIArray &res_types, int64_t &column_cnt) { int ret = OB_SUCCESS; @@ -15618,7 +15722,6 @@ int ObDMLResolver::resolve_table_values_for_select(const ParseNode &table_node, LOG_WARN("get unexpected null", K(ret), K(values_node), K(table_node.type_), K(table_node.num_child_), K(params_.expr_factory_)); } else { - ObSEArray res_types; for (int64_t i = 0; OB_SUCC(ret) && i < values_node->num_child_; i++) { ParseNode *vector_node = values_node->children_[i]; if (OB_ISNULL(vector_node) || @@ -15704,6 +15807,7 @@ int ObDMLResolver::resolve_table_values_for_select(const ParseNode &table_node, int ObDMLResolver::resolve_table_values_for_insert(const ParseNode &table_node, ObIArray &table_values, + ObIArray &res_types, int64_t &column_cnt) { int ret = OB_SUCCESS; @@ -15837,7 +15941,12 @@ int ObDMLResolver::resolve_table_values_for_insert(const ParseNode &table_node, if (OB_SUCC(ret)) { if (OB_FAIL(append(table_values, cur_values_vector))) { LOG_WARN("failed to append", K(ret)); - } else { + } else if (i == 0) { + for (int64_t k = 0; OB_SUCC(ret) && k < cur_values_vector.count(); k++) { + if (OB_FAIL(res_types.push_back(cur_values_vector.at(k)->get_result_type()))) { + LOG_WARN("failed to append", K(ret)); + } + } LOG_TRACE("succeed to resolve one row", K(cur_values_vector), K(table_values)); } } @@ -15918,12 +16027,15 @@ int ObDMLResolver::try_add_cast_to_values(const ObIArray &res_typ return ret; } -int ObDMLResolver::gen_values_table_column_items(const int64_t column_cnt, TableItem &table_item) +int ObDMLResolver::gen_values_table_column_items(const int64_t column_cnt, + const ObIArray &res_types, + TableItem &table_item) { int ret = OB_SUCCESS; if (OB_ISNULL(params_.expr_factory_) || OB_ISNULL(allocator_) || OB_ISNULL(get_stmt()) || OB_UNLIKELY(column_cnt <= 0 || table_item.table_values_.empty() || - table_item.table_values_.count() % column_cnt != 0)) { + table_item.table_values_.count() % column_cnt != 0 || + res_types.count() != column_cnt)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get unexpected error", K(column_cnt), K(params_.expr_factory_), K(table_item.table_values_), K(ret)); @@ -15936,7 +16048,7 @@ int ObDMLResolver::gen_values_table_column_items(const int64_t column_cnt, Table ret = OB_ERR_UNEXPECTED; LOG_WARN(("value desc is null")); } else { - column_expr->set_result_type(table_item.table_values_.at(i)->get_result_type()); + column_expr->set_result_type(res_types.at(i)); column_expr->set_result_flag(table_item.table_values_.at(i)->get_result_flag()); column_expr->set_ref_id(table_item.table_id_, i + OB_APP_MIN_COLUMN_ID); // compatible Mysql8.0, column name is column_0, column_1, ... @@ -15951,24 +16063,23 @@ int ObDMLResolver::gen_values_table_column_items(const int64_t column_cnt, Table MEMCPY(buf, tmp_col_name.ptr(), tmp_col_name.length()); ObString column_name(tmp_col_name.length(), buf); column_expr->set_column_attr(table_item.table_name_, column_name); - if (ob_is_enumset_tc(table_item.table_values_.at(i)->get_result_type().get_type()) - && OB_FAIL(column_expr->set_enum_set_values(table_item.table_values_.at(i)->get_enum_set_values()))) { - LOG_WARN("failed to set_enum_set_values", K(ret)); - } - if (OB_SUCC(ret)) { - if (OB_FAIL(column_expr->add_flag(IS_COLUMN))) { - LOG_WARN("failed to add flag IS_COLUMN", K(ret)); + if (ob_is_enumset_tc(column_expr->get_result_type().get_type()) || + column_expr->get_result_type().is_lob_storage()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("values stmt not support such column type", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "type of column in values table"); + } else if (OB_FAIL(column_expr->add_flag(IS_COLUMN))) { + LOG_WARN("failed to add flag IS_COLUMN", K(ret)); + } else { + ColumnItem column_item; + column_item.expr_ = column_expr; + column_item.table_id_ = column_expr->get_table_id(); + column_item.column_id_ = column_expr->get_column_id(); + column_item.column_name_ = column_expr->get_column_name(); + if (OB_FAIL(get_stmt()->add_column_item(column_item))) { + LOG_WARN("failed to add column item", K(ret)); } else { - ColumnItem column_item; - column_item.expr_ = column_expr; - column_item.table_id_ = column_expr->get_table_id(); - column_item.column_id_ = column_expr->get_column_id(); - column_item.column_name_ = column_expr->get_column_name(); - if (OB_FAIL(get_stmt()->add_column_item(column_item))) { - LOG_WARN("failed to add column item", K(ret)); - } else { - LOG_TRACE("succeed to gen table values desc", K(column_name), KPC(column_expr)); - } + LOG_TRACE("succeed to gen table values desc", K(column_name), KPC(column_expr)); } } } diff --git a/src/sql/resolver/dml/ob_dml_resolver.h b/src/sql/resolver/dml/ob_dml_resolver.h index a44d2c87c1..eb7f08c258 100644 --- a/src/sql/resolver/dml/ob_dml_resolver.h +++ b/src/sql/resolver/dml/ob_dml_resolver.h @@ -878,6 +878,7 @@ private: int resolve_pq_map_hint(const ParseNode &hint_node, ObOptHint *&opt_hint); int resolve_pq_distribute_hint(const ParseNode &hint_node, ObOptHint *&opt_hint); int resolve_pq_set_hint(const ParseNode &hint_node, ObOptHint *&opt_hint); + int resolve_pq_subquery_hint(const ParseNode &hint_node, ObOptHint *&opt_hint); int resolve_join_filter_hint(const ParseNode &join_node, ObOptHint *&opt_hint); int resolve_aggregation_hint(const ParseNode &hint_node, ObOptHint *&hint); int resolve_normal_transform_hint(const ParseNode &hint_node, ObTransHint *&hint); @@ -916,6 +917,8 @@ private: int resolve_table_check_constraint_items(const TableItem *table_item, const ObTableSchema *table_schema); + int resolve_table_constraint_items(const TableItem *table_item, + const ObTableSchema *table_schema); int find_table_index_infos(const ObString &index_name, const TableItem *table_item, bool &find_it, @@ -932,11 +935,16 @@ private: int resolve_values_table_item(const ParseNode &table_node, TableItem *&table_item); int resolve_table_values_for_select(const ParseNode &table_node, ObIArray &table_values, + ObIArray &res_types, int64_t &column_cnt); int resolve_table_values_for_insert(const ParseNode &table_node, ObIArray &table_values, + ObIArray &res_types, int64_t &column_cnt); - int gen_values_table_column_items(const int64_t column_cnt, TableItem &table_item); + + int gen_values_table_column_items(const int64_t column_cnt, + const ObIArray &res_types, + TableItem &table_item); int get_values_res_types(const ObIArray &cur_values_types, ObIArray &res_types); int try_add_cast_to_values(const ObIArray &res_types, diff --git a/src/sql/resolver/dml/ob_dml_stmt.cpp b/src/sql/resolver/dml/ob_dml_stmt.cpp index f6ef66f637..9f0694ad28 100644 --- a/src/sql/resolver/dml/ob_dml_stmt.cpp +++ b/src/sql/resolver/dml/ob_dml_stmt.cpp @@ -1045,7 +1045,7 @@ int ObDMLStmt::construct_join_table(const ObDMLStmt &other_stmt, * for or-expansion transformation * todo: do not update semi id in semi info now */ -int ObDMLStmt::update_stmt_table_id(const ObDMLStmt &other) +int ObDMLStmt::update_stmt_table_id(ObIAllocator *allocator, const ObDMLStmt &other) { int ret = OB_SUCCESS; if (OB_UNLIKELY(other.table_items_.count() != table_items_.count())) { @@ -1072,7 +1072,7 @@ int ObDMLStmt::update_stmt_table_id(const ObDMLStmt &other) other.table_items_.at(i)->is_generated_table() && NULL != table_items_.at(i)->ref_query_ && NULL != other.table_items_.at(i)->ref_query_ && - OB_FAIL(table_items_.at(i)->ref_query_->update_stmt_table_id( + OB_FAIL(table_items_.at(i)->ref_query_->update_stmt_table_id(allocator, *other.table_items_.at(i)->ref_query_))) { LOG_WARN("failed to update table id for generated table", K(ret)); } else { /*do nothing*/ } @@ -1087,7 +1087,7 @@ int ObDMLStmt::update_stmt_table_id(const ObDMLStmt &other) LOG_WARN("null point error", K(subquery_exprs_.at(i)), K(other.subquery_exprs_.at(i)), K(subquery_exprs_.at(i)->get_ref_stmt()), K(other.subquery_exprs_.at(i)->get_ref_stmt()), K(ret)); - } else if (OB_FAIL(subquery_exprs_.at(i)->get_ref_stmt()->update_stmt_table_id( + } else if (OB_FAIL(subquery_exprs_.at(i)->get_ref_stmt()->update_stmt_table_id(allocator, *other.subquery_exprs_.at(i)->get_ref_stmt()))) { LOG_WARN("failed to update table id for subquery exprs", K(ret)); } else { /*do nothing*/ } @@ -1116,7 +1116,8 @@ int ObDMLStmt::update_stmt_table_id(const ObDMLStmt &other) } else if (OB_FAIL(update_table_item_id(other, *other.table_items_.at(i), true, - *table_items_.at(i)))) { + *table_items_.at(i), + allocator))) { LOG_WARN("failed to update table id for table item", K(ret)); } else { /*do nothing*/ } } @@ -1270,7 +1271,7 @@ int ObDMLStmt::update_table_item_id_for_joined_table(const ObDMLStmt &other_stmt ret = OB_INVALID_ARGUMENT; LOG_WARN("null table item", K(other.left_table_), K(other.right_table_), K(current.left_table_), K(current.right_table_), K(ret)); - } else if (OB_FAIL(update_table_item_id(other_stmt, other, false, current))) { + } else if (OB_FAIL(update_table_item_id(other_stmt, other, false, current, NULL))) { LOG_WARN("failed to update table id", K(ret)); } else if (other.left_table_->is_joined_table() && current.left_table_->is_joined_table() && @@ -1291,27 +1292,29 @@ int ObDMLStmt::update_table_item_id_for_joined_table(const ObDMLStmt &other_stmt int ObDMLStmt::update_table_item_id(const ObDMLStmt &other, const TableItem &old_item, const bool has_bit_index, - TableItem &new_item) + TableItem &new_item, + ObIAllocator *allocator) { int ret = OB_SUCCESS; if (OB_ISNULL(query_ctx_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("null query ctx", K(ret)); - // } else if (OB_FAIL(get_qb_name(new_item.qb_name_))) { - // LOG_WARN("fail to get qb_name", K(ret), K(get_stmt_id())); - // do not update table item qb name } else { uint64_t old_table_id = old_item.table_id_; uint64_t new_table_id = query_ctx_->available_tb_id_--; int32_t old_bit_id = OB_INVALID_INDEX; int32_t new_bit_id = OB_INVALID_INDEX; new_item.table_id_ = new_table_id; - if (TableItem::TableType::BASE_TABLE == new_item.type_) { - new_item.type_ = TableItem::TableType::ALIAS_TABLE; - new_item.alias_name_ = ObString::make_string(""); - } if (has_bit_index) { - if (OB_FAIL(set_table_bit_index(new_table_id))) { + bool adjusted = false; + if (OB_ISNULL(allocator)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(ret), K(allocator)); + } else if (OB_FAIL(get_qb_name(new_item.qb_name_))) { + LOG_WARN("fail to get qb_name", K(ret), K(get_stmt_id())); + } else if (OB_FAIL(adjust_duplicated_table_name(*allocator, new_item, adjusted))) { + LOG_WARN("fail to update dup table name", K(ret), K(new_item)); + } else if (OB_FAIL(set_table_bit_index(new_table_id))) { LOG_WARN("failed to set table bit index", K(ret)); } else if (&new_item == &old_item) { /* do nothing */ @@ -1361,6 +1364,93 @@ int ObDMLStmt::update_table_item_id(const ObDMLStmt &other, return ret; } +// to keep object_name@qb_name unique in query, need adjust table alias name when update table item qb name +int ObDMLStmt::adjust_duplicated_table_name(ObIAllocator &allocator, + TableItem &table_item, + bool &adjusted) +{ + int ret = OB_SUCCESS; + adjusted = false; + const TableItem *cur_table = NULL; + bool find_dup = false; + for (int64_t i = 0; !find_dup && OB_SUCC(ret) && i < table_items_.count(); i++) { + if (OB_ISNULL(cur_table = table_items_.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("null table item", K(ret), K(i), K(table_items_)); + } else if (cur_table == &table_item) { + /* do nothing */ + } else if (0 != table_item.get_object_name().case_compare(cur_table->get_object_name())) { + /* do nothing */ + } else { + find_dup = true; + } + } + + if (OB_SUCC(ret) && find_dup) { + int64_t pos = 0; + // just to generate an unique alias name, use a minimal max name length value: OB_MAX_USER_TABLE_NAME_LENGTH_MYSQL = 64 + // ignore oracle mode max name length OB_MAX_USER_TABLE_NAME_LENGTH_ORACLE = 128 + char buf[OB_MAX_USER_TABLE_NAME_LENGTH_MYSQL + 1]; + const int64_t MAX_TIMES_FOR_GET_NO_DUP_ALIAS_NAME = 20; + int64_t buf_len = OB_MAX_USER_TABLE_NAME_LENGTH_MYSQL; + int32_t prefix_len = table_item.get_object_name().length(); + prefix_len = prefix_len > buf_len - 3 ? buf_len - 3 : prefix_len; + if (OB_FAIL(BUF_PRINTF("%.*s_", prefix_len, table_item.get_object_name().ptr()))) { + LOG_WARN("append name to buf error", K(ret)); + } else { + int64_t old_pos = pos; + int64_t id = 1; + for (; find_dup && OB_SUCC(ret) && id <= MAX_TIMES_FOR_GET_NO_DUP_ALIAS_NAME; id++) { + pos = old_pos; + if (OB_FAIL(BUF_PRINTF("%ld", id))) { + LOG_WARN("failed to print id", K(ret), K(id)); + } else { + find_dup = false; + for (int64_t i = 0; !find_dup && OB_SUCC(ret) && i < table_items_.count(); ++i) { + if (OB_ISNULL(table_items_.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("null table item", K(ret), K(i), K(table_items_)); + } else { + find_dup = (0 == table_items_.at(i)->get_object_name().case_compare(buf)); + } + } + } + } + if (OB_FAIL(ret)) { + } else if (find_dup) { + LOG_WARN("failed to update alias table name no dup", K(MAX_TIMES_FOR_GET_NO_DUP_ALIAS_NAME), + K(id), K(table_item.get_object_name())); + } else if (OB_FAIL(ob_write_string(allocator, ObString(pos, buf), table_item.alias_name_))) { + LOG_WARN("failed to write string", K(ret)); + } else { + adjusted = true; + } + } + } + return ret; +} + +int ObDMLStmt::adjust_duplicated_table_names(ObIAllocator &allocator, bool &adjusted) +{ + int ret = OB_SUCCESS; + adjusted = false; + bool is_adjusted = false; + TableItem *table_item = NULL; + for (int64_t i = 0; OB_SUCC(ret) && i < table_items_.count(); ++i) { + if (OB_ISNULL(table_item = table_items_.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpect null", K(ret), K(table_item)); + } else if (OB_FAIL(adjust_duplicated_table_name(allocator, + *table_item, + is_adjusted))) { + LOG_WARN("fail to update dup table name", K(ret), KPC(table_item)); + } else { + adjusted |= is_adjusted; + } + } + return ret; +} + bool ObDMLStmt::has_subquery() const { return subquery_exprs_.count() > 0; diff --git a/src/sql/resolver/dml/ob_dml_stmt.h b/src/sql/resolver/dml/ob_dml_stmt.h index d89fc81494..9f19408629 100644 --- a/src/sql/resolver/dml/ob_dml_stmt.h +++ b/src/sql/resolver/dml/ob_dml_stmt.h @@ -695,7 +695,8 @@ public: int iterate_joined_table_expr(JoinedTable *joined_table, ObStmtExprVisitor &visitor) const; - int update_stmt_table_id(const ObDMLStmt &other); + int update_stmt_table_id(ObIAllocator *allocator, const ObDMLStmt &other); + int adjust_duplicated_table_names(ObIAllocator &allocator, bool &adjusted); int set_table_item_qb_name(); int adjust_qb_name(ObIAllocator *allocator, const ObString &src_qb_name, @@ -1179,7 +1180,9 @@ protected: int update_table_item_id(const ObDMLStmt &other, const TableItem &old_item, const bool has_bit_index, - TableItem &new_item); + TableItem &new_item, + ObIAllocator *allocator); + int adjust_duplicated_table_name(ObIAllocator &allocator, TableItem &table_item, bool &adjusted); protected: /** diff --git a/src/sql/resolver/dml/ob_hint.cpp b/src/sql/resolver/dml/ob_hint.cpp index e9244185c7..3be7ae367e 100644 --- a/src/sql/resolver/dml/ob_hint.cpp +++ b/src/sql/resolver/dml/ob_hint.cpp @@ -927,8 +927,6 @@ ObItemType ObHint::get_hint_type(ObItemType type) case T_NO_DISTINCT_PUSHDOWN: return T_DISTINCT_PUSHDOWN; case T_NO_USE_HASH_SET: return T_USE_HASH_SET; case T_NO_USE_DISTRIBUTED_DML: return T_USE_DISTRIBUTED_DML; - case T_DYNAMIC_SAMPLING: return T_DYNAMIC_SAMPLING; - default: return type; } } @@ -999,6 +997,7 @@ const char* ObHint::get_hint_name(ObItemType type, bool is_enable_hint /* defaul case T_USE_HASH_SET: return is_enable_hint ? "USE_HASH_SET" : "NO_USE_HASH_SET"; case T_USE_DISTRIBUTED_DML: return is_enable_hint ? "USE_DISTRIBUTED_DML" : "NO_USE_DISTRIBUTED_DML"; case T_TABLE_DYNAMIC_SAMPLING: return "DYNAMIC_SAMPLING"; + case T_PQ_SUBQUERY: return "PQ_SUBQUERY"; default: return NULL; } } @@ -1073,7 +1072,6 @@ int ObHint::deep_copy_hint_contain_table(ObIAllocator *allocator, ObHint *&hint) case HINT_GROUPBY_PLACEMENT: DEEP_COPY_NORMAL_HINT(ObGroupByPlacementHint); break; case HINT_JOIN_FILTER: DEEP_COPY_NORMAL_HINT(ObJoinFilterHint); break; case HINT_WIN_MAGIC: DEEP_COPY_NORMAL_HINT(ObWinMagicHint); break; - case HINT_TABLE_DYNAMIC_SAMPLING: DEEP_COPY_NORMAL_HINT(ObTableDynamicSamplingHint); break; default: { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected hint type to deep copy", K(ret), K(hint_class_)); @@ -1224,6 +1222,29 @@ bool ObHint::is_expr_match_str(const ObRawExpr &expr, const ObString &str) return bret; } +int ObHint::print_table_list(const ObIArray &table_list, PlanText &plan_text) +{ + int ret = OB_SUCCESS; + if (!table_list.empty()) { + char *buf = plan_text.buf_; + int64_t &buf_len = plan_text.buf_len_; + int64_t &pos = plan_text.pos_; + for (int64_t i = 0; OB_SUCC(ret) && i < table_list.count(); ++i) { + const TablesInHint &cur_table = table_list.at(i); + if (cur_table.count() > 1 && OB_FAIL(BUF_PRINTF("("))) { + LOG_WARN("failed to do BUF_PRINTF", K(ret)); + } else if (OB_FAIL(ObTableInHint::print_join_tables_in_hint(plan_text, cur_table))) { + LOG_WARN("failed to print joined tables in hint", K(ret)); + } else if (cur_table.count() > 1 && OB_FAIL(BUF_PRINTF(")"))) { + LOG_WARN("failed to do BUF_PRINTF", K(ret)); + } else if (i < table_list.count() - 1 && OB_FAIL(BUF_PRINTF(" "))) { + LOG_WARN("failed to do BUF_PRINTF", K(ret)); + } + } + } + return ret; +} + int ObViewMergeHint::assign(const ObViewMergeHint &other) { int ret = OB_SUCCESS; @@ -1402,10 +1423,35 @@ bool QbNameList::is_subset(const ObIArray &qb_name_list) const return bret; } +int QbNameList::print_qb_names(PlanText &plan_text, const bool print_quote) const +{ + int ret = OB_SUCCESS; + if (!qb_names_.empty()) { + char *buf = plan_text.buf_; + int64_t &buf_len = plan_text.buf_len_; + int64_t &pos = plan_text.pos_; + if (print_quote && OB_FAIL(BUF_PRINTF("("))) { + LOG_WARN("failed to do BUF_PRINTF", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < qb_names_.count(); ++i) { + const ObString &qb_name = qb_names_.at(i); + if (OB_FAIL(BUF_PRINTF("\"%.*s\"", qb_name.length(), qb_name.ptr()))) { + LOG_WARN("failed to print qb name", K(ret)); + } else if (i != qb_names_.count() - 1 && OB_FAIL(BUF_PRINTF(" "))) { + LOG_WARN("failed to do BUF_PRINTF", K(ret)); + } + } + if (OB_SUCC(ret) && print_quote && OB_FAIL(BUF_PRINTF(")"))) { + LOG_WARN("failed to do BUF_PRINTF", K(ret)); + } + } + return ret; +} + int ObCountToExistsHint::assign(const ObCountToExistsHint &other) { int ret = OB_SUCCESS; - if (OB_FAIL(qb_name_list_.assign(other.qb_name_list_))) { + if (OB_FAIL(qb_names_.assign(other.qb_names_))) { LOG_WARN("failed to assign qb name list", K(ret)); } else if (OB_FAIL(ObTransHint::assign(other))) { LOG_WARN("fail to assign hint", K(ret)); @@ -1413,40 +1459,6 @@ int ObCountToExistsHint::assign(const ObCountToExistsHint &other) return ret; } -int ObCountToExistsHint::print_hint_desc(PlanText &plan_text) const -{ - int ret = OB_SUCCESS; - if (!qb_name_list_.empty()) { - char *buf = plan_text.buf_; - int64_t &buf_len = plan_text.buf_len_; - int64_t &pos = plan_text.pos_; - for (int64_t i = 0; OB_SUCC(ret) && i < qb_name_list_.count(); ++i) { - ObString qb_name = qb_name_list_.at(i); - if (OB_FAIL(BUF_PRINTF("\"%.*s\"", qb_name.length(), qb_name.ptr()))) { - LOG_WARN("failed to print qb name", K(ret)); - } else if (i != qb_name_list_.count() - 1) { - if (OB_FAIL(BUF_PRINTF(" "))) { - LOG_WARN("failed to do BUF_PRINTF", K(ret)); - } - } - } - } - return ret; -} - -bool ObCountToExistsHint::enable_count_to_exists(const ObString &qb_name) const -{ - bool bret = false; - if (is_enable_hint()) { - bool found = false; - for (int64_t i = 0; !found && i < qb_name_list_.count(); ++i) { - found = 0 == qb_name.case_compare(qb_name_list_.at(i)); - } - bret = found || qb_name_list_.empty(); - } - return bret; -} - int ObLeftToAntiHint::assign(const ObLeftToAntiHint &other) { int ret = OB_SUCCESS; @@ -1458,29 +1470,6 @@ int ObLeftToAntiHint::assign(const ObLeftToAntiHint &other) return ret; } -int ObLeftToAntiHint::print_hint_desc(PlanText &plan_text) const -{ - int ret = OB_SUCCESS; - if (!table_list_.empty()) { - char *buf = plan_text.buf_; - int64_t &buf_len = plan_text.buf_len_; - int64_t &pos = plan_text.pos_; - for (int64_t i = 0; OB_SUCC(ret) && i < table_list_.count(); ++i) { - single_or_joined_table cur_table = table_list_.at(i); - if (cur_table.count() > 1 && OB_FAIL(BUF_PRINTF("("))) { - LOG_WARN("failed to do BUF_PRINTF", K(ret)); - } else if (OB_FAIL(ObTableInHint::print_join_tables_in_hint(plan_text, cur_table))) { - LOG_WARN("failed to print joined tables in hint", K(ret)); - } else if (cur_table.count() > 1 && OB_FAIL(BUF_PRINTF(")"))) { - LOG_WARN("failed to do BUF_PRINTF", K(ret)); - } else if (i < table_list_.count() - 1 && OB_FAIL(BUF_PRINTF(" "))) { - LOG_WARN("failed to do BUF_PRINTF", K(ret)); - } - } - } - return ret; -} - int ObLeftToAntiHint::get_all_table_in_hint(ObIArray &all_tables) { int ret = OB_SUCCESS; @@ -1497,8 +1486,7 @@ bool ObLeftToAntiHint::enable_left_to_anti(ObCollationType cs_type, const TableI bool bret = false; if (is_enable_hint()) { for (int64_t i = 0; !bret && i < table_list_.count(); ++i) { - single_or_joined_table cur_table = table_list_.at(i); - bret = ObTableInHint::is_match_table_item(cs_type, cur_table, table); + bret = ObTableInHint::is_match_table_item(cs_type, table_list_.at(i), table); } bret |= table_list_.empty(); } @@ -1516,29 +1504,6 @@ int ObEliminateJoinHint::assign(const ObEliminateJoinHint &other) return ret; } -int ObEliminateJoinHint::print_hint_desc(PlanText &plan_text) const -{ - int ret = OB_SUCCESS; - if (!table_list_.empty()) { - char *buf = plan_text.buf_; - int64_t &buf_len = plan_text.buf_len_; - int64_t &pos = plan_text.pos_; - for (int64_t i = 0; OB_SUCC(ret) && i < table_list_.count(); ++i) { - single_or_joined_table cur_table = table_list_.at(i); - if (cur_table.count() > 1 && OB_FAIL(BUF_PRINTF("("))) { - LOG_WARN("failed to do BUF_PRINTF", K(ret)); - } else if (OB_FAIL(ObTableInHint::print_join_tables_in_hint(plan_text, cur_table))) { - LOG_WARN("failed to print joined tables in hint", K(ret)); - } else if (cur_table.count() > 1 && OB_FAIL(BUF_PRINTF(")"))) { - LOG_WARN("failed to do BUF_PRINTF", K(ret)); - } else if (i < table_list_.count() - 1 && OB_FAIL(BUF_PRINTF(" "))) { - LOG_WARN("failed to do BUF_PRINTF", K(ret)); - } - } - } - return ret; -} - int ObEliminateJoinHint::get_all_table_in_hint(ObIArray &all_tables) { int ret = OB_SUCCESS; @@ -1555,8 +1520,7 @@ bool ObEliminateJoinHint::enable_eliminate_join(ObCollationType cs_type, const T bool bret = false; if (is_enable_hint()) { for (int64_t i = 0; !bret && i < table_list_.count(); ++i) { - single_or_joined_table cur_table = table_list_.at(i); - bret = ObTableInHint::is_match_table_item(cs_type, cur_table, table); + bret = ObTableInHint::is_match_table_item(cs_type, table_list_.at(i), table); } bret |= table_list_.empty(); } @@ -1575,29 +1539,6 @@ int ObGroupByPlacementHint::assign(const ObGroupByPlacementHint &other) return ret; } -int ObGroupByPlacementHint::print_hint_desc(PlanText &plan_text) const -{ - int ret = OB_SUCCESS; - if (!table_list_.empty()) { - char *buf = plan_text.buf_; - int64_t &buf_len = plan_text.buf_len_; - int64_t &pos = plan_text.pos_; - for (int64_t i = 0; OB_SUCC(ret) && i < table_list_.count(); ++i) { - single_or_joined_table cur_table = table_list_.at(i); - if (cur_table.count() > 1 && OB_FAIL(BUF_PRINTF("("))) { - LOG_WARN("failed to do BUF_PRINTF", K(ret)); - } else if (OB_FAIL(ObTableInHint::print_join_tables_in_hint(plan_text, cur_table))) { - LOG_WARN("failed to print joined tables in hint", K(ret)); - } else if (cur_table.count() > 1 && OB_FAIL(BUF_PRINTF(")"))) { - LOG_WARN("failed to do BUF_PRINTF", K(ret)); - } else if (i < table_list_.count() - 1 && OB_FAIL(BUF_PRINTF(" "))) { - LOG_WARN("failed to do BUF_PRINTF", K(ret)); - } - } - } - return ret; -} - int ObGroupByPlacementHint::get_all_table_in_hint(ObIArray &all_tables) { int ret = OB_SUCCESS; @@ -1614,8 +1555,7 @@ bool ObGroupByPlacementHint::enable_groupby_placement(ObCollationType cs_type, c bool bret = false; if (is_enable_hint()) { for (int64_t i = 0; !bret && i < table_list_.count(); i++) { - single_or_joined_table cur_table = table_list_.at(i); - bret = ObTableInHint::is_match_table_item(cs_type, cur_table, table); + bret = ObTableInHint::is_match_table_item(cs_type, table_list_.at(i), table); } bret |= table_list_.empty(); } @@ -1633,8 +1573,7 @@ bool ObGroupByPlacementHint::enable_groupby_placement(ObCollationType cs_type, LOG_WARN("assign failed", K(ret)); } for (int64_t i = 0; !bret && i < table_list_.count(); i++) { - single_or_joined_table cur_table = table_list_.at(i); - bret = ObTableInHint::is_match_table_items(cs_type, cur_table, check_tables); + bret = ObTableInHint::is_match_table_items(cs_type, table_list_.at(i), check_tables); } bret |= table_list_.empty(); } @@ -1674,7 +1613,7 @@ int ObWinMagicHint::print_hint_desc(PlanText &plan_text) const LOG_WARN("table in hint count < 2", K(ret)); } for (int64_t i = 0; OB_SUCC(ret) && i < table_list_.count(); ++i) { - ObTableInHint cur_table = table_list_.at(i); + const ObTableInHint &cur_table = table_list_.at(i); if (table_list_.count() > 2 && i == 1 && OB_FAIL(BUF_PRINTF("("))) { LOG_WARN("failed to do BUF_PRINTF", K(ret)); } else if (OB_FAIL(cur_table.print_table_in_hint(plan_text))) { @@ -1991,7 +1930,7 @@ int ObJoinHint::print_hint_desc(PlanText &plan_text) const } else if (tables_.count() > 1 && OB_FAIL(BUF_PRINTF(")"))) { LOG_WARN("failed to print hint", K(ret)); } else if (T_PQ_DISTRIBUTE == hint_type_ && NULL != algo_str - && OB_FAIL(BUF_PRINTF(" %s", get_dist_algo_str()))) { + && OB_FAIL(BUF_PRINTF(" %s", algo_str))) { LOG_WARN("failed to print dist algo", K(ret)); } return ret; @@ -2314,6 +2253,34 @@ const char *ObPQSetHint::get_dist_method_str(const ObItemType dist_method) return NULL; }; +int ObPQSubqueryHint::assign(const ObPQSubqueryHint &other) +{ + int ret = OB_SUCCESS; + dist_algo_ = other.dist_algo_; + if (OB_FAIL(sub_qb_names_.assign(other.sub_qb_names_))) { + LOG_WARN("fail to assign subplan qb names", K(ret)); + } else if (OB_FAIL(ObOptHint::assign(other))) { + LOG_WARN("fail to assign hint", K(ret)); + } + return ret; +} + +int ObPQSubqueryHint::print_hint_desc(PlanText &plan_text) const +{ + int ret = OB_SUCCESS; + char *buf = plan_text.buf_; + int64_t &buf_len = plan_text.buf_len_; + int64_t &pos = plan_text.pos_; + const char *algo_str = NULL; + if (OB_FAIL(sub_qb_names_.print_qb_names(plan_text, true))) { + LOG_WARN("failed to print qb names", K(ret)); + } else if (NULL != (algo_str = ObJoinHint::get_dist_algo_str(get_dist_algo())) + && OB_FAIL(BUF_PRINTF(" %s", algo_str))) { + LOG_WARN("failed to print dist algo", K(ret)); + } + return ret; +} + int ObJoinOrderHint::print_hint_desc(PlanText &plan_text) const { int ret = OB_SUCCESS; diff --git a/src/sql/resolver/dml/ob_hint.h b/src/sql/resolver/dml/ob_hint.h index de9a68a75d..a7bcd64a99 100644 --- a/src/sql/resolver/dml/ob_hint.h +++ b/src/sql/resolver/dml/ob_hint.h @@ -377,6 +377,20 @@ struct ObLeadingTable { ObLeadingTable *right_table_; }; +struct QbNameList { + int assign(const QbNameList& other); + bool has_qb_name(const ObDMLStmt *stmt) const; + bool has_qb_name(const ObString &qb_name) const; + bool is_equal(const ObIArray &stmts) const; + bool is_equal(const ObIArray &qb_name_list) const; + bool is_subset(const ObIArray &stmts) const; + bool is_subset(const ObIArray &qb_name_list) const; + bool empty() const { return qb_names_.empty(); } + int print_qb_names(PlanText &plan_text, const bool print_quote) const; + TO_STRING_KV(K_(qb_names)); + common::ObSEArray qb_names_; +}; + class ObHint { public: @@ -408,6 +422,10 @@ public: static const int64_t MAX_EXPR_STR_LENGTH_IN_HINT = 1024; + // basic/generated table: size = 1 + // joined table: size > 1 + typedef ObSEArray TablesInHint; + ObHint(ObItemType hint_type = T_INVALID) : hint_class_(HINT_INVALID_CLASS), qb_name_(), @@ -445,6 +463,7 @@ public: int add_tables(ObIArray &tables, ObIArray &tables_ptr); static int get_expr_str_in_hint(ObIAllocator &allocator, const ObRawExpr &expr, ObString &str); static bool is_expr_match_str(const ObRawExpr &expr, const ObString &str); + static int print_table_list(const ObIArray &table_list, PlanText &plan_text); bool is_transform_outline_hint() const { return is_transform_hint() && (is_enable_hint() || is_materialize_hint()); }; bool is_transform_hint() const { return hint_class_ >= HINT_TRANSFORM && hint_class_ < HINT_OPTIMIZE; } bool is_view_merge_hint() const { return HINT_VIEW_MERGE == hint_class_; } @@ -468,7 +487,8 @@ public: bool is_table_parallel_hint() const { return HINT_TABLE_PARALLEL == hint_class_; } bool is_join_filter_hint() const { return HINT_JOIN_FILTER == hint_class_; } bool is_project_prune_hint() const { return T_PROJECT_PRUNE == hint_type_; } - bool is_table_dynamic_sampling_hint() const { return HINT_TABLE_DYNAMIC_SAMPLING == hint_class_; } + bool is_table_dynamic_sampling_hint() const { return T_TABLE_DYNAMIC_SAMPLING == hint_type_; } + bool is_pq_subquery_hint() const { return T_PQ_SUBQUERY == hint_type_; } VIRTUAL_TO_STRING_KV("hint_type", get_type_name(hint_type_), K_(hint_class), K_(qb_name), @@ -570,30 +590,28 @@ class ObCountToExistsHint : public ObTransHint public: ObCountToExistsHint(ObItemType hint_type) : ObTransHint(hint_type), - qb_name_list_() + qb_names_() { set_hint_class(HINT_COUNT_TO_EXISTS); } int assign(const ObCountToExistsHint &other); virtual ~ObCountToExistsHint() {} - virtual int print_hint_desc(PlanText &plan_text) const override; - common::ObIArray & get_qb_name_list() { return qb_name_list_; } - const common::ObIArray & get_qb_name_list() const { return qb_name_list_; } - bool enable_count_to_exists(const ObString &qb_name) const; + virtual int print_hint_desc(PlanText &plan_text) const override + { return qb_names_.print_qb_names(plan_text, false); } + common::ObIArray & get_qb_names() { return qb_names_.qb_names_; } + const common::ObIArray & get_qb_names() const { return qb_names_.qb_names_; } + bool enable_count_to_exists(const ObString &qb_name) const + { return is_enable_hint() && (qb_names_.has_qb_name(qb_name) || qb_names_.empty()); } - INHERIT_TO_STRING_KV("ObHint", ObHint, K_(qb_name_list)); + INHERIT_TO_STRING_KV("ObHint", ObHint, K_(qb_names)); private: - common::ObSEArray qb_name_list_; + QbNameList qb_names_; }; class ObLeftToAntiHint : public ObTransHint { public: - // basic/generated table: size = 1 - // joined table: size > 1 - typedef ObSEArray single_or_joined_table; - ObLeftToAntiHint(ObItemType hint_type) : ObTransHint(hint_type), table_list_() @@ -603,24 +621,20 @@ public: int assign(const ObLeftToAntiHint &other); virtual ~ObLeftToAntiHint() {} - virtual int print_hint_desc(PlanText &plan_text) const override; + virtual int print_hint_desc(PlanText &plan_text) const override { return ObHint::print_table_list(table_list_, plan_text); } virtual int get_all_table_in_hint(ObIArray &all_tables) override; - common::ObIArray & get_tb_name_list() { return table_list_; } - const common::ObIArray & get_tb_name_list() const { return table_list_; } + common::ObIArray & get_tb_name_list() { return table_list_; } + const common::ObIArray & get_tb_name_list() const { return table_list_; } bool enable_left_to_anti(ObCollationType cs_type, const TableItem &table) const; INHERIT_TO_STRING_KV("ObHint", ObHint, K_(table_list)); private: - common::ObSEArray table_list_; + common::ObSEArray table_list_; }; class ObEliminateJoinHint : public ObTransHint { public: - // basic/generated table: size = 1 - // joined table: size > 1 - typedef ObSEArray single_or_joined_table; - ObEliminateJoinHint(ObItemType hint_type) : ObTransHint(hint_type), table_list_() @@ -630,25 +644,21 @@ public: int assign(const ObEliminateJoinHint &other); virtual ~ObEliminateJoinHint() {} - virtual int print_hint_desc(PlanText &plan_text) const override; + virtual int print_hint_desc(PlanText &plan_text) const override { return ObHint::print_table_list(table_list_, plan_text); } virtual int get_all_table_in_hint(ObIArray &all_tables) override; - common::ObIArray & get_tb_name_list() { return table_list_; } - const common::ObIArray & get_tb_name_list() const { return table_list_; } + common::ObIArray & get_tb_name_list() { return table_list_; } + const common::ObIArray & get_tb_name_list() const { return table_list_; } bool enable_eliminate_join(ObCollationType cs_type, const TableItem &table) const; INHERIT_TO_STRING_KV("ObHint", ObHint, K_(table_list)); private: - common::ObSEArray table_list_; + common::ObSEArray table_list_; }; class ObGroupByPlacementHint : public ObTransHint { public: - // basic/generated table: size = 1 - // joined table: size > 1 - typedef ObSEArray single_or_joined_table; - ObGroupByPlacementHint(ObItemType hint_type) : ObTransHint(hint_type), table_list_() @@ -658,17 +668,17 @@ public: int assign(const ObGroupByPlacementHint &other); virtual ~ObGroupByPlacementHint() {} - virtual int print_hint_desc(PlanText &plan_text) const override; + virtual int print_hint_desc(PlanText &plan_text) const override { return ObHint::print_table_list(table_list_, plan_text); } virtual int get_all_table_in_hint(ObIArray &all_tables) override; - common::ObIArray & get_tb_name_list() { return table_list_; } - const common::ObIArray & get_tb_name_list() const { return table_list_; } + common::ObIArray & get_tb_name_list() { return table_list_; } + const common::ObIArray & get_tb_name_list() const { return table_list_; } bool enable_groupby_placement(ObCollationType cs_type, const TableItem &table) const; bool enable_groupby_placement(ObCollationType cs_type, const ObIArray &tables) const; INHERIT_TO_STRING_KV("ObHint", ObHint, K_(table_list)); private: - common::ObSEArray table_list_; + common::ObSEArray table_list_; }; class ObWinMagicHint : public ObTransHint { @@ -694,20 +704,6 @@ private: common::ObSEArray table_list_; }; - -struct QbNameList { - int assign(const QbNameList& other); - bool has_qb_name(const ObDMLStmt *stmt) const; - bool has_qb_name(const ObString &qb_name) const; - bool is_equal(const ObIArray &stmts) const; - bool is_equal(const ObIArray &qb_name_list) const; - bool is_subset(const ObIArray &stmts) const; - bool is_subset(const ObIArray &qb_name_list) const; - bool empty() const { return qb_names_.empty(); } - TO_STRING_KV(K_(qb_names)); - common::ObSEArray qb_names_; -}; - class ObMaterializeHint : public ObTransHint { public: @@ -932,6 +928,29 @@ private: common::ObString left_branch_; // qb_name for first branch of set, used for union distinct / intersect }; +class ObPQSubqueryHint : public ObOptHint +{ + public: + ObPQSubqueryHint(ObItemType hint_type = T_PQ_SUBQUERY) + : ObOptHint(hint_type), + dist_algo_(DistAlgo::DIST_INVALID_METHOD), + sub_qb_names_() + {} + int assign(const ObPQSubqueryHint &other); + virtual ~ObPQSubqueryHint() {} + virtual int print_hint_desc(PlanText &plan_text) const override; + DistAlgo get_dist_algo() const { return dist_algo_; } + void set_dist_algo(DistAlgo dist_algo) { dist_algo_ = dist_algo; } + common::ObIArray &get_sub_qb_names() { return sub_qb_names_.qb_names_; } + const common::ObIArray &get_sub_qb_names() const { return sub_qb_names_.qb_names_; } + bool is_match_subplans(const ObIArray &qb_names) const { return sub_qb_names_.is_equal(qb_names); } + + INHERIT_TO_STRING_KV("ObHint", ObHint, K_(dist_algo), K_(sub_qb_names)); +private: + DistAlgo dist_algo_; + QbNameList sub_qb_names_; +}; + class ObJoinOrderHint : public ObOptHint { public: ObJoinOrderHint(ObItemType hint_type = T_LEADING) diff --git a/src/sql/resolver/dml/ob_insert_resolver.cpp b/src/sql/resolver/dml/ob_insert_resolver.cpp index f5de5079d2..a38a2d78f3 100644 --- a/src/sql/resolver/dml/ob_insert_resolver.cpp +++ b/src/sql/resolver/dml/ob_insert_resolver.cpp @@ -421,9 +421,6 @@ int ObInsertResolver::resolve_insert_field(const ParseNode &insert_into, TableIt session_info_->set_table_name_hidden(old_flag); } OZ(column_namespace_checker_.add_reference_table(table_item)); - if (OB_SUCC(ret) && OB_FAIL(resolve_foreign_key_constraint(table_item))) { - LOG_WARN("failed to resolve foreign key constraint", K(ret), K(table_item->ref_id_)); - } if (OB_SUCC(ret)) { current_scope_ = T_INSERT_SCOPE; const ObTableSchema *table_schema = NULL; @@ -590,6 +587,10 @@ int ObInsertResolver::resolve_values(const ParseNode &value_node, } else { table_alias_node = alias_node; } + if (NULL != table_alias_node) { + view_name.assign_ptr(const_cast(table_alias_node->str_value_), + static_cast(table_alias_node->str_len_)); + } } if (OB_UNLIKELY(T_SELECT != value_node.type_)) { ret = OB_ERR_UNEXPECTED; @@ -608,17 +609,14 @@ int ObInsertResolver::resolve_values(const ParseNode &value_node, label_se_columns, *select_stmt))) { LOG_WARN("add label security columns to select item failed", K(ret)); - } else if (NULL != table_alias_node) { - view_name.assign_ptr(const_cast(table_alias_node->str_value_), - static_cast(table_alias_node->str_len_)); - } else if (OB_FAIL(insert_stmt->generate_anonymous_view_name(*allocator_, view_name))) { - LOG_WARN("failed to generate view name", K(ret)); + } else if (OB_FAIL(resolve_generate_table_item(select_stmt, view_name, sub_select_table))) { + LOG_WARN("failed to resolve generate table item", K(ret)); } if (OB_SUCC(ret) && is_mock) { ObString ori_table_name = table_item->table_name_; ObSEArray ori_column_names; - ObString row_alias_table_name = view_name; + ObString row_alias_table_name = sub_select_table->get_table_name(); ObSEArray row_alias_column_names; //1. check_table_and_column_name if (OB_FAIL(check_table_and_column_name(insert_stmt->get_values_desc(), @@ -644,8 +642,6 @@ int ObInsertResolver::resolve_values(const ParseNode &value_node, } if (OB_FAIL(ret)) { //do nothing - } else if (OB_FAIL(resolve_generate_table_item(select_stmt, view_name, sub_select_table))) { - LOG_WARN("failed to resolve generate table item", K(ret)); } else if (OB_FAIL(resolve_all_generated_table_columns(*sub_select_table, column_items))) { LOG_WARN("failed to resolve all generated table columns", K(ret)); diff --git a/src/sql/resolver/dml/ob_insert_stmt.cpp b/src/sql/resolver/dml/ob_insert_stmt.cpp index f929a750eb..fc08d5566a 100644 --- a/src/sql/resolver/dml/ob_insert_stmt.cpp +++ b/src/sql/resolver/dml/ob_insert_stmt.cpp @@ -448,5 +448,65 @@ int64_t ObInsertStmt::get_instead_of_trigger_column_count() const return column_count; } +int ObInsertStmt::check_pdml_disabled(const bool is_online_ddl, + bool &disable_pdml, bool &is_pk_auto_inc) const +{ + int ret = OB_SUCCESS; + disable_pdml = false; + is_pk_auto_inc = false; + if (!value_from_select()) { + disable_pdml = true; + } else if (is_online_ddl) { + disable_pdml = false; // keep online ddl use pdml + } else { + const common::ObIArray &column_conv_exprs = get_column_conv_exprs(); + const common::ObIArray &column_exprs = table_info_.column_exprs_; + if (OB_UNLIKELY(column_exprs.count() != column_conv_exprs.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected column count", K(ret), + K(column_exprs.count()), K(column_conv_exprs.count())); + } else { + for (int64_t i = 0; OB_SUCC(ret) && !disable_pdml && i < column_conv_exprs.count(); ++i) { + const ObColumnRefRawExpr *column_expr = column_exprs.at(i); + const ObRawExpr *column_conv_expr = column_conv_exprs.at(i); + if (OB_ISNULL(column_expr) || OB_ISNULL(column_conv_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null expr", K(ret)); + } else if (column_expr->is_rowkey_column() || column_expr->is_table_part_key_column()) { + const ObRawExpr *auto_inc_expr = NULL; + if (OB_FAIL(find_first_auto_inc_expr(column_conv_expr, auto_inc_expr))) { + LOG_WARN("fail to find first auto inc expr", K(ret)); + } else if (auto_inc_expr != NULL) { + disable_pdml = auto_inc_expr->get_param_count() > 0; // means the specified value exists + } + } + } + if (OB_SUCC(ret) && disable_pdml) { + is_pk_auto_inc = true; + } + } + } + LOG_TRACE("check insert pdml disabled", K(is_online_ddl), K(disable_pdml), K(is_pk_auto_inc)); + return ret; +} + +int ObInsertStmt::find_first_auto_inc_expr(const ObRawExpr *expr, const ObRawExpr *&auto_inc) const +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("expr is null", K(ret)); + } else if (T_FUN_SYS_AUTOINC_NEXTVAL == expr->get_expr_type()) { + auto_inc = expr; + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < expr->get_param_count(); ++i) { + if (OB_FAIL(find_first_auto_inc_expr(expr->get_param_expr(i), auto_inc))) { + LOG_WARN("fail to find first auto inc expr", K(ret), K(i), K(expr)); + } + } + } + return ret; +} + } // namespace sql } // namespace oceanbase diff --git a/src/sql/resolver/dml/ob_insert_stmt.h b/src/sql/resolver/dml/ob_insert_stmt.h index aa788359e6..0c70d189ce 100644 --- a/src/sql/resolver/dml/ob_insert_stmt.h +++ b/src/sql/resolver/dml/ob_insert_stmt.h @@ -73,6 +73,8 @@ public: int part_key_has_subquery(bool &has) const ; int part_key_has_auto_inc(bool &has) const; int part_key_is_updated(bool &is_updated) const; + int check_pdml_disabled(const bool is_online_ddl, bool &disable_pdml, bool &is_pk_auto_inc) const; + int find_first_auto_inc_expr(const ObRawExpr *expr, const ObRawExpr *&auto_inc) const; virtual int get_value_exprs(ObIArray &value_exprs) const override; // use this only when part_generated_col_dep_cols_.count() is not zero int get_values_desc_for_heap_table(common::ObIArray &arr) const; diff --git a/src/sql/resolver/dml/ob_merge_resolver.cpp b/src/sql/resolver/dml/ob_merge_resolver.cpp index fb675f59a8..3a5d374320 100644 --- a/src/sql/resolver/dml/ob_merge_resolver.cpp +++ b/src/sql/resolver/dml/ob_merge_resolver.cpp @@ -228,8 +228,6 @@ int ObMergeResolver::resolve_target_relation(const ParseNode *target_node) } else if (OB_ISNULL(table_item)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("table_item is NULL", K(ret)); - } else if (OB_FAIL(resolve_foreign_key_constraint(table_item))) { - LOG_WARN("failed to resolve foreign key constraint", K(ret), K(table_item->ref_id_)); } else if (OB_FAIL(column_namespace_checker_.add_reference_table(table_item))) { LOG_WARN("add reference table to column namespace checker failed", K(ret)); } else if (OB_FAIL(check_need_fired_trigger(table_item))) { @@ -444,6 +442,10 @@ int ObMergeResolver::resolve_generate_table(const ParseNode &table_node, select_resolver.set_is_sub_stmt(true); select_resolver.set_parent_namespace_resolver(parent_namespace_resolver_); select_resolver.set_current_view_level(current_view_level_); + ObString alias_name; + if (alias_node != NULL) { + alias_name.assign_ptr((char *)(alias_node->str_value_), static_cast(alias_node->str_len_)); + } if (OB_ISNULL(merge_stmt) || OB_ISNULL(allocator_)) { ret = OB_ERR_UNEXPECTED; @@ -458,11 +460,9 @@ int ObMergeResolver::resolve_generate_table(const ParseNode &table_node, } else if (OB_UNLIKELY(NULL == (item = merge_stmt->create_table_item(*allocator_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_ERROR("create table item failed", K(ret)); + } else if (alias_name.empty() && OB_FAIL(merge_stmt->generate_anonymous_view_name(*allocator_, alias_name))) { + LOG_WARN("failed to generate view name", K(ret)); } else { - ObString alias_name; - if (alias_node != NULL) { - alias_name.assign_ptr((char *)(alias_node->str_value_), static_cast(alias_node->str_len_)); - } item->ref_query_ = child_stmt; item->table_id_ = generate_table_id(); item->table_name_ = alias_name; diff --git a/src/sql/resolver/dml/ob_multi_table_insert_resolver.cpp b/src/sql/resolver/dml/ob_multi_table_insert_resolver.cpp index c76bad70eb..42ad5a907a 100644 --- a/src/sql/resolver/dml/ob_multi_table_insert_resolver.cpp +++ b/src/sql/resolver/dml/ob_multi_table_insert_resolver.cpp @@ -282,8 +282,6 @@ int ObMultiTableInsertResolver::resolve_multi_insert_subquey(const ParseNode &su LOG_WARN("add session id value to select item failed", K(ret)); } else if (OB_FAIL(add_new_sel_item_for_oracle_label_security_table(*select_stmt))) { LOG_WARN("add label security columns to select item failed", K(ret)); - } else if (OB_FAIL(insert_all_stmt->generate_anonymous_view_name(*allocator_, view_name))) { - LOG_WARN("failed to generate view name", K(ret)); } else if (OB_FAIL(resolve_generate_table_item(select_stmt, view_name, sub_select_table))) { LOG_WARN("failed to resolve generate table item", K(ret)); } else if (OB_FAIL(resolve_all_generated_table_columns(*sub_select_table, column_items))) { @@ -362,8 +360,6 @@ int ObMultiTableInsertResolver::resolve_insert_table_node(const ParseNode &inser LOG_WARN("a view is not appropriate here", K(ret)); } else if (OB_FAIL(column_namespace_checker_.add_reference_table(table_item))) { LOG_WARN("failed to resolve basic table"); - } else if (OB_FAIL(resolve_foreign_key_constraint(table_item))) { - LOG_WARN("failed to resolve foreign key constraint", K(ret), K(table_item->ref_id_)); } else if (OB_FAIL(check_need_fired_trigger(table_item))) { LOG_WARN("failed to check has need fired trigger", K(ret), K(table_item->ref_id_)); } else { diff --git a/src/sql/resolver/dml/ob_select_resolver.cpp b/src/sql/resolver/dml/ob_select_resolver.cpp index cd60e7d51d..e85727e203 100644 --- a/src/sql/resolver/dml/ob_select_resolver.cpp +++ b/src/sql/resolver/dml/ob_select_resolver.cpp @@ -34,6 +34,7 @@ #include "sql/rewrite/ob_transform_utils.h" #include "common/ob_smart_call.h" #include "sql/engine/expr/ob_expr_regexp_context.h" +#include "sql/engine/expr/ob_json_param_type.h" namespace oceanbase { using namespace common; @@ -1212,11 +1213,6 @@ int ObSelectResolver::resolve_normal_query(const ParseNode &parse_tree) } } } - - // add table_name for anonymous view(view in from) - if (OB_SUCC(ret) && OB_FAIL(add_name_for_anonymous_view())) { - LOG_WARN("fail to add name for anonymous view", K(ret)); - } return ret; } @@ -2103,6 +2099,9 @@ int ObSelectResolver::resolve_field_list(const ParseNode &node) ObString string_name(ptr_name); select_item.alias_name_ = string_name; select_item.is_real_alias_ = true; + } else if (T_FUN_SYS_JSON_QUERY == sel_expr->get_expr_type() + && OB_FAIL(add_alias_from_dot_notation(sel_expr, select_item))) { // deal dot notation without alias + LOG_WARN("fail to resolve alias in dot notation", K(ret)); } else { if (params_.is_prepare_protocol_ || !session_info_->get_local_ob_enable_plan_cache() @@ -2240,6 +2239,44 @@ inline bool ObSelectResolver::is_colum_without_alias(ParseNode *project_node) { return bret; } +int ObSelectResolver::add_alias_from_dot_notation(ObRawExpr *sel_expr, SelectItem& select_item) +{ + INIT_SUCC(ret); + int64_t pos = -1; + int64_t len = 0; + ObString path_str; + ObConstRawExpr* path_expr = NULL; + // whether is dot notation + if (OB_NOT_NULL(sel_expr->get_param_expr(JSN_QUE_MISMATCH)) + && JSN_QUERY_MISMATCH_DOT == static_cast(sel_expr->get_param_expr(JSN_QUE_MISMATCH))->get_value().get_int()) { + if (!select_item.alias_name_.empty()) { + select_item.is_real_alias_ = true; + } else if (OB_NOT_NULL(sel_expr->get_param_expr(JSN_QUE_PATH)) + && T_CHAR == sel_expr->get_param_expr(JSN_QUE_PATH)->get_expr_type()) { + path_expr = static_cast(sel_expr->get_param_expr(JSN_QUE_PATH)); + path_str = path_expr->get_value().get_string(); + pos = path_str.length() - 1; + len = 0; + char *buf = NULL; + while (pos >= 0 && path_str[pos] != '.') { + pos --; + } + len = path_str.length() - (pos + 1); + if (pos < 0) { + } else if (OB_ISNULL(buf = static_cast(allocator_->alloc(len)))) { + ret = common::OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory", K(ret), K(buf)); + } else { + MEMCPY(buf, path_str.ptr() + (pos + 1), len); + ObString alias_name(len, buf); + select_item.alias_name_ = alias_name; + select_item.is_real_alias_ = true; + } + } + } + return ret; +} + int ObSelectResolver::expand_target_list( const TableItem &table_item, ObIArray &target_list) { @@ -6705,34 +6742,6 @@ int ObSelectResolver::recursive_check_grouping_columns(ObSelectStmt *stmt, ObRaw return ret; } - -int ObSelectResolver::add_name_for_anonymous_view() -{ - int ret = OB_SUCCESS; - ObSelectStmt *stmt = get_select_stmt(); - if (OB_ISNULL(stmt)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null pointer", K(ret)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < stmt->get_table_size(); i++) { - TableItem *tmp_table = stmt->get_table_item(i); - if (OB_ISNULL(tmp_table)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected null pointer", K(ret)); - } else if (tmp_table->is_generated_table() - && tmp_table->alias_name_.empty() - && tmp_table->table_name_.empty()) { - // found anonymous view, generate new name. - if (OB_FAIL(stmt->generate_anonymous_view_name(*params_.allocator_, tmp_table->table_name_))) { - LOG_WARN("fail to generate view name", K(ret)); - } - } - } - } - - return ret; -} - int ObSelectResolver::resolve_shared_order_item(OrderItem &order_item, ObSelectStmt *select_stmt) { int ret = OB_SUCCESS; diff --git a/src/sql/resolver/dml/ob_select_resolver.h b/src/sql/resolver/dml/ob_select_resolver.h index c034b5f65e..6bcbe3ac67 100644 --- a/src/sql/resolver/dml/ob_select_resolver.h +++ b/src/sql/resolver/dml/ob_select_resolver.h @@ -341,9 +341,6 @@ private: int check_cube_items_valid(const common::ObIArray &cube_items); int recursive_check_grouping_columns(ObSelectStmt *stmt, ObRawExpr *expr); - int add_name_for_anonymous_view(); - int add_name_for_anonymous_view_recursive(TableItem *table_item); - int is_need_check_col_dup(const ObRawExpr *expr, bool &need_check); int resolve_shared_order_item(OrderItem &order_item, ObSelectStmt *select_stmt); @@ -351,6 +348,8 @@ private: int recursive_check_auto_gen_column_names(ObSelectStmt *select_stmt, bool in_outer_stmt); int recursive_update_column_name(ObSelectStmt *select_stmt, ObRawExpr *expr); int check_listagg_aggr_param_valid(ObAggFunRawExpr *aggr_expr); + + int add_alias_from_dot_notation(ObRawExpr *sel_expr, SelectItem& select_item); protected: // data members /*these member is only for with clause*/ diff --git a/src/sql/resolver/dml/ob_select_stmt.cpp b/src/sql/resolver/dml/ob_select_stmt.cpp index 80ebbcd8be..33d4033803 100644 --- a/src/sql/resolver/dml/ob_select_stmt.cpp +++ b/src/sql/resolver/dml/ob_select_stmt.cpp @@ -374,10 +374,10 @@ int ObSelectStmt::create_select_list_for_set_stmt(ObRawExprFactory &expr_factory return ret; } -int ObSelectStmt::update_stmt_table_id(const ObSelectStmt &other) +int ObSelectStmt::update_stmt_table_id(ObIAllocator *allocator, const ObSelectStmt &other) { int ret = OB_SUCCESS; - if (OB_FAIL(ObDMLStmt::update_stmt_table_id(other))) { + if (OB_FAIL(ObDMLStmt::update_stmt_table_id(allocator, other))) { LOG_WARN("failed to update stmt table id", K(ret)); } else if (OB_UNLIKELY(set_query_.count() != other.set_query_.count())) { ret = OB_ERR_UNEXPECTED; @@ -391,7 +391,7 @@ int ObSelectStmt::update_stmt_table_id(const ObSelectStmt &other) || OB_ISNULL(child_query = set_query_.at(i))) { ret = OB_ERR_UNEXPECTED; LOG_WARN("null statement", K(ret), K(child_query), K(other_child_query)); - } else if (OB_FAIL(SMART_CALL(child_query->update_stmt_table_id(*other_child_query)))) { + } else if (OB_FAIL(SMART_CALL(child_query->update_stmt_table_id(allocator, *other_child_query)))) { LOG_WARN("failed to update stmt table id", K(ret)); } else { /* do nothing*/ } } diff --git a/src/sql/resolver/dml/ob_select_stmt.h b/src/sql/resolver/dml/ob_select_stmt.h index 25a693622b..8abe5022c3 100644 --- a/src/sql/resolver/dml/ob_select_stmt.h +++ b/src/sql/resolver/dml/ob_select_stmt.h @@ -342,7 +342,7 @@ public: ObStmtExprVisitor &visitor); int iterate_rollup_items(ObIArray &rollup_items, ObStmtExprVisitor &visitor); int iterate_cube_items(ObIArray &cube_items, ObStmtExprVisitor &visitor); - int update_stmt_table_id(const ObSelectStmt &other); + int update_stmt_table_id(ObIAllocator *allocator, const ObSelectStmt &other); int64_t get_select_item_size() const { return select_items_.count(); } int64_t get_group_expr_size() const { return group_exprs_.count(); } int64_t get_rollup_expr_size() const { return rollup_exprs_.count(); } diff --git a/src/sql/resolver/dml/ob_sql_hint.cpp b/src/sql/resolver/dml/ob_sql_hint.cpp index 4102a68d1c..8141f64e08 100644 --- a/src/sql/resolver/dml/ob_sql_hint.cpp +++ b/src/sql/resolver/dml/ob_sql_hint.cpp @@ -824,40 +824,31 @@ int ObQueryHint::get_table_item_by_hint_table(const ObDMLStmt &stmt, { int ret = OB_SUCCESS; table_item = NULL; + bool implicit_match_allowed = table.qb_name_.empty(); const ObIArray &table_items = stmt.get_table_items(); - int64_t num = table_items.count(); - TableItem *item = NULL; - TableItem *final_item = NULL; - TableItem *tmp_item = NULL; - bool qb_name_matched = false; - bool tmp_qb_name_matched = false; - bool valid = true; - for (int64_t i = 0; OB_SUCC(ret) && NULL == final_item && i < num; ++i) { - if (OB_ISNULL(item = table_items.at(i)) || OB_UNLIKELY(item->qb_name_.empty())) { + TableItem *cur_table_item = NULL; + TableItem *explicit_matched = NULL; + TableItem *implicit_matched = NULL; + for (int64_t i = 0; OB_SUCC(ret) && NULL == explicit_matched && i < table_items.count(); ++i) { + if (OB_ISNULL(cur_table_item = table_items.at(i)) || OB_UNLIKELY(cur_table_item->qb_name_.empty())) { ret = OB_ERR_UNEXPECTED; // qb_name_ should not be empty - LOG_WARN("unexpected table item.", K(ret), K(item)); - } else if (!table.is_match_table_item(cs_type_, *item)) { + LOG_WARN("unexpected table item.", K(ret), K(cur_table_item)); + } else if (!table.is_match_table_item(cs_type_, *cur_table_item)) { /* do nothing */ - } else if (OB_FALSE_IT(tmp_qb_name_matched = (0 == item->qb_name_.case_compare(table.qb_name_)))) { - } else if (!table.db_name_.empty() && tmp_qb_name_matched) { - final_item = item; - } else if ((NULL == tmp_item) || (!qb_name_matched && tmp_qb_name_matched)) { - tmp_item = item; - qb_name_matched = tmp_qb_name_matched; - } else if (qb_name_matched && !tmp_qb_name_matched) { + } else if (0 == cur_table_item->qb_name_.case_compare(table.qb_name_)) { + explicit_matched = cur_table_item; + } else if (!implicit_match_allowed) { /* do nothing */ + } else if (NULL == implicit_matched) { + implicit_matched = cur_table_item; } else { - tmp_item = NULL; - qb_name_matched = false; - valid = false; + implicit_match_allowed = false; + implicit_matched = NULL; } } if (OB_SUCC(ret)) { - if (NULL != final_item) { - table_item = final_item; - } else if (valid && NULL != tmp_item) { - table_item = tmp_item; - } else { + table_item = NULL != explicit_matched ? explicit_matched : implicit_matched; + if (NULL == table_item) { LOG_TRACE("no table item matched hint table", K(table), K(table_items)); } } @@ -1315,7 +1306,8 @@ int ObStmtHint::merge_hint(ObHint &hint, || hint.is_join_hint() || hint.is_join_filter_hint() || hint.is_table_parallel_hint() - || hint.is_table_dynamic_sampling_hint()) { + || hint.is_table_dynamic_sampling_hint() + || hint.is_pq_subquery_hint()) { if (OB_FAIL(add_var_to_array_no_dup(other_opt_hints_, &hint))) { LOG_WARN("failed to add var to array", K(ret)); } @@ -1370,30 +1362,6 @@ int ObStmtHint::reset_explicit_trans_hint(ObItemType hint_type) return ret; } -int ObStmtHint::get_max_table_parallel(const ObDMLStmt &stmt, int64_t &max_table_parallel) const -{ - int ret = OB_SUCCESS; - max_table_parallel = ObGlobalHint::UNSET_PARALLEL; - const ObHint *hint = NULL; - bool matched = false; - if (OB_ISNULL(query_hint_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null", K(ret), K(query_hint_)); - } - for (int64_t i = 0; OB_SUCC(ret) && i < other_opt_hints_.count(); ++i) { - if (OB_NOT_NULL(hint = other_opt_hints_.at(i)) && hint->is_table_parallel_hint()) { - if (OB_FAIL(stmt.check_hint_table_matched_table_item(query_hint_->cs_type_, - static_cast(hint)->get_table(), matched))) { - LOG_WARN("failed to check hint table matched table item", K(ret)); - } else if (matched) { - max_table_parallel = std::max(max_table_parallel, - static_cast(hint)->get_parallel()); - } - } - } - return ret; -} - bool ObStmtHint::has_enable_hint(ObItemType hint_type) const { const ObHint *cur_hint = get_normal_hint(hint_type); @@ -1498,6 +1466,10 @@ int ObLogPlanHint::init_other_opt_hints(ObSqlSchemaGuard &schema_guard, *static_cast(hint)))) { LOG_WARN("failed to add dynamic sampling hint", K(ret)); } + } else if (hint->is_pq_subquery_hint()) { + if (OB_FAIL(normal_hints_.push_back(hint))) { + LOG_WARN("failed to push back", K(ret)); + } } else { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected hint type in other_opt_hints_", K(ret), K(*hint)); @@ -1979,6 +1951,43 @@ int ObLogPlanHint::check_valid_set_left_branch(const ObSelectStmt *select_stmt, return ret; } +int ObLogPlanHint::get_valid_pq_subquery_hint(const ObIArray &sub_qb_names, + const ObPQSubqueryHint *&explicit_hint, + const ObPQSubqueryHint *&implicit_hint) const +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; NULL == explicit_hint && i < normal_hints_.count(); ++i) { + if (NULL != normal_hints_.at(i) && normal_hints_.at(i)->get_hint_type() == T_PQ_SUBQUERY) { + const ObPQSubqueryHint *cur_hint = static_cast(normal_hints_.at(i)); + if (NULL == explicit_hint && cur_hint->is_match_subplans(sub_qb_names)) { + explicit_hint = cur_hint; + } else if (NULL == implicit_hint && cur_hint->get_sub_qb_names().empty()) { + implicit_hint = cur_hint; + } + } + } + return ret; +} + +DistAlgo ObLogPlanHint::get_valid_pq_subquery_dist_algo(const ObIArray &sub_qb_names, + const bool implicit_allowed) const +{ + int ret = OB_SUCCESS; // ignore this ret + DistAlgo dist_algo = DistAlgo::DIST_INVALID_METHOD; + const ObPQSubqueryHint *explicit_hint = NULL; + const ObPQSubqueryHint *implicit_hint = NULL; + if (OB_FAIL(get_valid_pq_subquery_hint(sub_qb_names, explicit_hint, implicit_hint))) { + LOG_WARN("failed to get valid subplan filter hint", K(ret)); + } else if (NULL != explicit_hint) { + dist_algo = explicit_hint->get_dist_algo(); + } else if (is_outline_data_) { + dist_algo = DistAlgo::DIST_BASIC_METHOD; + } else if (NULL != implicit_hint && implicit_allowed) { + dist_algo = implicit_hint->get_dist_algo(); + } + return dist_algo; +} + // generate spm evolution plan, throw a error code when can not get valid hint int ObLogPlanHint::check_status() const { diff --git a/src/sql/resolver/dml/ob_sql_hint.h b/src/sql/resolver/dml/ob_sql_hint.h index baaa9f4d9b..d594452c0c 100644 --- a/src/sql/resolver/dml/ob_sql_hint.h +++ b/src/sql/resolver/dml/ob_sql_hint.h @@ -219,7 +219,6 @@ struct ObStmtHint int merge_hint(ObHint &hint, ObHintMergePolicy policy, ObIArray &conflict_hints); int merge_normal_hint(ObHint &hint, ObHintMergePolicy policy, ObIArray &conflict_hints); int reset_explicit_trans_hint(ObItemType hint_type); - int get_max_table_parallel(const ObDMLStmt &stmt, int64_t &max_table_parallel) const; bool has_enable_hint(ObItemType hint_type) const; @@ -458,6 +457,11 @@ struct ObLogPlanHint bool &force_use_merge, bool &force_part_sort, bool &force_normal_sort) const; + int get_valid_pq_subquery_hint(const ObIArray &sub_qb_names, + const ObPQSubqueryHint *&explicit_hint, + const ObPQSubqueryHint *&implicit_hint) const; + DistAlgo get_valid_pq_subquery_dist_algo(const ObIArray &sub_qb_names, + const bool implicit_allowed) const; bool use_late_material() const { return has_enable_hint(T_USE_LATE_MATERIALIZATION); } bool no_use_late_material() const { return has_disable_hint(T_USE_LATE_MATERIALIZATION); } diff --git a/src/sql/resolver/dml/ob_update_resolver.cpp b/src/sql/resolver/dml/ob_update_resolver.cpp index bfb6a1fc74..6a9cc9a1f7 100644 --- a/src/sql/resolver/dml/ob_update_resolver.cpp +++ b/src/sql/resolver/dml/ob_update_resolver.cpp @@ -443,15 +443,13 @@ int ObUpdateResolver::resolve_table_list(const ParseNode &parse_tree) LOG_WARN("table node is null"); } else if (OB_FAIL(ObDMLResolver::resolve_table(*table_node, table_item))) { LOG_WARN("failed to resolve table", K(ret)); - } else if (OB_FAIL(resolve_foreign_key_constraint(table_item))) { - LOG_WARN("failed to resolve foreign key constraint", K(ret), K(table_item->ref_id_)); } else {/*do nothing*/} if (OB_SUCC(ret)) { if (OB_FAIL(column_namespace_checker_.add_reference_table(table_item))) { LOG_WARN("add reference table to namespace checker failed", K(ret)); } else if (OB_FAIL(update_stmt->add_from_item(table_item->table_id_, table_item->is_joined_table()))) { LOG_WARN("failed to add from item", K(ret)); - } else if (check_need_fired_trigger(table_item)) { + } else if (OB_FAIL(check_need_fired_trigger(table_item))) { LOG_WARN("failed to check need fired trigger", K(ret)); } else { /* diff --git a/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp b/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp index 4ec098fcfc..dd07de593d 100644 --- a/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_deduce_type.cpp @@ -373,6 +373,10 @@ int ObRawExprDeduceType::push_back_types(const ObRawExpr *param_expr, ObExprResT const ObPrecision prec = MAX(types.at(idx).get_precision(), max_prec); types.at(idx).set_precision(prec); types.at(idx).set_scale(0); + } else if (is_mysql_mode && ob_is_decimal_int_tc(types.at(idx).get_type())) { + // for decimal int type in mysql, reset calc accuracy to itself to avoid accuracy reuse + // during type deduce + types.at(idx).set_calc_accuracy(types.at(idx).get_accuracy()); } else if (!is_mysql_mode && (is_ddl_stmt || is_show_stmt) && types.at(idx).is_decimal_int() && param_expr->is_column_ref_expr()) { // If c1 and c2 are both ObDecimalIntType columns, result type of c1 + c2 is ObDecimalIntType. diff --git a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp index 74ca1dafa8..dbdad2880b 100644 --- a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp @@ -2054,6 +2054,7 @@ int ObRawExprResolverImpl::resolve_func_node_of_obj_access_idents(const ParseNod ObObjAccessIdent &access_ident = q_name.access_idents_.at(q_name.access_idents_.count() - 1); AccessNameType name_type = UNKNOWN; + access_ident.has_brackets_ = (1 == left_node.int16_values_[0]); if (!q_name.is_unknown()) { if (0 == access_ident.access_name_.case_compare("NEXT") || 0 == access_ident.access_name_.case_compare("PRIOR") @@ -2232,6 +2233,9 @@ int ObRawExprResolverImpl::resolve_left_node_of_obj_access_idents(const ParseNod || T_FUN_SYS_XMLPARSE == left_node.type_ || T_FUN_ORA_XMLAGG == left_node.type_) { OZ (resolve_func_node_of_obj_access_idents(left_node, q_name)); + } else if (left_node.type_ == T_LINK_NODE && left_node.value_ == 3) { + ret = OB_ERR_PARSER_SYNTAX; // array not in object access ref : array[1] + LOG_WARN("input invalid arguments", K(ret)); } else { ret = OB_ERR_UNEXPECTED; LOG_WARN("left node of obj access ref node not T_IDENT/T_QUESTIONMARK/T_FUN_SYS", K(ret), K(left_node.type_)); @@ -5700,7 +5704,7 @@ int ObRawExprResolverImpl::process_json_query_node(const ParseNode *node, ObRawE ObSysFunRawExpr *func_expr = NULL; if (OB_SUCC(ret)) { num = node->num_child_; - ctx_.expr_factory_.create_raw_expr(T_FUN_SYS, func_expr); + ctx_.expr_factory_.create_raw_expr(T_FUN_SYS_JSON_QUERY, func_expr); CK(OB_NOT_NULL(func_expr)); OX(func_expr->set_func_name(ObString::make_string("json_query"))); } @@ -5718,7 +5722,7 @@ int ObRawExprResolverImpl::process_json_query_node(const ParseNode *node, ObRawE } // pre check default returning type with item method if (OB_SUCC(ret)) { - if (returning_type->type_ == T_NULL) { + if (returning_type->type_ == T_NULL || returning_type->int16_values_[OB_NODE_CAST_TYPE_IDX] == T_JSON) { ObString path_str(node->children_[1]->text_len_, node->children_[1]->raw_text_); if (OB_FAIL(ObJsonPath::change_json_expr_res_type_if_need(ctx_.expr_factory_.get_allocator(), path_str, const_cast(*returning_type), OPT_JSON_QUERY))) { LOG_WARN("set return type by path item method fail", K(ret), K(path_str)); diff --git a/src/sql/resolver/expr/ob_raw_expr_util.cpp b/src/sql/resolver/expr/ob_raw_expr_util.cpp index a947957ec5..ad55b11d84 100644 --- a/src/sql/resolver/expr/ob_raw_expr_util.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_util.cpp @@ -1114,9 +1114,8 @@ int ObRawExprUtils::resolve_udf_param_exprs(ObResolverParams ¶ms, if (extern_params.at(i).second->same_as(*c_expr)) { ObRawExpr *rawexpr = extern_params.at(i).first; if (T_OBJ_ACCESS_REF == rawexpr->get_expr_type()) { - ObObjAccessRawExpr* obj = static_cast(rawexpr); - CK (OB_NOT_NULL(obj)); - OX (obj->set_write(true)); + OZ (pl::ObPLResolver::set_write_property( + rawexpr, *(params.expr_factory_), params.session_info_, params.schema_checker_->get_schema_guard(), true)); } break; } @@ -1152,9 +1151,8 @@ do { ObObjAccessRawExpr* obj = static_cast(iexpr); uint64_t pkg_id = OB_INVALID_ID; uint64_t var_id = OB_INVALID_ID; - CK (OB_NOT_NULL(obj)); - OX (obj->set_write(true)); - OZ (obj->formalize(params.session_info_)); + OZ (pl::ObPLResolver::set_write_property( + iexpr, *(params.expr_factory_), params.session_info_, params.schema_checker_->get_schema_guard(), true)); if (obj->get_access_idxs().count() > 0 && OB_NOT_NULL(obj->get_access_idxs().at(0).get_sysfunc_) && T_OP_GET_PACKAGE_VAR == obj->get_access_idxs().at(0).get_sysfunc_->get_expr_type()) { @@ -1167,10 +1165,33 @@ do { ObUDFParamDesc(ObUDFParamDesc::OBJ_ACCESS_OUT, var_id, OB_INVALID_ID, pkg_id))); } else if (T_QUESTIONMARK == iexpr->get_expr_type()) { ObConstRawExpr *c_expr = static_cast(iexpr); + pl::ObPLDataType param_type; CK (OB_NOT_NULL(c_expr)); CK (c_expr->get_value().is_unknown()); OZ (udf_raw_expr->add_param_desc( ObUDFParamDesc(ObUDFParamDesc::LOCAL_OUT, c_expr->get_value().get_unknown()))); + if (OB_FAIL(ret) || !pl::ObPLResolver::is_question_mark_value(iexpr, params.secondary_namespace_)) { + // do nothing ... + } else { + if (iparam->is_schema_routine_param()) { + ObRoutineParam *param = static_cast(iparam); + CK (OB_NOT_NULL(param)); + CK (OB_NOT_NULL(params.schema_checker_)); + CK (OB_NOT_NULL(params.schema_checker_->get_schema_guard())); + CK (OB_NOT_NULL(params.session_info_)); + CK (OB_NOT_NULL(params.allocator_)); + CK (OB_NOT_NULL(params.sql_proxy_)); + OZ (pl::ObPLDataType::transform_from_iparam(param, + *(params.schema_checker_->get_schema_guard()), + *(params.session_info_), + *(params.allocator_), + *(params.sql_proxy_), + param_type)); + } else { + param_type = iparam->get_pl_data_type(); + } + OZ (pl::ObPLResolver::set_question_mark_type(iexpr, params.secondary_namespace_, ¶m_type)); + } } else if (T_OP_GET_PACKAGE_VAR == iexpr->get_expr_type()) { const ObSysFunRawExpr *f_expr = static_cast(iexpr); uint64_t pkg_id = OB_INVALID_ID; diff --git a/src/sql/resolver/expr/ob_raw_expr_wrap_enum_set.cpp b/src/sql/resolver/expr/ob_raw_expr_wrap_enum_set.cpp index 2b91b10603..2933f627b9 100644 --- a/src/sql/resolver/expr/ob_raw_expr_wrap_enum_set.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_wrap_enum_set.cpp @@ -175,7 +175,20 @@ int ObRawExprWrapEnumSet::analyze_all_expr(ObDMLStmt &stmt) LOG_WARN("get relation exprs failed", K(ret)); } else if (OB_FAIL(stmt.get_child_stmts(child_stmts))) { LOG_WARN("get child stmt failed", K(ret)); - } else {/*do nothing*/} + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < stmt.get_table_size(); ++i) { + const TableItem *table_item = stmt.get_table_item(i); + if (OB_ISNULL(table_item)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table_item is null", K(i)); + } else if (table_item->is_temp_table()) { + if (OB_FAIL(child_stmts.push_back(table_item->ref_query_))) { + LOG_WARN("store child stmt failed", K(ret)); + } + } + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < relation_exprs.count(); ++i) { if (OB_FAIL(analyze_expr(relation_exprs.at(i)))) { LOG_WARN("failed to analyze expr", K(ret)); @@ -618,6 +631,7 @@ int ObRawExprWrapEnumSet::visit_right_expr(ObRawExpr &right_expr, int64_t row_di ret = OB_ERR_UNEXPECTED; LOG_ERROR("should not come here", K(right_expr), K(ret)); } + LOG_TRACE("succeed to visit right expr", K(root_type), K(row_dimension), K(right_expr)); return ret; } @@ -777,6 +791,7 @@ bool ObRawExprWrapEnumSet::can_wrap_type_to_str(const ObRawExpr &expr) const } } } + LOG_TRACE("succeed to check can wrap type to str", K(bret), K(expr)); return bret; } diff --git a/src/sql/resolver/ob_stmt_type.h b/src/sql/resolver/ob_stmt_type.h index 5f9de36981..1b323f1670 100644 --- a/src/sql/resolver/ob_stmt_type.h +++ b/src/sql/resolver/ob_stmt_type.h @@ -296,6 +296,12 @@ OB_STMT_TYPE_DEF_UNKNOWN_AT(T_DROP_MLOG, get_drop_mlog_stmt_need_privs, 296) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_FLUSH_PRIVILEGES, no_priv_needed, 298) // OB_STMT_TYPE_DEF_UNKNOWN_AT(T_TRANSFER_PARTITION, get_sys_tenant_alter_system_priv, 297) +// OB_STMT_TYPE_DEF_UNKNOWN_AT(T_ADD_LS_REPLICA, get_sys_tenant_alter_system_priv, 299) +// OB_STMT_TYPE_DEF_UNKNOWN_AT(T_REMOVE_LS_REPLICA, get_sys_tenant_alter_system_priv, 300) +// OB_STMT_TYPE_DEF_UNKNOWN_AT(T_MIGRATE_LS_REPLICA, get_sys_tenant_alter_system_priv, 301) +// OB_STMT_TYPE_DEF_UNKNOWN_AT(T_MODIFY_LS_REPLICA, get_sys_tenant_alter_system_priv, 302) +// OB_STMT_TYPE_DEF_UNKNOWN_AT(T_MODIFY_LS_PAXOS_REPLICA_NUM, get_sys_tenant_alter_system_priv, 303) +// OB_STMT_TYPE_DEF_UNKNOWN_AT(T_CANCEL_LS_REPLICA_TASK, get_sys_tenant_alter_system_priv, 304) OB_STMT_TYPE_DEF_UNKNOWN_AT(T_MAX, err_stmt_type_priv, 500) #endif diff --git a/src/sql/rewrite/ob_query_range.cpp b/src/sql/rewrite/ob_query_range.cpp index 30ffb31913..702796f693 100644 --- a/src/sql/rewrite/ob_query_range.cpp +++ b/src/sql/rewrite/ob_query_range.cpp @@ -1070,13 +1070,6 @@ int ObQueryRange::compute_range_size(const ObIArray &key_parts, return ret; } -int ObQueryRange::is_at_most_one_row(bool &is_one_row) const -{ - int ret = OB_SUCCESS; - is_one_row = table_graph_.is_precise_get_; - return ret; -} - int ObQueryRange::is_get(bool &is_range_get) const { return is_get(column_count_, is_range_get); diff --git a/src/sql/rewrite/ob_query_range.h b/src/sql/rewrite/ob_query_range.h index 3408aa0761..7a36f7b614 100644 --- a/src/sql/rewrite/ob_query_range.h +++ b/src/sql/rewrite/ob_query_range.h @@ -457,7 +457,6 @@ public: } return bret; } - int is_at_most_one_row(bool &is_one_row) const; int is_get(bool &is_get) const; int is_get(int64_t column_count, bool &is_get) const; bool is_precise_get() const { return table_graph_.is_precise_get_; } diff --git a/src/sql/rewrite/ob_transform_aggr_subquery.cpp b/src/sql/rewrite/ob_transform_aggr_subquery.cpp index 9304dd1636..339f93f2db 100644 --- a/src/sql/rewrite/ob_transform_aggr_subquery.cpp +++ b/src/sql/rewrite/ob_transform_aggr_subquery.cpp @@ -1409,7 +1409,7 @@ int ObTransformAggrSubquery::check_stmt_valid(ObDMLStmt &stmt, bool &is_valid) bool can_set_unique = false; if (stmt.is_set_stmt() || stmt.is_hierarchical_query() || !stmt.is_sel_del_upd()) { is_valid = false; - } else if (OB_FAIL(ObTransformUtils::check_can_set_stmt_unique(&stmt, can_set_unique))) { + } else if (OB_FAIL(StmtUniqueKeyProvider::check_can_set_stmt_unique(&stmt, can_set_unique))) { LOG_WARN("failed to check can set stmt unque", K(ret)); } else if (!can_set_unique) { is_valid = false; @@ -1856,7 +1856,8 @@ int ObTransformAggrSubquery::get_unique_keys(ObDMLStmt &stmt, LOG_WARN("transform context is invalid", K(ret), K(ctx_), K(query_hint)); } else if (is_first_trans) { // 第一次改写时需要生成所有from table的unique 可以 - if (OB_FAIL(ObTransformUtils::generate_unique_key(ctx_, &stmt, empty_ignore_tables, pkeys))) { + StmtUniqueKeyProvider unique_key_provider(false); + if (OB_FAIL(unique_key_provider.generate_unique_key(ctx_, &stmt, empty_ignore_tables, pkeys))) { LOG_WARN("failed to generate unique key", K(ret)); } } else if ((query_hint->has_outline_data() && stmt.get_table_items().count() < 1) || diff --git a/src/sql/rewrite/ob_transform_const_propagate.cpp b/src/sql/rewrite/ob_transform_const_propagate.cpp index f00633a00b..50cec561a3 100644 --- a/src/sql/rewrite/ob_transform_const_propagate.cpp +++ b/src/sql/rewrite/ob_transform_const_propagate.cpp @@ -1175,7 +1175,7 @@ int ObTransformConstPropagate::replace_internal(ObRawExpr *&cur_expr, if (OB_FAIL(ret)) { } else if (!can_replace) { // do nothing - } else if (OB_FAIL(check_need_cast_when_replace(cur_expr, parent_exprs, need_cast))) { + } else if (OB_FAIL(check_need_cast_when_replace(cur_expr, const_expr, parent_exprs, need_cast))) { LOG_WARN("failed to check need cast", K(ret)); } else if (need_cast && OB_FAIL(prepare_new_expr(expr_const_infos.at(i)))) { LOG_WARN("failed to prepare new expr", K(ret)); @@ -1298,7 +1298,7 @@ int ObTransformConstPropagate::do_remove_const_exec_param(ObRawExpr *&expr, ObRawExpr *cast_expr = ref_expr; bool need_cast = false; trans_happened = true; - if (OB_FAIL(check_need_cast_when_replace(expr, parent_exprs, need_cast))) { + if (OB_FAIL(check_need_cast_when_replace(expr, ref_expr, parent_exprs, need_cast))) { LOG_WARN("failed to check need cast", K(ret)); } else if (!need_cast && parent_exprs.count() != 0) { expr = ref_expr; @@ -1341,6 +1341,7 @@ int ObTransformConstPropagate::do_remove_const_exec_param(ObRawExpr *&expr, } int ObTransformConstPropagate::check_need_cast_when_replace(ObRawExpr *expr, + ObRawExpr *const_expr, ObIArray &parent_exprs, bool &need_cast) { @@ -1357,10 +1358,31 @@ int ObTransformConstPropagate::check_need_cast_when_replace(ObRawExpr *expr, need_cast = true; } else { ObRawExpr *parent_expr = parent_exprs.at(parent_exprs.count() - 1); - need_cast = !(IS_COMPARISON_OP(parent_expr->get_expr_type()) || - parent_expr->is_query_ref_expr() || - parent_expr->is_win_func_expr() || - T_OP_ROW == parent_expr->get_expr_type()); + if (OB_ISNULL(parent_expr) || OB_ISNULL(const_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else { + bool is_parent_cmp = IS_COMPARISON_OP(parent_expr->get_expr_type()); + // To adapt to the behavior of casting NULL values for hash compare + // cast need to be added above NULL when its' parent expr is CMP_OP. + bool need_cast_null = false; + if (is_parent_cmp && const_expr->get_expr_type() == T_NULL) { + for (int64_t i = 0; !need_cast_null && OB_SUCC(ret) && + i < parent_expr->get_param_count(); ++i) { + const ObRawExpr *param_expr = parent_expr->get_param_expr(i); + if (OB_ISNULL(param_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("param expr is null"); + } else if (ObDatumFuncs::is_null_aware_hash_type(param_expr->get_result_type().get_type())) { + need_cast_null = true; + } + } + } + need_cast = need_cast_null || !(is_parent_cmp || + parent_expr->is_query_ref_expr() || + parent_expr->is_win_func_expr() || + T_OP_ROW == parent_expr->get_expr_type()); + } } return ret; } diff --git a/src/sql/rewrite/ob_transform_const_propagate.h b/src/sql/rewrite/ob_transform_const_propagate.h index b8660eb49b..5de3c87b44 100644 --- a/src/sql/rewrite/ob_transform_const_propagate.h +++ b/src/sql/rewrite/ob_transform_const_propagate.h @@ -275,6 +275,7 @@ private: bool &trans_happened); int check_need_cast_when_replace(ObRawExpr *expr, + ObRawExpr *const_expr, ObIArray &parent_exprs, bool &need_cast); diff --git a/src/sql/rewrite/ob_transform_count_to_exists.cpp b/src/sql/rewrite/ob_transform_count_to_exists.cpp index e27789c7f1..0d7fb705d5 100644 --- a/src/sql/rewrite/ob_transform_count_to_exists.cpp +++ b/src/sql/rewrite/ob_transform_count_to_exists.cpp @@ -96,7 +96,7 @@ int ObTransformCountToExists::construct_transform_hint(ObDMLStmt &stmt, void *tr LOG_WARN("get unexpected null", K(ret), K(query_expr)); } else if (OB_FAIL(query_expr->get_ref_stmt()->get_qb_name(qb_name))) { LOG_WARN("failed to get qb name", K(ret)); - } else if (OB_FAIL(hint->get_qb_name_list().push_back(qb_name))) { + } else if (OB_FAIL(hint->get_qb_names().push_back(qb_name))) { LOG_WARN("failed to push back qb name", K(ret)); } } diff --git a/src/sql/rewrite/ob_transform_groupby_pullup.cpp b/src/sql/rewrite/ob_transform_groupby_pullup.cpp index ef515f9f94..eb6f4ace61 100644 --- a/src/sql/rewrite/ob_transform_groupby_pullup.cpp +++ b/src/sql/rewrite/ob_transform_groupby_pullup.cpp @@ -68,6 +68,8 @@ int ObTransformGroupByPullup::transform_one_stmt(common::ObIArraystmt_factory_, *ctx_->expr_factory_, @@ -80,7 +82,7 @@ int ObTransformGroupByPullup::transform_one_stmt(common::ObIArraycheck_if_contain_inner_table(contain_inner_table))) { LOG_WARN("failed to check if contain inner table", K(ret)); - } else if (OB_FAIL(ObTransformUtils::check_can_set_stmt_unique(stmt, has_unique_keys))) { + } else if (OB_FAIL(StmtUniqueKeyProvider::check_can_set_stmt_unique(stmt, has_unique_keys))) { LOG_WARN("failed to check stmt has unique keys", K(ret)); } else if (!has_unique_keys) { //如果当前stmt不能生成唯一键，do nothing @@ -704,7 +706,9 @@ int ObTransformGroupByPullup::get_trans_view(ObDMLStmt *stmt, ObSelectStmt *&vie return ret; } -int ObTransformGroupByPullup::do_groupby_pull_up(ObSelectStmt *stmt, PullupHelper &helper) +int ObTransformGroupByPullup::do_groupby_pull_up(ObSelectStmt *stmt, + PullupHelper &helper, + StmtUniqueKeyProvider &unique_key_provider) { int ret = OB_SUCCESS; ObSEArray unique_exprs; @@ -724,10 +728,10 @@ int ObTransformGroupByPullup::do_groupby_pull_up(ObSelectStmt *stmt, PullupHelpe LOG_WARN("subquery is null", K(*table_item), K(ret)); } else if (OB_FAIL(ignore_tables.add_member(stmt->get_table_bit_index(table_item->table_id_)))) { LOG_WARN("failed to add ignore table index", K(ret)); - } else if (OB_FAIL(ObTransformUtils::generate_unique_key(ctx_, - stmt, - ignore_tables, - unique_exprs))) { + } else if (OB_FAIL(unique_key_provider.generate_unique_key(ctx_, + stmt, + ignore_tables, + unique_exprs))) { LOG_WARN("failed to generated unique keys", K(ret)); } else if (OB_FAIL(append(stmt->get_group_exprs(), unique_exprs))) { LOG_WARN("failed to append group exprs", K(ret)); diff --git a/src/sql/rewrite/ob_transform_groupby_pullup.h b/src/sql/rewrite/ob_transform_groupby_pullup.h index 309abbc8ed..350e603ff2 100644 --- a/src/sql/rewrite/ob_transform_groupby_pullup.h +++ b/src/sql/rewrite/ob_transform_groupby_pullup.h @@ -127,7 +127,9 @@ private: ObRawExpr *&null_propagate_column, bool &is_valid); - int do_groupby_pull_up(ObSelectStmt *stmt, PullupHelper &helper); + int do_groupby_pull_up(ObSelectStmt *stmt, + PullupHelper &helper, + StmtUniqueKeyProvider &unique_key_provider); int get_trans_view(ObDMLStmt *stmt, ObSelectStmt *&view_stmt); diff --git a/src/sql/rewrite/ob_transform_or_expansion.cpp b/src/sql/rewrite/ob_transform_or_expansion.cpp index 8daeca8bd5..3dbca023ee 100644 --- a/src/sql/rewrite/ob_transform_or_expansion.cpp +++ b/src/sql/rewrite/ob_transform_or_expansion.cpp @@ -160,11 +160,15 @@ int ObTransformOrExpansion::transform_in_where_conditon(ObIArray &pa ctx.expand_exprs_.reuse(); ObSelectStmt *transformed_union_stmt = NULL; trans_stmt = upper_stmt; + StmtUniqueKeyProvider unique_key_provider; + try_trans_helper2.unique_key_provider_ = &unique_key_provider; if (reached_max_times_for_or_expansion()) { /*do nothing*/ OPT_TRACE("retry count reached max times:", try_times_); @@ -261,7 +267,8 @@ int ObTransformOrExpansion::transform_in_semi_info(ObIArray &pa } else if (INVALID_OR_EXPAND_TYPE == ctx.or_expand_type_) { /*do nothing*/ } else if (OB_FAIL(transform_or_expansion(spj_stmt, semi_info->semi_id_, i, - false, ctx, transformed_union_stmt))) { + false, ctx, transformed_union_stmt, + unique_key_provider))) { LOG_WARN("failed to do transformation", K(ret)); } else if (OB_FAIL(merge_stmt(trans_stmt, spj_stmt, transformed_union_stmt))) { LOG_WARN("failed to merge stmt", K(ret)); @@ -419,12 +426,14 @@ int ObTransformOrExpansion::try_do_transform_inner_join(ObIArray ObSEArray dummy_exprs; ObTryTransHelper try_trans_helper1; ObTryTransHelper try_trans_helper2; + StmtUniqueKeyProvider unique_key_provider1; + try_trans_helper1.unique_key_provider_ = &unique_key_provider1; if (OB_ISNULL(stmt) || OB_ISNULL(joined_table) || OB_ISNULL(ctx_) || OB_ISNULL(ctx_->stmt_factory_) || OB_ISNULL(ctx_->expr_factory_)) { ret = OB_ERR_UNEXPECTED; @@ -521,6 +532,7 @@ int ObTransformOrExpansion::try_do_transform_left_join(ObIArray dummy_exprs, cur_joined_table, trans_infos))) { LOG_WARN("failed to get conds trans infos", K(ret)); } else if (OB_FAIL(add_select_item_to_ref_query(ref_query, not_null_side_table->table_id_, + unique_key_provider1, left_unique_pos, right_flag_pos))) { LOG_WARN("failed to set stmt unique", K(ret)); } else if (OB_FAIL(try_trans_helper2.fill_helper(stmt->get_query_ctx()))) { @@ -533,12 +545,14 @@ int ObTransformOrExpansion::try_do_transform_left_join(ObIArray ctx.expand_exprs_.reuse(); ObSelectStmt *trans_ref_query = NULL; ObDMLStmt *trans_stmt = origin_trans_stmt; + StmtUniqueKeyProvider unique_key_provider2; + try_trans_helper2.unique_key_provider_ = &unique_key_provider2; if (reached_max_times_for_or_expansion()) { /*do nothing*/ OPT_TRACE("retry count reached max times:", try_times_); } else if (OB_FAIL(transform_or_expansion(ref_query, joined_table->table_id_, trans_infos.at(i).pos_, false, ctx, - trans_ref_query))) { + trans_ref_query, unique_key_provider2))) { LOG_WARN("failed to do transformation", K(ret)); } else if (OB_FAIL(do_transform_for_left_join(trans_ref_query, left_unique_pos, right_flag_pos))) { @@ -634,7 +648,7 @@ int ObTransformOrExpansion::check_child_table_valid(JoinedTable *cur_table, not_null_side_table = child_stmt->is_set_distinct() ? not_null_side_table : NULL; } else if (child_stmt->is_hierarchical_query() || child_stmt->has_rollup()) { not_null_side_table = NULL; - } else if (OB_FAIL(ObTransformUtils::check_can_set_stmt_unique(child_stmt, can_set_unique))) { + } else if (OB_FAIL(StmtUniqueKeyProvider::check_can_set_stmt_unique(child_stmt, can_set_unique))) { LOG_WARN("failed to check can set stmt unique", K(ret)); } else if (!can_set_unique) { not_null_side_table = NULL; @@ -678,6 +692,7 @@ int ObTransformOrExpansion::get_table_not_on_null_side(TableItem *cur_table, // add left table unique key, right table falg expr to stmt select int ObTransformOrExpansion::add_select_item_to_ref_query(ObSelectStmt *stmt, const uint64_t flag_table_id, + StmtUniqueKeyProvider &unique_key_provider, ObSqlBitSet<> &left_unique_pos, ObSqlBitSet<> &right_flag_pos) { @@ -703,8 +718,8 @@ int ObTransformOrExpansion::add_select_item_to_ref_query(ObSelectStmt *stmt, LOG_WARN("failed to get select exprs", K(ret)); } else if (OB_FAIL(stmt->get_table_rel_ids(*right_table, right_tables))) { LOG_WARN("faield to get table rel ids", K(ret), K(*right_table)); - } else if (OB_FAIL(ObTransformUtils::generate_unique_key(ctx_, stmt, right_tables, - left_unique_exprs))) { + } else if (OB_FAIL(unique_key_provider.generate_unique_key(ctx_, stmt, right_tables, + left_unique_exprs))) { LOG_WARN("faield to get table rel ids", K(ret), K(*right_table)); } else if (OB_ISNULL(flag_table = stmt->get_table_item_by_id(flag_table_id))) { LOG_WARN("faield to get table item", K(ret), K(flag_table), K(flag_table_id)); @@ -728,12 +743,9 @@ int ObTransformOrExpansion::add_select_item_to_ref_query(ObSelectStmt *stmt, select_list, right_flag_exprs))) { LOG_WARN("failed to create columns for view", K(ret)); } - } else if (!flag_table->is_basic_table()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpect table item type", K(*flag_table), K(ret)); - } else if (OB_FAIL(ObTransformUtils::generate_unique_key(ctx_, stmt, flag_table, - right_flag_exprs))) { - LOG_WARN("failed to generate unique key", K(ret)); + } else if (OB_FAIL(ObTransformUtils::generate_unique_key_for_basic_table(ctx_, stmt, flag_table, + right_flag_exprs))) { + LOG_WARN("failed to generate unique key for basic table", K(ret)); } else if (OB_UNLIKELY(right_flag_exprs.empty())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpect exprs", K(ret), K(right_flag_exprs)); @@ -1068,7 +1080,7 @@ int ObTransformOrExpansion::has_valid_condition(ObDMLStmt &stmt, OPT_TRACE("check conditions:", conds); if (NULL == expect_ordering) { /* do nothing */ - } else if (OB_FAIL(ObTransformUtils::check_can_set_stmt_unique(&stmt, can_set_distinct))) { + } else if (OB_FAIL(StmtUniqueKeyProvider::check_can_set_stmt_unique(&stmt, can_set_distinct))) { LOG_WARN("failed to check can set stmt unique", K(ret)); } else if (!can_set_distinct) { /* do nothing */ @@ -2061,7 +2073,8 @@ int ObTransformOrExpansion::transform_or_expansion(ObSelectStmt *stmt, const int64_t expr_pos, bool is_topk, ObCostBasedRewriteCtx &ctx, - ObSelectStmt *&trans_stmt) + ObSelectStmt *&trans_stmt, + StmtUniqueKeyProvider &unique_key_provider) { int ret = OB_SUCCESS; ObSelectStmt *copy_stmt = NULL; @@ -2106,7 +2119,7 @@ int ObTransformOrExpansion::transform_or_expansion(ObSelectStmt *stmt, } else if (!ctx.is_set_distinct_ && OB_FAIL(preprocess_or_condition(*view_stmt, trans_id, view_expr_pos))) { LOG_WARN("failed to preprocess or condition", K(ret)); } else if (ctx.is_set_distinct_ && !ctx.is_unique_ && - OB_FAIL(ObTransformUtils::recursive_set_stmt_unique(view_stmt, ctx_, true))) { + OB_FAIL(unique_key_provider.recursive_set_stmt_unique(view_stmt, ctx_, true))) { LOG_WARN("failed to set stmt unique", K(ret)); } else if (!is_topk && OB_FAIL(classify_or_expr(*view_stmt, conds_exprs->at(view_expr_pos)))) { @@ -2134,7 +2147,7 @@ int ObTransformOrExpansion::transform_or_expansion(ObSelectStmt *stmt, LOG_WARN("failed to recursive adjust statement id", K(ret)); } else if (OB_FAIL(get_expand_conds(*child_stmt, trans_id, conds_exprs))) { LOG_WARN("failed to get expand conds", K(ret)); - } else if (OB_FAIL(child_stmt->update_stmt_table_id(*view_stmt))) { + } else if (OB_FAIL(child_stmt->update_stmt_table_id(ctx_->allocator_, *view_stmt))) { //update stmt table id after find conds_exprs LOG_WARN("failed to update table id", K(ret)); } else if (OB_FAIL(adjust_or_expansion_stmt(conds_exprs, view_expr_pos, i, @@ -2355,7 +2368,7 @@ int ObTransformOrExpansion::has_valid_semi_anti_cond(ObDMLStmt &stmt, } else if (has_lob) { // do nothing OPT_TRACE("select expr has lob expr"); - } else if (OB_FAIL(ObTransformUtils::check_can_set_stmt_unique(&stmt, can_set_unique))) { + } else if (OB_FAIL(StmtUniqueKeyProvider::check_can_set_stmt_unique(&stmt, can_set_unique))) { LOG_WARN("failed to check can set stmt unique", K(ret)); } else if (!can_set_unique) { // do nothing @@ -2491,7 +2504,7 @@ int ObTransformOrExpansion::gather_transform_infos(ObSelectStmt *stmt, LOG_WARN("failed to check stmt has lob", K(ret)); } else if (has_lob) { can_set_distinct = false; - } else if (OB_FAIL(ObTransformUtils::check_can_set_stmt_unique(stmt, can_set_distinct))) { + } else if (OB_FAIL(StmtUniqueKeyProvider::check_can_set_stmt_unique(stmt, can_set_distinct))) { LOG_WARN("failed to check can set stmt unique", K(ret)); } diff --git a/src/sql/rewrite/ob_transform_or_expansion.h b/src/sql/rewrite/ob_transform_or_expansion.h index 7c9cea1e31..b3709a48c5 100644 --- a/src/sql/rewrite/ob_transform_or_expansion.h +++ b/src/sql/rewrite/ob_transform_or_expansion.h @@ -160,6 +160,7 @@ private: int add_select_item_to_ref_query(ObSelectStmt *stmt, const uint64_t flag_table_id, + StmtUniqueKeyProvider &unique_key_provider, ObSqlBitSet<> &left_unique_pos, ObSqlBitSet<> &right_flag_pos); @@ -298,7 +299,8 @@ private: const int64_t expr_pos, bool is_topk, ObCostBasedRewriteCtx &ctx, - ObSelectStmt *&trans_stmt); + ObSelectStmt *&trans_stmt, + StmtUniqueKeyProvider &unique_key_provider); int adjust_or_expansion_stmt(ObIArray *conds_exprs, const int64_t expr_pos, const int64_t param_pos, diff --git a/src/sql/rewrite/ob_transform_pre_process.cpp b/src/sql/rewrite/ob_transform_pre_process.cpp index 4680f283ca..5de8e76c93 100644 --- a/src/sql/rewrite/ob_transform_pre_process.cpp +++ b/src/sql/rewrite/ob_transform_pre_process.cpp @@ -60,15 +60,21 @@ int ObTransformPreProcess::transform_one_stmt(common::ObIArray int ret = OB_SUCCESS; trans_happened = false; bool is_happened = false; - if (OB_ISNULL(stmt)) { + if (OB_ISNULL(stmt) || OB_ISNULL(ctx_) || OB_ISNULL(ctx_->allocator_)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("stmt is NULL", K(ret)); + LOG_WARN("unexpected NULL", K(ret), K(stmt), K(ctx_)); } else if (OB_FAIL(ObTransformUtils::right_join_to_left(stmt))) { LOG_WARN("failed to transform right join as left", K(ret)); } else if (parent_stmts.empty() && lib::is_oracle_mode() && OB_FAIL(formalize_limit_expr(*stmt))) { LOG_WARN("formalize stmt fialed", K(ret)); + } else if (OB_FAIL(stmt->adjust_duplicated_table_names(*ctx_->allocator_, is_happened))) { + LOG_WARN("failed to adjust duplicated table names", K(ret)); } else { + trans_happened |= is_happened; + OPT_TRACE("adjust duplicated table name", is_happened); + LOG_TRACE("succeed to adjust duplicated table name", K(is_happened), K(ret)); + if (OB_SUCC(ret)) { if (OB_FAIL(flatten_conditions(stmt, is_happened))) { LOG_WARN("failed to flatten_condition", K(ret)); @@ -1494,7 +1500,7 @@ int ObTransformPreProcess::create_child_stmts_for_groupby_sets(ObSelectStmt *ori ctx_->src_hash_val_, i + 1))) { LOG_WARN("failed to recursive adjust statement id", K(ret)); - } else if (OB_FAIL(groupby_stmt->update_stmt_table_id(*origin_stmt))) { + } else if (OB_FAIL(groupby_stmt->update_stmt_table_id(ctx_->allocator_, *origin_stmt))) { LOG_WARN("failed to update stmt table id.", K(ret)); } else if (OB_FAIL(groupby_stmt->formalize_stmt(ctx_->session_info_))) { LOG_WARN("failed to formalized stmt.", K(ret)); @@ -2346,7 +2352,7 @@ int ObTransformPreProcess::create_and_mock_join_view(ObSelectStmt &stmt) ctx_->src_hash_val_, sub_num))) { LOG_WARN("failed to adjust statement id", K(ret)); - } else if (OB_FAIL(right_view_stmt->update_stmt_table_id(*left_view_stmt))) { + } else if (OB_FAIL(right_view_stmt->update_stmt_table_id(ctx_->allocator_, *left_view_stmt))) { LOG_WARN("failed to update stmt table id", K(ret)); } else { right_view_stmt->get_start_with_exprs().reset(); @@ -5153,7 +5159,7 @@ int ObTransformPreProcess::transform_insert_only_merge_into(ObDMLStmt* stmt, ObD ctx_->src_hash_val_, 1))) { LOG_WARN("failed to recursive adjust statement id", K(ret)); - } else if (OB_FAIL(ref_stmt->update_stmt_table_id(*target_table->ref_query_))) { + } else if (OB_FAIL(ref_stmt->update_stmt_table_id(ctx_->allocator_, *target_table->ref_query_))) { LOG_WARN("failed to update table id", K(ret)); } else if (OB_FALSE_IT(inner_target_table->ref_query_ = ref_stmt)) { } else if (OB_FAIL(ref_stmt->adjust_subquery_list())) { @@ -8802,7 +8808,7 @@ int ObTransformPreProcess::expand_full_outer_join(ObSelectStmt *&ref_query) ctx_->src_hash_val_, sub_num))) { LOG_WARN("failed to recursive adjust statement id", K(ret)); - } else if (OB_FAIL(right_stmt->update_stmt_table_id(*left_stmt))) { + } else if (OB_FAIL(right_stmt->update_stmt_table_id(ctx_->allocator_, *left_stmt))) { LOG_WARN("failed to updatew table id in stmt.", K(ret)); } else if (right_stmt->get_joined_tables().count() != 1) { ret = OB_ERR_UNEXPECTED; diff --git a/src/sql/rewrite/ob_transform_predicate_move_around.cpp b/src/sql/rewrite/ob_transform_predicate_move_around.cpp index ffef4c5b29..98360401d0 100644 --- a/src/sql/rewrite/ob_transform_predicate_move_around.cpp +++ b/src/sql/rewrite/ob_transform_predicate_move_around.cpp @@ -1258,12 +1258,8 @@ int ObTransformPredicateMoveAround::pushdown_predicates( pushdown_preds))) { LOG_WARN("recursive pushdown preds into set stmt failed", K(ret)); } else {/*do nothing*/} - } else if (stmt->is_hierarchical_query() || enable_no_pred_deduce) { + } else if (stmt->is_hierarchical_query()) { // do not transform for current level stmt, but need call function to transform child query - if (enable_no_pred_deduce) { - LOG_TRACE("NO PRED DEDUCE"); - OPT_TRACE("hint disable transform"); - } ObArray dummy_preds; for (int64_t i = 0; OB_SUCC(ret) && i < stmt->get_table_size(); ++i) { TableItem *table = stmt->get_table_item(i); @@ -1276,6 +1272,13 @@ int ObTransformPredicateMoveAround::pushdown_predicates( LOG_WARN("failed to push down predicates", K(ret)); } else {/*do nothing*/} } + } else if (enable_no_pred_deduce) { + // do not transform for current level stmt, but need call function to transform child query + LOG_TRACE("NO PRED DEDUCE"); + OPT_TRACE("hint disable transform"); + if (OB_FAIL(pushdown_into_tables_skip_current_level_stmt(*stmt))) { + LOG_WARN("failed to pushdown predicates into tables skip current level stmt", K(ret)); + } } else { const uint64_t pushdown_pred_count = pushdown_preds.count(); bool is_happened = false; @@ -1383,6 +1386,67 @@ int ObTransformPredicateMoveAround::pushdown_predicates( return ret; } +// when hint disabled transform for current level stmt, need try transform child stmt in table item in specific ordering. +int ObTransformPredicateMoveAround::pushdown_into_tables_skip_current_level_stmt(ObDMLStmt &stmt) +{ + int ret = OB_SUCCESS; + ObArray dummy_preds; + TableItem *table_item = NULL; + ObIArray &from_items = stmt.get_from_items(); + ObIArray &semi_infos = stmt.get_semi_infos(); + for (int64_t i = 0; OB_SUCC(ret) && i < from_items.count(); ++i) { + if (OB_ISNULL(table_item = stmt.get_table_item(from_items.at(i)))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("params have null", K(ret), K(table_item)); + } else if (table_item->is_generated_table()) { + if (OB_FAIL(SMART_CALL(pushdown_predicates(table_item->ref_query_, dummy_preds)))) { + LOG_WARN("failed to push down predicates", K(ret)); + } + } else if (!table_item->is_joined_table()) { + /* do nothing */ + } else if (OB_FAIL(pushdown_into_joined_table_skip_current_level_stmt(table_item))) { + LOG_WARN("failed to push down predicates", K(ret)); + } + } + + for (int64_t i = 0; OB_SUCC(ret) && i < semi_infos.count(); ++i) { + if (OB_ISNULL(semi_infos.at(i)) || + OB_ISNULL(table_item = stmt.get_table_item_by_id(semi_infos.at(i)->right_table_id_))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("params have null", K(ret), K(i), K(table_item), K(semi_infos)); + } else if (!table_item->is_generated_table()) { + /* do nothing */ + } else if (OB_FAIL(SMART_CALL(pushdown_predicates(table_item->ref_query_, dummy_preds)))) { + LOG_WARN("failed to push down predicates", K(ret)); + } + } + return ret; +} + +int ObTransformPredicateMoveAround::pushdown_into_joined_table_skip_current_level_stmt(TableItem *table_item) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(table_item)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(table_item)); + } else if (table_item->is_generated_table()) { + ObArray dummy_preds; + if (OB_FAIL(SMART_CALL(pushdown_predicates(table_item->ref_query_, dummy_preds)))) { + LOG_WARN("failed to push down predicates", K(ret)); + } + } else if (!table_item->is_joined_table()) { + /* do nothing */ + } else { + JoinedTable *joined_table = static_cast(table_item); + if (OB_FAIL(SMART_CALL(pushdown_into_joined_table_skip_current_level_stmt(joined_table->left_table_)))) { + LOG_WARN("failed to push down predicates", K(ret)); + } else if (OB_FAIL(SMART_CALL(pushdown_into_joined_table_skip_current_level_stmt(joined_table->right_table_)))) { + LOG_WARN("failed to push down predicates", K(ret)); + } + } + return ret; +} + int ObTransformPredicateMoveAround::store_all_preds(const ObDMLStmt &stmt, ObIArray> &all_preds) { diff --git a/src/sql/rewrite/ob_transform_predicate_move_around.h b/src/sql/rewrite/ob_transform_predicate_move_around.h index 588321e7b5..ecbced2e41 100644 --- a/src/sql/rewrite/ob_transform_predicate_move_around.h +++ b/src/sql/rewrite/ob_transform_predicate_move_around.h @@ -156,6 +156,9 @@ private: int pushdown_predicates(ObDMLStmt *stmt, ObIArray &predicates); + int pushdown_into_tables_skip_current_level_stmt(ObDMLStmt &stmt); + int pushdown_into_joined_table_skip_current_level_stmt(TableItem *table_item); + /** * @brief pushdown_into_set_stmt * 下推谓词到set stmt的左右子查询中 diff --git a/src/sql/rewrite/ob_transform_rule.cpp b/src/sql/rewrite/ob_transform_rule.cpp index d2e1561c1b..0c2c9528c2 100644 --- a/src/sql/rewrite/ob_transform_rule.cpp +++ b/src/sql/rewrite/ob_transform_rule.cpp @@ -923,7 +923,11 @@ int ObTryTransHelper::recover(ObQueryCtx *query_ctx) LOG_WARN("unexpected null query context", K(ret), K(query_ctx)); } else if (OB_FAIL(query_ctx->query_hint_.recover_qb_names(qb_name_counts_, query_ctx->stmt_count_))) { LOG_WARN("failed to revover qb names", K(ret)); + } else if (NULL != unique_key_provider_ + && OB_FAIL(unique_key_provider_->recover_useless_unique_for_temp_table())) { + LOG_WARN("failed to recover useless unique for temp table", K(ret)); } else { + unique_key_provider_ = NULL; query_ctx->available_tb_id_ = available_tb_id_; query_ctx->subquery_count_ = subquery_count_; query_ctx->temp_table_count_ = temp_table_count_; diff --git a/src/sql/rewrite/ob_transform_rule.h b/src/sql/rewrite/ob_transform_rule.h index da36a1b571..2458205581 100644 --- a/src/sql/rewrite/ob_transform_rule.h +++ b/src/sql/rewrite/ob_transform_rule.h @@ -36,6 +36,7 @@ class ObStmtFactory; class ObPhysicalPlan; class ObCodeGeneratorImpl; class ObLogPlan; +class StmtUniqueKeyProvider; struct ObTransformerCtx { @@ -192,7 +193,11 @@ struct ObParentDMLStmt // use to keep view name/stmt id/qb name stable after copy stmt and try transform struct ObTryTransHelper { - ObTryTransHelper() : available_tb_id_(0), subquery_count_(0), temp_table_count_(0) + ObTryTransHelper() : + available_tb_id_(0), + subquery_count_(0), + temp_table_count_(0), + unique_key_provider_(NULL) {} int fill_helper(const ObQueryCtx *query_ctx); @@ -203,6 +208,7 @@ struct ObTryTransHelper int64_t subquery_count_; int64_t temp_table_count_; ObSEArray qb_name_counts_; + StmtUniqueKeyProvider *unique_key_provider_; }; // record context param values or array/list size diff --git a/src/sql/rewrite/ob_transform_semi_to_inner.cpp b/src/sql/rewrite/ob_transform_semi_to_inner.cpp index 4cbfcc6260..2b306501b0 100644 --- a/src/sql/rewrite/ob_transform_semi_to_inner.cpp +++ b/src/sql/rewrite/ob_transform_semi_to_inner.cpp @@ -715,7 +715,8 @@ int ObTransformSemiToInner::collect_unique_property_of_from_items(ObTransformerC if (OB_ISNULL(table)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpect null table", K(ret), K(table)); - } else if (OB_FAIL(ObTransformUtils::generate_unique_key(ctx, stmt, table, pkeys))) { + // lijinmao todo: fix bug here, need generate unique from table except basic table + } else if (OB_FAIL(ObTransformUtils::generate_unique_key_for_basic_table(ctx, stmt, table, pkeys))) { LOG_WARN("failed to generate unique key", K(ret)); } else if (pkeys.empty()) { ret = OB_ERR_UNEXPECTED; diff --git a/src/sql/rewrite/ob_transform_temp_table.cpp b/src/sql/rewrite/ob_transform_temp_table.cpp index d4a0c0e10f..42c176fe93 100644 --- a/src/sql/rewrite/ob_transform_temp_table.cpp +++ b/src/sql/rewrite/ob_transform_temp_table.cpp @@ -1149,7 +1149,7 @@ int ObTransformTempTable::create_temp_table(ObDMLStmt &root_stmt, if (OB_FAIL(ObTransformUtils::deep_copy_stmt(*ctx_->stmt_factory_, *ctx_->expr_factory_, table->ref_query_, temp_table_stmt))) { LOG_WARN("failed to deep copy stmt", K(ret)); - } else if (OB_FAIL(temp_table_stmt->update_stmt_table_id(*table->ref_query_))) { + } else if (OB_FAIL(temp_table_stmt->update_stmt_table_id(ctx_->allocator_, *table->ref_query_))) { LOG_WARN("failed to update table id", K(ret)); } else if (OB_FAIL(stmt->generate_view_name(*ctx_->allocator_, temp_table->table_name_, diff --git a/src/sql/rewrite/ob_transform_utils.cpp b/src/sql/rewrite/ob_transform_utils.cpp index 9c5dbda3ff..9778465def 100644 --- a/src/sql/rewrite/ob_transform_utils.cpp +++ b/src/sql/rewrite/ob_transform_utils.cpp @@ -2309,27 +2309,7 @@ int ObTransformUtils::is_column_expr_not_null(ObNotNullContext &ctx, LOG_WARN("failed to check expr not null", K(ret)); } } else if (table->is_values_table()) { - int64_t idx = expr->get_column_id() - OB_APP_MIN_COLUMN_ID; - int64_t column_cnt = ctx.stmt_->get_column_size(table->table_id_); - if (OB_UNLIKELY(idx >= column_cnt || column_cnt == 0 || table->table_values_.empty() || - table->table_values_.count() % column_cnt != 0)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(idx), KPC(table), K(column_cnt)); - } else { - is_not_null = true; - int64_t row_count = table->table_values_.count() / column_cnt; - for (int64_t i = 0; OB_SUCC(ret) && is_not_null && i < row_count; ++i) { - if (OB_UNLIKELY(column_cnt * i + idx >= table->table_values_.count())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("get unexpected error", K(ret), K(i), K(idx), KPC(table), K(column_cnt)); - } else if (OB_FAIL(is_expr_not_null(ctx, - table->table_values_.at(column_cnt * i + idx), - is_not_null, - constraints))) { - LOG_WARN("failed to check expr not null", K(ret)); - } else {/*do nothing*/} - } - } + // It is temporarily considered that the values table is not satisfied. } else { // do nothing } @@ -5769,10 +5749,10 @@ int ObTransformUtils::find_relation_expr(ObDMLStmt *stmt, return ret; } -int ObTransformUtils::generate_unique_key(ObTransformerCtx *ctx, - ObDMLStmt *stmt, - TableItem *item, - ObIArray &unique_keys) +int ObTransformUtils::generate_unique_key_for_basic_table(ObTransformerCtx *ctx, + ObDMLStmt *stmt, + TableItem *item, + ObIArray &unique_keys) { int ret = OB_SUCCESS; const ObTableSchema *table_schema = NULL; @@ -5814,10 +5794,10 @@ int ObTransformUtils::generate_unique_key(ObTransformerCtx *ctx, return ret; } -int ObTransformUtils::generate_unique_key(ObTransformerCtx *ctx, - ObDMLStmt *stmt, - ObSqlBitSet<> &ignore_tables, - ObIArray &unique_keys) +int StmtUniqueKeyProvider::generate_unique_key(ObTransformerCtx *ctx, + ObDMLStmt *stmt, + ObSqlBitSet<> &ignore_tables, + ObIArray &unique_keys) { int ret = OB_SUCCESS; ObSqlBitSet<> from_rel_ids; @@ -5831,6 +5811,7 @@ int ObTransformUtils::generate_unique_key(ObTransformerCtx *ctx, for (int64_t i = 0; OB_SUCC(ret) && i < table_items.count(); ++i) { TableItem *table = table_items.at(i); int32_t idx = OB_INVALID_INDEX; + in_temp_table_ = false; if (OB_ISNULL(table)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("table item is null", K(table), K(ret)); @@ -5843,13 +5824,11 @@ int ObTransformUtils::generate_unique_key(ObTransformerCtx *ctx, } else if (ignore_tables.has_member(idx)) { //do nothing } else if (table->is_basic_table()) { - if (OB_FAIL(generate_unique_key(ctx, - stmt, - table, - unique_keys))) { - LOG_WARN("failed to generate unique key", K(ret)); + if (OB_FAIL(ObTransformUtils::generate_unique_key_for_basic_table(ctx, stmt, table, unique_keys))) { + LOG_WARN("failed to generate unique key for basic table", K(ret)); } } else if (table->is_generated_table() || table->is_temp_table()) { + in_temp_table_ = table->is_temp_table(); ObSelectStmt *view_stmt = NULL; ObSEArray stmt_unique_keys; ObSEArray column_exprs; @@ -5861,17 +5840,17 @@ int ObTransformUtils::generate_unique_key(ObTransformerCtx *ctx, false, &stmt_unique_keys))) { LOG_WARN("recursive set stmt unique failed", K(ret)); - } else if (OB_FAIL(create_columns_for_view(ctx, - *table, - stmt, - column_exprs))) { + } else if (OB_FAIL(ObTransformUtils::create_columns_for_view(ctx, + *table, + stmt, + column_exprs))) { //为view生成column exprs LOG_WARN("failed to create columns for view", K(ret)); - } else if (OB_FAIL(convert_select_expr_to_column_expr(stmt_unique_keys, - *view_stmt, - *stmt, - table->table_id_, - unique_keys))) { + } else if (OB_FAIL(ObTransformUtils::convert_select_expr_to_column_expr(stmt_unique_keys, + *view_stmt, + *stmt, + table->table_id_, + unique_keys))) { //找到view的unique keys对应的本层column expr LOG_WARN("failed to get stmt unique keys columns expr", K(ret)); } else { @@ -9295,10 +9274,10 @@ int ObTransformUtils::make_pushdown_limit_count(ObRawExprFactory &expr_factory, * 注意：使用之前需要调用check_can_set_stmt_unique函数确保select_stmt不存在union all, * 因为unoin all不能通过添加主键方式保证输出唯一 */ -int ObTransformUtils::recursive_set_stmt_unique(ObSelectStmt *select_stmt, - ObTransformerCtx *ctx, - bool ignore_check_unique,/*default false */ - ObIArray *unique_keys) +int StmtUniqueKeyProvider::recursive_set_stmt_unique(ObSelectStmt *select_stmt, + ObTransformerCtx *ctx, + bool ignore_check_unique,/*default false */ + ObIArray *unique_keys) { int ret = OB_SUCCESS; if (NULL != unique_keys) { @@ -9307,16 +9286,21 @@ int ObTransformUtils::recursive_set_stmt_unique(ObSelectStmt *select_stmt, bool is_unique = false; ObSqlBitSet<> origin_output_rel_ids; ObSEArray added_unique_keys; + const bool in_temp_table = in_temp_table_; + int64_t sel_item_count = 0; + int64_t col_item_count = 0; if (OB_ISNULL(select_stmt) || OB_ISNULL(ctx) || OB_ISNULL(ctx->allocator_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null", K(ret), K(select_stmt), K(ctx)); - } else if (!ignore_check_unique && (OB_FAIL(check_stmt_unique(select_stmt, - ctx->session_info_, - ctx->schema_checker_, - true /* strict */, - is_unique)))) { + } else if (!ignore_check_unique && (OB_FAIL(ObTransformUtils::check_stmt_unique(select_stmt, + ctx->session_info_, + ctx->schema_checker_, + true /* strict */, + is_unique)))) { LOG_WARN("failed to check stmt unique", K(ret)); } else if (is_unique) { + sel_item_count = select_stmt->get_select_item_size(); + col_item_count = select_stmt->get_column_size(); if (NULL != unique_keys && OB_FAIL(get_unique_keys_from_unique_stmt(select_stmt, ctx->expr_factory_, *unique_keys, @@ -9329,6 +9313,8 @@ int ObTransformUtils::recursive_set_stmt_unique(ObSelectStmt *select_stmt, ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected empty unique keys", K(ret), KPC(select_stmt)); } else { + sel_item_count = select_stmt->get_select_item_size(); + col_item_count = select_stmt->get_column_size(); ObIArray &table_items = select_stmt->get_table_items(); for (int64_t i = 0; OB_SUCC(ret) && i < table_items.count(); ++i) { TableItem *cur_table = table_items.at(i); @@ -9343,6 +9329,7 @@ int ObTransformUtils::recursive_set_stmt_unique(ObSelectStmt *select_stmt, } else if (!origin_output_rel_ids.has_member(bit_id)) { // semi join 中的表, 不会输出 } else if (cur_table->is_generated_table() || cur_table->is_temp_table()) { + in_temp_table_ = in_temp_table || cur_table->is_temp_table(); ObSelectStmt *view_stmt = NULL; ObSEArray stmt_unique_keys; ObSEArray column_exprs; @@ -9354,24 +9341,26 @@ int ObTransformUtils::recursive_set_stmt_unique(ObSelectStmt *select_stmt, false, &stmt_unique_keys)))) { LOG_WARN("recursive set stmt unique failed", K(ret)); - } else if (OB_FAIL(create_columns_for_view(ctx, - *cur_table, - select_stmt, - column_exprs))) { + } else if (OB_FAIL(ObTransformUtils::create_columns_for_view(ctx, + *cur_table, + select_stmt, + column_exprs))) { //为view生成column exprs LOG_WARN("failed to create columns for view", K(ret)); - } else if (OB_FAIL(convert_select_expr_to_column_expr(stmt_unique_keys, - *view_stmt, - *select_stmt, - cur_table->table_id_, - added_unique_keys))) { + } else if (OB_FAIL(ObTransformUtils::convert_select_expr_to_column_expr(stmt_unique_keys, + *view_stmt, + *select_stmt, + cur_table->table_id_, + added_unique_keys))) { //找到stmt unique keys对应的本层column expr LOG_WARN("failed to get stmt unique keys columns expr", K(ret)); } } else if (!cur_table->is_basic_table()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpect table item type", K(*cur_table), K(ret)); - } else if (OB_FAIL(generate_unique_key(ctx, select_stmt, cur_table, added_unique_keys))) { + } else if (OB_FAIL(ObTransformUtils::generate_unique_key_for_basic_table(ctx, select_stmt, + cur_table, + added_unique_keys))) { LOG_WARN("failed to generate unique key", K(ret)); } } @@ -9386,17 +9375,24 @@ int ObTransformUtils::recursive_set_stmt_unique(ObSelectStmt *select_stmt, LOG_WARN("failed to append unique keys", K(ret)); } else if (OB_FAIL(add_non_duplicated_select_expr(added_unique_keys, select_exprs))) { LOG_WARN("failed to add non-duplicated select expr", K(ret)); - } else if (OB_FAIL(create_select_item(*ctx->allocator_, added_unique_keys, select_stmt))) { + } else if (OB_FAIL(ObTransformUtils::create_select_item(*ctx->allocator_, added_unique_keys, select_stmt))) { LOG_WARN("failed to get tables primary keys", K(ret)); } else if (OB_FAIL(select_stmt->formalize_stmt(ctx->session_info_))) { LOG_WARN("failed to formalize stmt", K(ret)); } } + + if (OB_SUCC(ret)) { + in_temp_table_ = in_temp_table; + if (OB_FAIL(try_push_back_modified_info(select_stmt, sel_item_count, col_item_count))) { + LOG_WARN("failed to try push back modified info", K(ret)); + } + } return ret; } /* select_stmt is unique checked by ObTransformUtils::check_stmt_unique */ -int ObTransformUtils::get_unique_keys_from_unique_stmt(const ObSelectStmt *select_stmt, +int StmtUniqueKeyProvider::get_unique_keys_from_unique_stmt(const ObSelectStmt *select_stmt, ObRawExprFactory *expr_factory, ObIArray &unique_keys, ObIArray &added_unique_keys) @@ -9426,8 +9422,75 @@ int ObTransformUtils::get_unique_keys_from_unique_stmt(const ObSelectStmt *selec return ret; } -int ObTransformUtils::add_non_duplicated_select_expr(ObIArray &add_select_exprs, - ObIArray &org_select_exprs) +int StmtUniqueKeyProvider::recover_useless_unique_for_temp_table() +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(sel_stmts_.count() != sel_item_counts_.count() + || sel_stmts_.count() != col_item_counts_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected array count", K(ret), K(sel_stmts_.count()), + K(sel_item_counts_.count()), K(col_item_counts_.count())); + } else if (sel_stmts_.empty()) { + /* do nothing */ + } else { + ObSelectStmt *sel_stmt = NULL; + // recover select stmt from the end of the array + int64_t sel_item_count = 0; + int64_t col_item_count = 0; + for (int64_t i = sel_stmts_.count() - 1; OB_SUCC(ret) && i >= 0; --i) { + sel_item_count = sel_item_counts_.at(i); + col_item_count = col_item_counts_.at(i); + if (OB_ISNULL(sel_stmt = sel_stmts_.at(i)) + || OB_UNLIKELY(sel_item_count <= 0 || col_item_count < 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected params", K(ret), K(sel_stmt), K(sel_item_count), K(col_item_count)); + } else if (OB_UNLIKELY(sel_item_count >= sel_stmt->get_select_item_size() + || col_item_count > sel_stmt->get_column_size())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected params", K(ret), K(sel_item_count), K(sel_stmt->get_select_item_size()), + K(col_item_count), K(sel_stmt->get_column_size())); + } else { + ObOptimizerUtil::revert_items(sel_stmt->get_select_items(), sel_item_count); + ObOptimizerUtil::revert_items(sel_stmt->get_column_items(), col_item_count); + } + } + if (OB_SUCC(ret)) { + sel_stmts_.reuse(); + sel_item_counts_.reuse(); + col_item_counts_.reuse(); + } + } + return ret; +} + +int StmtUniqueKeyProvider::try_push_back_modified_info(ObSelectStmt *select_stmt, + int64_t sel_item_count, + int64_t col_item_count) +{ + int ret = OB_SUCCESS; + if (!for_costed_trans_) { + /* do nothing */ + } else if (OB_ISNULL(select_stmt) || + OB_UNLIKELY(sel_stmts_.count() != sel_item_counts_.count() + || sel_stmts_.count() != col_item_counts_.count())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected array count", K(ret), K(select_stmt), K(sel_stmts_.count()), + K(sel_item_counts_.count()), K(col_item_counts_.count())); + } else if (select_stmt->get_select_item_size() == sel_item_count + && select_stmt->get_column_size() == col_item_count) { + /* do nothing */ + } else if (OB_FAIL(sel_stmts_.push_back(select_stmt))) { + LOG_WARN("failed to push back array", K(ret), K(select_stmt)); + } else if (OB_FAIL(sel_item_counts_.push_back(sel_item_count))) { + LOG_WARN("failed to push back array", K(ret), K(sel_item_count)); + } else if (OB_FAIL(col_item_counts_.push_back(col_item_count))) { + LOG_WARN("failed to push back array", K(ret), K(col_item_count)); + } + return ret; +} + +int StmtUniqueKeyProvider::add_non_duplicated_select_expr(ObIArray &add_select_exprs, + ObIArray &org_select_exprs) { int ret = OB_SUCCESS; ObSEArray new_select_exprs; @@ -9441,8 +9504,8 @@ int ObTransformUtils::add_non_duplicated_select_expr(ObIArray &add_s return ret; } -int ObTransformUtils::check_can_set_stmt_unique(ObDMLStmt *stmt, - bool &can_set_unique) +int StmtUniqueKeyProvider::check_can_set_stmt_unique(ObDMLStmt *stmt, + bool &can_set_unique) { int ret = OB_SUCCESS; can_set_unique = false; @@ -14235,7 +14298,7 @@ int ObTransformUtils::expand_temp_table(ObTransformerCtx *ctx, ObDMLStmt::TempTa ctx->src_hash_val_, j))) { LOG_WARN("failed to recursive adjust statement id", K(ret)); - } else if (OB_FAIL(child_stmt->update_stmt_table_id(*temp_table_query))) { + } else if (OB_FAIL(child_stmt->update_stmt_table_id(ctx->allocator_, *temp_table_query))) { LOG_WARN("failed to update table id", K(ret)); } else if (OB_FAIL(upper_stmt->formalize_stmt_expr_reference(ctx->expr_factory_, ctx->session_info_))) { diff --git a/src/sql/rewrite/ob_transform_utils.h b/src/sql/rewrite/ob_transform_utils.h index f41cb0df16..92e3d3eedf 100644 --- a/src/sql/rewrite/ob_transform_utils.h +++ b/src/sql/rewrite/ob_transform_utils.h @@ -1038,21 +1038,10 @@ public: ObIArray &targets, ObIArray &parents); - static int generate_unique_key(ObTransformerCtx *ctx, - ObDMLStmt *stmt, - TableItem *item, - ObIArray &unique_keys); - - /** - * @brief generate_unique_key - * 可以对stmt生成唯一键，需要调用 - * check_can_set_stmt_unique - * 确认stmt有唯一键 - */ - static int generate_unique_key(ObTransformerCtx *ctx, - ObDMLStmt *stmt, - ObSqlBitSet<> &ignore_tables, - ObIArray &unique_keys); + static int generate_unique_key_for_basic_table(ObTransformerCtx *ctx, + ObDMLStmt *stmt, + TableItem *item, + ObIArray &unique_keys); static int check_loseless_join(ObDMLStmt *stmt, ObTransformerCtx *ctx, @@ -1365,17 +1354,6 @@ public: ObRawExpr *limit_offset, ObRawExpr *&pushdown_limit_count); - static int recursive_set_stmt_unique(ObSelectStmt *select_stmt, - ObTransformerCtx *ctx, - bool ignore_check_unique = false, - common::ObIArray *unique_keys = NULL); - static int get_unique_keys_from_unique_stmt(const ObSelectStmt *select_stmt, - ObRawExprFactory *expr_factory, - ObIArray &unique_keys, - ObIArray &added_unique_keys); - static int check_can_set_stmt_unique(ObDMLStmt *stmt, - bool &can_set_unique); - static int get_rel_ids_from_tables(const ObDMLStmt *stmt, const ObIArray &table_items, ObRelIds &rel_ids); @@ -1920,9 +1898,6 @@ private: ObIArray &from_exprs, ObIArray &view_exprs); - static int add_non_duplicated_select_expr(ObIArray &add_select_exprs, - ObIArray &org_select_exprs); - static int extract_shared_exprs(ObDMLStmt *parent, ObSelectStmt *view_stmt, ObIArray &common_exprs, @@ -1946,6 +1921,51 @@ private: bool &is_safe); }; +class StmtUniqueKeyProvider +{ +public: + StmtUniqueKeyProvider(bool for_costed_trans = true) : + for_costed_trans_(for_costed_trans), + in_temp_table_(false) + {} + virtual ~StmtUniqueKeyProvider() {} + + static int check_can_set_stmt_unique(ObDMLStmt *stmt, + bool &can_set_unique); + int recursive_set_stmt_unique(ObSelectStmt *select_stmt, + ObTransformerCtx *ctx, + bool ignore_check_unique = false, + common::ObIArray *unique_keys = NULL); + /** + * @brief generate_unique_key + * generate unique key for stmt, need call check_can_set_stmt_unique before to ensure unique key can be generated + */ + int generate_unique_key(ObTransformerCtx *ctx, + ObDMLStmt *stmt, + ObSqlBitSet<> &ignore_tables, + ObIArray &unique_keys); + int recover_useless_unique_for_temp_table(); +private: + int get_unique_keys_from_unique_stmt(const ObSelectStmt *select_stmt, + ObRawExprFactory *expr_factory, + ObIArray &unique_keys, + ObIArray &added_unique_keys); + int try_push_back_modified_info(ObSelectStmt *select_stmt, + int64_t sel_item_count, + int64_t col_item_count); + int add_non_duplicated_select_expr(ObIArray &add_select_exprs, + ObIArray &org_select_exprs); + +private: + DISALLOW_COPY_AND_ASSIGN(StmtUniqueKeyProvider); + const bool for_costed_trans_; + bool in_temp_table_; + // select items in temp tables may be appended by recursive_set_stmt_unique, store some information to recover them. + // ordering in array below must be maintained by this class + common::ObSEArray sel_stmts_; + common::ObSEArray sel_item_counts_; + common::ObSEArray col_item_counts_; +}; template int ObTransformUtils::replace_exprs(const common::ObIArray &other_exprs, diff --git a/src/sql/session/ob_basic_session_info.cpp b/src/sql/session/ob_basic_session_info.cpp index 556ae4127c..6f49946a71 100644 --- a/src/sql/session/ob_basic_session_info.cpp +++ b/src/sql/session/ob_basic_session_info.cpp @@ -1474,8 +1474,6 @@ int ObBasicSessionInfo::get_influence_plan_sys_var(ObSysVarInPC &sys_vars) const LOG_ERROR("influence plan system var is NULL", K(i), K(ret)); } else if (OB_FAIL(sys_vars.push_back(sys_vars_[index]->get_value()))) { LOG_WARN("influence plan system variables push failed", K(ret)); - } else { - LOG_WARN("luofan test get_influence_plan_sys_var", KPC(sys_vars_[index])); } } return ret; @@ -3625,7 +3623,7 @@ int ObBasicSessionInfo::replace_user_variables(const ObSessionValMap &user_var_m return ret; } -int ObBasicSessionInfo::replace_user_variable(const ObString &var, const ObSessionVariable &val) +int ObBasicSessionInfo::replace_user_variable(const ObString &var, const ObSessionVariable &val, bool need_track) { int ret = OB_SUCCESS; if (var.empty()) { @@ -3634,7 +3632,7 @@ int ObBasicSessionInfo::replace_user_variable(const ObString &var, const ObSessi } else if (OB_FAIL(user_var_val_map_.set_refactored(var, val))) { LOG_ERROR("fail to add variable", K(var), K(ret)); } else { - if (is_track_session_info()) { + if (need_track && is_track_session_info()) { if (OB_FAIL(track_user_var(var))) { LOG_WARN("fail to track user var", K(var), K(ret)); } diff --git a/src/sql/session/ob_basic_session_info.h b/src/sql/session/ob_basic_session_info.h index 631d97b7ca..816514c59e 100644 --- a/src/sql/session/ob_basic_session_info.h +++ b/src/sql/session/ob_basic_session_info.h @@ -983,7 +983,7 @@ public: ///@{ user variables related: sql::ObSessionValMap &get_user_var_val_map() {return user_var_val_map_;} const sql::ObSessionValMap &get_user_var_val_map() const {return user_var_val_map_;} - int replace_user_variable(const common::ObString &var, const ObSessionVariable &val); + int replace_user_variable(const common::ObString &var, const ObSessionVariable &val, bool need_track = true); int replace_user_variables(const ObSessionValMap &user_var_map); int remove_user_variable(const common::ObString &var); int get_user_variable(const common::ObString &var, ObSessionVariable &val) const; diff --git a/src/sql/session/ob_sql_session_info.cpp b/src/sql/session/ob_sql_session_info.cpp index 22b552c0b2..c76a22a71d 100644 --- a/src/sql/session/ob_sql_session_info.cpp +++ b/src/sql/session/ob_sql_session_info.cpp @@ -2237,6 +2237,7 @@ int ObSQLSessionInfo::replace_user_variable( OZ (reset_all_package_state_by_dbms_session(false)); } else if (is_package_variable && OB_NOT_NULL(get_pl_engine())) { OZ (set_package_variable(ctx, name, value.value_, true)); + OZ (ObBasicSessionInfo::replace_user_variable(name, value, false)); } else { OZ (ObBasicSessionInfo::replace_user_variable(name, value)); } diff --git a/src/sql/session/ob_sql_session_mgr.cpp b/src/sql/session/ob_sql_session_mgr.cpp index 1891b998c0..55ad5dc665 100644 --- a/src/sql/session/ob_sql_session_mgr.cpp +++ b/src/sql/session/ob_sql_session_mgr.cpp @@ -116,7 +116,7 @@ void ObTenantSQLSessionMgr::destroy() { } -int ObTenantSQLSessionMgr::mtl_init(ObTenantSQLSessionMgr *&t_session_mgr) +int ObTenantSQLSessionMgr::mtl_new(ObTenantSQLSessionMgr *&t_session_mgr) { int ret = OB_SUCCESS; t_session_mgr = OB_NEW(ObTenantSQLSessionMgr, ObMemAttr(MTL_ID(), "TSQLSessionMgr"), @@ -124,7 +124,14 @@ int ObTenantSQLSessionMgr::mtl_init(ObTenantSQLSessionMgr *&t_session_mgr) if (OB_ISNULL(t_session_mgr)) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to alloc tenant session manager", K(ret)); - } else if (OB_FAIL(t_session_mgr->init())) { + } + return ret; +} + +int ObTenantSQLSessionMgr::mtl_init(ObTenantSQLSessionMgr *&t_session_mgr) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(t_session_mgr->init())) { LOG_WARN("failed to init tenant session manager", K(ret)); } return ret; diff --git a/src/sql/session/ob_sql_session_mgr.h b/src/sql/session/ob_sql_session_mgr.h index d716233876..cb0fc7c065 100644 --- a/src/sql/session/ob_sql_session_mgr.h +++ b/src/sql/session/ob_sql_session_mgr.h @@ -376,6 +376,7 @@ public: int init(); void destroy(); + static int mtl_new(ObTenantSQLSessionMgr *&tenant_session_mgr); static int mtl_init(ObTenantSQLSessionMgr *&tenant_session_mgr); static void mtl_destroy(ObTenantSQLSessionMgr *&tenant_session_mgr); ObSQLSessionInfo *alloc_session(); diff --git a/src/storage/access/ob_index_sstable_estimator.cpp b/src/storage/access/ob_index_sstable_estimator.cpp index 8ee102aa90..5e85bdf9d6 100644 --- a/src/storage/access/ob_index_sstable_estimator.cpp +++ b/src/storage/access/ob_index_sstable_estimator.cpp @@ -75,27 +75,21 @@ int ObIndexBlockScanEstimator::estimate_row_count(ObPartitionEst &part_est) ObEstimatedResult result; lib::ObMemAttr mem_attr(MTL_ID(), "OB_STORAGE_EST"); - common::ObSEArray agg_projector; - common::ObSEArray agg_column_schema; if (OB_UNLIKELY(!context_.is_valid())) { ret = common::OB_INVALID_ARGUMENT; STORAGE_LOG(WARN, "estimate context is not valid", K(ret), K(context_)); } else if (OB_FAIL(allocator_.init(nullptr, OB_MALLOC_MIDDLE_BLOCK_SIZE, mem_attr))) { STORAGE_LOG(WARN, "Fail to init allocator", K(ret)); } else if (OB_FAIL(index_block_row_scanner_.init( - agg_projector, - agg_column_schema, context_.tablet_handle_.get_obj()->get_rowkey_read_info().get_datum_utils(), allocator_, context_.query_flag_, context_.sstable_.get_macro_offset()))) { - STORAGE_LOG(WARN, "Failed to init index block row scanner", K(ret), K(agg_projector), K(agg_column_schema)); + STORAGE_LOG(WARN, "Failed to init index block row scanner", K(ret)); } else if (OB_FAIL(context_.sstable_.get_index_tree_root(root_index_block_))) { STORAGE_LOG(WARN, "Failed to get index tree root", K(ret)); } else if (context_.sstable_.is_ddl_merge_sstable()) { index_block_row_scanner_.set_iter_param(&context_.sstable_, - context_.tablet_handle_.get_obj()->get_ls_id(), - context_.tablet_handle_.get_obj()->get_tablet_id(), context_.tablet_handle_.get_obj()); } if (OB_FAIL(ret)) { diff --git a/src/storage/access/ob_index_tree_prefetcher.cpp b/src/storage/access/ob_index_tree_prefetcher.cpp index 365e5ffd88..d40b413c50 100644 --- a/src/storage/access/ob_index_tree_prefetcher.cpp +++ b/src/storage/access/ob_index_tree_prefetcher.cpp @@ -49,7 +49,7 @@ void ObIndexTreePrefetcher::reset() void ObIndexTreePrefetcher::reuse() { - index_scanner_.reset(); + index_scanner_.reuse(); } int ObIndexTreePrefetcher::init( @@ -117,6 +117,18 @@ int ObIndexTreePrefetcher::init_basic_info( data_version_ = sstable_->get_data_version(); bool is_normal_query = !access_ctx_->query_flag_.is_daily_merge() && !access_ctx_->query_flag_.is_multi_version_minor_merge(); index_tree_height_ = sstable_meta_handle_.get_sstable_meta().get_index_tree_height(sstable.is_ddl_merge_sstable() && is_normal_query); + + if (index_scanner_.is_valid()) { + if (OB_ISNULL(iter_param_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid iter param", K(ret), KPC(iter_param_), K(lbt())); + } else { + const ObTablet *cur_tablet = OB_ISNULL(iter_param_->get_table_param_) ? nullptr : iter_param_->get_table_param_->tablet_iter_.get_tablet(); + index_scanner_.switch_context(sstable, cur_tablet, *datum_utils_, *access_ctx_); + } + } else if (OB_FAIL(init_index_scanner(index_scanner_))) { + LOG_WARN("Fail to init index_scanner", K(ret)); + } } return ret; } @@ -142,9 +154,7 @@ int ObIndexTreePrefetcher::single_prefetch(ObSSTableReadHandle &read_handle) OB_FAIL(lookup_in_cache(read_handle))) { LOG_WARN("Failed to lookup_in_cache", K(ret)); } else if (ObSSTableRowState::IN_BLOCK == read_handle.row_state_) { - if (OB_FAIL(init_index_scanner(index_scanner_))) { - LOG_WARN("Fail to init index scanner", K(ret)); - } else if (OB_FAIL(lookup_in_index_tree(read_handle, false))) { + if (OB_FAIL(lookup_in_index_tree(read_handle, false))) { LOG_WARN("Failed to lookup_in_block", K(ret)); } } @@ -259,16 +269,18 @@ int ObIndexTreePrefetcher::init_index_scanner(ObIndexBlockRowScanner &index_scan { int ret = OB_SUCCESS; if (OB_FAIL(index_scanner.init( - agg_projector_, - agg_column_schema_, *datum_utils_, *access_ctx_->stmt_allocator_, access_ctx_->query_flag_, sstable_->get_macro_offset(), sstable_->is_normal_cg_sstable()))) { LOG_WARN("init index scanner fail", K(ret), KPC(sstable_), KP(sstable_)); + } else if (OB_ISNULL(iter_param_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid iter param", K(ret), KPC(iter_param_), K(lbt())); } else { - index_scanner.set_iter_param(sstable_, access_ctx_->ls_id_, access_ctx_->tablet_id_); + const ObTablet *cur_tablet = OB_ISNULL(iter_param_->get_table_param_) ? nullptr : iter_param_->get_table_param_->tablet_iter_.get_tablet(); + index_scanner.set_iter_param(sstable_, cur_tablet); } return ret; } @@ -443,11 +455,6 @@ int ObIndexTreeMultiPrefetcher::switch_context( ext_read_handles_.at(i).reset(); } } - if (OB_SUCC(ret)) { - if (index_scanner_.is_valid()) { - index_scanner_.switch_context(sstable, *datum_utils_, access_ctx_->query_flag_, access_ctx_->ls_id_, access_ctx_->tablet_id_); - } - } } return ret; } @@ -484,10 +491,6 @@ int ObIndexTreeMultiPrefetcher::multi_prefetch() } else if (ObSSTableRowState::IN_BLOCK == read_handle.row_state_) { if (OB_FAIL(sstable_->get_index_tree_root(index_block_))) { LOG_WARN("Fail to get index block root", K(ret), KPC(sstable_), KP(sstable_)); - } - if (OB_FAIL(ret)) { - } else if (!index_scanner_.is_valid() && OB_FAIL(init_index_scanner(index_scanner_))) { - LOG_WARN("Fail to init index scanner", K(ret)); } else if (OB_FAIL(drill_down(ObIndexBlockRowHeader::DEFAULT_IDX_ROW_MACRO_ID, read_handle, false, is_rowkey_to_fetched))) { LOG_WARN("Fail to prefetch next level", K(ret), K(index_block_), K(read_handle), KPC(this)); } @@ -782,7 +785,13 @@ int ObIndexTreeMultiPassPrefetcher::s if (OB_SUCC(ret)) { for (int64_t level = 0; OB_SUCC(ret) && level < index_tree_height_; level++) { if (tree_handles_[level].index_scanner_.is_valid()) { - tree_handles_[level].index_scanner_.switch_context(sstable, *datum_utils_, access_ctx_->query_flag_, access_ctx_->ls_id_, access_ctx_->tablet_id_); + if (OB_ISNULL(iter_param_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid iter param", K(ret), KPC(iter_param_), K(lbt())); + } else { + const ObTablet *cur_tablet = OB_ISNULL(iter_param_->get_table_param_) ? nullptr : iter_param_->get_table_param_->tablet_iter_.get_tablet(); + tree_handles_[level].index_scanner_.switch_context(sstable, cur_tablet, *datum_utils_, *access_ctx_); + } } else if (OB_FAIL(init_index_scanner(tree_handles_[level].index_scanner_))) { LOG_WARN("Fail to init index_scanner", K(ret), K(level)); } diff --git a/src/storage/access/ob_index_tree_prefetcher.h b/src/storage/access/ob_index_tree_prefetcher.h index 2d47924b33..f11ec7dd77 100644 --- a/src/storage/access/ob_index_tree_prefetcher.h +++ b/src/storage/access/ob_index_tree_prefetcher.h @@ -194,8 +194,6 @@ protected: const ObTableIterParam *iter_param_; ObTableAccessContext *access_ctx_; const ObStorageDatumUtils *datum_utils_; - common::ObFixedArray agg_projector_; - common::ObFixedArray agg_column_schema_; static const int64_t DEFAULT_GET_MICRO_DATA_HANDLE_CNT = 2; ObIndexBlockRowScanner index_scanner_; ObMicroBlockDataHandle last_micro_block_handle_; diff --git a/src/storage/access/ob_multiple_merge.cpp b/src/storage/access/ob_multiple_merge.cpp index 5847977f6e..e4dd53742a 100644 --- a/src/storage/access/ob_multiple_merge.cpp +++ b/src/storage/access/ob_multiple_merge.cpp @@ -139,6 +139,7 @@ int ObMultipleMerge::init( } unprojected_row_.count_ = 0; get_table_param_ = &get_table_param; + access_param_->iter_param_.set_table_param(get_table_param_); const ObITableReadInfo *read_info = access_param_->iter_param_.get_read_info(); if (OB_SUCC(ret)) { if (OB_ISNULL(read_info)) { @@ -180,6 +181,7 @@ int ObMultipleMerge::switch_param( access_param_ = ¶m; access_ctx_ = &context; get_table_param_ = &get_table_param; + access_param_->iter_param_.set_table_param(get_table_param_); if (OB_FAIL(prepare_read_tables())) { STORAGE_LOG(WARN, "Failed to prepare read tables", K(ret), K(*this)); diff --git a/src/storage/access/ob_rows_info.cpp b/src/storage/access/ob_rows_info.cpp index 0a8eb07674..6e9d8f0d4e 100644 --- a/src/storage/access/ob_rows_info.cpp +++ b/src/storage/access/ob_rows_info.cpp @@ -205,10 +205,12 @@ int ObRowsInfo::check_duplicate(ObStoreRow *rows, const int64_t row_count, ObRel RowsCompare rows_cmp(*datum_utils_, min_key_, true, ret); std::sort(rowkeys_.begin(), rowkeys_.end(), rows_cmp); } - for (int64_t i = 0; i < row_count; i++) { - permutation_[rowkeys_[i].row_idx_] = i; + if (OB_SUCC(ret)) { + for (int64_t i = 0; i < row_count; i++) { + permutation_[rowkeys_[i].row_idx_] = i; + } + min_key_ = rowkeys_.at(0).marked_rowkey_.get_rowkey(); } - min_key_ = rowkeys_.at(0).marked_rowkey_.get_rowkey(); } } diff --git a/src/storage/access/ob_table_access_param.cpp b/src/storage/access/ob_table_access_param.cpp index f32266eadf..bc127676a1 100644 --- a/src/storage/access/ob_table_access_param.cpp +++ b/src/storage/access/ob_table_access_param.cpp @@ -31,6 +31,7 @@ ObTableIterParam::ObTableIterParam() tablet_id_(), read_info_(nullptr), rowkey_read_info_(nullptr), + get_table_param_(nullptr), cg_read_info_handle_(), out_cols_project_(NULL), agg_cols_project_(NULL), @@ -65,6 +66,7 @@ void ObTableIterParam::reset() tablet_id_.reset(); read_info_ = nullptr; rowkey_read_info_ = nullptr; + get_table_param_ = nullptr; cg_read_info_handle_.reset(); out_cols_project_ = NULL; agg_cols_project_ = NULL; diff --git a/src/storage/access/ob_table_access_param.h b/src/storage/access/ob_table_access_param.h index 8bd44d4008..f82fcb4cce 100644 --- a/src/storage/access/ob_table_access_param.h +++ b/src/storage/access/ob_table_access_param.h @@ -141,6 +141,8 @@ public: { return pd_storage_flag_.set_use_column_store(true); } OB_INLINE void set_not_use_column_store() { return pd_storage_flag_.set_use_column_store(false); } + OB_INLINE void set_table_param(const ObGetTableParam *get_table_param) + { get_table_param_ = get_table_param; } OB_INLINE bool use_uniform_format() const { return op_->enable_rich_format_ && @@ -159,6 +161,7 @@ public: uint32_t cg_idx_; const ObITableReadInfo *read_info_; const ObITableReadInfo *rowkey_read_info_; + const ObGetTableParam *get_table_param_; ObCGReadInfoHandle cg_read_info_handle_; //TODO(huronghui.hrh):temp solution const ObColumnParam *cg_col_param_; diff --git a/src/storage/blocksstable/encoding/ob_imicro_block_decoder.h b/src/storage/blocksstable/encoding/ob_imicro_block_decoder.h index 82144c1795..e180fb6a7c 100644 --- a/src/storage/blocksstable/encoding/ob_imicro_block_decoder.h +++ b/src/storage/blocksstable/encoding/ob_imicro_block_decoder.h @@ -30,7 +30,7 @@ public: ObIMicroBlockDecoder() : ObIMicroBlockReader() {} virtual ~ObIMicroBlockDecoder() {} virtual int compare_rowkey( - const ObDatumRowkey &rowkey, const int64_t index, int32_t &compare_result) = 0; + const ObDatumRowkey &rowkey, const int64_t index, int32_t &compare_result) override = 0; virtual int compare_rowkey(const ObDatumRange &range, const int64_t index, int32_t &start_key_compare_result, int32_t &end_key_compare_result) = 0; diff --git a/src/storage/blocksstable/index_block/ob_ddl_index_block_row_iterator.cpp b/src/storage/blocksstable/index_block/ob_ddl_index_block_row_iterator.cpp index bb9e7bc5b4..c17357750a 100644 --- a/src/storage/blocksstable/index_block/ob_ddl_index_block_row_iterator.cpp +++ b/src/storage/blocksstable/index_block/ob_ddl_index_block_row_iterator.cpp @@ -11,6 +11,7 @@ */ #define USING_LOG_PREFIX STORAGE +#include "storage/access/ob_rows_info.h" #include "storage/blocksstable/index_block/ob_ddl_index_block_row_iterator.h" #include "storage/ddl/ob_tablet_ddl_kv.h" #include "storage/ls/ob_ls.h" @@ -26,6 +27,7 @@ namespace blocksstable ObDDLIndexBlockRowIterator::ObDDLIndexBlockRowIterator() : is_iter_start_(false), is_iter_finish_(true), + is_normal_cg_(false), btree_iter_(), block_meta_tree_(nullptr), cur_tree_value_(nullptr) @@ -43,6 +45,7 @@ void ObDDLIndexBlockRowIterator::reset() ObIndexBlockRowIterator::reset(); is_iter_finish_ = true; is_iter_start_ = false; + is_normal_cg_ = false; btree_iter_.reset(); block_meta_tree_ = nullptr; cur_tree_value_ = nullptr; @@ -52,6 +55,7 @@ void ObDDLIndexBlockRowIterator::reuse() { is_iter_finish_ = true; is_iter_start_ = false; + is_normal_cg_ = false; btree_iter_.reset(); block_meta_tree_ = nullptr; cur_tree_value_ = nullptr; @@ -61,7 +65,6 @@ int ObDDLIndexBlockRowIterator::init(const ObMicroBlockData &idx_block_data, const ObStorageDatumUtils *datum_utils, ObIAllocator *allocator, const bool is_reverse_scan, - const bool set_iter_end, const ObIndexBlockIterParam &iter_param) { int ret = OB_SUCCESS; @@ -73,6 +76,7 @@ int ObDDLIndexBlockRowIterator::init(const ObMicroBlockData &idx_block_data, is_reverse_scan_ = is_reverse_scan; iter_step_ = is_reverse_scan_ ? -1 : 1; datum_utils_ = datum_utils; + is_normal_cg_ = iter_param.is_valid() ? iter_param.sstable_->is_normal_cg_sstable() : false; is_inited_ = true; } return ret; @@ -81,6 +85,7 @@ int ObDDLIndexBlockRowIterator::init(const ObMicroBlockData &idx_block_data, int ObDDLIndexBlockRowIterator::set_iter_param(const ObStorageDatumUtils *datum_utils, bool is_reverse_scan, const storage::ObBlockMetaTree *block_meta_tree, + const bool is_normal_cg, const int64_t iter_step) { int ret = OB_SUCCESS; @@ -92,6 +97,7 @@ int ObDDLIndexBlockRowIterator::set_iter_param(const ObStorageDatumUtils *datum_ is_reverse_scan_ = is_reverse_scan; iter_step_ = iter_step == INT64_MAX ? (is_reverse_scan_ ? -1 : 1) : iter_step; datum_utils_ = datum_utils; + is_normal_cg_ = is_normal_cg; is_inited_ = true; } return ret; @@ -171,6 +177,113 @@ int ObDDLIndexBlockRowIterator::locate_range(const ObDatumRange &range, return ret; } +int ObDDLIndexBlockRowIterator::locate_range() +{ + int ret = OB_SUCCESS; + ObDatumRange range; + range.set_start_key(ObDatumRowkey::MIN_ROWKEY); + range.set_end_key(ObDatumRowkey::MAX_ROWKEY); + range.set_left_open(); + range.set_right_open(); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("Iter not opened yet", K(ret), KPC(this)); + } else if (OB_ISNULL(block_meta_tree_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("block meta tree is null", K(ret)); + } else if (OB_FAIL(block_meta_tree_->locate_range(range, + *datum_utils_, + false, /*is_left_border*/ + false, /*is_right_border*/ + is_reverse_scan_, + btree_iter_, + cur_tree_value_))) { + if (OB_BEYOND_THE_RANGE != ret) { + LOG_WARN("block meta tree locate range failed", K(ret), K(range)); + } else { + is_iter_finish_ = true; + LOG_INFO("no data to locate", K(ret)); + ret = OB_SUCCESS; + } + } else if (OB_ISNULL(cur_tree_value_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("cur tree value is null", K(ret), KP(cur_tree_value_)); + } else { + is_iter_start_ = true; + is_iter_finish_ = false; + } + return ret; +} + +int ObDDLIndexBlockRowIterator::skip_to_next_valid_position(const ObDatumRowkey &rowkey) +{ + int ret = OB_SUCCESS; + storage::ObBlockMetaTreeValue *tmp_tree_value = nullptr; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("Iter not opened yet", K(ret), KPC(this)); + } else if (OB_ISNULL(block_meta_tree_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("block meta tree is null", K(ret)); + } else if (OB_FAIL(block_meta_tree_->skip_to_next_valid_position(rowkey, + *datum_utils_, + btree_iter_, + tmp_tree_value))) { + if (OB_UNLIKELY(OB_ITER_END != ret)) { + LOG_WARN("Failed to skip to next valid position in block meta tree", K(ret), K(rowkey)); + } else { + is_iter_finish_ = true; + } + } else { + cur_tree_value_ = tmp_tree_value; + } + return ret; +} + +int ObDDLIndexBlockRowIterator::find_rowkeys_belong_to_same_idx_row(ObMicroIndexInfo &idx_block_row, int64_t &rowkey_begin_idx, int64_t &rowkey_end_idx, const ObRowsInfo *&rows_info) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("Iter not opened yet", K(ret), KPC(this)); + } else if (OB_ISNULL(rows_info)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid rows info", K(ret)); + } else { + const ObDatumRowkey *cur_rowkey = cur_tree_value_->rowkey_; + bool is_decided = false; + for (; OB_SUCC(ret) && rowkey_begin_idx < rowkey_end_idx; ++rowkey_begin_idx) { + if (rows_info->is_row_skipped(rowkey_begin_idx)) { + continue; + } + const ObDatumRowkey &rowkey = rows_info->get_rowkey(rowkey_begin_idx); + int32_t cmp_ret = 0; + if (OB_ISNULL(cur_rowkey)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("null rowkey", K(ret), K(cur_tree_value_), KP(cur_rowkey)); + } else if (OB_FAIL(rowkey.compare(*cur_rowkey, *datum_utils_, cmp_ret, false))) { + LOG_WARN("Failed to compare rowkey", K(ret), K(rowkey), KPC(cur_rowkey)); + } + + if (OB_FAIL(ret)) { + } else if (cmp_ret > 0) { + idx_block_row.rowkey_end_idx_ = rowkey_begin_idx; + is_decided = true; + break; + } else if (cmp_ret == 0) { + idx_block_row.rowkey_end_idx_ = rowkey_begin_idx + 1; + is_decided = true; + break; + } + } + + if (OB_SUCC(ret) && !is_decided) { + idx_block_row.rowkey_end_idx_ = rowkey_begin_idx; + } + } + return ret; +} + int ObDDLIndexBlockRowIterator::check_blockscan(const ObDatumRowkey &rowkey, bool &can_blockscan) { int ret = OB_SUCCESS; @@ -255,7 +368,7 @@ int ObDDLIndexBlockRowIterator::get_next(const ObIndexBlockRowHeader *&idx_row_h cur_tree_value_ = tmp_tree_value; } if (OB_SUCC(ret)) { - row_offset = idx_row_parser_.get_row_offset(); + row_offset = is_normal_cg_ ? 0 : endkey->datums_[0].get_int(); is_scan_left_border = is_reverse_scan_ ? is_end_key : is_start_key; is_scan_right_border = is_reverse_scan_ ? is_start_key : is_end_key; } @@ -398,7 +511,6 @@ int ObDDLSStableAllRangeIterator::init(const ObMicroBlockData &idx_block_data, const ObStorageDatumUtils *datum_utils, ObIAllocator *allocator, const bool is_reverse_scan, - const bool set_iter_end, const ObIndexBlockIterParam &iter_param) { int ret = OB_SUCCESS; @@ -406,28 +518,15 @@ int ObDDLSStableAllRangeIterator::init(const ObMicroBlockData &idx_block_data, ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguement", K(ret), KPC(datum_utils), K(iter_param)); } else { - ObTablet *cur_tablet = nullptr; - ObTabletHandle tmp_tablet_handle; - if (OB_ISNULL(iter_param.tablet_)) { - //get tablet handle from ls - ObLSService *ls_service = MTL(ObLSService *); - ObLSHandle ls_handle; - if (OB_FAIL(ls_service->get_ls(iter_param.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls failed", K(ret), K(iter_param.ls_id_)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, - iter_param.tablet_id_, - tmp_tablet_handle, - ObMDSGetTabletMode::READ_ALL_COMMITED))) { - LOG_WARN("get tablet failed", K(ret), K(iter_param)); - } else { - cur_tablet = tmp_tablet_handle.get_obj(); + ObTablet *cur_tablet = const_cast(iter_param.tablet_); + if (iter_param.sstable_->is_normal_cg_sstable()) { + if (OB_FAIL(MTL(ObTenantCGReadInfoMgr *)->get_index_read_info(rowkey_read_info_))) { + LOG_WARN("failed to get index read info from ObTenantCGReadInfoMgr", K(ret)); } } else { - cur_tablet = const_cast(iter_param.tablet_); - } - - if (OB_SUCC(ret)) { rowkey_read_info_ = &cur_tablet->get_rowkey_read_info(); + } + if (OB_SUCC(ret)) { iter_param_ = iter_param; is_reverse_scan_ = is_reverse_scan; @@ -632,7 +731,6 @@ int ObDDLMergeEmptyIterator::init(const ObMicroBlockData &idx_block_data, const ObStorageDatumUtils *datum_utils, ObIAllocator *allocator, const bool is_reverse_scan, - const bool set_iter_end, const ObIndexBlockIterParam &iter_param) { is_inited_ = true; @@ -856,7 +954,6 @@ int ObDDLMergeBlockRowIterator::init(const ObMicroBlockData &idx_block_data, const ObStorageDatumUtils *datum_utils, ObIAllocator *allocator, const bool is_reverse_scan, - const bool set_iter_end, const ObIndexBlockIterParam &iter_param) { int ret = OB_SUCCESS; @@ -874,7 +971,6 @@ int ObDDLMergeBlockRowIterator::init(const ObMicroBlockData &idx_block_data, datum_utils, allocator, is_reverse_scan, - set_iter_end, iter_param, sst_index_iter))) { LOG_WARN("fail to init sstable index iter", K(ret), K(iters_), KPC(sst_index_iter)); @@ -956,7 +1052,6 @@ int ObDDLMergeBlockRowIterator::init_sstable_index_iter(const ObMicroBlockData & const ObStorageDatumUtils *datum_utils, ObIAllocator *allocator, const bool is_reverse_scan, - const bool set_iter_end, const ObIndexBlockIterParam &iter_param, ObIndexBlockRowIterator *&sst_index_iter) { @@ -1050,7 +1145,7 @@ int ObDDLMergeBlockRowIterator::init_sstable_index_iter(const ObMicroBlockData & if (OB_ISNULL(sst_index_iter)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("iter is null", K(idx_block_data.type_), K(ret)); - } else if (OB_FAIL(sst_index_iter->init(idx_block_data, datum_utils, allocator, is_reverse_scan, set_iter_end, iter_param))) { + } else if (OB_FAIL(sst_index_iter->init(idx_block_data, datum_utils, allocator, is_reverse_scan, iter_param))) { LOG_WARN("fail to init iter", K(ret), K(idx_block_data), KPC(sst_index_iter)); } } @@ -1063,33 +1158,13 @@ int ObDDLMergeBlockRowIterator::get_readable_ddl_kvs(const ObIndexBlockIterParam { int ret = OB_SUCCESS; // todo qilu :get DDLKV from ls or from tablet_handle now, opt this get DDLKV from MTL() after refactor ddl_kv_mgr - ObTablet *cur_tablet = nullptr; ddl_memtables.reset(); ObTabletHandle tmp_tablet_handle; if (OB_UNLIKELY(!iter_param.is_valid())) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid iter param", K(ret), K(iter_param)); + LOG_WARN("invalid iter param", K(ret), K(iter_param), K(lbt())); } else { - if (OB_ISNULL(iter_param.tablet_)) { - //get tablet handle from ls - ObLSService *ls_service = MTL(ObLSService *); - ObLSHandle ls_handle; - if (OB_FAIL(ls_service->get_ls(iter_param.ls_id_, ls_handle, ObLSGetMod::DDL_MOD))) { - LOG_WARN("get ls failed", K(ret), K(iter_param.ls_id_)); - } else if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, - iter_param.tablet_id_, - tmp_tablet_handle, - ObMDSGetTabletMode::READ_ALL_COMMITED))) { - LOG_WARN("get tablet failed", K(ret), K(iter_param)); - } else { - cur_tablet = tmp_tablet_handle.get_obj(); - } - } else { - cur_tablet = const_cast(iter_param.tablet_); - } - } - - if (OB_SUCC(ret)) { + ObTablet *cur_tablet = const_cast(iter_param.tablet_); const uint16_t sstable_cg_idx = iter_param.sstable_->get_key().get_column_group_id(); ObDDLKvMgrHandle ddl_kv_mgr_handle; ObArray ddl_kvs_handle; @@ -2015,7 +2090,6 @@ int ObDDLMergeBlockRowIterator::get_index_row_count(const ObDatumRange &range, datum_utils_, allocator_, is_reverse_scan_, - false/*set iter end*/, iter_param_))) { LOG_WARN("fail to init iter", K(ret), KPC(idx_block_data_), KPC(tmp_merge_iter)); } else if (OB_FAIL(tmp_merge_iter->locate_range(range, is_left_border, is_right_border, true/*is_normal_cg*/))) { diff --git a/src/storage/blocksstable/index_block/ob_ddl_index_block_row_iterator.h b/src/storage/blocksstable/index_block/ob_ddl_index_block_row_iterator.h index ead05e7a0b..767e752462 100644 --- a/src/storage/blocksstable/index_block/ob_ddl_index_block_row_iterator.h +++ b/src/storage/blocksstable/index_block/ob_ddl_index_block_row_iterator.h @@ -35,7 +35,6 @@ public: const ObStorageDatumUtils *datum_utils, ObIAllocator *allocator, const bool is_reverse_scan, - const bool set_iter_end, const ObIndexBlockIterParam &iter_param) override; virtual int get_current(const ObIndexBlockRowHeader *&idx_row_header, const ObDatumRowkey *&endkey) override; @@ -52,6 +51,9 @@ public: const bool is_left_border, const bool is_right_border, const bool is_normal_cg) override; + virtual int locate_range() override; + virtual int skip_to_next_valid_position(const ObDatumRowkey &rowkey) override; + virtual int find_rowkeys_belong_to_same_idx_row(ObMicroIndexInfo &idx_block_row, int64_t &rowkey_begin_idx, int64_t &rowkey_end_idx, const ObRowsInfo *&rows_info) override; virtual int check_blockscan(const ObDatumRowkey &rowkey, bool &can_blockscan) override; virtual bool end_of_block() const override; virtual int get_index_row_count(const ObDatumRange &range, @@ -60,19 +62,21 @@ public: int64_t &index_row_count) override; virtual void reset() override; virtual void reuse() override; + virtual void set_iter_end() override { is_iter_finish_ = true; } INHERIT_TO_STRING_KV("base iterator:", ObIndexBlockRowIterator, "format:", "ObDDLIndexBlockRowIterator", - K_(is_iter_start), K_(is_iter_finish), KP(cur_tree_value_), KP(block_meta_tree_)); + K_(is_iter_start), K_(is_iter_finish), KP(cur_tree_value_), KP(block_meta_tree_), K(is_normal_cg_)); public: int set_iter_param(const ObStorageDatumUtils *datum_utils, bool is_reverse_scan, const storage::ObBlockMetaTree *block_meta_tree, + const bool is_normal_cg, const int64_t iter_step = INT64_MAX); bool is_valid() { return OB_NOT_NULL(block_meta_tree_); } - void set_iter_end() { is_iter_finish_ = true; } int get_next_meta(const ObDataMacroBlockMeta *&meta); private: bool is_iter_start_; bool is_iter_finish_; + bool is_normal_cg_; DDLBtreeIterator btree_iter_; const storage::ObBlockMetaTree *block_meta_tree_; storage::ObBlockMetaTreeValue *cur_tree_value_; @@ -88,7 +92,6 @@ public: const ObStorageDatumUtils *datum_utils, ObIAllocator *allocator, const bool is_reverse_scan, - const bool set_iter_end, const ObIndexBlockIterParam &iter_param) override; virtual int get_current(const ObIndexBlockRowHeader *&idx_row_header, const ObDatumRowkey *&endkey) override; @@ -137,7 +140,6 @@ public: const ObStorageDatumUtils *datum_utils, ObIAllocator *allocator, const bool is_reverse_scan, - const bool set_iter_end, const ObIndexBlockIterParam &iter_param) override; virtual int get_current(const ObIndexBlockRowHeader *&idx_row_header, const ObDatumRowkey *&endkey) override; @@ -176,7 +178,6 @@ public: const ObStorageDatumUtils *datum_utils, ObIAllocator *allocator, const bool is_reverse_scan, - const bool set_iter_end, const ObIndexBlockIterParam &iter_param) override; virtual int get_current(const ObIndexBlockRowHeader *&idx_row_header, const ObDatumRowkey *&endkey) override; @@ -252,7 +253,6 @@ private: const ObStorageDatumUtils *datum_utils, ObIAllocator *allocator, const bool is_reverse_scan, - const bool set_iter_end, const ObIndexBlockIterParam &iter_param, ObIndexBlockRowIterator *&sst_index_iter); int init_ddl_kv_index_iters(const ObMicroBlockData &idx_block_data, diff --git a/src/storage/blocksstable/index_block/ob_index_block_row_scanner.cpp b/src/storage/blocksstable/index_block/ob_index_block_row_scanner.cpp index c090e630ad..003a33276d 100644 --- a/src/storage/blocksstable/index_block/ob_index_block_row_scanner.cpp +++ b/src/storage/blocksstable/index_block/ob_index_block_row_scanner.cpp @@ -294,9 +294,7 @@ int ObIndexBlockDataTransformer::get_reader( /****************** ObIndexBlockIterParam **********************/ ObIndexBlockIterParam::ObIndexBlockIterParam() : sstable_(nullptr), - tablet_(nullptr), - ls_id_(), - tablet_id_() + tablet_(nullptr) { } @@ -309,8 +307,6 @@ ObIndexBlockIterParam &ObIndexBlockIterParam::operator=(const ObIndexBlockIterPa { sstable_ = other.sstable_; tablet_ = other.tablet_; - ls_id_ = other.ls_id_; - tablet_id_ = other.tablet_id_; return *this; } @@ -319,8 +315,6 @@ int ObIndexBlockIterParam::assign(const ObIndexBlockIterParam &other) int ret = OB_SUCCESS; sstable_ = other.sstable_; tablet_ = other.tablet_; - ls_id_ = other.ls_id_; - tablet_id_ = other.tablet_id_; return ret; } @@ -328,13 +322,11 @@ void ObIndexBlockIterParam::reset() { sstable_ = nullptr; tablet_ = nullptr; - ls_id_.reset(); - tablet_id_.reset(); } bool ObIndexBlockIterParam::is_valid() const { - return OB_NOT_NULL(sstable_) && ((ls_id_.is_valid() && tablet_id_.is_valid()) || OB_NOT_NULL(tablet_)); + return OB_NOT_NULL(sstable_) && OB_NOT_NULL(tablet_); } /****************** ObIndexBlockRowIterator **********************/ @@ -404,13 +396,13 @@ void ObRAWIndexBlockRowIterator::reuse() current_ = ObIMicroBlockReaderInfo::INVALID_ROW_INDEX; start_ = ObIMicroBlockReaderInfo::INVALID_ROW_INDEX; end_ = ObIMicroBlockReaderInfo::INVALID_ROW_INDEX; + idx_row_parser_.reset(); } int ObRAWIndexBlockRowIterator::init(const ObMicroBlockData &idx_block_data, const ObStorageDatumUtils *datum_utils, ObIAllocator *allocator, const bool is_reverse_scan, - const bool set_iter_end, const ObIndexBlockIterParam &iter_param) { int ret = OB_SUCCESS; @@ -502,6 +494,93 @@ int ObRAWIndexBlockRowIterator::locate_range(const ObDatumRange &range, return ret; } +int ObRAWIndexBlockRowIterator::locate_range() +{ + int ret = OB_SUCCESS; + int64_t row_count = 0; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("Iter not opened yet", K(ret), KPC(this)); + } else if (OB_FAIL(micro_reader_->get_row_count(row_count))) { + LOG_WARN("Failed to get row count", K(ret), K(micro_reader_)); + } else { + start_ = 0; + end_ = row_count - 1; + current_ = 0; + } + return ret; +} + +int ObRAWIndexBlockRowIterator::skip_to_next_valid_position(const ObDatumRowkey &rowkey) +{ + int ret = OB_SUCCESS; + bool equal = false; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("Iter not opened yet", K(ret), KPC(this)); + } else if (OB_FAIL(micro_reader_->find_bound(rowkey, true, current_, current_, equal))) { + LOG_WARN("Failed to skip to next valid position in micro block reader", K(ret), K(current_), K(rowkey)); + } else if (current_ == (end_ + 1)) { + ret = OB_ITER_END; + } + return ret; +} + +int ObRAWIndexBlockRowIterator::find_rowkeys_belong_to_same_idx_row(ObMicroIndexInfo &idx_block_row, int64_t &rowkey_begin_idx, int64_t &rowkey_end_idx, const ObRowsInfo *&rows_info) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("Iter not opened yet", K(ret), KPC(this)); + } else if (OB_ISNULL(rows_info)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid rows info", K(ret)); + } else { + bool is_decided = false; + for (; OB_SUCC(ret) && rowkey_begin_idx < rowkey_end_idx; ++rowkey_begin_idx) { + if (rows_info->is_row_skipped(rowkey_begin_idx)) { + continue; + } + const ObDatumRowkey &rowkey = rows_info->get_rowkey(rowkey_begin_idx); + int32_t cmp_ret = 0; + if (OB_FAIL(compare_rowkey(rowkey, cmp_ret))) { + LOG_WARN("fail to cmp rowkey in iter", K(ret), K(*this), K(rowkey)); + } else { + cmp_ret = -cmp_ret; + } + + if (OB_FAIL(ret)) { + } else if (cmp_ret > 0) { + idx_block_row.rowkey_end_idx_ = rowkey_begin_idx; + is_decided = true; + break; + } else if (cmp_ret == 0) { + idx_block_row.rowkey_end_idx_ = rowkey_begin_idx + 1; + is_decided = true; + break; + } + } + + if (OB_SUCC(ret) && !is_decided) { + idx_block_row.rowkey_end_idx_ = rowkey_begin_idx; + } + } + return ret; +} + +int ObRAWIndexBlockRowIterator::compare_rowkey(const ObDatumRowkey &rowkey, int32_t &cmp_ret) +{ + int ret = OB_SUCCESS; + cmp_ret = 0; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("Iter not opened yet", K(ret), KPC(this)); + } else if (OB_FAIL(micro_reader_->compare_rowkey(rowkey, current_, cmp_ret))) { + LOG_WARN("Failed to compare rowkey", K(ret), K(rowkey)); + } + return ret; +} + int ObRAWIndexBlockRowIterator::check_blockscan(const ObDatumRowkey &rowkey, bool &can_blockscan) { int ret = OB_SUCCESS; @@ -709,7 +788,6 @@ int ObTFMIndexBlockRowIterator::init(const ObMicroBlockData &idx_block_data, const ObStorageDatumUtils *datum_utils, ObIAllocator *allocator, const bool is_reverse_scan, - const bool set_iter_end, const ObIndexBlockIterParam &iter_param) { int ret = OB_SUCCESS; @@ -726,11 +804,6 @@ int ObTFMIndexBlockRowIterator::init(const ObMicroBlockData &idx_block_data, is_reverse_scan_ = is_reverse_scan; iter_step_ = is_reverse_scan_ ? -1 : 1; datum_utils_ = datum_utils; - if (set_iter_end) { - current_ = 0; - start_ = 0; - end_ = idx_data_header_->row_cnt_ - 1; - } is_inited_ = true; } return ret; @@ -857,6 +930,19 @@ int ObTFMIndexBlockRowIterator::locate_range(const ObDatumRange &range, return ret; } +int ObTFMIndexBlockRowIterator::locate_range() +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("Iter not opened yet", K(ret), KPC(this)); + } else { + start_ = 0; + end_ = idx_data_header_->row_cnt_ - 1; + current_ = 0; + } + return ret; +} int ObTFMIndexBlockRowIterator::check_blockscan(const ObDatumRowkey &rowkey, bool &can_blockscan) { @@ -1134,79 +1220,67 @@ int ObTFMIndexBlockRowIterator::get_cur_row_id_range(const ObCSRange &parent_row return ret; } -int ObTFMIndexBlockRowIterator::skip_to_next_valid_position(ObMicroIndexInfo &idx_block_row, - int64_t &rowkey_begin_idx, - int64_t &rowkey_end_idx, - const ObRowsInfo *&rows_info) +int ObTFMIndexBlockRowIterator::skip_to_next_valid_position(const ObDatumRowkey &rowkey) { int ret = OB_SUCCESS; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("Iter not opened yet", K(ret), KPC(this)); } else { - for (; rowkey_begin_idx < rowkey_end_idx; ++rowkey_begin_idx) { - if (!rows_info->is_row_skipped(rowkey_begin_idx)) { - break; - } - } - - if (rowkey_begin_idx == rowkey_end_idx) { + ObDatumComparor cmp(*datum_utils_, ret, false, true, false); + const ObDatumRowkey *first = idx_data_header_->rowkey_array_ + current_; + const ObDatumRowkey *last = idx_data_header_->rowkey_array_ + end_ + 1; + const ObDatumRowkey *found = std::lower_bound(first, last, rowkey, cmp); + if (OB_FAIL(ret)) { + LOG_WARN("Failed to get lower bound of rowkey", K(ret), K(rowkey), KPC_(idx_data_header)); + } else if (found == last) { ret = OB_ITER_END; } else { - const ObDatumRowkey &rowkey = rows_info->get_rowkey(rowkey_begin_idx); - ObDatumComparor cmp(*datum_utils_, ret); - const ObDatumRowkey *first = idx_data_header_->rowkey_array_ + current_; - const ObDatumRowkey *last = idx_data_header_->rowkey_array_ + end_ + 1; - const ObDatumRowkey *found = std::lower_bound(first, last, rowkey, cmp); - if (OB_FAIL(ret)) { - LOG_WARN("Failed to get lower bound of rowkey", K(ret), K(rowkey), KPC(this)); - } else if (found == last) { - ret = OB_ITER_END; - } else { - current_= found - idx_data_header_->rowkey_array_; - idx_block_row.rows_info_ = rows_info; - idx_block_row.rowkey_begin_idx_ = rowkey_begin_idx++; - if (OB_FAIL(find_rowkeys_belong_to_same_idx_row(idx_block_row.rowkey_end_idx_, rowkey_begin_idx, rowkey_end_idx, rows_info))) { - LOG_WARN("Failed to find rowkeys belong to same index row", K(ret), K(rowkey_begin_idx), K(rowkey_end_idx), KPC(rows_info)); - } - } + current_ = found - idx_data_header_->rowkey_array_; } } return ret; } -int ObTFMIndexBlockRowIterator::find_rowkeys_belong_to_same_idx_row(int64_t &rowkey_idx, - int64_t &rowkey_begin_idx, - int64_t &rowkey_end_idx, - const ObRowsInfo *&rows_info) +int ObTFMIndexBlockRowIterator::find_rowkeys_belong_to_same_idx_row(ObMicroIndexInfo &idx_block_row, int64_t &rowkey_begin_idx, int64_t &rowkey_end_idx, const ObRowsInfo *&rows_info) { int ret = OB_SUCCESS; - const ObDatumRowkey *cur_rowkey = idx_data_header_->rowkey_array_ + current_; - bool is_decided = false; if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("Iter not opened yet", K(ret), KPC(this)); + } else if (OB_ISNULL(rows_info)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid rows info", K(ret)); } else { + const ObDatumRowkey *cur_rowkey = idx_data_header_->rowkey_array_ + current_; + bool is_decided = false; for (; OB_SUCC(ret) && rowkey_begin_idx < rowkey_end_idx; ++rowkey_begin_idx) { if (rows_info->is_row_skipped(rowkey_begin_idx)) { continue; } const ObDatumRowkey &rowkey = rows_info->get_rowkey(rowkey_begin_idx); - int cmp_ret = 0; - if (OB_FAIL(rowkey.compare(*cur_rowkey, *datum_utils_, cmp_ret))) { + int32_t cmp_ret = 0; + if (OB_ISNULL(cur_rowkey)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("null rowkey", K(ret), K(current_), KP(cur_rowkey)); + } else if (OB_FAIL(rowkey.compare(*cur_rowkey, *datum_utils_, cmp_ret, false))) { LOG_WARN("Failed to compare rowkey", K(ret), K(rowkey), KPC(cur_rowkey)); + } + + if (OB_FAIL(ret)) { } else if (cmp_ret > 0) { - rowkey_idx = rowkey_begin_idx; + idx_block_row.rowkey_end_idx_ = rowkey_begin_idx; is_decided = true; break; } else if (cmp_ret == 0) { - rowkey_idx = rowkey_begin_idx + 1; + idx_block_row.rowkey_end_idx_ = rowkey_begin_idx + 1; is_decided = true; break; } } - if (!is_decided) { - rowkey_idx = rowkey_begin_idx; + + if (OB_SUCC(ret) && !is_decided) { + idx_block_row.rowkey_end_idx_ = rowkey_begin_idx; } } return ret; @@ -1214,8 +1288,7 @@ int ObTFMIndexBlockRowIterator::find_rowkeys_belong_to_same_idx_row(int64_t &row /****************** ObIndexBlockRowScanner **********************/ ObIndexBlockRowScanner::ObIndexBlockRowScanner() - : query_range_(nullptr), agg_projector_(nullptr), agg_column_schema_(nullptr), - macro_id_(), allocator_(nullptr), raw_iter_(nullptr), transformed_iter_(nullptr), + : query_range_(nullptr), macro_id_(), allocator_(nullptr), raw_iter_(nullptr), transformed_iter_(nullptr), ddl_iter_(nullptr), ddl_merge_iter_(nullptr), iter_(nullptr), datum_utils_(nullptr), range_idx_(0), nested_offset_(0), rowkey_begin_idx_(0), rowkey_end_idx_(0), index_format_(ObIndexFormat::INVALID), parent_row_range_(), is_get_(false), is_reverse_scan_(false), @@ -1302,8 +1375,6 @@ void ObIndexBlockRowScanner::reset() } int ObIndexBlockRowScanner::init( - const ObIArray &agg_projector, - const ObIArray &agg_column_schema, const ObStorageDatumUtils &datum_utils, ObIAllocator &allocator, const common::ObQueryFlag &query_flag, @@ -1317,12 +1388,7 @@ int ObIndexBlockRowScanner::init( } else if (OB_UNLIKELY(!datum_utils.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid datum utils", K(ret), K(datum_utils)); - } else if (OB_UNLIKELY(agg_projector.count() != agg_column_schema.count())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("Agg meta count not same", K(ret), K(agg_projector), K(agg_column_schema)); } else { - agg_projector_ = &agg_projector; - agg_column_schema_ = &agg_column_schema; allocator_ = &allocator; is_reverse_scan_ = query_flag.is_reverse_scan(); datum_utils_ = &datum_utils; @@ -1350,7 +1416,7 @@ int ObIndexBlockRowScanner::open( ret = OB_INVALID_ARGUMENT; LOG_WARN("Invalid argument to open an index micro block", K(ret), K(macro_id), K(idx_block_data), K(rowkey), K_(is_normal_cg), KP(idx_info)); - } else if (OB_FAIL(init_by_micro_data(idx_block_data, false/*set iter finish*/))) { + } else if (OB_FAIL(init_by_micro_data(idx_block_data))) { LOG_WARN("Fail to init scanner by micro data", K(ret), K(idx_block_data), K(index_format_)); } else if (OB_ISNULL(iter_)) { ret = OB_ERR_UNEXPECTED; @@ -1404,11 +1470,13 @@ int ObIndexBlockRowScanner::open( ret = OB_INVALID_ARGUMENT; LOG_WARN("Invalid argument to open an index micro block", K(ret), K(macro_id), K(idx_block_data), KP(rows_info)); - } else if (OB_FAIL(init_by_micro_data(idx_block_data, true/*set iter finish*/))) { + } else if (OB_FAIL(init_by_micro_data(idx_block_data))) { LOG_WARN("Fail to init scanner by micro data", K(ret), K(idx_block_data)); - } else if (OB_ISNULL(iter_) || ObIndexFormat::TRANSFORMED != index_format_) { + } else if (OB_ISNULL(iter_)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("Unexpected index format or iter is null", K(index_format_), K(ret), KPC(iter_)); + LOG_WARN("iter is null", K(index_format_), K(ret), KPC(iter_)); + } else if (OB_FAIL(iter_->locate_range())) { + LOG_WARN("fail to locate range", K(ret), KPC(iter_)); } else { macro_id_ = macro_id; rows_info_ = rows_info; @@ -1436,7 +1504,7 @@ int ObIndexBlockRowScanner::open( || !idx_block_data.is_index_block() || (is_normal_cg_ && nullptr == idx_info))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("Invalid argument to open an index micro block", K(ret), K(idx_block_data), K(range), K_(is_normal_cg), KP(idx_info)); - } else if (OB_FAIL(init_by_micro_data(idx_block_data, false/*set iter finish*/))) { + } else if (OB_FAIL(init_by_micro_data(idx_block_data))) { LOG_WARN("Fail to init scanner by micro data", K(ret), K(idx_block_data)); } else if (OB_ISNULL(iter_)) { ret = OB_ERR_UNEXPECTED; @@ -1472,14 +1540,14 @@ int ObIndexBlockRowScanner::get_next( LOG_WARN("Not inited", K(ret)); } else if (end_of_block()) { ret = OB_ITER_END; - } else if (is_multi_check && OB_FAIL(iter_->skip_to_next_valid_position(idx_block_row, rowkey_begin_idx_, rowkey_end_idx_, rows_info_))) { + } else if (is_multi_check && OB_FAIL(skip_to_next_valid_position(idx_block_row))) { if (OB_UNLIKELY(OB_ITER_END != ret)) { LOG_WARN("Failed to skip to next valid position", K(ret), K(rowkey_begin_idx_), K(rowkey_end_idx_), KPC(rows_info_)); } else if (OB_ISNULL(iter_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("iter is null", K(index_format_), K(ret)); } else { - iter_->reuse(); + iter_->set_iter_end(); } } else if (OB_FAIL(get_next_idx_row(idx_block_row))) { LOG_WARN("Failed to get next idx row", K(ret), K(is_multi_check)); @@ -1488,13 +1556,9 @@ int ObIndexBlockRowScanner::get_next( } void ObIndexBlockRowScanner::set_iter_param(const blocksstable::ObSSTable *sstable, - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, const ObTablet *tablet) { iter_param_.sstable_ = sstable; - iter_param_.ls_id_ = ls_id; - iter_param_.tablet_id_ = tablet_id; iter_param_.tablet_ = tablet; } @@ -1563,7 +1627,7 @@ int ObIndexBlockRowScanner::check_blockscan( return ret; } -int ObIndexBlockRowScanner::init_by_micro_data(const ObMicroBlockData &idx_block_data, bool set_iter_end) +int ObIndexBlockRowScanner::init_by_micro_data(const ObMicroBlockData &idx_block_data) { int ret = OB_SUCCESS; void *iter_buf = nullptr; @@ -1632,7 +1696,7 @@ int ObIndexBlockRowScanner::init_by_micro_data(const ObMicroBlockData &idx_block if (OB_ISNULL(iter_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("iter is null", K(index_format_), K(ret)); - } else if (OB_FAIL(iter_->init(idx_block_data, datum_utils_, allocator_, is_reverse_scan_, set_iter_end, iter_param_))) { + } else if (OB_FAIL(iter_->init(idx_block_data, datum_utils_, allocator_, is_reverse_scan_, iter_param_))) { LOG_WARN("fail to init iter", K(ret), K(idx_block_data), KPC(iter_)); } } @@ -1730,20 +1794,17 @@ const ObDatumRowkey &ObIndexBlockRowScanner::get_end_key() const } void ObIndexBlockRowScanner::switch_context(const ObSSTable &sstable, + const ObTablet *tablet, const ObStorageDatumUtils &datum_utils, - const common::ObQueryFlag &query_flag, - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id) + ObTableAccessContext &access_ctx) { nested_offset_ = sstable.get_macro_offset(); datum_utils_ = &datum_utils; is_normal_cg_ = sstable.is_normal_cg_sstable(); - is_reverse_scan_ = query_flag.is_reverse_scan(); - is_normal_query_ = !query_flag.is_daily_merge() && !query_flag.is_multi_version_minor_merge(); + is_reverse_scan_ = access_ctx.query_flag_.is_reverse_scan(); + is_normal_query_ = !access_ctx.query_flag_.is_daily_merge() && !access_ctx.query_flag_.is_multi_version_minor_merge(); iter_param_.sstable_ = &sstable; - iter_param_.ls_id_ = ls_id; - iter_param_.tablet_id_ = tablet_id; - iter_param_.tablet_ = nullptr; + iter_param_.tablet_ = tablet; int ret = OB_SUCCESS; if (OB_NOT_NULL(iter_)) { ObStorageDatumUtils *switch_datum_utils = const_cast(datum_utils_); @@ -1817,5 +1878,31 @@ void ObIndexBlockRowScanner::skip_index_rows() } } +int ObIndexBlockRowScanner::skip_to_next_valid_position(ObMicroIndexInfo &idx_block_row) +{ + int ret = OB_SUCCESS; + skip_index_rows(); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("Not inited", K(ret)); + } else if (rowkey_begin_idx_ == rowkey_end_idx_) { + ret = OB_ITER_END; + } else if (OB_ISNULL(iter_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("iter is null", K(index_format_), K(ret)); + } else if (OB_FAIL(iter_->skip_to_next_valid_position(rows_info_->get_rowkey(rowkey_begin_idx_)))) { + if (OB_ITER_END != ret) { + LOG_WARN("fail to skip to next valid position", K(ret), K(rowkey_begin_idx_), K(rowkey_end_idx_), KPC(rows_info_), KPC(iter_)); + } + } else { + idx_block_row.rows_info_ = rows_info_; + idx_block_row.rowkey_begin_idx_ = rowkey_begin_idx_; + if (OB_FAIL(iter_->find_rowkeys_belong_to_same_idx_row(idx_block_row, rowkey_begin_idx_, rowkey_end_idx_, rows_info_))) { + LOG_WARN("Failed to find rowkeys belong to same index row", K(ret), KPC(iter_)); + } + } + return ret; +} + } // namespace blocksstable } // namespace oceanbase diff --git a/src/storage/blocksstable/index_block/ob_index_block_row_scanner.h b/src/storage/blocksstable/index_block/ob_index_block_row_scanner.h index cb6e0bbcdf..6246f264b3 100644 --- a/src/storage/blocksstable/index_block/ob_index_block_row_scanner.h +++ b/src/storage/blocksstable/index_block/ob_index_block_row_scanner.h @@ -117,14 +117,11 @@ public: int assign(const ObIndexBlockIterParam &other); void reset(); bool is_valid() const; - TO_STRING_KV(KP(sstable_), KP(tablet_), K(ls_id_), K(tablet_id_)); + TO_STRING_KV(KP(sstable_), KP(tablet_)); public: const ObSSTable *sstable_; - // pass ls_id + table_id or directly pass tablet const ObTablet *tablet_; - share::ObLSID ls_id_; - common::ObTabletID tablet_id_; }; class ObIndexBlockRowIterator @@ -138,7 +135,6 @@ public: const ObStorageDatumUtils *datum_utils, ObIAllocator *allocator, const bool is_reverse_scan, - const bool set_iter_end, const ObIndexBlockIterParam &iter_param) = 0; virtual int get_current(const ObIndexBlockRowHeader *&idx_row_header, const ObDatumRowkey *&endkey) = 0; @@ -155,6 +151,7 @@ public: const bool is_left_border, const bool is_right_border, const bool is_normal_cg) = 0; + virtual int locate_range() { return OB_NOT_SUPPORTED; } virtual int check_blockscan(const ObDatumRowkey &rowkey, bool &can_blockscan) = 0; virtual bool end_of_block() const = 0; virtual int get_index_row_count(const ObDatumRange &range, @@ -173,16 +170,15 @@ public: const ObCSRange &parent_row_range, bool &is_certain, int64_t &found_idx) { return OB_NOT_SUPPORTED; } - virtual int skip_to_next_valid_position(ObMicroIndexInfo &idx_block_row, - int64_t &rowkey_begin_idx, - int64_t &rowkey_end_idx, - const ObRowsInfo *&rows_info) { return OB_NOT_SUPPORTED; } + virtual int skip_to_next_valid_position(const ObDatumRowkey &rowkey) { return OB_NOT_SUPPORTED; } + virtual int find_rowkeys_belong_to_same_idx_row(ObMicroIndexInfo &idx_block_row, int64_t &rowkey_begin_idx, int64_t &rowkey_end_idx, const ObRowsInfo *&rows_info) { return OB_NOT_SUPPORTED; } virtual int advance_to_border(const ObDatumRowkey &rowkey, const bool is_left_border, const bool is_right_border, const ObCSRange &parent_row_range, ObCSRange &cs_range) { return OB_NOT_SUPPORTED; } virtual void get_end_key(const ObDatumRowkey *&rowkey) {} + virtual void set_iter_end() {} public: virtual int switch_context(ObStorageDatumUtils *datum_utils) { @@ -190,7 +186,7 @@ public: return OB_SUCCESS; } bool is_inited() { return is_inited_; } - VIRTUAL_TO_STRING_KV(K(is_inited_), K(is_reverse_scan_), K(iter_step_), K(idx_row_parser_), KPC(datum_utils_)); + VIRTUAL_TO_STRING_KV(K(is_inited_), K(is_reverse_scan_), K(iter_step_), KPC(datum_utils_)); protected: bool is_inited_; @@ -210,7 +206,6 @@ public: const ObStorageDatumUtils *datum_utils, ObIAllocator *allocator, const bool is_reverse_scan, - const bool set_iter_end, const ObIndexBlockIterParam &iter_param) override; virtual int get_current(const ObIndexBlockRowHeader *&idx_row_header, const ObDatumRowkey *&endkey) override; @@ -227,6 +222,10 @@ public: const bool is_left_border, const bool is_right_border, const bool is_normal_cg) override; + virtual int locate_range() override; + virtual int skip_to_next_valid_position(const ObDatumRowkey &rowkey) override; + virtual int find_rowkeys_belong_to_same_idx_row(ObMicroIndexInfo &idx_block_row, int64_t &rowkey_begin_idx, int64_t &rowkey_end_idx, const ObRowsInfo *&rows_info) override; + virtual void set_iter_end() override { current_ = ObIMicroBlockReader::INVALID_ROW_INDEX; } virtual int check_blockscan(const ObDatumRowkey &rowkey, bool &can_blockscan) override; virtual bool end_of_block() const override; virtual int get_index_row_count(const ObDatumRange &range, @@ -240,6 +239,7 @@ public: private: int init_datum_row(const ObStorageDatumUtils &datum_utils, ObIAllocator *allocator); bool is_in_border(bool is_reverse_scan, bool is_left_border, bool is_right_border); + int compare_rowkey(const ObDatumRowkey &rowkey, int32_t &cmp_ret); protected: int64_t current_; int64_t start_; // inclusive @@ -260,7 +260,6 @@ public: const ObStorageDatumUtils *datum_utils, ObIAllocator *allocator, const bool is_reverse_scan, - const bool set_iter_end, const ObIndexBlockIterParam &iter_param) override; virtual int get_current(const ObIndexBlockRowHeader *&idx_row_header, const ObDatumRowkey *&endkey) override; @@ -277,6 +276,7 @@ public: const bool is_left_border, const bool is_right_border, const bool is_normal_cg) override; + virtual int locate_range() override; virtual int check_blockscan(const ObDatumRowkey &rowkey, bool &can_blockscan) override; virtual void reset() override; virtual void reuse() override; @@ -289,10 +289,8 @@ public: const ObCSRange &parent_row_range, bool &is_certain, int64_t &found_idx) override; - virtual int skip_to_next_valid_position(ObMicroIndexInfo &idx_block_row, - int64_t &rowkey_begin_idx, - int64_t &rowkey_end_idx, - const ObRowsInfo *&rows_info) override; + virtual int skip_to_next_valid_position(const ObDatumRowkey &rowkey) override; + virtual int find_rowkeys_belong_to_same_idx_row(ObMicroIndexInfo &idx_block_row, int64_t &rowkey_begin_idx, int64_t &rowkey_end_idx, const ObRowsInfo *&rows_info) override; virtual int get_idx_row_header_in_target_idx(const int64_t idx, const ObIndexBlockRowHeader *&idx_row_header) override; virtual int advance_to_border(const ObDatumRowkey &rowkey, @@ -306,10 +304,6 @@ public: private: int get_cur_row_id_range(const ObCSRange &parent_row_range, ObCSRange &cs_range); - int find_rowkeys_belong_to_same_idx_row(int64_t &rowkey_idx, - int64_t &rowkey_begin_idx, - int64_t &rowkey_end_idx, - const ObRowsInfo *&rows_info); private: const ObIndexBlockDataHeader *idx_data_header_; @@ -324,8 +318,6 @@ public: void reset(); int init( - const ObIArray &agg_projector, - const ObIArray &agg_column_schema, const ObStorageDatumUtils &datum_utils, ObIAllocator &allocator, const common::ObQueryFlag &query_flag, @@ -356,9 +348,7 @@ public: ObMicroIndexInfo &idx_block_row, const bool is_multi_check = false); void set_iter_param(const ObSSTable *sstable, - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id, - const ObTablet *tablet = nullptr); + const ObTablet *tablet); bool end_of_block() const; bool is_ddl_merge_type() const; int get_index_row_count(int64_t &index_row_count) const; @@ -381,17 +371,16 @@ public: OB_INLINE bool is_valid() const { return is_inited_; } OB_INLINE bool is_ddl_merge_scan() const { return index_format_ == ObIndexFormat::DDL_MERGE; } void switch_context(const ObSSTable &sstable, + const ObTablet *tablet, const ObStorageDatumUtils &datum_utils, - const common::ObQueryFlag &query_flag, - const share::ObLSID &ls_id, - const common::ObTabletID &tablet_id); + ObTableAccessContext &access_ctx); TO_STRING_KV(K_(index_format), KP_(raw_iter), KP_(transformed_iter), KP_(ddl_iter), KP_(ddl_merge_iter), KPC_(iter), K_(range_idx), K_(is_get), K_(is_reverse_scan), K_(is_left_border), K_(is_right_border), K_(rowkey_begin_idx), K_(rowkey_end_idx), K_(is_inited), K_(macro_id), KPC_(datum_utils), K_(is_normal_cg), K_(parent_row_range), K_(filter_constant_type), K_(is_normal_query), K_(iter_param)); private: - int init_by_micro_data(const ObMicroBlockData &idx_block_data, bool set_iter_end); + int init_by_micro_data(const ObMicroBlockData &idx_block_data); int locate_key(const ObDatumRowkey &rowkey); int init_datum_row(); int read_curr_idx_row(const ObIndexBlockRowHeader *&idx_row_header, const ObDatumRowkey *&endkey); @@ -405,6 +394,7 @@ private: ObCSRange &cs_range); int get_next_idx_row(ObMicroIndexInfo &idx_block_row); void skip_index_rows(); + int skip_to_next_valid_position(ObMicroIndexInfo &idx_block_row); private: union { const ObDatumRowkey *rowkey_; @@ -412,8 +402,6 @@ private: const ObRowsInfo *rows_info_; const void *query_range_; }; - const ObIArray *agg_projector_; - const ObIArray *agg_column_schema_; MacroBlockId macro_id_; ObIAllocator *allocator_; ObRAWIndexBlockRowIterator *raw_iter_; diff --git a/src/storage/blocksstable/index_block/ob_sstable_sec_meta_iterator.cpp b/src/storage/blocksstable/index_block/ob_sstable_sec_meta_iterator.cpp index d8e20cedb7..6f4a2a1bd5 100644 --- a/src/storage/blocksstable/index_block/ob_sstable_sec_meta_iterator.cpp +++ b/src/storage/blocksstable/index_block/ob_sstable_sec_meta_iterator.cpp @@ -103,6 +103,7 @@ int ObSSTableSecMetaIterator::open( } if (OB_FAIL(ret) || is_prefetch_end_) { } else if (is_ddl_mem_sstable) { + const bool is_normal_cg = sstable.is_normal_cg_sstable(); const ObMicroBlockData &root_block = sstable_meta_hdl_.get_sstable_meta().get_root_info().get_block_data(); if (ObMicroBlockData::DDL_BLOCK_TREE != root_block.type_ || nullptr == root_block.buf_) { ret = OB_ERR_UNEXPECTED; @@ -110,7 +111,7 @@ int ObSSTableSecMetaIterator::open( } else { block_meta_tree_ = reinterpret_cast(const_cast(root_block.buf_)); const int64_t step = max(1, sample_step); - if (OB_FAIL(ddl_iter_.set_iter_param(const_cast(&rowkey_read_info.get_datum_utils()), is_reverse_scan, block_meta_tree_, step))) { + if (OB_FAIL(ddl_iter_.set_iter_param(const_cast(&rowkey_read_info.get_datum_utils()), is_reverse_scan, block_meta_tree_, is_normal_cg, step))) { LOG_WARN("fail to set ddl iter param", K(ret)); } else if (OB_FAIL(ddl_iter_.locate_range(query_range, true, /*is_left_border*/ diff --git a/src/storage/blocksstable/ob_imicro_block_reader.h b/src/storage/blocksstable/ob_imicro_block_reader.h index 65df2acbae..cd084a6fd2 100644 --- a/src/storage/blocksstable/ob_imicro_block_reader.h +++ b/src/storage/blocksstable/ob_imicro_block_reader.h @@ -381,6 +381,10 @@ public: const int64_t begin_idx, int64_t &row_idx, bool &equal) = 0; + virtual int compare_rowkey( + const ObDatumRowkey &rowkey, + const int64_t index, + int32_t &compare_result) = 0; static int filter_white_filter( const sql::ObWhiteFilterExecutor &filter, const common::ObObjMeta &obj_meta, diff --git a/src/storage/blocksstable/ob_micro_block_reader.cpp b/src/storage/blocksstable/ob_micro_block_reader.cpp index 7b754c6f37..9fda19cd30 100644 --- a/src/storage/blocksstable/ob_micro_block_reader.cpp +++ b/src/storage/blocksstable/ob_micro_block_reader.cpp @@ -441,6 +441,27 @@ int ObMicroBlockReader::init( return ret; } +int ObMicroBlockReader::compare_rowkey( + const ObDatumRowkey &rowkey, + const int64_t idx, + int32_t &compare_result) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + LOG_WARN("Not inited", K(ret)); + } else if (OB_UNLIKELY(!rowkey.is_valid() || idx < 0 || idx >= row_count_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid argument", K(ret), K(rowkey), K(idx), K_(row_count)); + } else if (OB_FAIL(flat_row_reader_.compare_meta_rowkey(rowkey, + *datum_utils_, + data_begin_ + index_data_[idx], + index_data_[idx + 1] - index_data_[idx], + compare_result))) { + LOG_WARN("Failed to compare rowkey", K(ret), K(rowkey), K_(row_count), K(idx)); + } + return ret; +} int ObMicroBlockReader::find_bound( const ObDatumRowkey &key, diff --git a/src/storage/blocksstable/ob_micro_block_reader.h b/src/storage/blocksstable/ob_micro_block_reader.h index eac2f9b1a4..84f545178e 100644 --- a/src/storage/blocksstable/ob_micro_block_reader.h +++ b/src/storage/blocksstable/ob_micro_block_reader.h @@ -134,6 +134,10 @@ public: const int32_t col_offset, const int64_t row_index, ObStorageDatum &datum) override; + virtual int compare_rowkey( + const ObDatumRowkey &rowkey, + const int64_t index, + int32_t &compare_result) override; virtual int find_bound( const ObDatumRowkey &key, const bool lower_bound, diff --git a/src/storage/blocksstable/ob_shared_macro_block_manager.cpp b/src/storage/blocksstable/ob_shared_macro_block_manager.cpp index ce6339e8e7..9860c8d8c6 100644 --- a/src/storage/blocksstable/ob_shared_macro_block_manager.cpp +++ b/src/storage/blocksstable/ob_shared_macro_block_manager.cpp @@ -717,58 +717,10 @@ int ObSharedMacroBlockMgr::create_new_sstable( ObSSTableMetaHandle meta_handle; if (OB_FAIL(old_table.get_meta(meta_handle))) { LOG_WARN("get meta handle fail", K(ret), K(old_table)); - } else { - const ObSSTableBasicMeta &basic_meta = meta_handle.get_sstable_meta().get_basic_meta(); - param.filled_tx_scn_ = basic_meta.filled_tx_scn_; - param.ddl_scn_ = basic_meta.ddl_scn_; - param.table_key_ = old_table.get_key(); - param.sstable_logic_seq_ = meta_handle.get_sstable_meta().get_sstable_seq(); - param.table_mode_ = basic_meta.table_mode_; - param.index_type_ = static_cast(basic_meta.index_type_); - param.schema_version_ = basic_meta.schema_version_; - param.create_snapshot_version_ = basic_meta.create_snapshot_version_; - param.progressive_merge_round_ = basic_meta.progressive_merge_round_; - param.progressive_merge_step_ = basic_meta.progressive_merge_step_; - param.rowkey_column_cnt_ = basic_meta.rowkey_column_count_; - param.recycle_version_ = basic_meta.recycle_version_; - param.latest_row_store_type_ = basic_meta.latest_row_store_type_; - param.is_ready_for_read_ = true; - - ObSSTableMergeRes::fill_addr_and_data(res.root_desc_, - param.root_block_addr_, param.root_block_data_); - ObSSTableMergeRes::fill_addr_and_data(res.data_root_desc_, - param.data_block_macro_meta_addr_, param.data_block_macro_meta_); - param.root_row_store_type_ = res.root_row_store_type_; - param.data_index_tree_height_ = res.root_desc_.height_; - param.index_blocks_cnt_ = res.index_blocks_cnt_; - param.data_blocks_cnt_ = res.data_blocks_cnt_; - param.micro_block_cnt_ = res.micro_block_cnt_; - param.use_old_macro_block_count_ = res.use_old_macro_block_count_; - param.row_count_ = res.row_count_; - param.column_cnt_ = res.data_column_cnt_; - param.data_checksum_ = res.data_checksum_; - param.occupy_size_ = res.occupy_size_; - param.original_size_ = res.original_size_; - param.max_merged_trans_version_ = res.max_merged_trans_version_; - param.contain_uncommitted_row_ = res.contain_uncommitted_row_; - param.compressor_type_ = res.compressor_type_; - param.encrypt_id_ = res.encrypt_id_; - param.master_key_id_ = res.master_key_id_; - param.data_block_ids_ = res.data_block_ids_; - param.is_meta_root_ = res.data_root_desc_.is_meta_root_; - param.nested_offset_ = block_info.nested_offset_; - param.nested_size_ = block_info.nested_size_; - param.other_block_ids_ = res.other_block_ids_; - MEMCPY(param.encrypt_key_, res.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); - - if (param.table_key_.is_major_sstable()) { - if (OB_FAIL(param.column_checksums_.assign(res.data_column_checksums_))) { - LOG_WARN("fail to fill column checksum", K(ret), K(res)); - } - } - } - if (OB_FAIL(ret)) { - // do nothing + } else if (OB_FAIL(param.init_for_small_sstable(res, old_table.get_key(), + meta_handle.get_sstable_meta(), block_info))) { + LOG_WARN("fail to init create sstable param for small sst", + K(ret), K(res), K(old_table.get_key()), K(meta_handle.get_sstable_meta()), K(block_info)); } else if (OB_UNLIKELY(!param.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid args", K(ret), K(param)); diff --git a/src/storage/column_store/ob_co_sstable_row_scanner.cpp b/src/storage/column_store/ob_co_sstable_row_scanner.cpp index d4744773c9..91fe945eaf 100644 --- a/src/storage/column_store/ob_co_sstable_row_scanner.cpp +++ b/src/storage/column_store/ob_co_sstable_row_scanner.cpp @@ -411,15 +411,17 @@ int ObCOSSTableRowScanner::construct_cg_iter_params_for_single_row( } else { ObTableIterParam* cg_param = nullptr; const common::ObIArray *access_cgs = nullptr; - const int64_t schema_rowkey_cnt = row_param.get_read_info()->get_schema_rowkey_count(); + const int64_t rowkey_cnt = row_param.get_read_info()->get_rowkey_count(); + const ObColumnIndexArray &cols_index = row_param.get_read_info()->get_columns_index(); if (OB_ISNULL(access_cgs = row_param.get_read_info()->get_cg_idxs())) { ret = OB_ERR_UNEXPECTED; LOG_WARN("Unexpected null access cg index", K(ret)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < row_param.out_cols_project_->count(); ++i) { const int32_t col_offset = row_param.out_cols_project_->at(i); + const int32_t col_index = cols_index.at(col_offset); sql::ObExpr* expr = row_param.output_exprs_ == nullptr ? nullptr : row_param.output_exprs_->at(i); - if ((nullptr == expr || !is_group_idx_expr(expr)) && col_offset >= schema_rowkey_cnt) { + if ((nullptr == expr || !is_group_idx_expr(expr)) && col_index >= rowkey_cnt) { int32_t cg_idx = access_cgs->at(col_offset); if (OB_FAIL(cg_param_pool_->get_iter_param(cg_idx, row_param, expr, cg_param))) { LOG_WARN("Fail to get cg iter param", K(ret), K(i), K(cg_idx), K(row_param), KPC(access_cgs)); @@ -450,7 +452,6 @@ int ObCOSSTableRowScanner::construct_cg_iter_params( } else { ObTableIterParam* cg_param = nullptr; const common::ObIArray *access_cgs = nullptr; - const int64_t schema_rowkey_cnt = row_param.get_read_info()->get_schema_rowkey_count(); // Assert only one column in one column group if (row_param.enable_pd_aggregate()) { if (OB_FAIL(construct_cg_agg_iter_params(row_param, context, iter_params))) { @@ -470,7 +471,6 @@ int ObCOSSTableRowScanner::construct_cg_iter_params( } else { int32_t idx = 0; for (int64_t i = 0; OB_SUCC(ret) && i < row_param.output_exprs_->count(); ++i) { - const int32_t col_offset = row_param.out_cols_project_->at(i); const bool need_iter_param = (nullptr == row_param.output_sel_mask_ || row_param.output_sel_mask_->at(i)); if (!is_group_idx_expr(row_param.output_exprs_->at(i)) && need_iter_param) { int32_t cg_idx = access_cgs->at(row_param.out_cols_project_->at(i)); diff --git a/src/storage/compaction/ob_compaction_diagnose.cpp b/src/storage/compaction/ob_compaction_diagnose.cpp index d0ef179513..2c797d4cfc 100644 --- a/src/storage/compaction/ob_compaction_diagnose.cpp +++ b/src/storage/compaction/ob_compaction_diagnose.cpp @@ -539,6 +539,41 @@ ADD_DIAGNOSE_INFO(merge_type, ls_id, tablet_id, status, time, __VA_ARGS__) #define ADD_COMMON_DIAGNOSE_INFO(merge_type, status, time, ...) \ ADD_DIAGNOSE_INFO(merge_type, UNKNOW_LS_ID, UNKNOW_TABLET_ID, status, time, __VA_ARGS__) +#define ADD_MAJOR_WAIT_SCHEDULE(time, info) \ + if (ObTimeUtility::current_time_ns() > time) { \ + if (DIAGNOSE_TABELT_MAX_COUNT > diagnose_tablet_count_[COMPACTION_DIAGNOSE_MAJOR_NOT_SCHEDULE] \ + && can_add_diagnose_info() \ + && OB_TMP_FAIL(ADD_DIAGNOSE_INFO_FOR_TABLET( \ + MAJOR_MERGE, \ + gen_diagnose_status(compaction_scn), \ + ObTimeUtility::fast_current_time(), \ + "major not schedule for long time", info, \ + "max_receive_medium_snapshot", max_sync_medium_scn, \ + "compaction_scn", compaction_scn, \ + "tablet_snapshot", tablet.get_snapshot_version(), \ + "last_major_scn", last_major_snapshot_version))) { \ + LOG_WARN("failed to add diagnose info", K(ret), K(ls_id), K(tablet_id)); \ + } \ + ++diagnose_tablet_count_[COMPACTION_DIAGNOSE_MAJOR_NOT_SCHEDULE]; \ + } + +#define ADD_MEDIUM_WAIT_SCHEDULE(time, info) \ + if (ObTimeUtility::current_time_ns() > time) { \ + if (DIAGNOSE_TABELT_MAX_COUNT > diagnose_tablet_count_[COMPACTION_DIAGNOSE_MEDIUM_NOT_SCHEDULE] \ + && can_add_diagnose_info() \ + && OB_TMP_FAIL(ADD_DIAGNOSE_INFO_FOR_TABLET( \ + MEDIUM_MERGE, \ + gen_diagnose_status(max_sync_medium_scn), \ + ObTimeUtility::fast_current_time(), \ + "medium not schedule for long time", info,\ + "max_receive_medium_scn", max_sync_medium_scn, \ + "tablet_snapshot", tablet.get_snapshot_version(), \ + "last_major_scn", last_major_snapshot_version))) { \ + LOG_WARN("failed to add diagnose info", K(ret), K(ls_id), K(tablet_id)); \ + } \ + ++diagnose_tablet_count_[COMPACTION_DIAGNOSE_MEDIUM_NOT_SCHEDULE]; \ + } + const char *ObCompactionDiagnoseInfo::ObDiagnoseStatusStr[DIA_STATUS_MAX] = { "NOT_SCHEDULE", "RUNNING", @@ -721,8 +756,6 @@ int ObCompactionDiagnoseMgr::get_and_set_suspect_info( LOG_WARN("failed to add dignose info", K(ret), K(tmp_str)); } else if (OB_HASH_NOT_EXIST != ret) { LOG_WARN("failed get suspect info", K(ret), K(ls_id)); - } else { - ret = OB_SUCCESS; } return ret; } @@ -844,9 +877,8 @@ void ObCompactionDiagnoseMgr::diagnose_tenant_ls( LOG_WARN_RET(tmp_ret, "failed to add dignose info about weak read ts", K(tmp_ret), K(compaction_scn)); } // check ls suspect info for memtable freezing // ls level - if (OB_TMP_FAIL(get_and_set_suspect_info(MINI_MERGE, ls_id, UNKNOW_TABLET_ID))) { - LOG_WARN_RET(tmp_ret, "failed to add dignose info about ls suspect info for memtable freezing", K(tmp_ret)); - } + (void) get_and_set_suspect_info(MINI_MERGE, ls_id, UNKNOW_TABLET_ID); + // check ls suspect info for ls locality change if (is_leader && MTL(ObTenantMediumChecker*)->locality_cache_empty()) { if (can_add_diagnose_info() @@ -1270,8 +1302,8 @@ int ObCompactionDiagnoseMgr::diagnose_tablet_mini_merge( tablet))) { LOG_WARN("diagnose failed", K(tmp_ret), K(ls_id), "tablet_id", tablet.get_tablet_meta().tablet_id_, KPC(latest_sstable)); } - } else if (OB_TMP_FAIL(get_and_set_suspect_info(MINI_MERGE, ls_id, tablet_id))) { // mini compaction finish, but memtable have not release - LOG_WARN("failed to add dignose info about memtable release", K(tmp_ret)); + } else { + (void) get_and_set_suspect_info(MINI_MERGE, ls_id, tablet_id); } } return ret; @@ -1315,44 +1347,25 @@ int ObCompactionDiagnoseMgr::diagnose_tablet_major_merge( } else { LOG_TRACE("diagnose tablet major merge", K(ls_id), K(tablet_id), K(compaction_scn), K(max_sync_medium_scn), K(last_major_snapshot_version)); if (last_major_snapshot_version < compaction_scn) { - // max_sync_medium_scn > last_major_snapshot_version means last compaction is not finished, - // this will be diagnosed in diagnose_tablet_medium_merge - if (max_sync_medium_scn < compaction_scn - && max_sync_medium_scn == last_major_snapshot_version) { - // now last compaction finish - if (OB_TMP_FAIL(get_and_set_suspect_info(MEDIUM_MERGE, ls_id, tablet_id))) { - LOG_WARN("failed get major merge suspect info", K(ret), K(ls_id)); - } - if (OB_HASH_NOT_EXIST == tmp_ret - && ObTimeUtility::current_time_ns() > compaction_scn + WAIT_MEDIUM_SCHEDULE_INTERVAL * 2) { - if (DIAGNOSE_TABELT_MAX_COUNT > diagnose_tablet_count_[COMPACTION_DIAGNOSE_MAJOR_NOT_SCHEDULE] - && can_add_diagnose_info() - && OB_TMP_FAIL(ADD_DIAGNOSE_INFO_FOR_TABLET( - MAJOR_MERGE, - gen_diagnose_status(compaction_scn), - ObTimeUtility::fast_current_time(), - "major not schedule for long time, interval", static_cast(WAIT_MEDIUM_SCHEDULE_INTERVAL * 2 / NS_TIME), - "max_receive_medium_snapshot", max_sync_medium_scn, - "compaction_scn", compaction_scn))) { - LOG_WARN("failed to add diagnose info", K(ret), K(ls_id), K(tablet_id)); - } - ++diagnose_tablet_count_[COMPACTION_DIAGNOSE_MAJOR_NOT_SCHEDULE]; - } - } else if (max_sync_medium_scn >= compaction_scn) { - if (tablet.get_snapshot_version() < compaction_scn) { // wait mini compaction or tablet freeze - if (ObTimeUtility::current_time_ns() > compaction_scn + WAIT_MEDIUM_SCHEDULE_INTERVAL) { - if (DIAGNOSE_TABELT_MAX_COUNT > diagnose_tablet_count_[COMPACTION_DIAGNOSE_MAJOR_NOT_SCHEDULE] && can_add_diagnose_info()) { - ADD_DIAGNOSE_INFO_FOR_TABLET( - MAJOR_MERGE, - gen_diagnose_status(compaction_scn), - ObTimeUtility::fast_current_time(), - "major wait for freeze, interval", static_cast(WAIT_MEDIUM_SCHEDULE_INTERVAL / NS_TIME), - "compaction_scn", compaction_scn, - "tablet_snapshot", tablet.get_snapshot_version()); + if (max_sync_medium_scn < compaction_scn) { + // max_sync_medium_scn > last_major_snapshot_version means last compaction is not finished, + // this will be diagnosed in diagnose_tablet_medium_merge + if (max_sync_medium_scn == last_major_snapshot_version) { + // now last compaction finish + if (OB_HASH_NOT_EXIST == get_and_set_suspect_info(MEDIUM_MERGE, ls_id, tablet_id)) { + const char *info = "medium info behind major"; + if (0 == last_major_snapshot_version) { + info = "medium info behind major & no major sstable"; } + ADD_MAJOR_WAIT_SCHEDULE(compaction_scn + WAIT_MEDIUM_SCHEDULE_INTERVAL * 2, info); } - ++diagnose_tablet_count_[COMPACTION_DIAGNOSE_MAJOR_NOT_SCHEDULE]; } + } else if (tablet.get_snapshot_version() < compaction_scn) { // wait mini compaction or tablet freeze + const char* info = "major wait for freeze"; + ADD_MAJOR_WAIT_SCHEDULE(compaction_scn + WAIT_MEDIUM_SCHEDULE_INTERVAL, info); + } else if (0 == last_major_snapshot_version) { + const char* info = "no major sstable"; + ADD_MAJOR_WAIT_SCHEDULE(compaction_scn + WAIT_MEDIUM_SCHEDULE_INTERVAL, info); } if (OB_TMP_FAIL(diagnose_tablet_merge( MEDIUM_MERGE, @@ -1388,18 +1401,11 @@ int ObCompactionDiagnoseMgr::diagnose_tablet_medium_merge( if (!diagnose_major_flag || (diagnose_major_flag && max_sync_medium_scn < compaction_scn)) { if (max_sync_medium_scn > last_major_snapshot_version) { if (tablet.get_snapshot_version() < max_sync_medium_scn) { // wait mini compaction or tablet freeze - if (ObTimeUtility::current_time_ns() > max_sync_medium_scn + WAIT_MEDIUM_SCHEDULE_INTERVAL) { - if (DIAGNOSE_TABELT_MAX_COUNT > diagnose_tablet_count_[COMPACTION_DIAGNOSE_MEDIUM_NOT_SCHEDULE] && can_add_diagnose_info()) { - ADD_DIAGNOSE_INFO_FOR_TABLET( - MEDIUM_MERGE, - gen_diagnose_status(max_sync_medium_scn), - ObTimeUtility::fast_current_time(), - "medium wait for freeze, interval", static_cast(WAIT_MEDIUM_SCHEDULE_INTERVAL / NS_TIME), - "max_receive_medium_scn", max_sync_medium_scn, - "tablet_snapshot", tablet.get_snapshot_version()); - } - ++diagnose_tablet_count_[COMPACTION_DIAGNOSE_MEDIUM_NOT_SCHEDULE]; - } + const char *info = "medium wait for freeze"; + ADD_MEDIUM_WAIT_SCHEDULE(max_sync_medium_scn + WAIT_MEDIUM_SCHEDULE_INTERVAL, info); + } else if (0 == last_major_snapshot_version) { + const char *info = "no major sstable"; + ADD_MEDIUM_WAIT_SCHEDULE(max_sync_medium_scn + WAIT_MEDIUM_SCHEDULE_INTERVAL, info); } else if (OB_TMP_FAIL(diagnose_tablet_merge( MEDIUM_MERGE, ls_id, diff --git a/src/storage/compaction/ob_tablet_merge_info.cpp b/src/storage/compaction/ob_tablet_merge_info.cpp index a112fd412a..91446b4970 100644 --- a/src/storage/compaction/ob_tablet_merge_info.cpp +++ b/src/storage/compaction/ob_tablet_merge_info.cpp @@ -131,101 +131,17 @@ int ObTabletMergeInfo::build_create_sstable_param(const ObBasicTabletMergeCtx &c const bool is_main_table) { int ret = OB_SUCCESS; - const ObStaticMergeParam &static_param = ctx.static_param_; if (OB_UNLIKELY(!ctx.is_valid() || !res.is_valid() || (nullptr != cg_schema && (!cg_schema->is_valid() || column_group_idx < 0)))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid merge ctx", K(ret), K(ctx), K(res), K(column_group_idx), KPC(cg_schema)); } else if (OB_UNLIKELY(nullptr != cg_schema && cg_schema->column_cnt_ != res.data_column_cnt_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("table column cnt is unexpected mismatched!", K(ret), KPC(cg_schema), K(res)); - } else if (OB_FAIL(static_param.schema_->get_stored_column_count_in_sstable(param.full_column_cnt_))) { - LOG_WARN("fail to get_stored_column_count_in_sstable", K(ret), KPC(cg_schema), K(res)); - } else { - ObITable::TableKey table_key; - table_key.table_type_ = ctx.get_merged_table_type(cg_schema, is_main_table); - table_key.tablet_id_ = ctx.get_tablet_id(); - table_key.column_group_idx_ = (nullptr == cg_schema) ? 0 : column_group_idx; - if (is_major_or_meta_merge_type(static_param.get_merge_type())) { - table_key.version_range_.snapshot_version_ = static_param.version_range_.snapshot_version_; - } else { - table_key.scn_range_ = static_param.scn_range_; - } - param.table_key_ = table_key; - - if (ObITable::TableType::COLUMN_ORIENTED_SSTABLE == table_key.table_type_ || - ObITable::TableType::COLUMN_ORIENTED_META_SSTABLE == table_key.table_type_) { - param.co_base_type_ = cg_schema->is_all_column_group() - ? ObCOSSTableBaseType::ALL_CG_TYPE - : ObCOSSTableBaseType::ROWKEY_CG_TYPE; - } - - param.sstable_logic_seq_ = static_param.sstable_logic_seq_; - param.filled_tx_scn_ = ctx.get_merge_scn(); - - param.table_mode_ = ctx.get_schema()->get_table_mode_struct(); - param.index_type_ = ctx.get_schema()->get_index_type(); - if (nullptr != cg_schema && !cg_schema->is_rowkey_column_group() && !cg_schema->is_all_column_group()) { - param.rowkey_column_cnt_ = 0; - } else { - param.column_group_cnt_ = static_param.schema_->get_column_group_count(); - param.rowkey_column_cnt_ = static_param.schema_->get_rowkey_column_num() - + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); - } - param.latest_row_store_type_ = ctx.get_schema()->get_row_store_type(); - if (is_minor_merge_type(static_param.get_merge_type())) { - param.recycle_version_ = static_param.version_range_.base_version_; - } else { - param.recycle_version_ = 0; - } - param.schema_version_ = static_param.schema_version_; - param.create_snapshot_version_ = static_param.create_snapshot_version_; - param.progressive_merge_round_ = static_param.progressive_merge_round_; - param.progressive_merge_step_ = std::min( - static_param.progressive_merge_num_, static_param.progressive_merge_step_ + 1); - - ObSSTableMergeRes::fill_addr_and_data(res.root_desc_, - param.root_block_addr_, param.root_block_data_); - ObSSTableMergeRes::fill_addr_and_data(res.data_root_desc_, - param.data_block_macro_meta_addr_, param.data_block_macro_meta_); - param.is_meta_root_ = res.data_root_desc_.is_meta_root_; - param.root_row_store_type_ = res.root_row_store_type_; - param.data_index_tree_height_ = res.root_desc_.height_; - param.index_blocks_cnt_ = res.index_blocks_cnt_; - param.data_blocks_cnt_ = res.data_blocks_cnt_; - param.is_empty_co_table_ = is_main_table ? (0 == param.data_blocks_cnt_) : false; - param.micro_block_cnt_ = res.micro_block_cnt_; - param.use_old_macro_block_count_ = res.use_old_macro_block_count_; - param.row_count_ = res.row_count_; - param.column_cnt_ = res.data_column_cnt_; - param.data_checksum_ = res.data_checksum_; - param.occupy_size_ = res.occupy_size_; - param.original_size_ = res.original_size_; - if ((0 == res.row_count_ && 0 == res.max_merged_trans_version_) - || (nullptr != cg_schema && !cg_schema->has_multi_version_column())) { - // empty mini table merged forcely - param.max_merged_trans_version_ = static_param.version_range_.snapshot_version_; - } else { - param.max_merged_trans_version_ = res.max_merged_trans_version_; - } - param.contain_uncommitted_row_ = res.contain_uncommitted_row_; - param.compressor_type_ = res.compressor_type_; - param.encrypt_id_ = res.encrypt_id_; - param.master_key_id_ = res.master_key_id_; - param.nested_size_ = res.nested_size_; - param.nested_offset_ = res.nested_offset_; - param.data_block_ids_ = res.data_block_ids_; - param.other_block_ids_ = res.other_block_ids_; - param.ddl_scn_.set_min(); - MEMCPY(param.encrypt_key_, res.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); - if (is_major_or_meta_merge_type(static_param.get_merge_type())) { - if (OB_FAIL(param.column_checksums_.assign(res.data_column_checksums_))) { - LOG_WARN("fail to fill column checksum", K(ret), K(res)); - } - } - - if (OB_SUCC(ret) && ctx.get_tablet_id().is_ls_tx_data_tablet()) { + } else if (OB_FAIL(param.init_for_merge(ctx, res, cg_schema, column_group_idx, is_main_table))) { + LOG_WARN("fail to init create sstable param for merge", + K(ret), K(ctx), K(res), KPC(cg_schema), K(column_group_idx), K(is_main_table)); + } else if (ctx.get_tablet_id().is_ls_tx_data_tablet()) { ret = record_start_tx_scn_for_tx_data(ctx, param); - } } return ret; } diff --git a/src/storage/compaction/ob_tenant_tablet_scheduler.cpp b/src/storage/compaction/ob_tenant_tablet_scheduler.cpp index d34a18ac81..ba12372d67 100644 --- a/src/storage/compaction/ob_tenant_tablet_scheduler.cpp +++ b/src/storage/compaction/ob_tenant_tablet_scheduler.cpp @@ -1678,6 +1678,7 @@ int ObTenantTabletScheduler::schedule_tablet_medium( K(could_major_merge), K(enable_adaptive_compaction), K(tablet_could_schedule_merge)); if (OB_FAIL(ret) || !is_leader || 0 >= last_major_snapshot_version) { // follower or no major: do nothing + need_diagnose = true; } else if (OB_FAIL(tablet.read_medium_info_list(tmp_allocator, medium_list))) { LOG_WARN("failed to load medium info list", K(ret), K(tablet_id)); } else if (FALSE_IT(time_guard.click(ObCompactionScheduleTimeGuard::READ_MEDIUM_INFO))){ diff --git a/src/storage/ddl/ob_complement_data_task.cpp b/src/storage/ddl/ob_complement_data_task.cpp index 39734f16d9..dce8617dd5 100644 --- a/src/storage/ddl/ob_complement_data_task.cpp +++ b/src/storage/ddl/ob_complement_data_task.cpp @@ -1265,6 +1265,7 @@ int ObComplementWriteTask::append_lob( slice_info.data_tablet_id_ = param_->dest_tablet_id_; slice_info.slice_id_ = iterator.get_lob_slice_id(); slice_info.context_id_ = context_->context_id_; + slice_info.src_tenant_id_ = param_->orig_tenant_id_; if (OB_FAIL(MTL(ObTenantDirectLoadMgr *)->fill_lob_sstable_slice(lob_allocator, slice_info, iterator.get_lob_id_cache(), lob_column_idxs, col_types, write_row_))) { LOG_WARN("fill batch lob sstable slice failed", K(ret), K(slice_info), K(write_row_)); diff --git a/src/storage/ddl/ob_ddl_merge_task.cpp b/src/storage/ddl/ob_ddl_merge_task.cpp index 4df596f4f3..3a0ab8c852 100644 --- a/src/storage/ddl/ob_ddl_merge_task.cpp +++ b/src/storage/ddl/ob_ddl_merge_task.cpp @@ -36,6 +36,7 @@ #include "storage/column_store/ob_column_oriented_sstable.h" #include "storage/ddl/ob_direct_insert_sstable_ctx_new.h" #include "storage/column_store/ob_column_oriented_sstable.h" +#include "storage/compaction/ob_tenant_tablet_scheduler.h" using namespace oceanbase::observer; using namespace oceanbase::share::schema; @@ -208,6 +209,37 @@ int ObDDLTableMergeTask::init(const ObDDLTableMergeDagParam &ddl_dag_param, cons return ret; } +int wait_lob_tablet_major_exist(ObLSHandle &ls_handle, ObTablet &tablet) +{ + int ret = OB_SUCCESS; + ObTabletBindingMdsUserData ddl_data; + const ObTabletMeta &tablet_meta = tablet.get_tablet_meta(); + ObTenantDirectLoadMgr *tenant_direct_load_mgr = MTL(ObTenantDirectLoadMgr *); + ObTabletDirectLoadMgrHandle direct_load_mgr_handle; + ObDDLTableMergeDagParam param; + bool is_major_sstable_exist = false; + if (OB_FAIL(tablet.ObITabletMdsInterface::get_ddl_data(share::SCN::max_scn(), ddl_data))) { + LOG_WARN("failed to get ddl data from tablet", K(ret), K(tablet_meta)); + } else if (ddl_data.lob_meta_tablet_id_.is_valid()) { + ObTabletHandle lob_tablet_handle; + const ObTabletID lob_tablet_id = ddl_data.lob_meta_tablet_id_; + if (OB_FAIL(ObDDLUtil::ddl_get_tablet(ls_handle, lob_tablet_id, lob_tablet_handle, ObMDSGetTabletMode::READ_ALL_COMMITED))) { + LOG_WARN("get lob tablet handle failed", K(ret), K(lob_tablet_id)); + } else { + bool is_major_sstable_exist = lob_tablet_handle.get_obj()->get_major_table_count() > 0 + || lob_tablet_handle.get_obj()->get_tablet_meta().table_store_flag_.with_major_sstable(); + if (!is_major_sstable_exist) { + ret = OB_EAGAIN; + int tmp_ret = OB_SUCCESS; + if (OB_TMP_FAIL(compaction::ObTenantTabletScheduler::schedule_tablet_ddl_major_merge(ls_handle.get_ls()->get_ls_id(), lob_tablet_handle))) { + LOG_WARN("schedule ddl major merge for lob tablet failed", K(tmp_ret), K(lob_tablet_id)); + } + } + } + } + return ret; +} + int ObDDLTableMergeTask::process() { int ret = OB_SUCCESS; @@ -286,6 +318,12 @@ int ObDDLTableMergeTask::process() LOG_WARN("get tablet direct load mgr failed", K(ret), K(merge_param_)); } else if (OB_FAIL(tablet_mgr_hdl.get_full_obj()->prepare_major_merge_param(ddl_param))) { LOG_WARN("preare full direct load sstable param failed", K(ret)); + } else if (merge_param_.is_commit_ && OB_FAIL(wait_lob_tablet_major_exist(ls_handle, *tablet_handle.get_obj()))) { + if (OB_EAGAIN != ret) { + LOG_WARN("wait lob tablet major sstable exist faild", K(ret), K(merge_param_)); + } else { + LOG_INFO("need wait lob tablet major sstable exist", K(ret), K(merge_param_)); + } } else if (merge_param_.start_scn_ > SCN::min_scn() && merge_param_.start_scn_ < ddl_param.start_scn_) { ret = OB_TASK_EXPIRED; LOG_INFO("ddl merge task expired, do nothing", K(merge_param_), "new_start_scn", ddl_param.start_scn_); @@ -452,20 +490,17 @@ int ObTabletDDLUtil::prepare_index_data_desc(ObTablet &tablet, const uint64_t data_format_version, const ObSSTable *first_ddl_sstable, const SCN &end_scn, + const ObStorageSchema *storage_schema, ObWholeDataStoreDesc &data_desc) { int ret = OB_SUCCESS; data_desc.reset(); ObLSService *ls_service = MTL(ObLSService *); - ObArenaAllocator tmp_arena("DDLIdxDescTmp", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); const ObTabletID &tablet_id = tablet.get_tablet_meta().tablet_id_; const ObLSID &ls_id = tablet.get_tablet_meta().ls_id_; - ObStorageSchema *storage_schema = nullptr; - if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid() || snapshot_version <= 0 || data_format_version <= 0)) { + if (OB_UNLIKELY(!ls_id.is_valid() || !tablet_id.is_valid() || snapshot_version <= 0 || data_format_version <= 0 || OB_ISNULL(storage_schema))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(data_format_version)); - } else if (OB_FAIL(tablet.load_storage_schema(tmp_arena, storage_schema))) { - LOG_WARN("fail to get storage schema", K(ret)); + LOG_WARN("invalid argument", K(ret), K(ls_id), K(tablet_id), K(snapshot_version), K(data_format_version), KP(storage_schema)); } else if (cg_idx >= 0) { const ObIArray &cg_schemas = storage_schema->get_column_groups(); if (cg_idx >= cg_schemas.count()) { @@ -503,7 +538,6 @@ int ObTabletDDLUtil::prepare_index_data_desc(ObTablet &tablet, } } } - ObTabletObjLoadHelper::free(tmp_arena, storage_schema); LOG_DEBUG("prepare_index_data_desc", K(ret), K(data_desc)); return ret; } @@ -512,6 +546,7 @@ int ObTabletDDLUtil::create_ddl_sstable(ObTablet &tablet, const ObTabletDDLParam &ddl_param, const ObIArray &meta_array, const ObSSTable *first_ddl_sstable, + const ObStorageSchema *storage_schema, common::ObArenaAllocator &allocator, ObTableHandleV2 &sstable_handle) { @@ -520,9 +555,9 @@ int ObTabletDDLUtil::create_ddl_sstable(ObTablet &tablet, ObIndexBlockRebuilder index_block_rebuilder; ObWholeDataStoreDesc data_desc(true/*is_ddl*/); int64_t macro_block_column_count = 0; - if (OB_UNLIKELY(!ddl_param.is_valid())) { + if (OB_UNLIKELY(!ddl_param.is_valid() || OB_ISNULL(storage_schema))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ddl_param)); + LOG_WARN("invalid argument", K(ret), K(ddl_param), KP(storage_schema)); } else if (OB_FAIL(ObTabletDDLUtil::prepare_index_data_desc( tablet, ddl_param.table_key_.is_column_store_sstable() ? ddl_param.table_key_.get_column_group_id() : -1/*negative value means row store*/, @@ -530,6 +565,7 @@ int ObTabletDDLUtil::create_ddl_sstable(ObTablet &tablet, ddl_param.data_format_version_, first_ddl_sstable, ddl_param.table_key_.get_end_scn(), + storage_schema, data_desc))) { LOG_WARN("prepare data store desc failed", K(ret), K(ddl_param)); } else if (FALSE_IT(macro_block_column_count = meta_array.empty() ? 0 : meta_array.at(0)->get_meta_val().column_count_)) { @@ -557,7 +593,7 @@ int ObTabletDDLUtil::create_ddl_sstable(ObTablet &tablet, if (OB_FAIL(index_block_rebuilder.close())) { LOG_WARN("close index block rebuilder failed", K(ret)); } else if (OB_FAIL(ObTabletDDLUtil::create_ddl_sstable(tablet, &sstable_index_builder, ddl_param, first_ddl_sstable, - macro_block_column_count, allocator, sstable_handle))) { + macro_block_column_count, storage_schema, allocator, sstable_handle))) { LOG_WARN("create ddl sstable failed", K(ret), K(ddl_param)); } } @@ -572,171 +608,36 @@ int ObTabletDDLUtil::create_ddl_sstable( const ObTabletDDLParam &ddl_param, const ObSSTable *first_ddl_sstable, const int64_t macro_block_column_count, + const ObStorageSchema *storage_schema, common::ObArenaAllocator &allocator, ObTableHandleV2 &sstable_handle) { int ret = OB_SUCCESS; - ObArenaAllocator tmp_arena("CreateDDLSstTmp", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); - ObStorageSchema *storage_schema = nullptr; - SMART_VAR(ObSSTableMergeRes, res) { - if (OB_UNLIKELY(nullptr == sstable_index_builder || !ddl_param.is_valid())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), KP(sstable_index_builder), K(ddl_param)); - } else if (OB_FAIL(tablet.load_storage_schema(tmp_arena, storage_schema))) { - LOG_WARN("failed to load storage schema", K(ret), K(tablet.get_tablet_meta())); - } else { - int64_t column_count = 0; - int64_t full_column_cnt = 0; // only used for co sstable - share::schema::ObTableMode table_mode = storage_schema->get_table_mode_struct(); - share::schema::ObIndexType index_type = storage_schema->get_index_type(); - int64_t rowkey_column_cnt = storage_schema->get_rowkey_column_num() + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); - common::ObRowStoreType row_store_type = storage_schema->get_row_store_type(); - if (nullptr != first_ddl_sstable) { - ObSSTableMetaHandle meta_handle; - if (OB_FAIL(first_ddl_sstable->get_meta(meta_handle))) { - LOG_WARN("get sstable meta handle fail", K(ret), KPC(first_ddl_sstable)); - } else { - column_count = meta_handle.get_sstable_meta().get_column_count(); - table_mode = meta_handle.get_sstable_meta().get_basic_meta().table_mode_; - index_type = static_cast(meta_handle.get_sstable_meta().get_basic_meta().index_type_); - rowkey_column_cnt = meta_handle.get_sstable_meta().get_basic_meta().rowkey_column_count_; - row_store_type = meta_handle.get_sstable_meta().get_basic_meta().latest_row_store_type_; - if (first_ddl_sstable->is_co_sstable()) { - const ObCOSSTableV2 *first_co_sstable = static_cast(first_ddl_sstable); - if (OB_ISNULL((first_co_sstable))) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("first co sstable is null", K(ret), KP(first_co_sstable), KPC(first_ddl_sstable)); - } else { - full_column_cnt = first_co_sstable->get_cs_meta().full_column_cnt_; - } - } - } - } else if (ddl_param.table_key_.is_column_store_sstable()) { - if (ddl_param.table_key_.is_normal_cg_sstable()) { - rowkey_column_cnt = 0; - column_count = 1; - } else { // co sstable with all cg or rowkey cg - const ObIArray &cg_schemas = storage_schema->get_column_groups(); - const int64_t cg_idx = ddl_param.table_key_.get_column_group_id(); - if (cg_idx < 0 || cg_idx >= cg_schemas.count()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected column group index", K(ret), K(cg_idx)); - } else if (OB_FAIL(storage_schema->get_stored_column_count_in_sstable(full_column_cnt))) { // set full_column_cnt in first ddl sstable - LOG_WARN("fail to get stored column count in sstable", K(ret)); - } else if (cg_schemas.at(cg_idx).is_rowkey_column_group()) { - column_count = rowkey_column_cnt; - } else { - column_count = full_column_cnt; - if (macro_block_column_count > 0 && macro_block_column_count < column_count) { - LOG_INFO("use macro block column count", K(ddl_param), K(macro_block_column_count), K(column_count)); - column_count = macro_block_column_count; - full_column_cnt = macro_block_column_count; - } - } - } - } else { // row store sstable - if (OB_FAIL(storage_schema->get_stored_column_count_in_sstable(column_count))) { - LOG_WARN("fail to get stored column count in sstable", K(ret)); - } else if (macro_block_column_count > 0 && macro_block_column_count < column_count) { - LOG_INFO("use macro block column count", K(ddl_param), K(macro_block_column_count), K(column_count)); - column_count = macro_block_column_count; - } + if (OB_UNLIKELY(nullptr == sstable_index_builder || !ddl_param.is_valid() || OB_ISNULL(storage_schema))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), KP(sstable_index_builder), K(ddl_param), KP(storage_schema)); + } else { + const int64_t create_schema_version_on_tablet = tablet.get_tablet_meta().create_schema_version_; + ObTabletCreateSSTableParam param; + if (OB_FAIL(param.init_for_ddl(sstable_index_builder, ddl_param, first_ddl_sstable, + *storage_schema, macro_block_column_count, create_schema_version_on_tablet))) { + LOG_WARN("fail to init param for ddl", + K(ret), K(macro_block_column_count), K(create_schema_version_on_tablet), + KPC(sstable_index_builder), K(ddl_param), + KPC(first_ddl_sstable), KPC(storage_schema)); + } else if (ddl_param.table_key_.is_co_sstable()) { + if (OB_FAIL(ObTabletCreateDeleteHelper::create_sstable(param, allocator, sstable_handle))) { + LOG_WARN("create sstable failed", K(ret), K(param)); } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(sstable_index_builder->close(res))) { - LOG_WARN("close sstable index builder close failed", K(ret)); - } else if (ddl_param.table_key_.is_normal_cg_sstable() // index builder of cg sstable cannot get trans_version from row, manually set it - && FALSE_IT(res.max_merged_trans_version_ = ddl_param.snapshot_version_)) { - } else if (OB_UNLIKELY((ddl_param.table_key_.is_major_sstable() || - ddl_param.table_key_.is_ddl_sstable()) && - res.row_count_ > 0 && - res.max_merged_trans_version_ != ddl_param.snapshot_version_)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("max_merged_trans_version_ in res is different from ddl snapshot version", K(ret), - K(res), K(ddl_param)); - } else { - const int64_t create_schema_version_on_tablet = tablet.get_tablet_meta().create_schema_version_; - ObTabletCreateSSTableParam param; - param.table_key_ = ddl_param.table_key_; - param.table_mode_ = table_mode; - param.index_type_ = index_type; - param.rowkey_column_cnt_ = rowkey_column_cnt; - param.schema_version_ = create_schema_version_on_tablet; - param.latest_row_store_type_ = row_store_type; - param.create_snapshot_version_ = ddl_param.snapshot_version_; - param.ddl_scn_ = ddl_param.start_scn_; - ObSSTableMergeRes::fill_addr_and_data(res.root_desc_, - param.root_block_addr_, param.root_block_data_); - ObSSTableMergeRes::fill_addr_and_data(res.data_root_desc_, - param.data_block_macro_meta_addr_, param.data_block_macro_meta_); - param.is_meta_root_ = res.data_root_desc_.is_meta_root_; - param.root_row_store_type_ = res.root_row_store_type_; - param.data_index_tree_height_ = res.root_desc_.height_; - param.index_blocks_cnt_ = res.index_blocks_cnt_; - param.data_blocks_cnt_ = res.data_blocks_cnt_; - param.micro_block_cnt_ = res.micro_block_cnt_; - param.use_old_macro_block_count_ = res.use_old_macro_block_count_; - param.row_count_ = res.row_count_; - param.column_cnt_ = column_count; - param.full_column_cnt_ = full_column_cnt; - param.data_checksum_ = res.data_checksum_; - param.occupy_size_ = res.occupy_size_; - param.original_size_ = res.original_size_; - param.max_merged_trans_version_ = ddl_param.snapshot_version_; - param.contain_uncommitted_row_ = res.contain_uncommitted_row_; - param.compressor_type_ = res.compressor_type_; - param.encrypt_id_ = res.encrypt_id_; - param.master_key_id_ = res.master_key_id_; - param.nested_size_ = res.nested_size_; - param.nested_offset_ = res.nested_offset_; - param.data_block_ids_ = res.data_block_ids_; - param.other_block_ids_ = res.other_block_ids_; - MEMCPY(param.encrypt_key_, res.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); - if (ddl_param.table_key_.is_co_sstable()) { - param.column_group_cnt_ = storage_schema->get_column_group_count(); - // only set true when build empty major sstable. ddl co sstable must set false and fill cg sstables - param.is_empty_co_table_ = ddl_param.table_key_.is_major_sstable() && 0 == param.data_blocks_cnt_; - const int64_t base_cg_idx = ddl_param.table_key_.get_column_group_id(); - if (base_cg_idx < 0 || base_cg_idx >= storage_schema->get_column_group_count()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("invalid column group index", K(ret), K(ddl_param.table_key_)); - } else { - const ObStorageColumnGroupSchema &base_cg_schema = storage_schema->get_column_groups().at(base_cg_idx); - if (base_cg_schema.is_all_column_group()) { - param.co_base_type_ = ObCOSSTableBaseType::ALL_CG_TYPE; - } else if (base_cg_schema.is_rowkey_column_group()) { - param.co_base_type_ = ObCOSSTableBaseType::ROWKEY_CG_TYPE; - } else { - ret = OB_ERR_SYS; - LOG_WARN("unknown type of base cg schema", K(ret), K(base_cg_idx)); - } - } - } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(param.column_checksums_.assign(res.data_column_checksums_))) { - LOG_WARN("fail to fill column checksum for empty major", K(ret), K(param)); - } else if (OB_UNLIKELY(param.column_checksums_.count() != column_count)) { - // we have corrected the col_default_checksum_array_ in prepare_index_data_desc - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected column checksums", K(ret), K(column_count), K(param)); - } else { - if (ddl_param.table_key_.is_co_sstable()) { - if (OB_FAIL(ObTabletCreateDeleteHelper::create_sstable(param, allocator, sstable_handle))) { - LOG_WARN("create sstable failed", K(ret), K(param)); - } - } else { - if (OB_FAIL(ObTabletCreateDeleteHelper::create_sstable(param, allocator, sstable_handle))) { - LOG_WARN("create sstable failed", K(ret), K(param)); - } - } - if (OB_SUCC(ret)) { - LOG_INFO("create ddl sstable success", K(ddl_param), K(sstable_handle), - "create_schema_version", create_schema_version_on_tablet); - } - } + } else { + if (OB_FAIL(ObTabletCreateDeleteHelper::create_sstable(param, allocator, sstable_handle))) { + LOG_WARN("create sstable failed", K(ret), K(param)); } } - ObTabletObjLoadHelper::free(tmp_arena, storage_schema); + if (OB_SUCC(ret)) { + LOG_INFO("create ddl sstable success", K(ddl_param), K(sstable_handle), + "create_schema_version", create_schema_version_on_tablet); + } } return ret; } @@ -745,48 +646,42 @@ int ObTabletDDLUtil::update_ddl_table_store( ObLS &ls, ObTablet &tablet, const ObTabletDDLParam &ddl_param, + const ObStorageSchema *storage_schema, common::ObArenaAllocator &allocator, blocksstable::ObSSTable *sstable) { int ret = OB_SUCCESS; - if (OB_UNLIKELY(!ddl_param.is_valid() || nullptr == sstable)) { + if (OB_UNLIKELY(!ddl_param.is_valid() || OB_ISNULL(storage_schema) || OB_ISNULL(sstable))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ddl_param), KP(sstable)); + LOG_WARN("invalid argument", K(ret), K(ddl_param), KP(storage_schema), KP(sstable)); } else { - ObArenaAllocator allocator("DDLUtil_update", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); - ObStorageSchema *tablet_storage_schema = nullptr; - if (OB_FAIL(tablet.load_storage_schema(allocator, tablet_storage_schema))) { - LOG_WARN("fail to load storage schema failed", K(ret)); + const bool is_major_sstable = ddl_param.table_key_.is_major_sstable(); + const int64_t rebuild_seq = ls.get_rebuild_seq(); + const int64_t snapshot_version = is_major_sstable ? max(ddl_param.snapshot_version_, tablet.get_snapshot_version()) + : tablet.get_snapshot_version(); + const int64_t multi_version_start = is_major_sstable ? max(ddl_param.snapshot_version_, tablet.get_multi_version_start()) + : 0; + ObTabletHandle new_tablet_handle; + ObUpdateTableStoreParam table_store_param(sstable, + snapshot_version, + multi_version_start, + rebuild_seq, + storage_schema, + is_major_sstable, // update_with_major_flag + /*DDL does not have verification between replicas, + So using medium merge to force verification between replicas*/ + compaction::MEDIUM_MERGE, + is_major_sstable// need report checksum + ); + table_store_param.ddl_info_.keep_old_ddl_sstable_ = !is_major_sstable; + table_store_param.ddl_info_.data_format_version_ = ddl_param.data_format_version_; + table_store_param.ddl_info_.ddl_commit_scn_ = ddl_param.commit_scn_; + table_store_param.ddl_info_.ddl_checkpoint_scn_ = sstable->is_ddl_dump_sstable() ? sstable->get_end_scn() : ddl_param.commit_scn_; + if (OB_FAIL(ls.update_tablet_table_store(ddl_param.table_key_.get_tablet_id(), table_store_param, new_tablet_handle))) { + LOG_WARN("failed to update tablet table store", K(ret), K(ddl_param.table_key_), K(table_store_param)); } else { - const bool is_major_sstable = ddl_param.table_key_.is_major_sstable(); - const int64_t rebuild_seq = ls.get_rebuild_seq(); - const int64_t snapshot_version = is_major_sstable ? max(ddl_param.snapshot_version_, tablet.get_snapshot_version()) - : tablet.get_snapshot_version(); - const int64_t multi_version_start = is_major_sstable ? max(ddl_param.snapshot_version_, tablet.get_multi_version_start()) - : 0; - ObTabletHandle new_tablet_handle; - ObUpdateTableStoreParam table_store_param(sstable, - snapshot_version, - multi_version_start, - rebuild_seq, - tablet_storage_schema, - is_major_sstable, // update_with_major_flag - /*DDL does not have verification between replicas, - So using medium merge to force verification between replicas*/ - compaction::MEDIUM_MERGE, - is_major_sstable// need report checksum - ); - table_store_param.ddl_info_.keep_old_ddl_sstable_ = !is_major_sstable; - table_store_param.ddl_info_.data_format_version_ = ddl_param.data_format_version_; - table_store_param.ddl_info_.ddl_commit_scn_ = ddl_param.commit_scn_; - table_store_param.ddl_info_.ddl_checkpoint_scn_ = sstable->is_ddl_dump_sstable() ? sstable->get_end_scn() : ddl_param.commit_scn_; - if (OB_FAIL(ls.update_tablet_table_store(ddl_param.table_key_.get_tablet_id(), table_store_param, new_tablet_handle))) { - LOG_WARN("failed to update tablet table store", K(ret), K(ddl_param.table_key_), K(table_store_param)); - } else { - LOG_INFO("ddl update table store success", K(ddl_param), K(table_store_param), KPC(sstable)); - } + LOG_INFO("ddl update table store success", K(ddl_param), K(table_store_param), KPC(sstable)); } - ObTabletObjLoadHelper::free(allocator, tablet_storage_schema); } return ret; } @@ -960,6 +855,7 @@ int compact_sstables( ObIArray &sstables, const ObTabletDDLParam &ddl_param, const ObITableReadInfo &read_info, + const ObStorageSchema *storage_schema, ObArenaAllocator &allocator, ObTableHandleV2 &sstable_handle) { @@ -967,7 +863,7 @@ int compact_sstables( ObArenaAllocator arena("compact_sst", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); ObBlockMetaTree meta_tree; ObArray sorted_metas; - if (OB_FAIL(meta_tree.init(tablet, ddl_param.table_key_, ddl_param.start_scn_, ddl_param.data_format_version_))) { + if (OB_FAIL(meta_tree.init(tablet, ddl_param.table_key_, ddl_param.start_scn_, ddl_param.data_format_version_, storage_schema))) { LOG_WARN("init meta tree failed", K(ret), K(ddl_param)); } else if (OB_FAIL(get_sorted_meta_array(sstables, read_info, meta_tree, arena, sorted_metas))) { LOG_WARN("get sorted meta array failed", K(ret), K(read_info), K(sstables)); @@ -976,6 +872,7 @@ int compact_sstables( ddl_param, sorted_metas, sstables.empty() ? nullptr : sstables.at(0)/*first ddl sstable*/, + storage_schema, allocator, sstable_handle))) { LOG_WARN("create sstable failed", K(ret), K(ddl_param), K(sstables)); @@ -989,6 +886,7 @@ int compact_co_ddl_sstable( ObTableStoreIterator &ddl_sstable_iter, const ObIArray &frozen_ddl_kvs, const ObTabletDDLParam &ddl_param, + const ObStorageSchema *storage_schema, common::ObArenaAllocator &allocator, ObTablesHandleArray &compacted_cg_sstable_handles, ObTableHandleV2 &co_sstable_handle) @@ -997,13 +895,9 @@ int compact_co_ddl_sstable( compacted_cg_sstable_handles.reset(); co_sstable_handle.reset(); const ObITableReadInfo *cg_index_read_info = nullptr; - ObArenaAllocator arena("compact_co_ddl", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); - ObStorageSchema *storage_schema = nullptr; - if (OB_UNLIKELY(ddl_sstable_iter.count() == 0 && frozen_ddl_kvs.count() == 0)) { + if (OB_UNLIKELY(ddl_sstable_iter.count() == 0 && frozen_ddl_kvs.count() == 0) || OB_ISNULL(storage_schema)) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(ddl_sstable_iter.count()), K(frozen_ddl_kvs.count())); - } else if (OB_FAIL(tablet.load_storage_schema(arena, storage_schema))) { - LOG_WARN("load storage schema failed", K(ret), K(ddl_param)); + LOG_WARN("invalid argument", K(ret), K(ddl_sstable_iter.count()), K(frozen_ddl_kvs.count()), KP(storage_schema)); } else { const int64_t base_cg_idx = ddl_param.table_key_.get_column_group_id(); ObArray base_sstables; @@ -1013,7 +907,7 @@ int compact_co_ddl_sstable( LOG_WARN("get base sstable from ddl sstables failed", K(ret), K(ddl_sstable_iter), K(base_cg_idx)); } else if (OB_FAIL(get_sstables(frozen_ddl_kvs, base_cg_idx, base_sstables))) { LOG_WARN("get base sstable from ddl kv array failed", K(ret), K(frozen_ddl_kvs), K(base_cg_idx)); - } else if (OB_FAIL(compact_sstables(tablet, base_sstables, ddl_param, tablet.get_rowkey_read_info(), allocator, co_sstable_handle))) { + } else if (OB_FAIL(compact_sstables(tablet, base_sstables, ddl_param, tablet.get_rowkey_read_info(), storage_schema, allocator, co_sstable_handle))) { LOG_WARN("compact base sstable failed", K(ret)); } else { // empty major co sstable, no need fill cg sstables @@ -1036,7 +930,7 @@ int compact_co_ddl_sstable( LOG_WARN("get current cg sstables failed", K(ret)); } else if (OB_FAIL(get_sstables(frozen_ddl_kvs, cur_cg_idx, cur_cg_sstables))) { LOG_WARN("get current cg sstables failed", K(ret)); - } else if (OB_FAIL(compact_sstables(tablet, cur_cg_sstables, cg_ddl_param, *cg_index_read_info, allocator, target_table_handle))) { + } else if (OB_FAIL(compact_sstables(tablet, cur_cg_sstables, cg_ddl_param, *cg_index_read_info, storage_schema, allocator, target_table_handle))) { LOG_WARN("compact cg sstable failed", K(ret), K(cur_cg_idx), K(cur_cg_sstables.count()), K(cg_ddl_param), KPC(cg_index_read_info)); } else if (OB_FAIL(compacted_cg_sstable_handles.add_table(target_table_handle))) { LOG_WARN("push back compacted cg sstable failed", K(ret), K(i), KP(target_table_handle.get_table())); @@ -1052,7 +946,6 @@ int compact_co_ddl_sstable( } } } - ObTabletObjLoadHelper::free(arena, storage_schema); LOG_INFO("compact_co_ddl_sstable", K(ret), K(ddl_sstable_iter), K(ddl_param), KP(&tablet), KPC(co_sstable_handle.get_table())); return ret; } @@ -1062,6 +955,7 @@ int compact_ro_ddl_sstable( ObTableStoreIterator &ddl_sstable_iter, const ObIArray &frozen_ddl_kvs, const ObTabletDDLParam &ddl_param, + const ObStorageSchema *storage_schema, common::ObArenaAllocator &allocator, ObTableHandleV2 &ro_sstable_handle) { @@ -1077,7 +971,7 @@ int compact_ro_ddl_sstable( LOG_WARN("get base sstable from ddl sstables failed", K(ret), K(ddl_sstable_iter), K(base_cg_idx)); } else if (OB_FAIL(get_sstables(frozen_ddl_kvs, base_cg_idx, base_sstables))) { LOG_WARN("get base sstable from ddl kv array failed", K(ret), K(frozen_ddl_kvs), K(base_cg_idx)); - } else if (OB_FAIL(compact_sstables(tablet, base_sstables, ddl_param, tablet.get_rowkey_read_info(), allocator, ro_sstable_handle))) { + } else if (OB_FAIL(compact_sstables(tablet, base_sstables, ddl_param, tablet.get_rowkey_read_info(), storage_schema, allocator, ro_sstable_handle))) { LOG_WARN("compact base sstable failed", K(ret)); } } @@ -1100,9 +994,12 @@ int ObTabletDDLUtil::compact_ddl_kv( ObArray sorted_metas; bool is_data_continue = true; ObTablesHandleArray compacted_cg_sstable_handles; // for tmp hold handle of macro block until the tablet updated + ObStorageSchema *storage_schema = nullptr; if (OB_UNLIKELY(!ddl_param.is_valid() || (0 == ddl_sstable_iter.count() && frozen_ddl_kvs.empty()))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(ddl_param), K(ddl_sstable_iter.count()), K(frozen_ddl_kvs.count())); + } else if (OB_FAIL(tablet.load_storage_schema(arena, storage_schema))) { + LOG_WARN("load storage schema failed", K(ret), K(ddl_param)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < frozen_ddl_kvs.count(); ++i) { if (OB_FAIL(frozen_ddl_kvs.at(i).get_obj()->close())) { @@ -1121,22 +1018,23 @@ int ObTabletDDLUtil::compact_ddl_kv( if (OB_FAIL(ret)) { } else if (ddl_param.table_key_.is_co_sstable()) { - if (OB_FAIL(compact_co_ddl_sstable(tablet, ddl_sstable_iter, frozen_ddl_kvs, ddl_param, allocator, compacted_cg_sstable_handles, compacted_sstable_handle))) { + if (OB_FAIL(compact_co_ddl_sstable(tablet, ddl_sstable_iter, frozen_ddl_kvs, ddl_param, storage_schema, allocator, compacted_cg_sstable_handles, compacted_sstable_handle))) { LOG_WARN("compact co ddl sstable failed", K(ret), K(ddl_param)); } } else { - if (OB_FAIL(compact_ro_ddl_sstable(tablet, ddl_sstable_iter, frozen_ddl_kvs, ddl_param, allocator, compacted_sstable_handle))) { + if (OB_FAIL(compact_ro_ddl_sstable(tablet, ddl_sstable_iter, frozen_ddl_kvs, ddl_param, storage_schema, allocator, compacted_sstable_handle))) { LOG_WARN("compact co ddl sstable failed", K(ret), K(ddl_param)); } } if (OB_SUCC(ret)) { // update table store - if (OB_FAIL(update_ddl_table_store(ls, tablet, ddl_param, allocator, static_cast(compacted_sstable_handle.get_table())))) { + if (OB_FAIL(update_ddl_table_store(ls, tablet, ddl_param, storage_schema, allocator, static_cast(compacted_sstable_handle.get_table())))) { LOG_WARN("update ddl table store failed", K(ret)); } else { LOG_INFO("compact ddl sstable success", K(ddl_param)); } } } + ObTabletObjLoadHelper::free(arena, storage_schema); return ret; } diff --git a/src/storage/ddl/ob_ddl_merge_task.h b/src/storage/ddl/ob_ddl_merge_task.h index bdc6e923c3..b9e72c91be 100644 --- a/src/storage/ddl/ob_ddl_merge_task.h +++ b/src/storage/ddl/ob_ddl_merge_task.h @@ -95,6 +95,7 @@ public: const uint64_t data_format_version, const blocksstable::ObSSTable *first_ddl_sstable, const share::SCN &end_scn, + const ObStorageSchema *storage_schema, blocksstable::ObWholeDataStoreDesc &data_desc); static int create_ddl_sstable( @@ -102,6 +103,7 @@ public: const ObTabletDDLParam &ddl_param, const ObIArray &meta_array, const blocksstable::ObSSTable *first_ddl_sstable, + const ObStorageSchema *storage_schema, common::ObArenaAllocator &allocator, ObTableHandleV2 &sstable_handle); @@ -109,6 +111,7 @@ public: ObLS &ls, ObTablet &tablet, const ObTabletDDLParam &ddl_param, + const ObStorageSchema *storage_schema, common::ObArenaAllocator &allocator, blocksstable::ObSSTable *sstable); @@ -158,6 +161,7 @@ private: const ObTabletDDLParam &ddl_param, const blocksstable::ObSSTable *first_ddl_sstable, const int64_t macro_block_column_count, + const ObStorageSchema *storage_schema, common::ObArenaAllocator &allocator, ObTableHandleV2 &sstable_handle); diff --git a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp index db93a84751..83d1ff3935 100644 --- a/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp +++ b/src/storage/ddl/ob_direct_insert_sstable_ctx_new.cpp @@ -1329,7 +1329,7 @@ int ObTabletDirectLoadMgr::fill_lob_sstable_slice( ObDirectLoadSliceWriter *slice_writer = nullptr; const int64_t trans_version = is_full_direct_load(direct_load_type_) ? table_key_.get_snapshot_version() : INT64_MAX; ObBatchSliceWriteInfo info(tablet_id_, ls_id_, trans_version, direct_load_type_, sqc_build_ctx_.build_param_.runtime_only_param_.trans_id_, - sqc_build_ctx_.build_param_.runtime_only_param_.seq_no_); + sqc_build_ctx_.build_param_.runtime_only_param_.seq_no_, slice_info.src_tenant_id_); if (OB_FAIL(lob_mgr_handle_.get_obj()->get_sqc_build_ctx().slice_mgr_map_.get_refactored(slice_info.slice_id_, slice_writer))) { LOG_WARN("get refactored failed", K(ret), K(slice_info), K(sqc_build_ctx_.slice_mgr_map_.size())); @@ -1390,7 +1390,7 @@ int ObTabletDirectLoadMgr::fill_lob_sstable_slice( ObDirectLoadSliceWriter *slice_writer = nullptr; const int64_t trans_version = is_full_direct_load(direct_load_type_) ? table_key_.get_snapshot_version() : INT64_MAX; ObBatchSliceWriteInfo info(tablet_id_, ls_id_, trans_version, direct_load_type_, sqc_build_ctx_.build_param_.runtime_only_param_.trans_id_, - sqc_build_ctx_.build_param_.runtime_only_param_.seq_no_); + sqc_build_ctx_.build_param_.runtime_only_param_.seq_no_, slice_info.src_tenant_id_); if (OB_FAIL(lob_mgr_handle_.get_obj()->get_sqc_build_ctx().slice_mgr_map_.get_refactored(slice_info.slice_id_, slice_writer))) { LOG_WARN("get refactored failed", K(ret), K(slice_info), K(sqc_build_ctx_.slice_mgr_map_.size())); @@ -2706,7 +2706,7 @@ int ObTabletFullDirectLoadMgr::init_ddl_table_store( param.ddl_info_.ddl_execution_id_ = execution_id_; param.ddl_info_.data_format_version_ = data_format_version_; if (OB_FAIL(ObTabletDDLUtil::create_ddl_sstable(*tablet_handle.get_obj(), ddl_param, empty_meta_array, nullptr/*first_ddl_sstable*/, - tmp_arena, sstable_handle))) { + storage_schema, tmp_arena, sstable_handle))) { LOG_WARN("create empty ddl sstable failed", K(ret)); } else if (ddl_param.table_key_.is_co_sstable()) { // add empty cg sstables @@ -2719,7 +2719,8 @@ int ObTabletFullDirectLoadMgr::init_ddl_table_store( cg_ddl_param.table_key_.column_group_idx_ = static_cast(i); if (table_key_.get_column_group_id() == i) { // skip base cg idx - } else if (OB_FAIL(ObTabletDDLUtil::create_ddl_sstable(*tablet_handle.get_obj(), cg_ddl_param, empty_meta_array, nullptr/*first_ddl_sstable*/, tmp_arena, cur_handle))) { + } else if (OB_FAIL(ObTabletDDLUtil::create_ddl_sstable(*tablet_handle.get_obj(), cg_ddl_param, empty_meta_array, + nullptr/*first_ddl_sstable*/, storage_schema, tmp_arena, cur_handle))) { LOG_WARN("create empty cg sstable failed", K(ret), K(i), K(cg_ddl_param)); } else if (OB_FAIL(empty_cg_sstable_handles.add_table(cur_handle))) { LOG_WARN("add table handle failed", K(ret), K(i), K(cur_handle)); @@ -2760,6 +2761,8 @@ int ObTabletFullDirectLoadMgr::update_major_sstable() int ret = OB_SUCCESS; ObLSHandle ls_handle; ObTabletHandle tablet_handle; + ObArenaAllocator tmp_arena("DDLUpdateTblTmp", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); + ObStorageSchema *storage_schema = nullptr; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); @@ -2770,14 +2773,20 @@ int ObTabletFullDirectLoadMgr::update_major_sstable() tablet_handle, ObMDSGetTabletMode::READ_WITHOUT_CHECK))) { LOG_WARN("get tablet handle failed", K(ret), K(ls_id_), K(tablet_id_)); + } else if (OB_FAIL(tablet_handle.get_obj()->load_storage_schema(tmp_arena, storage_schema))) { + LOG_WARN("load storage schema failed", K(ret), K(ls_id_), K(tablet_id_)); } else { - SCN ddl_commit_scn = get_commit_scn(tablet_handle.get_obj()->get_tablet_meta()); - if (OB_ISNULL(ls_handle.get_ls()->get_tablet_svr())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("ls tablet service is null", K(ret), K(ls_id_)); - } else if (OB_FAIL(ls_handle.get_ls()->get_tablet_svr()->update_tablet_ddl_commit_scn(tablet_id_, ddl_commit_scn))) { - LOG_WARN("update ddl commit scn failed", K(ret), K(ls_id_), K(tablet_id_), K(ddl_commit_scn)); + ObTabletHandle new_tablet_handle; + ObUpdateTableStoreParam param(tablet_handle.get_obj()->get_snapshot_version(), + ObVersionRange::MIN_VERSION, // multi_version_start + storage_schema, + ls_handle.get_ls()->get_rebuild_seq()); + param.ddl_info_.keep_old_ddl_sstable_ = true; + param.ddl_info_.ddl_commit_scn_ = get_commit_scn(tablet_handle.get_obj()->get_tablet_meta()); + if (OB_FAIL(ls_handle.get_ls()->update_tablet_table_store(tablet_id_, param, new_tablet_handle))) { + LOG_WARN("failed to update tablet table store", K(ret), K(ls_id_), K(tablet_id_), K(param)); } } + ObTabletObjLoadHelper::free(tmp_arena, storage_schema); return ret; } diff --git a/src/storage/ddl/ob_direct_load_struct.cpp b/src/storage/ddl/ob_direct_load_struct.cpp index 0ef25d5c30..fa91c6ac0c 100644 --- a/src/storage/ddl/ob_direct_load_struct.cpp +++ b/src/storage/ddl/ob_direct_load_struct.cpp @@ -619,6 +619,7 @@ int ObDirectLoadSliceWriter::prepare_iters( const int64_t seq_no, const int64_t timeout_ts, const int64_t lob_inrow_threshold, + const uint64_t src_tenant_id, ObLobMetaRowIterator *&row_iter) { int ret = OB_SUCCESS; @@ -652,8 +653,9 @@ int ObDirectLoadSliceWriter::prepare_iters( lob_storage_param.inrow_threshold_ = lob_inrow_threshold; int64_t unused_affected_rows = 0; if (OB_FAIL(ObInsertLobColumnHelper::insert_lob_column( - allocator, nullptr, ls_id, tablet_id, lob_id, cs_type, lob_storage_param, datum, timeout_ts, true/*has_lob_header*/, *meta_write_iter_))) { - LOG_WARN("fail to insert_lob_col", K(ret), K(ls_id), K(tablet_id), K(lob_id)); + allocator, nullptr, ls_id, tablet_id, lob_id, cs_type, lob_storage_param, datum, + timeout_ts, true/*has_lob_header*/, src_tenant_id, *meta_write_iter_))) { + LOG_WARN("fail to insert_lob_col", K(ret), K(ls_id), K(tablet_id), K(lob_id), K(src_tenant_id)); } else if (OB_FAIL(row_iterator_->init(meta_write_iter_, trans_id, trans_version, seq_no))) { LOG_WARN("fail to lob meta row iterator", K(ret), K(trans_id), K(trans_version), K(seq_no)); @@ -711,7 +713,7 @@ int ObDirectLoadSliceWriter::fill_lob_into_memtable( lob_storage_param.inrow_threshold_ = lob_inrow_threshold; if (OB_FAIL(ObInsertLobColumnHelper::insert_lob_column( allocator, info.ls_id_, info.data_tablet_id_, col_types.at(i).get_collation_type(), - lob_storage_param, datum, timeout_ts, true/*has_lob_header*/, MTL_ID()))) { + lob_storage_param, datum, timeout_ts, true/*has_lob_header*/, info.src_tenant_id_))) { LOG_WARN("fail to insert_lob_col", K(ret), K(datum)); } } @@ -748,7 +750,7 @@ int ObDirectLoadSliceWriter::fill_lob_into_macro_block( ObLobMetaRowIterator *row_iter = nullptr; if (OB_FAIL(prepare_iters(allocator, iter_allocator, datum, info.ls_id_, info.data_tablet_id_, info.trans_version_, col_types.at(i).get_collation_type(), lob_id, - info.trans_id_, info.seq_no_, timeout_ts, lob_inrow_threshold, row_iter))) { + info.trans_id_, info.seq_no_, timeout_ts, lob_inrow_threshold, info.src_tenant_id_, row_iter))) { LOG_WARN("fail to prepare iters", K(ret), KP(row_iter), K(datum)); } else { while (OB_SUCC(ret)) { diff --git a/src/storage/ddl/ob_direct_load_struct.h b/src/storage/ddl/ob_direct_load_struct.h index 17e9bcaeda..b8590cfd50 100644 --- a/src/storage/ddl/ob_direct_load_struct.h +++ b/src/storage/ddl/ob_direct_load_struct.h @@ -52,20 +52,23 @@ public: trans_version_(0), direct_load_type_(), trans_id_(), - seq_no_(0) + seq_no_(0), + src_tenant_id_(0) { } ObBatchSliceWriteInfo(const common::ObTabletID &tablet_id, const share::ObLSID &ls_id, const int64_t &trans_version, - const ObDirectLoadType &direct_load_type, const transaction::ObTransID &trans_id, const int64_t &seq_no) + const ObDirectLoadType &direct_load_type, const transaction::ObTransID &trans_id, const int64_t &seq_no, + const uint64_t src_tenant_id) : data_tablet_id_(tablet_id), ls_id_(ls_id), trans_version_(trans_version), direct_load_type_(direct_load_type), trans_id_(trans_id), - seq_no_(seq_no) + seq_no_(seq_no), + src_tenant_id_(src_tenant_id) { } ~ObBatchSliceWriteInfo() = default; - TO_STRING_KV(K(ls_id_), K(data_tablet_id_), K(trans_version_), K(direct_load_type_)); + TO_STRING_KV(K(ls_id_), K(data_tablet_id_), K(trans_version_), K(direct_load_type_), K(src_tenant_id_)); public: common::ObTabletID data_tablet_id_; share::ObLSID ls_id_; @@ -73,6 +76,7 @@ public: ObDirectLoadType direct_load_type_; transaction::ObTransID trans_id_; int64_t seq_no_; // + uint64_t src_tenant_id_; }; struct ObTabletDirectLoadMgrKey final @@ -103,11 +107,11 @@ struct ObDirectLoadSliceInfo final public: ObDirectLoadSliceInfo() : is_full_direct_load_(false), is_lob_slice_(false), ls_id_(), data_tablet_id_(), slice_id_(-1), - context_id_(0) + context_id_(0), src_tenant_id_(MTL_ID()) { } ~ObDirectLoadSliceInfo() = default; - bool is_valid() const { return ls_id_.is_valid() && data_tablet_id_.is_valid() && slice_id_ >= 0 && context_id_ >= 0; } - TO_STRING_KV(K_(is_full_direct_load), K_(is_lob_slice), K_(ls_id), K_(data_tablet_id), K_(slice_id), K_(context_id)); + bool is_valid() const { return ls_id_.is_valid() && data_tablet_id_.is_valid() && slice_id_ >= 0 && context_id_ >= 0 && src_tenant_id_ > 0; } + TO_STRING_KV(K_(is_full_direct_load), K_(is_lob_slice), K_(ls_id), K_(data_tablet_id), K_(slice_id), K_(context_id), K_(src_tenant_id)); public: bool is_full_direct_load_; bool is_lob_slice_; @@ -115,6 +119,7 @@ public: common::ObTabletID data_tablet_id_; int64_t slice_id_; int64_t context_id_; + uint64_t src_tenant_id_; DISALLOW_COPY_AND_ASSIGN(ObDirectLoadSliceInfo); }; @@ -527,6 +532,7 @@ private: const int64_t seq_no, const int64_t timeout_ts, const int64_t lob_inrow_threshold, + const uint64_t src_tenant_id, ObLobMetaRowIterator *&row_iter); private: bool is_inited_; diff --git a/src/storage/ddl/ob_tablet_ddl_kv.cpp b/src/storage/ddl/ob_tablet_ddl_kv.cpp index fdf3413c03..11e41e0d0d 100644 --- a/src/storage/ddl/ob_tablet_ddl_kv.cpp +++ b/src/storage/ddl/ob_tablet_ddl_kv.cpp @@ -54,7 +54,8 @@ ObBlockMetaTree::~ObBlockMetaTree() int ObBlockMetaTree::init(ObTablet &tablet, const ObITable::TableKey &table_key, const share::SCN &ddl_start_scn, - const uint64_t data_format_version) + const uint64_t data_format_version, + const ObStorageSchema *storage_schema) { int ret = OB_SUCCESS; const ObMemAttr mem_attr(MTL_ID(), "BlockMetaTree"); @@ -64,9 +65,9 @@ int ObBlockMetaTree::init(ObTablet &tablet, if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("init twice", K(ret)); - } else if (OB_UNLIKELY(!table_key.is_valid() || data_format_version <= 0)) { + } else if (OB_UNLIKELY(!table_key.is_valid() || data_format_version <= 0 || OB_ISNULL(storage_schema))) { ret = OB_INVALID_ARGUMENT; - LOG_WARN("invalid argument", K(ret), K(table_key)); + LOG_WARN("invalid argument", K(ret), K(table_key), KP(storage_schema)); } else if (FALSE_IT(arena_.set_attr(mem_attr))) { } else if (OB_FAIL(block_tree_.init())) { LOG_WARN("init block tree failed", K(ret)); @@ -80,6 +81,7 @@ int ObBlockMetaTree::init(ObTablet &tablet, data_format_version, static_cast(first_ddl_sstable), table_key.get_end_scn(), + storage_schema, data_desc_))) { LOG_WARN("prepare data store desc failed", K(ret), K(table_key), K(data_format_version)); } else { @@ -673,6 +675,42 @@ int ObBlockMetaTree::locate_range(const blocksstable::ObDatumRange &range, return ret; } +int ObBlockMetaTree::skip_to_next_valid_position(const blocksstable::ObDatumRowkey &rowkey, + const blocksstable::ObStorageDatumUtils &datum_utils, + blocksstable::DDLBtreeIterator &iter, + ObBlockMetaTreeValue *&tree_value) const +{ + int ret = OB_SUCCESS; + tree_value = nullptr; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else { + int cmp_ret = 0; + while (OB_SUCC(ret)) { + ObDatumRowkeyWrapper rowkey_wrapper; + ObBlockMetaTreeValue *tmp_tree_value = nullptr; + if (OB_FAIL(iter.get_next(rowkey_wrapper, tmp_tree_value))) { + if (OB_ITER_END != ret) { + LOG_WARN("get next failed", K(ret)); + } + // just return ITER_END + } else if (OB_FAIL(rowkey_wrapper.rowkey_->compare(rowkey, datum_utils, cmp_ret, false/*need_compare_datum_cnt*/))) { + LOG_WARN("fail to cmp rowkey", K(ret), K(rowkey), K(rowkey_wrapper)); + } else if(cmp_ret >= 0) { //lower bound + if (OB_ISNULL(tmp_tree_value)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tree_value is null", K(ret), KP(tmp_tree_value)); + } else { + tree_value = tmp_tree_value; + } + break; + } + } + } + return ret; +} + int ObBlockMetaTree::get_next_tree_value(blocksstable::DDLBtreeIterator &iter, const int64_t step, ObBlockMetaTreeValue *&tree_value) const @@ -788,7 +826,11 @@ int ObDDLMemtable::init( LOG_WARN("invalid argument", K(ret), K(table_key), K(ddl_start_scn), K(data_format_version)); } else { ObTabletCreateSSTableParam sstable_param; - if (OB_FAIL(block_meta_tree_.init(tablet, table_key, ddl_start_scn, data_format_version))) { + ObStorageSchema *storage_schema = nullptr; + ObArenaAllocator arena("init_ddl_memt", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); + if (OB_FAIL(tablet.load_storage_schema(arena, storage_schema))) { + LOG_WARN("load storage schema failed", K(ret), KPC(this)); + } else if (OB_FAIL(block_meta_tree_.init(tablet, table_key, ddl_start_scn, data_format_version, storage_schema))) { LOG_WARN("init mem index sstable failed", K(ret), K(table_key), K(ddl_start_scn)); } else if (OB_FAIL(init_sstable_param(tablet, table_key, ddl_start_scn, sstable_param))) { LOG_WARN("init sstable param failed", K(ret)); @@ -797,6 +839,7 @@ int ObDDLMemtable::init( } else { is_inited_ = true; } + ObTabletObjLoadHelper::free(arena, storage_schema); } return ret; } @@ -836,10 +879,11 @@ int ObDDLMemtable::init_ddl_index_iterator(const blocksstable::ObStorageDatumUti blocksstable::ObDDLIndexBlockRowIterator *ddl_kv_index_iter) { int ret = OB_SUCCESS; + const bool is_normal_cg = is_normal_cg_sstable(); if (OB_ISNULL(datum_utils) || OB_UNLIKELY(!datum_utils->is_valid()) || OB_ISNULL(ddl_kv_index_iter)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguement", K(ret), KP(ddl_kv_index_iter), KPC(datum_utils)); - } else if (OB_FAIL(ddl_kv_index_iter->set_iter_param(datum_utils, is_reverse_scan, &block_meta_tree_))) { + } else if (OB_FAIL(ddl_kv_index_iter->set_iter_param(datum_utils, is_reverse_scan, &block_meta_tree_, is_normal_cg))) { LOG_WARN("fail to set ddl iter param", K(ret)); } return ret; diff --git a/src/storage/ddl/ob_tablet_ddl_kv.h b/src/storage/ddl/ob_tablet_ddl_kv.h index c1e9401f04..abcd6d3218 100644 --- a/src/storage/ddl/ob_tablet_ddl_kv.h +++ b/src/storage/ddl/ob_tablet_ddl_kv.h @@ -72,7 +72,8 @@ public: int init(ObTablet &tablet, const ObITable::TableKey &table_key, const share::SCN &ddl_start_scn, - const uint64_t data_format_version); + const uint64_t data_format_version, + const ObStorageSchema *storage_schema); void destroy(); void destroy_tree_value(); int insert_macro_block(const ObDDLMacroHandle ¯o_handle, @@ -89,6 +90,10 @@ public: const bool is_reverse_scan, blocksstable::DDLBtreeIterator &iter, ObBlockMetaTreeValue *&cur_tree_value) const; + int skip_to_next_valid_position(const blocksstable::ObDatumRowkey &rowkey, + const blocksstable::ObStorageDatumUtils &datum_utils, + blocksstable::DDLBtreeIterator &iter, + ObBlockMetaTreeValue *&tree_value) const; int get_next_tree_value(blocksstable::DDLBtreeIterator &iter, const int64_t step, ObBlockMetaTreeValue *&tree_value) const; diff --git a/src/storage/direct_load/ob_direct_load_external_table.cpp b/src/storage/direct_load/ob_direct_load_external_table.cpp index cf585e25e3..dd73715025 100644 --- a/src/storage/direct_load/ob_direct_load_external_table.cpp +++ b/src/storage/direct_load/ob_direct_load_external_table.cpp @@ -103,6 +103,11 @@ int ObDirectLoadExternalTable::init(const ObDirectLoadExternalTableCreateParam & return ret; } +void ObDirectLoadExternalTable::release_data() +{ + fragments_.reset(); +} + int ObDirectLoadExternalTable::copy(const ObDirectLoadExternalTable &other) { int ret = OB_SUCCESS; diff --git a/src/storage/direct_load/ob_direct_load_external_table.h b/src/storage/direct_load/ob_direct_load_external_table.h index 5d482f5166..54c72c3a27 100644 --- a/src/storage/direct_load/ob_direct_load_external_table.h +++ b/src/storage/direct_load/ob_direct_load_external_table.h @@ -59,6 +59,7 @@ public: const common::ObTabletID &get_tablet_id() const override { return meta_.tablet_id_; } int64_t get_row_count() const override { return meta_.row_count_; } bool is_valid() const override { return is_inited_; } + void release_data() override; int copy(const ObDirectLoadExternalTable &other); const ObDirectLoadExternalTableMeta &get_meta() const { return meta_; } const ObDirectLoadExternalFragmentArray &get_fragments() const { return fragments_; } diff --git a/src/storage/direct_load/ob_direct_load_fast_heap_table.h b/src/storage/direct_load/ob_direct_load_fast_heap_table.h index 1c8c22bd9c..a3a5b88d7e 100644 --- a/src/storage/direct_load/ob_direct_load_fast_heap_table.h +++ b/src/storage/direct_load/ob_direct_load_fast_heap_table.h @@ -54,6 +54,7 @@ public: const common::ObTabletID &get_tablet_id() const override { return meta_.tablet_id_; } int64_t get_row_count() const override { return meta_.row_count_; } bool is_valid() const override { return is_inited_; } + void release_data() override { /*do nothing*/ } const ObDirectLoadFastHeapTableMeta &get_meta() const { return meta_; } TO_STRING_KV(K_(meta)); private: diff --git a/src/storage/direct_load/ob_direct_load_i_table.h b/src/storage/direct_load/ob_direct_load_i_table.h index 31a2a2c9d9..3f9ae728fc 100644 --- a/src/storage/direct_load/ob_direct_load_i_table.h +++ b/src/storage/direct_load/ob_direct_load_i_table.h @@ -27,6 +27,7 @@ public: virtual const common::ObTabletID &get_tablet_id() const = 0; virtual int64_t get_row_count() const = 0; virtual bool is_valid() const = 0; + virtual void release_data() = 0; TO_STRING_EMPTY(); }; diff --git a/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp b/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp index 4bf0e05c0a..b4594433ac 100644 --- a/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp +++ b/src/storage/direct_load/ob_direct_load_insert_table_ctx.cpp @@ -188,6 +188,7 @@ int ObDirectLoadInsertTabletContext::close() LOG_WARN("fail to close tablet direct load", KR(ret), K(param_.ls_id_), K(param_.tablet_id_)); } else { + handle_.reset(); is_open_ = false; } } diff --git a/src/storage/direct_load/ob_direct_load_multiple_heap_table.cpp b/src/storage/direct_load/ob_direct_load_multiple_heap_table.cpp index 6b422c2cc3..9304ee9ae5 100644 --- a/src/storage/direct_load/ob_direct_load_multiple_heap_table.cpp +++ b/src/storage/direct_load/ob_direct_load_multiple_heap_table.cpp @@ -168,6 +168,12 @@ int ObDirectLoadMultipleHeapTable::init(const ObDirectLoadMultipleHeapTableCreat return ret; } +void ObDirectLoadMultipleHeapTable::release_data() +{ + index_file_handle_.reset(); + data_fragments_.reset(); +} + int ObDirectLoadMultipleHeapTable::copy(const ObDirectLoadMultipleHeapTable &other) { int ret = OB_SUCCESS; diff --git a/src/storage/direct_load/ob_direct_load_multiple_heap_table.h b/src/storage/direct_load/ob_direct_load_multiple_heap_table.h index f6d544faf0..11994498d6 100644 --- a/src/storage/direct_load/ob_direct_load_multiple_heap_table.h +++ b/src/storage/direct_load/ob_direct_load_multiple_heap_table.h @@ -95,6 +95,7 @@ public: const common::ObTabletID &get_tablet_id() const override { return tablet_id_; } int64_t get_row_count() const override { return meta_.row_count_; } bool is_valid() const override { return is_inited_; } + void release_data() override; int copy(const ObDirectLoadMultipleHeapTable &other); const ObDirectLoadMultipleHeapTableMeta &get_meta() const { return meta_; } const ObDirectLoadTmpFileHandle &get_index_file_handle() const { return index_file_handle_; } diff --git a/src/storage/direct_load/ob_direct_load_multiple_sstable.cpp b/src/storage/direct_load/ob_direct_load_multiple_sstable.cpp index 39b792c4a2..bac003f42e 100644 --- a/src/storage/direct_load/ob_direct_load_multiple_sstable.cpp +++ b/src/storage/direct_load/ob_direct_load_multiple_sstable.cpp @@ -173,6 +173,11 @@ int ObDirectLoadMultipleSSTable::init(const ObDirectLoadMultipleSSTableCreatePar return ret; } +void ObDirectLoadMultipleSSTable::release_data() +{ + fragments_.reset(); +} + int ObDirectLoadMultipleSSTable::copy(const ObDirectLoadMultipleSSTable &other) { int ret = OB_SUCCESS; diff --git a/src/storage/direct_load/ob_direct_load_multiple_sstable.h b/src/storage/direct_load/ob_direct_load_multiple_sstable.h index d21feea6f0..34c1912a2c 100644 --- a/src/storage/direct_load/ob_direct_load_multiple_sstable.h +++ b/src/storage/direct_load/ob_direct_load_multiple_sstable.h @@ -100,6 +100,7 @@ public: const common::ObTabletID &get_tablet_id() const override { return tablet_id_; } int64_t get_row_count() const override { return meta_.row_count_; } bool is_valid() const override { return is_inited_; } + void release_data() override; int copy(const ObDirectLoadMultipleSSTable &other); bool is_empty() const { return 0 == meta_.row_count_; } const ObDirectLoadMultipleSSTableMeta &get_meta() const { return meta_; } diff --git a/src/storage/direct_load/ob_direct_load_range_splitter.cpp b/src/storage/direct_load/ob_direct_load_range_splitter.cpp index ab2ed5a340..e197bbd077 100644 --- a/src/storage/direct_load/ob_direct_load_range_splitter.cpp +++ b/src/storage/direct_load/ob_direct_load_range_splitter.cpp @@ -885,7 +885,11 @@ int ObDirectLoadMultipleMergeRangeSplitter::get_rowkeys_by_multiple( } } if (OB_SUCC(ret) && nullptr != last_rowkey_) { - if (last_rowkey_->tablet_id_.compare(tablet_id) == 0) { + const int cmp_ret = last_rowkey_->tablet_id_.compare(tablet_id); + if (OB_UNLIKELY(cmp_ret < 0)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected tablet id", KR(ret), KPC(last_rowkey_), K(tablet_id)); + } else if (cmp_ret == 0) { // the same tablet ObDatumRowkey rowkey; ObDatumRowkey copied_rowkey; @@ -916,9 +920,8 @@ int ObDirectLoadMultipleMergeRangeSplitter::get_rowkeys_by_multiple( } } } - } else { - // the different tablet - abort_unless(last_rowkey_->tablet_id_.compare(tablet_id) > 0); + } else { /* cmp_ret > 0 */ + // no rowkey for current tablet_id, do nothing } } return ret; diff --git a/src/storage/direct_load/ob_direct_load_sstable.cpp b/src/storage/direct_load/ob_direct_load_sstable.cpp index 340a2f4003..ffdd5dc723 100644 --- a/src/storage/direct_load/ob_direct_load_sstable.cpp +++ b/src/storage/direct_load/ob_direct_load_sstable.cpp @@ -159,6 +159,11 @@ int ObDirectLoadSSTable::init(ObDirectLoadSSTableCreateParam ¶m) return ret; } +void ObDirectLoadSSTable::release_data() +{ + fragments_.reset(); +} + int ObDirectLoadSSTable::copy(const ObDirectLoadSSTable &other) { int ret = OB_SUCCESS; diff --git a/src/storage/direct_load/ob_direct_load_sstable.h b/src/storage/direct_load/ob_direct_load_sstable.h index 345da21635..531278992f 100644 --- a/src/storage/direct_load/ob_direct_load_sstable.h +++ b/src/storage/direct_load/ob_direct_load_sstable.h @@ -152,6 +152,7 @@ public: const common::ObTabletID &get_tablet_id() const override { return meta_.tablet_id_; } bool is_empty() const { return 0 == meta_.row_count_; } bool is_valid() const override { return is_inited_; } + void release_data() override; int copy(const ObDirectLoadSSTable &other); const common::ObArray &get_fragment_array() const { diff --git a/src/storage/high_availability/ob_finish_transfer.cpp b/src/storage/high_availability/ob_finish_transfer.cpp index 1d3e7eb3c9..cfe9596b94 100644 --- a/src/storage/high_availability/ob_finish_transfer.cpp +++ b/src/storage/high_availability/ob_finish_transfer.cpp @@ -316,7 +316,7 @@ int ObTxFinishTransfer::do_tx_transfer_doing_(const ObTransferTaskID &task_id, c } } } - if (OB_TMP_FAIL(record_server_event_(ret, is_ready, tmp_round))) { + if (OB_TMP_FAIL(record_server_event_(ret, is_ready, tmp_round, start_scn))) { LOG_WARN("failed to record server event", K(tmp_ret), K(ret), K(is_ready)); } return ret; @@ -1050,12 +1050,15 @@ int ObTxFinishTransfer::select_transfer_task_for_update_(const ObTransferTaskID int ObTxFinishTransfer::record_server_event_( const int32_t result, const bool is_ready, - const int64_t round) const + const int64_t round, + const share::SCN &start_scn) const { int ret = OB_SUCCESS; ObSqlString extra_info_str; const share::ObTransferStatus doing_status(ObTransferStatus::DOING); const share::ObTransferStatus finish_status(ObTransferStatus::COMPLETED); + const int64_t start_scn_ts = start_scn.convert_to_ts(); + const int64_t elapsed_us_from_start_scn = ObTimeUtility::current_time() - start_scn_ts; if (OB_SUCCESS == result) { if (is_ready) { if (OB_FAIL(extra_info_str.append_fmt("msg:\"transfer doing success\";"))) { @@ -1070,6 +1073,8 @@ int ObTxFinishTransfer::record_server_event_( if (OB_SUCC(ret)) { if (OB_FAIL(extra_info_str.append_fmt("round:%ld;", round))) { LOG_WARN("fail to printf retry time", K(ret)); + } else if (OB_FAIL(extra_info_str.append_fmt("elapsed_us_from_start_scn:%ld;", elapsed_us_from_start_scn))) { + LOG_WARN("fail to printf retry time", K(ret)); } else { if (OB_SUCCESS == result && is_ready) { if (OB_FAIL(write_server_event_(result, extra_info_str, finish_status))) { @@ -1079,6 +1084,9 @@ int ObTxFinishTransfer::record_server_event_( if (REACH_TENANT_TIME_INTERVAL(10 * 1000 * 1000)) { if (OB_FAIL(write_server_event_(result, extra_info_str, doing_status))) { LOG_WARN("fail to write server event", K(ret), K(result), K(extra_info_str)); + } else if (elapsed_us_from_start_scn > TASK_EXECUTE_LONG_WARNING_THRESHOLD) { + LOG_ERROR("transfer task stuck at doing stage for too long, may be wait log replay or transfer backfill too slow", + K(start_scn), K(elapsed_us_from_start_scn), K_(task_id)); } } } @@ -1100,5 +1108,6 @@ int ObTxFinishTransfer::write_server_event_(const int32_t result, const ObSqlStr extra_info.ptr()); return ret; } + } // namespace storage } // namespace oceanbase diff --git a/src/storage/high_availability/ob_finish_transfer.h b/src/storage/high_availability/ob_finish_transfer.h index c0c4f88b29..2026d77861 100644 --- a/src/storage/high_availability/ob_finish_transfer.h +++ b/src/storage/high_availability/ob_finish_transfer.h @@ -261,14 +261,16 @@ private: int record_server_event_( const int32_t result, const bool is_ready, - const int64_t round) const; + const int64_t round, + const share::SCN &start_scn) const; int write_server_event_( const int32_t result, const ObSqlString &extra_info, const share::ObTransferStatus &status) const; private: - static const int64_t DEFAULT_WAIT_INTERVAL_US = 10 * 1000; // 10ms + static const int64_t DEFAULT_WAIT_INTERVAL_US = 10_ms; + static const int64_t TASK_EXECUTE_LONG_WARNING_THRESHOLD = 60_min; private: bool is_inited_; diff --git a/src/storage/high_availability/ob_physical_copy_task.cpp b/src/storage/high_availability/ob_physical_copy_task.cpp index 46cc747b1f..ef2c7c56df 100644 --- a/src/storage/high_availability/ob_physical_copy_task.cpp +++ b/src/storage/high_availability/ob_physical_copy_task.cpp @@ -1060,7 +1060,6 @@ int ObSSTableCopyFinishTask::build_create_sstable_param_( const blocksstable::ObSSTableMergeRes &res, ObTabletCreateSSTableParam ¶m) { - //TODO(lingchuan) this param maker ObTablet class will be better to be safeguard int ret = OB_SUCCESS; if (!is_inited_) { ret = OB_INVALID_ARGUMENT; @@ -1068,48 +1067,8 @@ int ObSSTableCopyFinishTask::build_create_sstable_param_( } else if (OB_UNLIKELY(OB_ISNULL(tablet) || !res.is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("build create sstable param get invalid argument", K(ret), KP(tablet), K(res)); - } else { - param.table_key_ = sstable_param_->table_key_; - param.sstable_logic_seq_ = sstable_param_->basic_meta_.sstable_logic_seq_; - param.schema_version_ = sstable_param_->basic_meta_.schema_version_; - param.table_mode_ = sstable_param_->basic_meta_.table_mode_; - param.index_type_ = static_cast(sstable_param_->basic_meta_.index_type_); - param.create_snapshot_version_ = sstable_param_->basic_meta_.create_snapshot_version_; - param.progressive_merge_round_ = sstable_param_->basic_meta_.progressive_merge_round_; - param.progressive_merge_step_ = sstable_param_->basic_meta_.progressive_merge_step_; - param.latest_row_store_type_ = sstable_param_->basic_meta_.latest_row_store_type_; - - ObSSTableMergeRes::fill_addr_and_data(res.root_desc_, - param.root_block_addr_, param.root_block_data_); - ObSSTableMergeRes::fill_addr_and_data(res.data_root_desc_, - param.data_block_macro_meta_addr_, param.data_block_macro_meta_); - param.is_meta_root_ = res.data_root_desc_.is_meta_root_; - param.root_row_store_type_ = res.root_row_store_type_; - param.data_index_tree_height_ = res.root_desc_.height_; - param.index_blocks_cnt_ = res.index_blocks_cnt_; - param.data_blocks_cnt_ = res.data_blocks_cnt_; - param.micro_block_cnt_ = res.micro_block_cnt_; - param.use_old_macro_block_count_ = res.use_old_macro_block_count_; - param.row_count_ = res.row_count_; - param.column_cnt_ = res.data_column_cnt_; - param.data_checksum_ = res.data_checksum_; - param.occupy_size_ = res.occupy_size_; - param.original_size_ = res.original_size_; - param.max_merged_trans_version_ = res.max_merged_trans_version_; - param.contain_uncommitted_row_ = res.contain_uncommitted_row_; - param.compressor_type_ = res.compressor_type_; - param.encrypt_id_ = res.encrypt_id_; - param.master_key_id_ = res.master_key_id_; - param.nested_size_ = res.nested_size_; - param.nested_offset_ = res.nested_offset_; - param.data_block_ids_ = res.data_block_ids_; - param.other_block_ids_ = res.other_block_ids_; - param.rowkey_column_cnt_ = sstable_param_->basic_meta_.rowkey_column_count_; - param.ddl_scn_ = sstable_param_->basic_meta_.ddl_scn_; - MEMCPY(param.encrypt_key_, res.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); - if (OB_FAIL(param.column_checksums_.assign(sstable_param_->column_checksums_))) { - LOG_WARN("fail to fill column checksum", K(ret), KPC_(sstable_param)); - } + } else if (OB_FAIL(param.init_for_ha(*sstable_param_, res))) { + LOG_WARN("fail to init create sstable param", K(ret), KPC(sstable_param_), K(res)); } return ret; } @@ -1117,7 +1076,6 @@ int ObSSTableCopyFinishTask::build_create_sstable_param_( int ObSSTableCopyFinishTask::build_create_sstable_param_( ObTabletCreateSSTableParam ¶m) { - //TODO(lingchuan) this param maker ObTablet class will be better to be safeguard //using sstable meta to create sstable int ret = OB_SUCCESS; if (!is_inited_) { @@ -1126,41 +1084,8 @@ int ObSSTableCopyFinishTask::build_create_sstable_param_( } else if (0 != sstable_param_->basic_meta_.data_macro_block_count_) { ret = OB_ERR_UNEXPECTED; LOG_WARN("sstable param has data macro block, can not build sstable from basic meta", K(ret), KPC(sstable_param_)); - } else { - param.table_key_ = sstable_param_->table_key_; - param.sstable_logic_seq_ = sstable_param_->basic_meta_.sstable_logic_seq_; - param.schema_version_ = sstable_param_->basic_meta_.schema_version_; - param.create_snapshot_version_ = sstable_param_->basic_meta_.create_snapshot_version_; - param.table_mode_ = sstable_param_->basic_meta_.table_mode_; - param.index_type_ = static_cast(sstable_param_->basic_meta_.index_type_); - param.progressive_merge_round_ = sstable_param_->basic_meta_.progressive_merge_round_; - param.progressive_merge_step_ = sstable_param_->basic_meta_.progressive_merge_step_; - param.is_ready_for_read_ = true; - param.root_row_store_type_ = sstable_param_->basic_meta_.root_row_store_type_; - param.latest_row_store_type_ = sstable_param_->basic_meta_.latest_row_store_type_; - param.index_blocks_cnt_ = sstable_param_->basic_meta_.index_macro_block_count_; - param.data_blocks_cnt_ = sstable_param_->basic_meta_.data_macro_block_count_; - param.micro_block_cnt_ = sstable_param_->basic_meta_.data_micro_block_count_; - param.use_old_macro_block_count_ = sstable_param_->basic_meta_.use_old_macro_block_count_; - param.row_count_ = sstable_param_->basic_meta_.row_count_; - param.column_cnt_ = sstable_param_->basic_meta_.column_cnt_; - param.data_checksum_ = sstable_param_->basic_meta_.data_checksum_; - param.occupy_size_ = sstable_param_->basic_meta_.occupy_size_; - param.original_size_ = sstable_param_->basic_meta_.original_size_; - param.max_merged_trans_version_ = sstable_param_->basic_meta_.max_merged_trans_version_; - param.ddl_scn_ = sstable_param_->basic_meta_.ddl_scn_; - param.filled_tx_scn_ = sstable_param_->basic_meta_.filled_tx_scn_; - param.contain_uncommitted_row_ = sstable_param_->basic_meta_.contain_uncommitted_row_; - param.compressor_type_ = sstable_param_->basic_meta_.compressor_type_; - param.encrypt_id_ = sstable_param_->basic_meta_.encrypt_id_; - param.master_key_id_ = sstable_param_->basic_meta_.master_key_id_; - param.root_block_addr_.set_none_addr(); - param.data_block_macro_meta_addr_.set_none_addr(); - param.rowkey_column_cnt_ = sstable_param_->basic_meta_.rowkey_column_count_; - MEMCPY(param.encrypt_key_, sstable_param_->basic_meta_.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); - if (OB_FAIL(param.column_checksums_.assign(sstable_param_->column_checksums_))) { - LOG_WARN("fail to assign column checksums", K(ret), KPC(sstable_param_)); - } + } else if (OB_FAIL(param.init_for_ha(*sstable_param_))) { + LOG_WARN("fail to init create sstable param", K(ret), KPC(sstable_param_)); } return ret; } diff --git a/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp b/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp index e3b97b6353..d450a06e65 100644 --- a/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp +++ b/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp @@ -941,7 +941,7 @@ int ObStorageHATabletsBuilder::get_minor_scn_range_( if (OB_SUCC(ret)) { //need copy src all minor sstables for tablet meta merge, do not need calculate sstable version range. //here set end scn just for compatible - if (GET_MIN_CLUSTER_VERSION() <= CLUSTER_VERSION_4_2_1_1) { + if (GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_4_2_2_0) { scn_range.start_scn_ = ObTabletMeta::INIT_CLOG_CHECKPOINT_SCN; scn_range.end_scn_ = sstables.empty() ? tablet->get_tablet_meta().clog_checkpoint_scn_ : sstables.at(0).get_sstable()->get_start_scn(); } else { diff --git a/src/storage/lob/ob_lob_manager.cpp b/src/storage/lob/ob_lob_manager.cpp index 3edec2cb82..d37cef9807 100644 --- a/src/storage/lob/ob_lob_manager.cpp +++ b/src/storage/lob/ob_lob_manager.cpp @@ -802,7 +802,7 @@ int ObLobManager::compare(ObLobLocatorV2& lob_left, ObLobCompareParams& cmp_params, int64_t& result) { INIT_SUCC(ret); - ObArenaAllocator tmp_allocator(ObModIds::OB_LOB_READER, OB_MALLOC_MIDDLE_BLOCK_SIZE, MTL_ID()); + ObArenaAllocator tmp_allocator("ObLobCmp", OB_MALLOC_MIDDLE_BLOCK_SIZE, MTL_ID()); ObLobManager *lob_mngr = MTL(ObLobManager*); if (OB_ISNULL(lob_mngr)) { ret = OB_ERR_UNEXPECTED; @@ -2584,13 +2584,30 @@ int ObLobManager::write_outrow_inner(ObLobAccessParam& param, ObLobQueryIter *it // do append => [old_data][padding][data] post_data = old_data; } else { - // combine data and old data - // [old_data][data] + // here has four situation + // [old][new][old] --> cover part of old data + // [new_data][old_data] --> cover front part + // [old_data][new_data] --> cover back part + // [new_data] --> full cover old data int64_t offset_byte_len = ObCharset::charpos(param.coll_type_, old_data.ptr(), old_data.length(), param.offset_); - post_data.assign_ptr(old_data.ptr(), offset_byte_len); + if (offset_byte_len > 0) { // offset is not 0, must have some old data at front + post_data.assign_ptr(old_data.ptr(), offset_byte_len); + } + if (param.offset_ + param.len_ < old_char_len) { // not full cover, must have some old data at back + int64_t end_byte_len = ObCharset::charpos(param.coll_type_, + old_data.ptr(), + old_data.length(), + param.offset_ + param.len_); + if (end_byte_len >= old_data.length()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get byte len is bigger then data length", K(ret), K(end_byte_len), K(old_data.length()), K(param)); + } else { + remain_buf.assign_ptr(old_data.ptr() + end_byte_len, old_data.length() - end_byte_len); + } + } } } @@ -2867,6 +2884,8 @@ int ObLobManager::replace_process_meta_info(ObLobAccessParam& param, read_data.length(), replace_byte_len, replace_byte_st); + } else { + replace_byte_len = OB_MIN(replace_byte_len, read_data.length() - replace_byte_st); } ObString temp_read_buf; uint32_t temp_read_size = ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE - replace_byte_st; @@ -2894,6 +2913,7 @@ int ObLobManager::replace_process_meta_info(ObLobAccessParam& param, // try copy data to meta int64_t max_byte = ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE - replace_byte_st; int64_t data_char_len = 0; + max_byte = OB_MIN(max_byte, temp_read_buf.length()); size_t data_by_len = ObCharset::max_bytes_charpos(param.coll_type_, temp_read_buf.ptr(), temp_read_buf.length(), max_byte, data_char_len); data_by_len = ob_lob_writer_length_validation(param.coll_type_, temp_read_buf.length(), data_by_len, data_char_len); MEMCPY(read_data.ptr() + replace_byte_st, temp_read_buf.ptr(), data_by_len); @@ -2901,6 +2921,7 @@ int ObLobManager::replace_process_meta_info(ObLobAccessParam& param, if (data_by_len == temp_read_buf.length()) { // try copy remain data to meta if data has copy fully max_byte = ObLobMetaUtil::LOB_OPER_PIECE_DATA_SIZE - read_data.length(); + max_byte = OB_MIN(max_byte, remain_data.length()); size_t rdata_by_len = ObCharset::max_bytes_charpos(param.coll_type_, remain_data.ptr(), remain_data.length(), max_byte, data_char_len); rdata_by_len = ob_lob_writer_length_validation(param.coll_type_, remain_data.length(), rdata_by_len, data_char_len); if (rdata_by_len >= remain_data.length()) { @@ -2927,6 +2948,8 @@ int ObLobManager::replace_process_meta_info(ObLobAccessParam& param, read_data.length(), by_len, by_st); + } else { + by_len = OB_MIN(by_len, read_data.length() - by_st); } // [0, by_len][by_len, 256K] // try copy data to meta in [by_len, 256K] @@ -2953,6 +2976,7 @@ int ObLobManager::replace_process_meta_info(ObLobAccessParam& param, } } else { int64_t data_char_len = 0; + max_byte = OB_MIN(max_byte, temp_read_buf.length()); size_t data_by_len = ObCharset::max_bytes_charpos(param.coll_type_, temp_read_buf.ptr(), temp_read_buf.length(), max_byte, data_char_len); data_by_len = ob_lob_writer_length_validation(param.coll_type_, temp_read_buf.length(), data_by_len, data_char_len); @@ -2970,6 +2994,8 @@ int ObLobManager::replace_process_meta_info(ObLobAccessParam& param, read_data.length(), by_len, by_st); + } else { + by_len = OB_MIN(by_len, read_data.length() - by_st); } // calc data ObString temp_read_buf; @@ -2997,6 +3023,7 @@ int ObLobManager::replace_process_meta_info(ObLobAccessParam& param, } else { // calc data int64_t data_char_len = 0; + max_byte = OB_MIN(max_byte, temp_read_buf.length()); size_t data_by_len = ObCharset::max_bytes_charpos(param.coll_type_, temp_read_buf.ptr(), temp_read_buf.length(), max_byte, data_char_len); data_by_len = ob_lob_writer_length_validation(param.coll_type_, temp_read_buf.length(), data_by_len, data_char_len); MEMMOVE(read_data.ptr() + data_by_len, read_data.ptr() + by_len, piece_byte_len - by_len); @@ -3047,6 +3074,7 @@ int ObLobManager::replace_process_meta_info(ObLobAccessParam& param, read_data.assign_buffer(tmp_buf.ptr(), tmp_buf.size()); int64_t data_char_len = 0; + max_byte = OB_MIN(max_byte, temp_read_buf.length()); size_t data_by_len = ObCharset::max_bytes_charpos(param.coll_type_, temp_read_buf.ptr(), temp_read_buf.length(), max_byte, data_char_len); data_by_len = ob_lob_writer_length_validation(param.coll_type_, temp_read_buf.length(), data_by_len, data_char_len); MEMCPY(read_data.ptr(), temp_read_buf.ptr(), data_by_len); @@ -3256,15 +3284,14 @@ int ObLobManager::erase_process_meta_info(ObLobAccessParam& param, ObLobMetaScan new_meta_row.byte_len_ = read_data.length(); } } else { + // here expect read_data read full lob meta data read_data.assign_buffer(tmp_buf, tmp_buff.size()); - if (OB_FAIL(get_real_data(param, result, read_data))) { - LOG_WARN("failed to write data to read buf.", K(ret), K(result)); - } else { - new_meta_row.byte_len_ -= (by_len); - new_meta_row.char_len_ -= (local_end - local_begin); - MEMMOVE(read_data.ptr() + by_st, read_data.ptr() + (by_st + by_len), piece_byte_len - (by_st + by_len)); - read_data.assign_ptr(read_data.ptr(), read_data.length() - by_len); - } + MEMCPY(read_data.ptr(), result.meta_result_.info_.lob_data_.ptr(), result.meta_result_.info_.lob_data_.length()); + read_data.set_length(result.meta_result_.info_.lob_data_.length()); + new_meta_row.byte_len_ -= (by_len); + new_meta_row.char_len_ -= (local_end - local_begin); + MEMMOVE(read_data.ptr() + by_st, read_data.ptr() + (by_st + by_len), piece_byte_len - (by_st + by_len)); + read_data.assign_ptr(read_data.ptr(), read_data.length() - by_len); } } else { del_piece = true; @@ -3319,15 +3346,14 @@ int ObLobManager::erase_process_meta_info(ObLobAccessParam& param, ObLobMetaScan new_meta_row.byte_len_ = read_data.length(); } } else { - read_data.assign_buffer(tmp_buf, result.meta_result_.info_.byte_len_); - if (OB_FAIL(get_real_data(param, result, read_data))) { - LOG_WARN("failed to write data to read buf.", K(ret), K(result)); - } else { - new_meta_row.char_len_ = piece_char_len - local_end; - new_meta_row.byte_len_ = read_data.length() - by_len; - MEMMOVE(read_data.ptr(), read_data.ptr() + by_len, read_data.length() - by_len); - read_data.assign_ptr(read_data.ptr(), read_data.length() - by_len); - } + // here expect read_data read full lob meta data + read_data.assign_buffer(tmp_buf, tmp_buff.size()); + MEMCPY(read_data.ptr(), result.meta_result_.info_.lob_data_.ptr(), result.meta_result_.info_.lob_data_.length()); + read_data.set_length(result.meta_result_.info_.lob_data_.length()); + new_meta_row.char_len_ = piece_char_len - local_end; + new_meta_row.byte_len_ = read_data.length() - by_len; + MEMMOVE(read_data.ptr(), read_data.ptr() + by_len, read_data.length() - by_len); + read_data.assign_ptr(read_data.ptr(), read_data.length() - by_len); } } } diff --git a/src/storage/lob/ob_lob_meta.cpp b/src/storage/lob/ob_lob_meta.cpp index daba2eff3c..1771f83412 100644 --- a/src/storage/lob/ob_lob_meta.cpp +++ b/src/storage/lob/ob_lob_meta.cpp @@ -424,6 +424,7 @@ ObLobMetaWriteIter::ObLobMetaWriteIter(ObIAllocator* allocator) allocator_(allocator), last_info_(), iter_(nullptr), + iter_fill_size_(0), read_param_(nullptr), lob_common_(nullptr), is_end_(false) @@ -446,6 +447,7 @@ ObLobMetaWriteIter::ObLobMetaWriteIter( allocator_(allocator), last_info_(), iter_(nullptr), + iter_fill_size_(0), read_param_(nullptr), lob_common_(nullptr), is_end_(false) @@ -654,6 +656,8 @@ int ObLobMetaWriteIter::try_fill_data( row.info_.byte_len_ += by_len; row.info_.char_len_ += char_len; OB_ASSERT(row.info_.byte_len_ >= row.info_.char_len_); + offset_ += by_len; + iter_fill_size_ += by_len; } return ret; } @@ -846,8 +850,8 @@ int ObLobMetaWriteIter::get_next_row(ObLobMetaWriteResult &row) ret = try_fill_data(row, post_data_, post_data_.length(), true, use_inner_buffer, fill_full); } else if (!iter->is_end()) { ret = try_fill_data(row, use_inner_buffer, fill_full); - } else if (remain_buf_.length() > offset_ - post_data_.length() - padding_size_) { - ret = try_fill_data(row, remain_buf_, post_data_.length() + padding_size_, false, use_inner_buffer, fill_full); + } else if (remain_buf_.length() > offset_ - post_data_.length() - padding_size_ - iter_fill_size_) { + ret = try_fill_data(row, remain_buf_, post_data_.length() + padding_size_ + iter_fill_size_, false, use_inner_buffer, fill_full); } else { ret = OB_ITER_END; } @@ -911,6 +915,7 @@ void ObLobMetaWriteIter::reuse() remain_buf_.reset(); last_info_.reset(); iter_ = nullptr; + iter_fill_size_ = 0; lob_common_ = nullptr; is_end_ = false; } diff --git a/src/storage/lob/ob_lob_meta.h b/src/storage/lob/ob_lob_meta.h index 63bd575fce..3ddac377c1 100644 --- a/src/storage/lob/ob_lob_meta.h +++ b/src/storage/lob/ob_lob_meta.h @@ -164,6 +164,7 @@ private: ObIAllocator* allocator_; ObLobMetaInfo last_info_; void *iter_; // ObLobQueryIter + uint64_t iter_fill_size_; void *read_param_; // ObLobAccessParam void* lob_common_; // ObLobCommon bool is_end_; diff --git a/src/storage/lob/ob_lob_util.cpp b/src/storage/lob/ob_lob_util.cpp index 6776d51371..400b2d2c02 100644 --- a/src/storage/lob/ob_lob_util.cpp +++ b/src/storage/lob/ob_lob_util.cpp @@ -221,6 +221,7 @@ int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator, blocksstable::ObStorageDatum &datum, const int64_t timeout_ts, const bool has_lob_header, + const uint64_t src_tenant_id, ObLobMetaWriteIter &iter) { int ret = OB_SUCCESS; @@ -272,6 +273,7 @@ int ObInsertLobColumnHelper::insert_lob_column(ObIAllocator &allocator, lob_param.offset_ = 0; lob_param.spec_lob_id_ = lob_id; lob_param.inrow_threshold_ = lob_storage_param.inrow_threshold_; + lob_param.src_tenant_id_ = src_tenant_id; if (!src.is_valid()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid src lob locator.", K(ret)); diff --git a/src/storage/lob/ob_lob_util.h b/src/storage/lob/ob_lob_util.h index 85334bbb3b..783c91bb09 100644 --- a/src/storage/lob/ob_lob_util.h +++ b/src/storage/lob/ob_lob_util.h @@ -249,6 +249,7 @@ public: blocksstable::ObStorageDatum &datum, const int64_t timeout_ts, const bool has_lob_header, + const uint64_t src_tenant_id, ObLobMetaWriteIter &iter); }; diff --git a/src/storage/ls/ob_ls_tablet_service.cpp b/src/storage/ls/ob_ls_tablet_service.cpp index b841eeea04..e62e1ec950 100644 --- a/src/storage/ls/ob_ls_tablet_service.cpp +++ b/src/storage/ls/ob_ls_tablet_service.cpp @@ -173,6 +173,8 @@ int ObLSTabletService::offline() LOG_WARN("not inited", K(ret), K_(is_inited)); } else if (OB_FAIL(offline_build_tablet_without_memtable_())) { LOG_WARN("failed to offline build tablet without memtable", K(ret)); + } else if (OB_FAIL(offline_gc_tablet_for_create_or_transfer_in_abort_())) { + LOG_WARN("failed to offline_gc_tablet_for_create_or_transfer_in_abort", K(ret)); } else if (OB_FAIL(offline_destroy_memtable_and_mds_table_())) { LOG_WARN("failed to offline destroy memtable and mds table", K(ret)); } else { @@ -1439,6 +1441,21 @@ int ObLSTabletService::update_tablet_release_memtable_for_offline( return ret; } +int ObLSTabletService::ObUpdateDDLCommitSCN::modify_tablet_meta(ObTabletMeta &meta) +{ + int ret = OB_SUCCESS; + if (!ddl_commit_scn_.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(ddl_commit_scn_)); + } else if (meta.ddl_commit_scn_.is_valid_and_not_min() && ddl_commit_scn_ != meta.ddl_commit_scn_) { + ret = OB_ERR_SYS; + LOG_WARN("ddl commit scn already set", K(ret), K(meta), K(ddl_commit_scn_)); + } else { + meta.ddl_commit_scn_ = ddl_commit_scn_; + } + return ret; +} + int ObLSTabletService::update_tablet_ddl_commit_scn( const common::ObTabletID &tablet_id, const SCN ddl_commit_scn) @@ -1452,7 +1469,7 @@ int ObLSTabletService::update_tablet_ddl_commit_scn( if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("not inited", K(ret), K_(is_inited)); - } else if (OB_UNLIKELY(!tablet_id.is_valid() || !ddl_commit_scn.is_valid())) { + } else if (OB_UNLIKELY(!tablet_id.is_valid() || !ddl_commit_scn.is_valid_and_not_min())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(tablet_id), K(ddl_commit_scn)); } else if (OB_FAIL(ObTabletCreateDeleteHelper::get_tablet(key, old_handle))) { @@ -2697,6 +2714,8 @@ int ObLSTabletService::direct_insert_rows( ObTableLoadStore store(table_ctx); if (OB_FAIL(store.init())) { LOG_WARN("fail to init store", KR(ret)); + } else if (OB_FAIL(store.px_check_for_write(tablet_id))) { + LOG_WARN("fail to check for write", KR(ret), K(tablet_id)); } while (OB_SUCC(ret) && OB_SUCC(get_next_rows(row_iter, rows, row_count))) { @@ -3901,9 +3920,12 @@ int ObLSTabletService::insert_rows_to_tablet( run_ctx.dml_param_.tz_info_)) { LOG_WARN("extract rowkey failed"); } else { + int tmp_ret = OB_SUCCESS; ObString index_name = "PRIMARY"; if (data_table.is_index_table()) { data_table.get_index_name(index_name); + } else if (lib::is_oracle_mode() && OB_TMP_FAIL(data_table.get_primary_key_name(index_name))) { + LOG_WARN("Failed to get pk name", K(ret), K(tmp_ret)); } LOG_USER_ERROR(OB_ERR_PRIMARY_KEY_DUPLICATE, rowkey_buffer, index_name.length(), index_name.ptr()); @@ -6552,6 +6574,69 @@ int ObLSTabletService::check_tablet_no_active_memtable(const ObIArrayget_ls_id())); + ObTabletIDArray deleted_tablets; + ObLSTabletIterator tablet_iter(ObMDSGetTabletMode::READ_WITHOUT_CHECK); + bool tablet_status_is_written = false; + ObTabletCreateDeleteMdsUserData data; + bool is_finish = false; + // get deleted_tablets + if (OB_FAIL(build_tablet_iter(tablet_iter))) { + LOG_WARN("failed to build ls tablet iter", KR(ret), KPC(this)); + } else { + ObTabletHandle tablet_handle; + ObTablet *tablet = NULL; + while (OB_SUCC(ret)) { + if (OB_FAIL(tablet_iter.get_next_tablet(tablet_handle))) { + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + break; + } else { + LOG_WARN("failed to get tablet", KR(ret), KPC(this), K(tablet_handle)); + } + } else if (OB_UNLIKELY(!tablet_handle.is_valid())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid tablet handle", KR(ret), KPC(this), K(tablet_handle)); + } else if (OB_ISNULL(tablet = tablet_handle.get_obj())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("tablet is NULL", KR(ret)); + } else if (tablet->is_ls_inner_tablet()) { + // skip ls inner tablet + } else if (tablet->is_empty_shell()) { + // skip empty shell + } else if (OB_FAIL(tablet->check_tablet_status_written(tablet_status_is_written))) { + LOG_WARN("failed to check mds written", KR(ret), KPC(tablet)); + } else if (OB_FAIL(tablet->ObITabletMdsInterface::get_latest_tablet_status(data, is_finish))) { + if (OB_EMPTY_RESULT == ret) { + ret = OB_SUCCESS; + if (tablet_status_is_written) { + if (OB_FAIL(deleted_tablets.push_back(tablet->get_tablet_id()))) { + LOG_WARN("failed to push_back", KR(ret)); + } else { + LOG_INFO("tablet need be gc", KPC(tablet)); + } + } + } + } + } + } + + // gc deleted_tablets + if (OB_SUCC(ret)) { + for (int64_t i = 0; OB_SUCC(ret) && i < deleted_tablets.count(); ++i) { + const ObTabletID tablet_id = deleted_tablets.at(i); + if (OB_FAIL(inner_remove_tablet(ls_->get_ls_id(), tablet_id))) { + LOG_WARN("failed to remove tablet", K(ret), K(tablet_id)); + } else { + LOG_INFO("gc tablet finish", K(ret), K(tablet_id)); + } + } + } + return ret; +} } // namespace storage } // namespace oceanbase diff --git a/src/storage/ls/ob_ls_tablet_service.h b/src/storage/ls/ob_ls_tablet_service.h index f869eaea6c..cd485a8f0d 100644 --- a/src/storage/ls/ob_ls_tablet_service.h +++ b/src/storage/ls/ob_ls_tablet_service.h @@ -483,11 +483,7 @@ private: public: explicit ObUpdateDDLCommitSCN(const share::SCN ddl_commit_scn) : ddl_commit_scn_(ddl_commit_scn) {} virtual ~ObUpdateDDLCommitSCN() = default; - virtual int modify_tablet_meta(ObTabletMeta &meta) override - { - meta.ddl_commit_scn_ = ddl_commit_scn_; - return OB_SUCCESS; - } + virtual int modify_tablet_meta(ObTabletMeta &meta) override; private: const share::SCN ddl_commit_scn_; DISALLOW_COPY_AND_ASSIGN(ObUpdateDDLCommitSCN); @@ -561,6 +557,7 @@ private: ObTabletHandle &handle); int delete_all_tablets(); int offline_build_tablet_without_memtable_(); + int offline_gc_tablet_for_create_or_transfer_in_abort_(); int offline_destroy_memtable_and_mds_table_(); private: static int check_real_leader_for_4377_(const ObLSID ls_id); diff --git a/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp b/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp index e02ccaf654..bcc6d879b1 100644 --- a/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp +++ b/src/storage/memtable/mvcc/ob_mvcc_trans_ctx.cpp @@ -1434,7 +1434,11 @@ int ObTransCallbackMgr::get_callback_list_stat(ObIArray &s } } else if (OB_SUCC(stats.prepare_allocate(get_callback_list_count()))) { CALLBACK_LISTS_FOREACH(idx, list) { - ret = list->get_stat_for_display(stats.at(idx)); + if (list->get_appended() > 0) { + ret = list->get_stat_for_display(stats.at(idx)); + } else { + stats.at(idx).id_ = -1; // mark as invalid + } } } rwlock_.unlock(); diff --git a/src/storage/memtable/mvcc/ob_tx_callback_list.cpp b/src/storage/memtable/mvcc/ob_tx_callback_list.cpp index da5138883e..6a3a17e4ba 100644 --- a/src/storage/memtable/mvcc/ob_tx_callback_list.cpp +++ b/src/storage/memtable/mvcc/ob_tx_callback_list.cpp @@ -778,7 +778,7 @@ int ObTxCallbackList::get_stat_for_display(ObTxCallbackListStat &stat) const { int ret = OB_SUCCESS; #define _ASSIGN_STAT_(x) stat.x = x; - LST_DO(_ASSIGN_STAT_, (), id_, sync_scn_, checksum_scn_, length_, logged_, removed_); + LST_DO(_ASSIGN_STAT_, (), id_, sync_scn_, length_, logged_, removed_, branch_removed_); #undef __ASSIGN_STAT_ return ret; } diff --git a/src/storage/meta_mem/ob_tablet_pointer.cpp b/src/storage/meta_mem/ob_tablet_pointer.cpp index fbb76da315..8ec53b808f 100644 --- a/src/storage/meta_mem/ob_tablet_pointer.cpp +++ b/src/storage/meta_mem/ob_tablet_pointer.cpp @@ -523,6 +523,11 @@ void ObTabletPointer::set_tablet_status_written() mds_table_handler_.set_tablet_status_written(); } +void ObTabletPointer::reset_tablet_status_written() +{ + mds_table_handler_.reset_tablet_status_written(); +} + bool ObTabletPointer::is_tablet_status_written() const { return mds_table_handler_.is_tablet_status_written(); @@ -537,6 +542,7 @@ int ObTabletPointer::release_memtable_and_mds_table_for_ls_offline(const ObTable { int ret = OB_SUCCESS; mds::MdsTableHandle mds_table; + reset_tablet_status_written(); if (tablet_id.is_ls_inner_tablet()) { LOG_INFO("skip inner tablet", K(tablet_id)); } else if (OB_FAIL(protected_memtable_mgr_handle_.reset())) { diff --git a/src/storage/meta_mem/ob_tablet_pointer.h b/src/storage/meta_mem/ob_tablet_pointer.h index 793765c10b..ed0d260e14 100644 --- a/src/storage/meta_mem/ob_tablet_pointer.h +++ b/src/storage/meta_mem/ob_tablet_pointer.h @@ -116,6 +116,7 @@ public: // interfaces forward to mds_table_handler_ void mark_mds_table_deleted(); void set_tablet_status_written(); + void reset_tablet_status_written(); bool is_tablet_status_written() const; int try_release_mds_nodes_below(const share::SCN &scn); int try_gc_mds_table(); diff --git a/src/storage/multi_data_source/mds_table_handler.h b/src/storage/multi_data_source/mds_table_handler.h index 9b44951fbc..9f934cd368 100644 --- a/src/storage/multi_data_source/mds_table_handler.h +++ b/src/storage/multi_data_source/mds_table_handler.h @@ -42,6 +42,7 @@ public: int try_release_nodes_below(const share::SCN &scn); void reset() { this->~ObMdsTableHandler(); } void set_tablet_status_written() { ATOMIC_CAS(&(is_written_), false, true); } + void reset_tablet_status_written() { ATOMIC_STORE(&(is_written_), false); } void mark_removed_from_t3m(ObTabletPointer *pointer); bool is_tablet_status_written() const { return ATOMIC_LOAD(&(is_written_)); } TO_STRING_KV(K_(mds_table_handle)); @@ -68,4 +69,4 @@ private: } } } -#endif \ No newline at end of file +#endif diff --git a/src/storage/tablelock/ob_table_lock_rpc_processor.cpp b/src/storage/tablelock/ob_table_lock_rpc_processor.cpp index 682ef8b8eb..481d8970d6 100644 --- a/src/storage/tablelock/ob_table_lock_rpc_processor.cpp +++ b/src/storage/tablelock/ob_table_lock_rpc_processor.cpp @@ -71,6 +71,10 @@ int check_exist(const ObLockTaskBatchRequest &arg, } else if (OB_FAIL(tablet_handle.get_obj()->ObITabletMdsInterface::get_latest_tablet_status( data, is_commited))) { LOG_WARN("failed to get CreateDeleteMdsUserData", KR(ret)); + if (OB_EMPTY_RESULT == ret) { + // tablet is creating + ret = OB_TABLET_NOT_EXIST; + } } else if (FALSE_IT(tablet_status = data.get_tablet_status())) { } else if (ObTabletStatus::NORMAL == tablet_status || ObTabletStatus::TRANSFER_OUT == tablet_status diff --git a/src/storage/tablet/ob_tablet.cpp b/src/storage/tablet/ob_tablet.cpp index c8373984e7..2f8934ce4b 100644 --- a/src/storage/tablet/ob_tablet.cpp +++ b/src/storage/tablet/ob_tablet.cpp @@ -1166,7 +1166,7 @@ int ObTablet::init_with_new_snapshot_version( LOG_WARN("fail to copy table store", K(ret), KPC(old_table_store)); } else if (OB_FAIL(try_update_start_scn())) { LOG_WARN("failed to update start scn", K(ret), KPC(old_table_store)); - } else if (OB_FAIL(mds_data_.init_with_update_medium_info(allocator, old_mds_data))) { + } else if (OB_FAIL(mds_data_.init_for_evict_medium_info(allocator, old_mds_data, 0/*finish_medium_scn*/))) { LOG_WARN("failed to init mds data", K(ret), K(old_mds_data)); } else if (OB_FAIL(table_store_cache_.init(table_store_addr_.get_ptr()->get_major_sstables(), table_store_addr_.get_ptr()->get_minor_sstables(), @@ -7423,5 +7423,38 @@ int ObTablet::get_protected_memtable_mgr_handle(ObProtectedMemtableMgrHandle *&h return ret; } +int ObTablet::get_ready_for_read_param(ObReadyForReadParam ¶m) const +{ + int ret = OB_SUCCESS; + param.ddl_commit_scn_ = tablet_meta_.ddl_commit_scn_; + param.clog_checkpoint_scn_ = tablet_meta_.clog_checkpoint_scn_; + return ret; +} + +int ObTablet::check_ready_for_read_if_need(const ObTablet &old_tablet) +{ + int ret = OB_SUCCESS; + ObReadyForReadParam old_param; + ObReadyForReadParam new_param; + if (OB_FAIL(old_tablet.get_ready_for_read_param(old_param))) { + LOG_WARN("fail to get ready for read param", K(ret), K(old_tablet)); + } else if (OB_FAIL(get_ready_for_read_param(new_param))) { + LOG_WARN("fail to get ready for read param", K(ret), KPC(this)); + } else if (old_param == new_param) { + // no change, nothing to do + } else if (table_store_addr_.is_memory_object()) { + if (OB_FAIL(table_store_addr_.get_ptr()->check_ready_for_read(new_param))) { + if (OB_SIZE_OVERFLOW == ret) { + ObTabletTableStore::diagnose_table_count_unsafe(*this); + } else { + LOG_WARN("table store check ready for read failed", K(ret), KPC(this)); + } + } + } else { + // invalid the cache to force reload table store from disk, for updating the ready_for_read flag + table_store_addr_.addr_.inc_seq(); + } + return ret; +} } // namespace storage } // namespace oceanbase diff --git a/src/storage/tablet/ob_tablet.h b/src/storage/tablet/ob_tablet.h index 5b6723f9fd..2a1e040e1a 100644 --- a/src/storage/tablet/ob_tablet.h +++ b/src/storage/tablet/ob_tablet.h @@ -265,6 +265,7 @@ public: bool is_empty_shell() const; // major merge or medium merge call bool is_data_complete() const; + int get_ready_for_read_param(ObReadyForReadParam &parm) const; // serialize & deserialize // TODO: change the impl of serialize and get_serialize_size after rebase @@ -822,6 +823,7 @@ private: ObUpdateTabletPointerParam ¶m, const bool need_tablet_attr = true) const; int calc_tablet_attr(ObTabletAttr &attr) const; + int check_ready_for_read_if_need(const ObTablet &old_tablet); private: // ObTabletDDLKvMgr::MAX_DDL_KV_CNT_IN_STORAGE // Array size is too large, need to shrink it if possible diff --git a/src/storage/tablet/ob_tablet_create_sstable_param.cpp b/src/storage/tablet/ob_tablet_create_sstable_param.cpp index 2198172312..aa1dd96464 100644 --- a/src/storage/tablet/ob_tablet_create_sstable_param.cpp +++ b/src/storage/tablet/ob_tablet_create_sstable_param.cpp @@ -19,7 +19,11 @@ #include "storage/ob_sstable_struct.h" #include "storage/blocksstable/ob_macro_block_struct.h" #include "storage/blocksstable/ob_block_sstable_struct.h" +#include "storage/blocksstable/index_block/ob_index_block_builder.h" #include "storage/column_store/ob_column_oriented_sstable.h" +#include "storage/blocksstable/ob_shared_macro_block_manager.h" +#include "storage/compaction/ob_basic_tablet_merge_ctx.h" +#include "storage/ddl/ob_direct_load_struct.h" namespace oceanbase { @@ -134,5 +138,352 @@ bool ObTabletCreateSSTableParam::is_block_meta_valid(const storage::ObMetaDiskAd return addr.is_valid() && (!addr.is_memory() || (data.is_valid() && data.size_ == addr.size())); } +int ObTabletCreateSSTableParam::inner_init_with_merge_res(const blocksstable::ObSSTableMergeRes &res) +{ + int ret = OB_SUCCESS; + ObSSTableMergeRes::fill_addr_and_data(res.root_desc_, + root_block_addr_, root_block_data_); + ObSSTableMergeRes::fill_addr_and_data(res.data_root_desc_, + data_block_macro_meta_addr_, data_block_macro_meta_); + root_row_store_type_ = res.root_row_store_type_; + data_index_tree_height_ = res.root_desc_.height_; + index_blocks_cnt_ = res.index_blocks_cnt_; + data_blocks_cnt_ = res.data_blocks_cnt_; + micro_block_cnt_ = res.micro_block_cnt_; + use_old_macro_block_count_ = res.use_old_macro_block_count_; + row_count_ = res.row_count_; + data_checksum_ = res.data_checksum_; + occupy_size_ = res.occupy_size_; + original_size_ = res.original_size_; + contain_uncommitted_row_ = res.contain_uncommitted_row_; + compressor_type_ = res.compressor_type_; + encrypt_id_ = res.encrypt_id_; + master_key_id_ = res.master_key_id_; + is_meta_root_ = res.data_root_desc_.is_meta_root_; + STATIC_ASSERT(ARRAYSIZEOF(encrypt_key_) == share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH, + "ObTabletCreateSSTableParam encrypt_key_ array size mismatch OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH"); + STATIC_ASSERT(ARRAYSIZEOF(res.encrypt_key_) == share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH, + "ObSSTableMergeRes encrypt_key_ array size mismatch OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH"); + MEMCPY(encrypt_key_, res.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); + + if (OB_FAIL(data_block_ids_.assign(res.data_block_ids_))) { + LOG_WARN("fail to fill data block ids", K(ret), K(res.data_block_ids_)); + } else if (OB_FAIL(other_block_ids_.assign(res.other_block_ids_))) { + LOG_WARN("fail to fill other block ids", K(ret), K(res.other_block_ids_)); + } + return ret; +} + +int ObTabletCreateSSTableParam::init_for_small_sstable(const blocksstable::ObSSTableMergeRes &res, + const ObITable::TableKey &table_key, + const blocksstable::ObSSTableMeta &sstable_meta, + const blocksstable::ObBlockInfo &block_info) +{ + int ret = OB_SUCCESS; + const blocksstable::ObSSTableBasicMeta &basic_meta = sstable_meta.get_basic_meta(); + filled_tx_scn_ = basic_meta.filled_tx_scn_; + ddl_scn_ = basic_meta.ddl_scn_; + table_key_ = table_key; + sstable_logic_seq_ = sstable_meta.get_sstable_seq(); + table_mode_ = basic_meta.table_mode_; + index_type_ = static_cast(basic_meta.index_type_); + schema_version_ = basic_meta.schema_version_; + create_snapshot_version_ = basic_meta.create_snapshot_version_; + progressive_merge_round_ = basic_meta.progressive_merge_round_; + progressive_merge_step_ = basic_meta.progressive_merge_step_; + rowkey_column_cnt_ = basic_meta.rowkey_column_count_; + recycle_version_ = basic_meta.recycle_version_; + latest_row_store_type_ = basic_meta.latest_row_store_type_; + is_ready_for_read_ = true; + column_cnt_ = res.data_column_cnt_; + max_merged_trans_version_ = res.max_merged_trans_version_; + nested_offset_ = block_info.nested_offset_; + nested_size_ = block_info.nested_size_; + if (OB_FAIL(inner_init_with_merge_res(res))) { + LOG_WARN("fail to inner init with merge res", K(ret), K(res)); + } else if (table_key_.is_major_sstable()) { + if (OB_FAIL(column_checksums_.assign(res.data_column_checksums_))) { + LOG_WARN("fail to fill column checksum", K(ret), K(res.data_column_checksums_)); + } + } + + return ret; +} + +int ObTabletCreateSSTableParam::init_for_merge(const compaction::ObBasicTabletMergeCtx &ctx, + const blocksstable::ObSSTableMergeRes &res, + const ObStorageColumnGroupSchema *cg_schema, + const int64_t column_group_idx, + const bool is_main_table) +{ + int ret = OB_SUCCESS; + const compaction::ObStaticMergeParam &static_param = ctx.static_param_; + if (OB_FAIL(static_param.schema_->get_stored_column_count_in_sstable(full_column_cnt_))) { + LOG_WARN("fail to get_stored_column_count_in_sstable", K(ret), KPC(cg_schema), K(res)); + } else { + ObITable::TableKey table_key; + table_key.table_type_ = ctx.get_merged_table_type(cg_schema, is_main_table); + table_key.tablet_id_ = ctx.get_tablet_id(); + table_key.column_group_idx_ = (nullptr == cg_schema) ? 0 : column_group_idx; + if (is_major_or_meta_merge_type(static_param.get_merge_type())) { + table_key.version_range_.snapshot_version_ = static_param.version_range_.snapshot_version_; + } else { + table_key.scn_range_ = static_param.scn_range_; + } + table_key_ = table_key; + + if (ObITable::TableType::COLUMN_ORIENTED_SSTABLE == table_key.table_type_ || + ObITable::TableType::COLUMN_ORIENTED_META_SSTABLE == table_key.table_type_) { + co_base_type_ = cg_schema->is_all_column_group() + ? ObCOSSTableBaseType::ALL_CG_TYPE + : ObCOSSTableBaseType::ROWKEY_CG_TYPE; + } + + sstable_logic_seq_ = static_param.sstable_logic_seq_; + filled_tx_scn_ = ctx.get_merge_scn(); + + table_mode_ = ctx.get_schema()->get_table_mode_struct(); + index_type_ = ctx.get_schema()->get_index_type(); + if (nullptr != cg_schema && !cg_schema->is_rowkey_column_group() && !cg_schema->is_all_column_group()) { + rowkey_column_cnt_ = 0; + } else { + column_group_cnt_ = static_param.schema_->get_column_group_count(); + rowkey_column_cnt_ = static_param.schema_->get_rowkey_column_num() + + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); + } + latest_row_store_type_ = ctx.get_schema()->get_row_store_type(); + if (is_minor_merge_type(static_param.get_merge_type())) { + recycle_version_ = static_param.version_range_.base_version_; + } else { + recycle_version_ = 0; + } + schema_version_ = static_param.schema_version_; + create_snapshot_version_ = static_param.create_snapshot_version_; + progressive_merge_round_ = static_param.progressive_merge_round_; + progressive_merge_step_ = std::min( + static_param.progressive_merge_num_, static_param.progressive_merge_step_ + 1); + is_empty_co_table_ = is_main_table ? (0 == res.data_blocks_cnt_) : false; + column_cnt_ = res.data_column_cnt_; + if ((0 == res.row_count_ && 0 == res.max_merged_trans_version_) + || (nullptr != cg_schema && !cg_schema->has_multi_version_column())) { + // empty mini table merged forcely + max_merged_trans_version_ = static_param.version_range_.snapshot_version_; + } else { + max_merged_trans_version_ = res.max_merged_trans_version_; + } + nested_size_ = res.nested_size_; + nested_offset_ = res.nested_offset_; + ddl_scn_.set_min(); + + if (OB_FAIL(inner_init_with_merge_res(res))) { + LOG_WARN("fail to init with merge res", K(ret), K(res.data_block_ids_)); + } else if (is_major_or_meta_merge_type(static_param.get_merge_type())) { + if (OB_FAIL(column_checksums_.assign(res.data_column_checksums_))) { + LOG_WARN("fail to fill column checksum", K(ret), K(res.data_column_checksums_)); + } + } + } + return ret; +} + +int ObTabletCreateSSTableParam::init_for_ddl(blocksstable::ObSSTableIndexBuilder *sstable_index_builder, + const ObTabletDDLParam &ddl_param, + const blocksstable::ObSSTable *first_ddl_sstable, + const ObStorageSchema &storage_schema, + const int64_t macro_block_column_count, + const int64_t create_schema_version_on_tablet) +{ + int ret = OB_SUCCESS; + SMART_VAR(blocksstable::ObSSTableMergeRes, res) { + int64_t column_count = 0; + int64_t full_column_cnt = 0; // only used for co sstable + share::schema::ObTableMode table_mode = storage_schema.get_table_mode_struct(); + share::schema::ObIndexType index_type = storage_schema.get_index_type(); + int64_t rowkey_column_cnt = storage_schema.get_rowkey_column_num() + ObMultiVersionRowkeyHelpper::get_extra_rowkey_col_cnt(); + common::ObRowStoreType row_store_type = storage_schema.get_row_store_type(); + if (nullptr != first_ddl_sstable) { + blocksstable::ObSSTableMetaHandle meta_handle; + if (OB_FAIL(first_ddl_sstable->get_meta(meta_handle))) { + LOG_WARN("get sstable meta handle fail", K(ret), KPC(first_ddl_sstable)); + } else { + column_count = meta_handle.get_sstable_meta().get_column_count(); + table_mode = meta_handle.get_sstable_meta().get_basic_meta().table_mode_; + index_type = static_cast(meta_handle.get_sstable_meta().get_basic_meta().index_type_); + rowkey_column_cnt = meta_handle.get_sstable_meta().get_basic_meta().rowkey_column_count_; + row_store_type = meta_handle.get_sstable_meta().get_basic_meta().latest_row_store_type_; + if (first_ddl_sstable->is_co_sstable()) { + const ObCOSSTableV2 *first_co_sstable = static_cast(first_ddl_sstable); + if (OB_ISNULL((first_co_sstable))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("first co sstable is null", K(ret), KP(first_co_sstable), KPC(first_ddl_sstable)); + } else { + full_column_cnt = first_co_sstable->get_cs_meta().full_column_cnt_; + } + } + } + } else if (ddl_param.table_key_.is_column_store_sstable()) { + if (ddl_param.table_key_.is_normal_cg_sstable()) { + rowkey_column_cnt = 0; + column_count = 1; + } else { // co sstable with all cg or rowkey cg + const ObIArray &cg_schemas = storage_schema.get_column_groups(); + const int64_t cg_idx = ddl_param.table_key_.get_column_group_id(); + if (cg_idx < 0 || cg_idx >= cg_schemas.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected column group index", K(ret), K(cg_idx)); + } else if (OB_FAIL(storage_schema.get_stored_column_count_in_sstable(full_column_cnt))) { // set full_column_cnt in first ddl sstable + LOG_WARN("fail to get stored column count in sstable", K(ret)); + } else if (cg_schemas.at(cg_idx).is_rowkey_column_group()) { + column_count = rowkey_column_cnt; + } else { + column_count = full_column_cnt; + if (macro_block_column_count > 0 && macro_block_column_count < column_count) { + LOG_INFO("use macro block column count", K(ddl_param), K(macro_block_column_count), K(column_count)); + column_count = macro_block_column_count; + full_column_cnt = macro_block_column_count; + } + } + } + } else { // row store sstable + if (OB_FAIL(storage_schema.get_stored_column_count_in_sstable(column_count))) { + LOG_WARN("fail to get stored column count in sstable", K(ret)); + } else if (macro_block_column_count > 0 && macro_block_column_count < column_count) { + LOG_INFO("use macro block column count", K(ddl_param), K(macro_block_column_count), K(column_count)); + column_count = macro_block_column_count; + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(sstable_index_builder->close(res))) { + LOG_WARN("close sstable index builder close failed", K(ret)); + } else if (ddl_param.table_key_.is_normal_cg_sstable() // index builder of cg sstable cannot get trans_version from row, manually set it + && FALSE_IT(res.max_merged_trans_version_ = ddl_param.snapshot_version_)) { + } else if (OB_UNLIKELY((ddl_param.table_key_.is_major_sstable() || + ddl_param.table_key_.is_ddl_sstable()) && + res.row_count_ > 0 && + res.max_merged_trans_version_ != ddl_param.snapshot_version_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("max_merged_trans_version_ in res is different from ddl snapshot version", K(ret), + K(res), K(ddl_param)); + } else { + table_key_ = ddl_param.table_key_; + table_mode_ = table_mode; + index_type_ = index_type; + rowkey_column_cnt_ = rowkey_column_cnt; + schema_version_ = create_schema_version_on_tablet; + latest_row_store_type_ = row_store_type; + create_snapshot_version_ = ddl_param.snapshot_version_; + ddl_scn_ = ddl_param.start_scn_; + column_cnt_ = column_count; + full_column_cnt_ = full_column_cnt; + max_merged_trans_version_ = ddl_param.snapshot_version_; + nested_size_ = res.nested_size_; + nested_offset_ = res.nested_offset_; + + if (OB_FAIL(inner_init_with_merge_res(res))) { + LOG_WARN("fail to inner init with merge res", K(ret), K(res)); + } else if (ddl_param.table_key_.is_co_sstable()) { + column_group_cnt_ = storage_schema.get_column_group_count(); + // only set true when build empty major sstable. ddl co sstable must set false and fill cg sstables + is_empty_co_table_ = ddl_param.table_key_.is_major_sstable() && 0 == data_blocks_cnt_; + const int64_t base_cg_idx = ddl_param.table_key_.get_column_group_id(); + if (base_cg_idx < 0 || base_cg_idx >= storage_schema.get_column_group_count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid column group index", K(ret), K(ddl_param.table_key_)); + } else { + const ObStorageColumnGroupSchema &base_cg_schema = storage_schema.get_column_groups().at(base_cg_idx); + if (base_cg_schema.is_all_column_group()) { + co_base_type_ = ObCOSSTableBaseType::ALL_CG_TYPE; + } else if (base_cg_schema.is_rowkey_column_group()) { + co_base_type_ = ObCOSSTableBaseType::ROWKEY_CG_TYPE; + } else { + ret = OB_ERR_SYS; + LOG_WARN("unknown type of base cg schema", K(ret), K(base_cg_idx)); + } + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(column_checksums_.assign(res.data_column_checksums_))) { + LOG_WARN("fail to fill column checksum for empty major", K(ret), K(res.data_column_checksums_)); + } else if (OB_UNLIKELY(column_checksums_.count() != column_count)) { + // we have corrected the col_default_checksum_array_ in prepare_index_data_desc + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected column checksums", K(ret), K(column_count), KPC(this)); + } + } + } + return ret; +} + +int ObTabletCreateSSTableParam::init_for_ha( + const blocksstable::ObMigrationSSTableParam &sstable_param, + const blocksstable::ObSSTableMergeRes &res) +{ + int ret = OB_SUCCESS; + + table_key_ = sstable_param.table_key_; + sstable_logic_seq_ = sstable_param.basic_meta_.sstable_logic_seq_; + schema_version_ = sstable_param.basic_meta_.schema_version_; + table_mode_ = sstable_param.basic_meta_.table_mode_; + index_type_ = static_cast(sstable_param.basic_meta_.index_type_); + create_snapshot_version_ = sstable_param.basic_meta_.create_snapshot_version_; + progressive_merge_round_ = sstable_param.basic_meta_.progressive_merge_round_; + progressive_merge_step_ = sstable_param.basic_meta_.progressive_merge_step_; + latest_row_store_type_ = sstable_param.basic_meta_.latest_row_store_type_; + column_cnt_ = res.data_column_cnt_; + nested_size_ = res.nested_size_; + nested_offset_ = res.nested_offset_; + max_merged_trans_version_ = res.max_merged_trans_version_; + rowkey_column_cnt_ = sstable_param.basic_meta_.rowkey_column_count_; + ddl_scn_ = sstable_param.basic_meta_.ddl_scn_; + if (OB_FAIL(inner_init_with_merge_res(res))) { + LOG_WARN("fail to inner init with merge res", K(ret), K(res)); + } else if (OB_FAIL(column_checksums_.assign(sstable_param.column_checksums_))) { + LOG_WARN("fail to fill column checksum", K(ret), K(sstable_param)); + } + + return ret; +} + +int ObTabletCreateSSTableParam::init_for_ha(const blocksstable::ObMigrationSSTableParam &sstable_param) +{ + int ret = OB_SUCCESS; + table_key_ = sstable_param.table_key_; + sstable_logic_seq_ = sstable_param.basic_meta_.sstable_logic_seq_; + schema_version_ = sstable_param.basic_meta_.schema_version_; + create_snapshot_version_ = sstable_param.basic_meta_.create_snapshot_version_; + table_mode_ = sstable_param.basic_meta_.table_mode_; + index_type_ = static_cast(sstable_param.basic_meta_.index_type_); + progressive_merge_round_ = sstable_param.basic_meta_.progressive_merge_round_; + progressive_merge_step_ = sstable_param.basic_meta_.progressive_merge_step_; + is_ready_for_read_ = true; + root_row_store_type_ = sstable_param.basic_meta_.root_row_store_type_; + latest_row_store_type_ = sstable_param.basic_meta_.latest_row_store_type_; + index_blocks_cnt_ = sstable_param.basic_meta_.index_macro_block_count_; + data_blocks_cnt_ = sstable_param.basic_meta_.data_macro_block_count_; + micro_block_cnt_ = sstable_param.basic_meta_.data_micro_block_count_; + use_old_macro_block_count_ = sstable_param.basic_meta_.use_old_macro_block_count_; + row_count_ = sstable_param.basic_meta_.row_count_; + column_cnt_ = sstable_param.basic_meta_.column_cnt_; + data_checksum_ = sstable_param.basic_meta_.data_checksum_; + occupy_size_ = sstable_param.basic_meta_.occupy_size_; + original_size_ = sstable_param.basic_meta_.original_size_; + max_merged_trans_version_ = sstable_param.basic_meta_.max_merged_trans_version_; + ddl_scn_ = sstable_param.basic_meta_.ddl_scn_; + filled_tx_scn_ = sstable_param.basic_meta_.filled_tx_scn_; + contain_uncommitted_row_ = sstable_param.basic_meta_.contain_uncommitted_row_; + compressor_type_ = sstable_param.basic_meta_.compressor_type_; + encrypt_id_ = sstable_param.basic_meta_.encrypt_id_; + master_key_id_ = sstable_param.basic_meta_.master_key_id_; + root_block_addr_.set_none_addr(); + data_block_macro_meta_addr_.set_none_addr(); + rowkey_column_cnt_ = sstable_param.basic_meta_.rowkey_column_count_; + MEMCPY(encrypt_key_, sstable_param.basic_meta_.encrypt_key_, share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH); + if (OB_FAIL(column_checksums_.assign(sstable_param.column_checksums_))) { + LOG_WARN("fail to assign column checksums", K(ret), K(sstable_param)); + } + + return ret; +} + } // namespace storage } // namespace oceanbase diff --git a/src/storage/tablet/ob_tablet_create_sstable_param.h b/src/storage/tablet/ob_tablet_create_sstable_param.h index b3950b0d11..452eb4a890 100644 --- a/src/storage/tablet/ob_tablet_create_sstable_param.h +++ b/src/storage/tablet/ob_tablet_create_sstable_param.h @@ -23,8 +23,23 @@ namespace oceanbase { +namespace blocksstable +{ +struct ObSSTableMergeRes; +struct ObBlockInfo; +class ObSSTableMeta; +class ObMigrationSSTableParam; +class ObSSTable; +class ObSSTableIndexBuilder; +} +namespace compaction +{ +struct ObBasicTabletMergeCtx; +} namespace storage { +struct ObTabletDDLParam; + struct ObTabletCreateSSTableParam final { public: @@ -37,6 +52,34 @@ public: bool is_block_meta_valid(const ObMetaDiskAddr &addr, const blocksstable::ObMicroBlockData &data) const; + // Without checking the validity of the input parameters, necessary to ensure the correctness of the method call. + int init_for_small_sstable(const blocksstable::ObSSTableMergeRes &res, + const ObITable::TableKey &table_key, + const blocksstable::ObSSTableMeta &sstable_meta, + const blocksstable::ObBlockInfo &block_info); + + // Without checking the validity of the input parameters, necessary to ensure the correctness of the method call. + int init_for_merge(const compaction::ObBasicTabletMergeCtx &ctx, + const blocksstable::ObSSTableMergeRes &res, + const ObStorageColumnGroupSchema *cg_schema, + const int64_t column_group_idx, + const bool is_main_table); + + // Without checking the validity of the input parameters, necessary to ensure the correctness of the method call. + int init_for_ddl(blocksstable::ObSSTableIndexBuilder *sstable_index_builder, + const ObTabletDDLParam &ddl_param, + const blocksstable::ObSSTable *first_ddl_sstable, + const ObStorageSchema &storage_schema, + const int64_t macro_block_column_count, + const int64_t create_schema_version_on_tablet); + + // Without checking the validity of the input parameters, necessary to ensure the correctness of the method call. + int init_for_ha(const blocksstable::ObMigrationSSTableParam &migration_param, + const blocksstable::ObSSTableMergeRes &res); + + // Without checking the validity of the input parameters, necessary to ensure the correctness of the method call. + int init_for_ha(const blocksstable::ObMigrationSSTableParam &migration_param); + TO_STRING_KV(K_(table_key), K_(sstable_logic_seq), K_(schema_version), @@ -81,6 +124,7 @@ public: KPHEX_(encrypt_key, sizeof(encrypt_key_))); private: static const int64_t DEFAULT_MACRO_BLOCK_CNT = 64; + int inner_init_with_merge_res(const blocksstable::ObSSTableMergeRes &res); public: ObITable::TableKey table_key_; int16_t sstable_logic_seq_; diff --git a/src/storage/tablet/ob_tablet_memtable_mgr.cpp b/src/storage/tablet/ob_tablet_memtable_mgr.cpp index c3740d8a9e..961985a90d 100644 --- a/src/storage/tablet/ob_tablet_memtable_mgr.cpp +++ b/src/storage/tablet/ob_tablet_memtable_mgr.cpp @@ -165,7 +165,7 @@ int ObTabletMemtableMgr::reset_storage_recorder() return ret; } -inline int ObTabletMemtableMgr::try_resolve_boundary_on_create_memtable_( +inline int ObTabletMemtableMgr::try_resolve_boundary_on_create_memtable_for_leader_( memtable::ObMemtable *last_frozen_memtable, memtable::ObMemtable *new_memtable) { @@ -192,14 +192,23 @@ inline int ObTabletMemtableMgr::try_resolve_boundary_on_create_memtable_( double_check = !double_check; } while (!can_resolve && double_check); - if (can_resolve) { + + if (write_ref > 0) { + // NB: for the leader, if the write ref on the frozen memtable is greater + // than 0, we cannot create a new memtable. Otherwise we may finish the + // write on the new memtable before finishing the write on the frozen + // memtable and cause the writes and callbacks on memtable_ctx out of order. + ret = OB_EAGAIN; + TRANS_LOG(INFO, "last frozen's write flag is not 0 during create new memtable", + KPC(last_frozen_memtable), KPC(new_memtable)); + } else if (can_resolve) { last_frozen_memtable->set_resolved_active_memtable_left_boundary(true); last_frozen_memtable->resolve_right_boundary(); TRANS_LOG(INFO, "[resolve_right_boundary] in create_memtable on leader", KPC(last_frozen_memtable)); if (new_memtable != last_frozen_memtable) { new_memtable->resolve_left_boundary(last_frozen_memtable->get_end_scn()); } - } else if (unsubmitted_cnt > 0 || write_ref > 0) { + } else if (unsubmitted_cnt > 0) { new_memtable->set_logging_blocked(); TRANS_LOG(INFO, "set new memtable logging blocked", KPC(last_frozen_memtable), KPC(new_memtable)); } @@ -307,7 +316,7 @@ int ObTabletMemtableMgr::create_memtable(const SCN clog_checkpoint_scn, } } // for leader, decide the right boundary of frozen memtable - else if (OB_FAIL(try_resolve_boundary_on_create_memtable_(last_frozen_memtable, memtable))) { + else if (OB_FAIL(try_resolve_boundary_on_create_memtable_for_leader_(last_frozen_memtable, memtable))) { TRANS_LOG(WARN, "try resolve boundary fail", K(ret)); } } else { diff --git a/src/storage/tablet/ob_tablet_memtable_mgr.h b/src/storage/tablet/ob_tablet_memtable_mgr.h index 7cd8e4a2ba..a97f0a1ee4 100644 --- a/src/storage/tablet/ob_tablet_memtable_mgr.h +++ b/src/storage/tablet/ob_tablet_memtable_mgr.h @@ -126,7 +126,7 @@ private: int get_first_frozen_memtable_(ObTableHandleV2 &handle) const; void clean_tail_memtable_(); int get_last_frozen_memtable_(ObTableHandleV2 &handle) const; - int try_resolve_boundary_on_create_memtable_(memtable::ObMemtable *last_frozen_memtable, + int try_resolve_boundary_on_create_memtable_for_leader_(memtable::ObMemtable *last_frozen_memtable, memtable::ObMemtable *new_memtable); int resolve_left_boundary_for_active_memtable(memtable::ObIMemtable *memtable, share::SCN start_scn, diff --git a/src/storage/tablet/ob_tablet_persister.cpp b/src/storage/tablet/ob_tablet_persister.cpp index d80a4e76cd..cb95d48cc5 100644 --- a/src/storage/tablet/ob_tablet_persister.cpp +++ b/src/storage/tablet/ob_tablet_persister.cpp @@ -404,6 +404,8 @@ int ObTabletPersister::modify_and_fill_tablet( } else if (FALSE_IT(new_handle.get_obj()->set_next_tablet_guard(old_tablet.next_tablet_guard_))) { } else if (OB_FAIL(modifier.modify_tablet_meta(new_handle.get_obj()->tablet_meta_))) { LOG_WARN("fail to modify tablet meta", K(ret), KPC(new_handle.get_obj())); + } else if (OB_FAIL(new_handle.get_obj()->check_ready_for_read_if_need(old_tablet))) { + LOG_WARN("fail to check ready for read if need", K(ret), K(old_tablet), K(new_handle)); } else { time_stats->click("transform_and_modify"); } diff --git a/src/storage/tablet/ob_tablet_table_store.cpp b/src/storage/tablet/ob_tablet_table_store.cpp index 7df89edeb8..5ec7a5d237 100644 --- a/src/storage/tablet/ob_tablet_table_store.cpp +++ b/src/storage/tablet/ob_tablet_table_store.cpp @@ -704,7 +704,7 @@ int ObTabletTableStore::calculate_read_tables( if (OB_FAIL(iterator.add_table(meta_major_tables_.at(0)))) { LOG_WARN("failed to add meta major table to iterator", K(ret), K(meta_major_tables_)); } - } else if (!is_major_sstable_empty(tablet)) { + } else if (!is_major_sstable_empty(tablet.get_tablet_meta().ddl_commit_scn_)) { if (!major_tables_.empty()) { for (int64_t i = major_tables_.count() - 1; OB_SUCC(ret) && i >= 0; --i) { if (major_tables_[i]->get_snapshot_version() <= snapshot_version) { @@ -773,7 +773,7 @@ int ObTabletTableStore::calculate_read_tables( } } else { // not find base table if (!allow_no_ready_read) { - if (is_major_sstable_empty(tablet)) { + if (is_major_sstable_empty(tablet.get_tablet_meta().ddl_commit_scn_)) { ret = OB_REPLICA_NOT_READABLE; LOG_WARN("no base table, not allow no ready read, tablet is not readable", K(ret), K(snapshot_version), K(allow_no_ready_read), K(PRINT_TS(*this))); @@ -950,7 +950,8 @@ int ObTabletTableStore::get_read_tables( if (OB_UNLIKELY(snapshot_version < 0)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), K(snapshot_version)); - } else if (OB_UNLIKELY(is_major_sstable_empty(tablet) && minor_tables_.empty() && allow_no_ready_read)) { + } else if (OB_UNLIKELY(is_major_sstable_empty(tablet.get_tablet_meta().ddl_commit_scn_) + && minor_tables_.empty() && allow_no_ready_read)) { if (memtables_.empty()) { LOG_INFO("no table in table store, cannot read", K(ret), K(*this)); } else if (OB_FAIL(iterator.add_tables(memtables_))) { @@ -1487,10 +1488,10 @@ int ObTabletTableStore::build_meta_major_table( return ret; } -bool ObTabletTableStore::is_major_sstable_empty(const ObTablet &tablet) const +bool ObTabletTableStore::is_major_sstable_empty(const share::SCN &ddl_commit_scn) const { return major_tables_.empty() - && !tablet.get_tablet_meta().ddl_commit_scn_.is_valid_and_not_min(); // ddl logic major sstable require commit scn valid + && !ddl_commit_scn.is_valid_and_not_min(); // ddl logic major sstable require commit scn valid } int ObTabletTableStore::get_ddl_major_sstables(ObIArray &ddl_major_sstables) const @@ -1643,6 +1644,28 @@ int ObTabletTableStore::build_memtable_array(const ObTablet &tablet) } int ObTabletTableStore::check_ready_for_read(const ObTablet &tablet) +{ + int ret = OB_SUCCESS; + ObReadyForReadParam param; + if (OB_FAIL(tablet.get_ready_for_read_param(param))) { + LOG_WARN("fail to get ready for read param", K(ret), K(tablet)); + } else if (OB_FAIL(check_ready_for_read(param))) { + if (OB_SIZE_OVERFLOW == ret) { + diagnose_table_count_unsafe(tablet); + } else { + LOG_WARN("fail to check ready for read", K(ret), K(tablet)); + } + } + return ret; +} + +void ObTabletTableStore::diagnose_table_count_unsafe(const ObTablet &tablet) +{ + compaction::ObPartitionMergePolicy::diagnose_table_count_unsafe(compaction::MAJOR_MERGE, ObDiagnoseTabletType::TYPE_SPECIAL, tablet); + MTL(concurrency_control::ObMultiVersionGarbageCollector *)->report_sstable_overflow(); +} + +int ObTabletTableStore::check_ready_for_read(const ObReadyForReadParam ¶m) { int ret = OB_SUCCESS; is_ready_for_read_ = false; @@ -1650,30 +1673,25 @@ int ObTabletTableStore::check_ready_for_read(const ObTablet &tablet) if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret), KPC(this)); - } else if (is_major_sstable_empty(tablet)) { + } else if (is_major_sstable_empty(param.ddl_commit_scn_)) { LOG_INFO("no valid major sstable, not ready for read", K(*this)); } else if (OB_FAIL(check_continuous())) { LOG_WARN("failed to check continuous of tables", K(ret)); } else if (minor_tables_.count() + 1 > MAX_SSTABLE_CNT_IN_STORAGE) { ret = OB_SIZE_OVERFLOW; - LOG_WARN("Too Many sstables in table store", K(ret), KPC(this), K(tablet)); - compaction::ObPartitionMergePolicy::diagnose_table_count_unsafe( - compaction::MAJOR_MERGE, ObDiagnoseTabletType::TYPE_SPECIAL, tablet); - MTL(concurrency_control::ObMultiVersionGarbageCollector *)->report_sstable_overflow(); + LOG_WARN("Too Many sstables in table store", K(ret), KPC(this)); } else if (get_table_count() > ObTabletTableStore::MAX_SSTABLE_CNT) { ret = OB_SIZE_OVERFLOW; - LOG_WARN("Too Many sstables, cannot add another sstable any more", K(ret), KPC(this), K(tablet)); - compaction::ObPartitionMergePolicy::diagnose_table_count_unsafe(compaction::MAJOR_MERGE, ObDiagnoseTabletType::TYPE_SPECIAL, tablet); - MTL(concurrency_control::ObMultiVersionGarbageCollector *)->report_sstable_overflow(); + LOG_WARN("Too Many sstables, cannot add another sstable any more", K(ret), KPC(this)); } else if (minor_tables_.empty()) { is_ready_for_read_ = true; } else { - const SCN &clog_checkpoint_scn = tablet.get_clog_checkpoint_scn(); + const SCN &clog_checkpoint_scn = param.clog_checkpoint_scn_; const SCN &last_minor_end_scn = minor_tables_.get_boundary_table(true/*last*/)->get_end_scn(); if (OB_UNLIKELY(clog_checkpoint_scn != last_minor_end_scn)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("last minor table's end_scn must be equal to clog_checkpoint_scn", - K(ret), K(last_minor_end_scn), K(clog_checkpoint_scn), KPC(this), K(tablet)); + K(ret), K(last_minor_end_scn), K(clog_checkpoint_scn), KPC(this)); } else { is_ready_for_read_ = true; } diff --git a/src/storage/tablet/ob_tablet_table_store.h b/src/storage/tablet/ob_tablet_table_store.h index 9fdf39be26..fe418124d5 100644 --- a/src/storage/tablet/ob_tablet_table_store.h +++ b/src/storage/tablet/ob_tablet_table_store.h @@ -30,6 +30,42 @@ class ObTableStoreIterator; class ObCachedTableHandle; class ObStorageMetaHandle; +class ObReadyForReadParam final +{ +// if you want to add a member variable, please add here. +#define LIST_MEMBERS \ + DO_SOMETHING(share::SCN, ddl_commit_scn_) \ + DO_SOMETHING(share::SCN, clog_checkpoint_scn_) +public: + ObReadyForReadParam() = default; + ~ObReadyForReadParam() = default; + int assign(const ObReadyForReadParam &other) + { + int ret = OB_SUCCESS; +#define DO_SOMETHING(type, name) name = other.name; + LIST_MEMBERS +#undef DO_SOMETHING + return ret; + } + bool operator==(const ObReadyForReadParam &other) const + { + bool is_equal = false; + // check for equality; +#define DO_SOMETHING(type, name) is_equal = is_equal && (name == other.name); + LIST_MEMBERS +#undef DO_SOMETHING + return is_equal; + } + bool operator!=(const ObReadyForReadParam &other) const { return !(*this == other); } +public: +// define member variables +#define DO_SOMETHING(type, name) type name; + LIST_MEMBERS +#undef DO_SOMETHING +private: + DISALLOW_COPY_AND_ASSIGN(ObReadyForReadParam); +}; + class ObTabletTableStore : public ObIStorageMetaObj { public: @@ -145,6 +181,8 @@ public: blocksstable::ObSSTableMetaHandle &meta_handle, blocksstable::ObSSTable *sstable, int64_t &remain_size); + int check_ready_for_read(const ObReadyForReadParam ¶m); + static void diagnose_table_count_unsafe(const ObTablet &tablet); int64_t to_string(char *buf, const int64_t buf_len) const; // Load sstable with @addr, loaded object lifetime guaranteed by @handle @@ -302,7 +340,7 @@ private: const blocksstable::ObSSTable *new_sstable, const bool keep_old_ddl_sstable, const ObTabletTableStore &old_store); - bool is_major_sstable_empty(const ObTablet &tablet) const; + bool is_major_sstable_empty(const share::SCN &ddl_commit_scn) const; int get_ddl_major_sstables(ObIArray &ddl_major_sstables) const; int inner_replace_sstables( diff --git a/src/storage/tenant_snapshot/ob_tenant_snapshot_service.cpp b/src/storage/tenant_snapshot/ob_tenant_snapshot_service.cpp index 14385398c4..47ea77e7da 100644 --- a/src/storage/tenant_snapshot/ob_tenant_snapshot_service.cpp +++ b/src/storage/tenant_snapshot/ob_tenant_snapshot_service.cpp @@ -144,8 +144,7 @@ int ObTenantSnapshotService::wait_() int ret = OB_SUCCESS; if (IS_NOT_INIT) { - ret = OB_NOT_INIT; - LOG_WARN("ObTenantSnapshotService not inited", KR(ret), KPC(this)); + LOG_INFO("ObTenantSnapshotService not inited", KPC(this)); } else if (ATOMIC_LOAD(&is_running_)) { ret = OB_ERR_UNEXPECTED; LOG_ERROR("ObTenantSnapshotService is running when wait function is called", KR(ret), KPC(this)); diff --git a/src/storage/tx/ob_ctx_tx_data.cpp b/src/storage/tx/ob_ctx_tx_data.cpp index 6761ee6156..962b054139 100644 --- a/src/storage/tx/ob_ctx_tx_data.cpp +++ b/src/storage/tx/ob_ctx_tx_data.cpp @@ -69,6 +69,7 @@ void ObCtxTxData::reset() tx_data_guard_.reset(); read_only_ = false; recovered_from_tx_table_ = false; + max_replayed_rollback_scn_.set_min(); } void ObCtxTxData::destroy() diff --git a/src/storage/tx/ob_ctx_tx_data.h b/src/storage/tx/ob_ctx_tx_data.h index d3318795e6..89b7c437b4 100644 --- a/src/storage/tx/ob_ctx_tx_data.h +++ b/src/storage/tx/ob_ctx_tx_data.h @@ -38,6 +38,8 @@ public: bool is_read_only() const { return read_only_; } bool has_recovered_from_tx_table() const { return recovered_from_tx_table_; } + share::SCN get_max_replayed_rollback_scn() const { return max_replayed_rollback_scn_; } + void set_max_replayed_rollback_scn(const share::SCN &scn) { max_replayed_rollback_scn_ = scn; } int insert_into_tx_table(); int recover_tx_data(storage::ObTxData *tmp_tx_data); int deep_copy_tx_data_out(storage::ObTxDataGuard &tmp_tx_data_guard); @@ -109,6 +111,17 @@ private: storage::ObTxDataGuard tx_data_guard_; bool read_only_; bool recovered_from_tx_table_; + // record the max replayed rollback to end_scn + // used in replay of RollbackToLog + // when replay multiple RollbackToLog in parallell, tx-data inserted into + // tx-data-table with end_scn out of order, in order to ensure the invariant + // of tx-data with larger end_scn contains the tx-data with smaller end_scn + // we rewrite the tx-data by delete and insert the tx-data with same end_scn + // + // this is a temporary solution for the problem, in the comming refine names as + // `shared contents of tx-data`, which can ensure the tx-data has been inserted + // into tx-data memtable was refresh with the latest content replayed out. + share::SCN max_replayed_rollback_scn_; // lock for tx_data_ pointer RWLock lock_; }; diff --git a/src/storage/tx/ob_dup_table_base.cpp b/src/storage/tx/ob_dup_table_base.cpp index f16de335b3..7ab02985c6 100644 --- a/src/storage/tx/ob_dup_table_base.cpp +++ b/src/storage/tx/ob_dup_table_base.cpp @@ -1204,6 +1204,150 @@ int ObDupTableRpc::init(rpc::frame::ObReqTransport *req_transport, return ret; } + +int ObTxRedoSyncRetryTask::ObTxRedoSyncIterFunc::operator()( + common::hash::HashSetTypes::pair_type &redo_sync_hash_pair) +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + + if (ls_handle_.is_valid()) { + if (redo_sync_hash_pair.first.ls_id_ != ls_handle_.get_ls()->get_ls_id()) { + ls_handle_.reset(); + } + } + + // tmp_ret = OB_SUCCESS; + ObPartTransCtx *tx_ctx = nullptr; + + if (!ls_handle_.is_valid() && OB_SUCC(ret) && OB_SUCCESS == tmp_ret) { + if (OB_TMP_FAIL( + MTL(ObLSService *) + ->get_ls(redo_sync_hash_pair.first.ls_id_, ls_handle_, ObLSGetMod::TRANS_MOD))) { + TRANS_LOG(WARN, "get ls failed", K(ret), K(redo_sync_hash_pair.first)); + } + } + + if (OB_SUCC(ret) && OB_SUCCESS == tmp_ret) { + if (OB_TMP_FAIL(ls_handle_.get_ls()->get_tx_ctx(redo_sync_hash_pair.first.tx_id_, + false /*for_replay*/, tx_ctx))) { + TRANS_LOG(WARN, "get tx ctx failed", K(ret), K(tmp_ret), K(redo_sync_hash_pair.first), + K(ls_handle_), KP(tx_ctx)); + } else if (OB_TMP_FAIL(tx_ctx->dup_table_tx_redo_sync(false/*need_retry_by_task*/))) { + TRANS_LOG(WARN, "dup table redo sync failed", K(ret), K(tmp_ret), + K(redo_sync_hash_pair.first), K(ls_handle_), KP(tx_ctx)); + if (tmp_ret == OB_EAGAIN) { + tmp_ret = OB_SUCCESS; + } + } + + if (OB_NOT_NULL(tx_ctx)) { + (void)ls_handle_.get_ls()->revert_tx_ctx(tx_ctx); + } + } + + TRANS_LOG(INFO, "iter tx redo sync by task", K(redo_sync_hash_pair.first), K(tmp_ret), K(ret), + KP(this)); + + if (OB_SUCCESS != tmp_ret) { + int tmp_ret2 = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret2 = del_list_.push_back(redo_sync_hash_pair.first))) { + TRANS_LOG(WARN, "push back into del_list failed", K(ret), K(tmp_ret), K(tmp_ret2), + K(redo_sync_hash_pair.first)); + } + } + + return ret; +} + +void ObTxRedoSyncRetryTask::ObTxRedoSyncIterFunc::remove_unused_redo_sync_key() +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + RedoSyncKey tmp_key; + if (del_list_.size() > 0) { + while (OB_SUCC(del_list_.pop_front(tmp_key))) { + if (OB_TMP_FAIL(redo_sync_retry_set_.erase_refactored(tmp_key))) { + TRANS_LOG(WARN, "erase from hash map failed", K(ret), K(tmp_ret),K(tmp_key)); + } else { + // DUP_TABLE_LOG(INFO, "erase from hash map succ", K(ret), K(tmp_key)); + } + } + if (OB_ENTRY_NOT_EXIST == ret) { + // when pop all list, rewrite ret code + TRANS_LOG(DEBUG, "end del in while loop", K(ret)); + ret = OB_SUCCESS; + } + } +} + +int ObTxRedoSyncRetryTask::iter_tx_retry_redo_sync() +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + share::ObLSID last_ls_id; + last_ls_id.reset(); + ObLSHandle ls_handle; + TransModulePageAllocator allocator; + + ObTxRedoSyncIterFunc redo_sync_func(redo_sync_retry_set_, allocator); + + if (OB_FAIL(redo_sync_retry_set_.foreach_refactored(redo_sync_func))) { + TRANS_LOG(WARN, "retry to start tx redo sync failed", K(ret), K(redo_sync_retry_set_.size())); + } + + redo_sync_func.remove_unused_redo_sync_key(); + + if (OB_SUCC(ret) && !redo_sync_retry_set_.empty() && ATOMIC_BCAS(&in_thread_pool_, false, true)) { + set_retry_interval_us(5 * 1000, 5 * 1000); + if (OB_TMP_FAIL(MTL(ObTransService *)->push(this))) { + ATOMIC_BCAS(&in_thread_pool_, true, false); + TRANS_LOG(WARN, "push redo sync task failed", K(ret), K(in_thread_pool_)); + } + } + + return ret; +} +int ObTxRedoSyncRetryTask::push_back_redo_sync_object(ObTransID tx_id, share::ObLSID ls_id) +{ + int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; + + if (!redo_sync_retry_set_.created()) { + if (OB_FAIL(redo_sync_retry_set_.create(100))) { + TRANS_LOG(WARN, "alloc a redo sync set failed", K(ret), K(tx_id), K(ls_id)); + } + } + + if (OB_SUCC(ret)) { + RedoSyncKey redo_sync_key; + redo_sync_key.tx_id_ = tx_id; + redo_sync_key.ls_id_ = ls_id; + if (OB_FAIL(redo_sync_retry_set_.set_refactored(redo_sync_key, 0))) { + if (ret == OB_HASH_EXIST) { + ret = OB_SUCCESS; + } else { + TRANS_LOG(WARN, "insert redo_sync_set into hash_set failed", K(ret), K(tx_id), K(ls_id)); + } + } + } + + if (OB_SUCC(ret) && !redo_sync_retry_set_.empty() && ATOMIC_BCAS(&in_thread_pool_, false, true)) { + set_retry_interval_us(5 * 1000, 5 * 1000); + if (OB_TMP_FAIL(MTL(ObTransService *)->push(this))) { + ATOMIC_BCAS(&in_thread_pool_, true, false); + TRANS_LOG(WARN, "push redo sync task failed", K(ret), K(tx_id), K(ls_id), K(in_thread_pool_)); + } + } + + TRANS_LOG(INFO, "push redo sync task succ", K(ret), KP(this), K(tx_id), K(ls_id), + K(in_thread_pool_)); + + return ret; +} + + + } // namespace transaction namespace obrpc diff --git a/src/storage/tx/ob_dup_table_base.h b/src/storage/tx/ob_dup_table_base.h index 871fecbd06..3dd4958ea7 100644 --- a/src/storage/tx/ob_dup_table_base.h +++ b/src/storage/tx/ob_dup_table_base.h @@ -1369,6 +1369,70 @@ int ObDupTableRpc::post_msg(const common::ObAddr dst, DupTableMsgType &msg) return ret; } +class ObTxRedoSyncRetryTask : public ObTransTask +{ +public: + struct RedoSyncKey + { + share::ObLSID ls_id_; + ObTransID tx_id_; + + int hash(uint64_t& res) const { res = ls_id_.hash() + tx_id_.hash(); return OB_SUCCESS; } + uint64_t hash() const { return ls_id_.hash() + tx_id_.hash(); } + bool operator==(const RedoSyncKey &other) const + { + return ls_id_ == other.ls_id_ && tx_id_ == other.tx_id_; + } + TO_STRING_KV(K(ls_id_), K(tx_id_)); + }; + + typedef common::hash::ObHashSet + RedoSyncRetrySet; + +class ObTxRedoSyncIterFunc +{ +public: + ObTxRedoSyncIterFunc(RedoSyncRetrySet &redo_sync_set, TransModulePageAllocator &trans_page_alloc) + : del_list_(trans_page_alloc), ls_handle_(), redo_sync_retry_set_(redo_sync_set) + {} + int operator()(common::hash::HashSetTypes::pair_type &redo_sync_hash_pair); + + void remove_unused_redo_sync_key(); + + void reset() + { + del_list_.destroy(); + ls_handle_.reset(); + } + +private: + ObList del_list_; + ObLSHandle ls_handle_; + RedoSyncRetrySet &redo_sync_retry_set_; +}; + +public: + ObTxRedoSyncRetryTask() : ObTransTask(ObTransRetryTaskType::DUP_TABLE_TX_REDO_SYNC_RETRY_TASK) + { + reset(); + } + ~ObTxRedoSyncRetryTask() { destroy(); } + void reset() + { + redo_sync_retry_set_.destroy(); + in_thread_pool_ = false; + } + void destroy() { reset(); } + int init(); + int iter_tx_retry_redo_sync(); + int push_back_redo_sync_object(ObTransID tx_id, share::ObLSID ls_id); + void clear_in_thread_pool_flag() { ATOMIC_STORE(&in_thread_pool_, false);} + +public: + RedoSyncRetrySet redo_sync_retry_set_; + bool in_thread_pool_; +}; + } // namespace transaction } // namespace oceanbase diff --git a/src/storage/tx/ob_trans_ctx_mgr_v4.cpp b/src/storage/tx/ob_trans_ctx_mgr_v4.cpp index f978a3f53f..2bdca9da3e 100644 --- a/src/storage/tx/ob_trans_ctx_mgr_v4.cpp +++ b/src/storage/tx/ob_trans_ctx_mgr_v4.cpp @@ -251,7 +251,7 @@ int ObLSTxCtxMgr::process_callback_(ObTxCommitCallback *&cb_list) const int ret = OB_SUCCESS; ObTxCommitCallback *next = NULL; for (ObTxCommitCallback *iter = cb_list; iter != NULL; iter = next) { - ObTxCommitCallback *next = iter->get_link_next(); + next = iter->get_link_next(); iter->callback(); } return ret; diff --git a/src/storage/tx/ob_trans_define.h b/src/storage/tx/ob_trans_define.h index 0608387f52..48e6b02737 100644 --- a/src/storage/tx/ob_trans_define.h +++ b/src/storage/tx/ob_trans_define.h @@ -1212,7 +1212,8 @@ public: static const int64_t END_TRANS_CB_TASK = 0; static const int64_t ADVANCE_LS_CKPT_TASK = 1; static const int64_t STANDBY_CLEANUP_TASK = 2; - static const int64_t MAX = 3; + static const int64_t DUP_TABLE_TX_REDO_SYNC_RETRY_TASK = 3; + static const int64_t MAX = 4; public: static bool is_valid(const int64_t task_type) { return task_type > UNKNOWN && task_type < MAX; } diff --git a/src/storage/tx/ob_trans_define_v4.h b/src/storage/tx/ob_trans_define_v4.h index 4534480583..6d8930fbee 100644 --- a/src/storage/tx/ob_trans_define_v4.h +++ b/src/storage/tx/ob_trans_define_v4.h @@ -487,6 +487,7 @@ protected: struct COMPAT_FOR_EXEC { uint64_t v_; uint64_t get_serialize_v_() const; + TO_STRING_KV(K_(v)); NEED_SERIALIZE_AND_DESERIALIZE; } compat_for_exec_; struct @@ -784,7 +785,7 @@ LST_DO(DEF_FREE_ROUTE_DECODE, (;), static, dynamic, parts, extra); int64_t estimate_state_size(); bool is_static_changed() { return state_change_flags_.STATIC_CHANGED_; } bool is_dynamic_changed() { return state_ > State::IDLE && state_change_flags_.DYNAMIC_CHANGED_; } - bool is_parts_changed() { return state_change_flags_.PARTS_CHANGED_; }; + bool is_parts_changed() { return state_ > State::IDLE && state_change_flags_.PARTS_CHANGED_; }; bool is_extra_changed() { return state_change_flags_.EXTRA_CHANGED_; }; void set_explicit() { flags_.EXPLICIT_ = true; } void clear_interrupt() { flags_.INTERRUPTED_ = false; } diff --git a/src/storage/tx/ob_trans_part_ctx.cpp b/src/storage/tx/ob_trans_part_ctx.cpp index bdc4914f54..2d9393d825 100644 --- a/src/storage/tx/ob_trans_part_ctx.cpp +++ b/src/storage/tx/ob_trans_part_ctx.cpp @@ -581,9 +581,9 @@ int ObPartTransCtx::handle_timeout(const int64_t delay) if (exec_info_.is_dup_tx_) { if (!is_sub2pc() && ObTxState::REDO_COMPLETE == exec_info_.state_) { if (OB_SUCCESS != (tmp_ret = dup_table_tx_redo_sync_())) { - TRANS_LOG(WARN, "dup table tx redo sync error", K(tmp_ret)); - } - } else if (ObTxState::PRE_COMMIT == exec_info_.state_) { + TRANS_LOG(WARN, "dup table tx redo sync error", K(tmp_ret)); + } + } else if (ObTxState::PRE_COMMIT == exec_info_.state_) { if (OB_SUCCESS != (tmp_ret = dup_table_tx_pre_commit_())) { TRANS_LOG(WARN, "dup table tx pre commit error", K(tmp_ret)); } @@ -864,12 +864,26 @@ int ObPartTransCtx::commit(const ObTxCommitParts &parts, if (OB_FAIL(one_phase_commit_())) { TRANS_LOG(WARN, "start sp coimit fail", K(ret), KPC(this)); } + + if ((OB_SUCC(ret) || OB_EAGAIN == ret) && exec_info_.is_dup_tx_) { + set_2pc_upstream_(ls_id_); + if (OB_FAIL(two_phase_commit())) { + TRANS_LOG(WARN, "start dist commit for dup table fail", K(ret), KPC(this)); + if (OB_EAGAIN == ret) { + ret = OB_SUCCESS; + } + } + } + } else { exec_info_.trans_type_ = TransType::DIST_TRANS; // set 2pc upstream to self set_2pc_upstream_(ls_id_); if (OB_FAIL(two_phase_commit())) { - TRANS_LOG(WARN, "start dist coimit fail", K(ret), KPC(this)); + TRANS_LOG(WARN, "start dist commit fail", K(ret), KPC(this)); + if (OB_EAGAIN == ret) { + ret = OB_SUCCESS; + } } } } @@ -5023,6 +5037,7 @@ int ObPartTransCtx::replay_commit_info(const ObTxCommitInfoLog &commit_info_log, const bool pre_barrier) { int ret = OB_SUCCESS; + int tmp_ret = OB_SUCCESS; common::ObTimeGuard timeguard("replay_commit_info", 10 * 1000); // const int64_t start = ObTimeUtility::fast_current_time(); bool need_replay = true; @@ -5103,6 +5118,11 @@ int ObPartTransCtx::replay_commit_info(const ObTxCommitInfoLog &commit_info_log, if (OB_FAIL(dup_table_before_preapre_(timestamp))) { TRANS_LOG(WARN, "set commit_info scn as before_prepare_version failed", K(ret), KPC(this)); } + + // if (OB_TMP_FAIL(post_redo_sync_ts_response_(timestamp))) { + // TRANS_LOG(WARN, "post dup table redo sync response failed", K(ret), K(tmp_ret), K(timestamp), + // KPC(this)); + // } } if (OB_SUCC(ret)) { @@ -7043,7 +7063,7 @@ int ObPartTransCtx::search_unsubmitted_dup_table_redo_() // return mt_ctx_.get_redo_generator().search_unsubmitted_dup_tablet_redo(); } -int ObPartTransCtx::dup_table_tx_redo_sync_() +int ObPartTransCtx::dup_table_tx_redo_sync_(const bool need_retry_by_task) { int ret = OB_SUCCESS; int tmp_ret = OB_SUCCESS; @@ -7066,36 +7086,44 @@ int ObPartTransCtx::dup_table_tx_redo_sync_() TRANS_LOG(WARN, "can not execute redo sync on a follower", KPC(this)); } else if (OB_FAIL(errism_dup_table_redo_sync_())) { TRANS_LOG(WARN, "errsim for dup table redo sync", K(ret), KPC(this)); - } else if (is_dup_table_redo_sync_completed_()) { - ret = OB_SUCCESS; - redo_sync_finish = true; - bool no_need_submit_log = false; - if (OB_TMP_FAIL(drive_self_2pc_phase(ObTxState::PREPARE))) { - TRANS_LOG(WARN, "do prepare failed after redo sync", K(tmp_ret), KPC(this)); - } else { - } - } else if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->check_redo_sync_completed( - trans_id_, exec_info_.max_applied_log_ts_, redo_sync_finish, - tmp_max_read_version))) { - TRANS_LOG(WARN, "check redo sync completed failed", K(ret), K(redo_sync_finish), - K(tmp_max_read_version), KPC(this)); - } else if (redo_sync_finish) { - if (!tmp_max_read_version.is_valid()) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "invalid dup table follower's max_read_version", K(ret), + } else { + if (is_dup_table_redo_sync_completed_()) { + ret = OB_SUCCESS; + redo_sync_finish = true; + bool no_need_submit_log = false; + if (OB_TMP_FAIL(drive_self_2pc_phase(ObTxState::PREPARE))) { + TRANS_LOG(WARN, "do prepare failed after redo sync", K(tmp_ret), KPC(this)); + } + } else if (OB_FAIL(ls_tx_ctx_mgr_->get_ls_log_adapter()->check_redo_sync_completed( + trans_id_, exec_info_.max_applied_log_ts_, redo_sync_finish, + tmp_max_read_version))) { + TRANS_LOG(WARN, "check redo sync completed failed", K(ret), K(redo_sync_finish), K(tmp_max_read_version), KPC(this)); + } else if (redo_sync_finish) { + if (!tmp_max_read_version.is_valid()) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "invalid dup table follower's max_read_version", K(ret), + K(tmp_max_read_version), KPC(this)); + } else { + dup_table_follower_max_read_version_ = tmp_max_read_version; + /* + * drive into prepare state in the next do_prepare operation + * */ + TRANS_LOG(INFO, "finish redo sync for dup table trx", K(ret), K(redo_sync_finish), + K(dup_table_follower_max_read_version_), KPC(this)); + + if (OB_TMP_FAIL(drive_self_2pc_phase(ObTxState::PREPARE))) { + TRANS_LOG(WARN, "do prepare failed after redo sync", K(tmp_ret), KPC(this)); + } + } } else { - dup_table_follower_max_read_version_ = tmp_max_read_version; - /* - * drive into prepare state in the next do_prepare operation - * */ - TRANS_LOG(INFO, "finish redo sync for dup table trx", K(ret), K(redo_sync_finish), + if (need_retry_by_task) { + (void)MTL(ObTransService *)->retry_redo_sync_by_task(get_trans_id(), get_ls_id()); + } + ret = OB_EAGAIN; + TRANS_LOG(INFO, "redo sync will retry", K(ret), K(redo_sync_finish), K(tmp_max_read_version), K(dup_table_follower_max_read_version_), KPC(this)); } - } else { - ret = OB_EAGAIN; - TRANS_LOG(INFO, "redo sync need retry", K(ret), K(redo_sync_finish), K(tmp_max_read_version), - K(dup_table_follower_max_read_version_), KPC(this)); } return ret; @@ -7327,6 +7355,44 @@ int ObPartTransCtx::retry_dup_trx_before_prepare(const share::SCN &before_prepar return ret; } +int ObPartTransCtx::dup_table_tx_redo_sync(const bool need_retry_by_task) +{ + int ret = OB_SUCCESS; + + if (OB_SUCCESS == lock_.try_lock()) { + CtxLockGuard guard(lock_, false); + + if (!is_leader_()) { + ret = OB_NOT_MASTER; + TRANS_LOG(WARN, "redo sync need not execute on a follower", K(ret), KPC(this)); + } else if (is_exiting_) { + ret = OB_TRANS_IS_EXITING; + TRANS_LOG(WARN, "redo sync need not execute on a exiting tx_ctx", K(ret), KPC(this)); + } else if (get_downstream_state() >= ObTxState::PREPARE) { + ret = OB_STATE_NOT_MATCH; + TRANS_LOG(WARN, "redo sync need not execute on a prepared tx_ctx", K(ret), KPC(this)); + } else { + if (OB_FAIL(dup_table_tx_redo_sync_(need_retry_by_task))) { + TRANS_LOG(WARN, "execute dup table redo sync failed", K(ret), KPC(this)); + } + + if (OB_SUCC(ret)) { + if (!is_dup_table_redo_sync_completed_()) { + ret = OB_EAGAIN; + // TRANS_LOG(INFO, "redo sync need retry", K(ret), KPC(this)); + } else { + ret = OB_TRANS_HAS_DECIDED; + } + } + } + } else { + TRANS_LOG(INFO, "The redo sync task can not acquire ctx lock. Waiting the next retry.", K(ret), + K(get_trans_id()), K(get_ls_id())); + ret = OB_EAGAIN; + } + + return ret; +} // int ObPartTransCtx::merge_tablet_modify_record(const common::ObTabletID &tablet_id) // { // int ret = OB_SUCCESS; @@ -7636,9 +7702,11 @@ int ObPartTransCtx::start_access(const ObTxDesc &tx_desc, ObTxSEQ &data_scn, con } last_request_ts_ = ObClockGenerator::getClock(); TRANS_LOG(TRACE, "start_access", K(ret), K(data_scn.support_branch()), K(data_scn), KPC(this)); + common::ObTraceIdAdaptor trace_id; + trace_id.set(ObCurTraceId::get()); REC_TRANS_TRACE_EXT(tlog_, start_access, OB_ID(ret), ret, - OB_ID(trace_id), *ObCurTraceId::get(), + OB_ID(trace_id), trace_id, OB_ID(opid), tx_desc.op_sn_, OB_ID(data_seq), data_scn.cast_to_int(), OB_ID(pending), pending_write, @@ -7766,14 +7834,17 @@ int ObPartTransCtx::rollback_to_savepoint_(const ObTxSEQ from_scn, /* * Follower: * 1. add UndoAction into tx_ctx's tx_data - * 2. insert UndoAction into tx_data_table + * 2. insert tx-data into tx_data_table * Leader: * 1. submit 'RollbackToLog' * 2. add UndoAction into tx_ctx's tx_data - * 3. insert UndoAction into tx_data_table after log sync success + * 3. insert tx-data into tx_data_table after log sync success */ + bool need_update_tx_data = false; ObTxDataGuard tmp_tx_data_guard; + ObTxDataGuard update_tx_data_guard; tmp_tx_data_guard.reset(); + update_tx_data_guard.reset(); if (is_follower_()) { /* Follower */ ObUndoAction undo_action(from_scn, to_scn); // _NOTICE_ must load Undo(s) from TxDataTable before overwriten @@ -7785,9 +7856,62 @@ int ObPartTransCtx::rollback_to_savepoint_(const ObTxSEQ from_scn, TRANS_LOG(WARN, "recrod undo info fail", K(ret), K(from_scn), K(to_scn), KPC(this)); } else if (OB_FAIL(ctx_tx_data_.deep_copy_tx_data_out(tmp_tx_data_guard))) { TRANS_LOG(WARN, "deep copy tx data failed", KR(ret), K(*this)); - } else if (FALSE_IT(tmp_tx_data_guard.tx_data()->end_scn_ = share::SCN::max(replay_scn, exec_info_.max_applying_log_ts_))) { - } else if (OB_FAIL(ctx_tx_data_.insert_tmp_tx_data(tmp_tx_data_guard.tx_data()))) { - TRANS_LOG(WARN, "insert to tx table failed", KR(ret), K(*this)); + } + + // + // when multiple branch-level savepoints were replayed out of order, to ensure + // tx-data with larger end_scn include all undo-actions of others before + // + // we do deleting in frozen memtable and updating (which with largest end_scn) in active memtable + // because distinguish frozen/active memtable is not easy, just always do those two actions. + // + // following is an illusion of this strategy: + // + // assume rollback to logs with scn of: 80 90 110 + // + // and frozen scn is 100 + // + // case 1: replay order: 110, 90, 80 + // case 2: replay order: 110, 80, 90 + // case 3: replay order: 90, 110, 80 + // case 4: replay order: 90, 80, 110 + // + // the operations of each case: + // case 1: insert 110 -> [insert 90, update 110] -> [insert 80, delete 90, udpate 110] + // case 2: insert 110 -> [insert 80 update 110] -> [insert 90, delete 80, update 110] + // case 3: insert 90 -> insert 110 -> [insert 80, delete 90, update 110] + // case 4: insert 90 -> [insert 80, update 90] -> insert 100 + // + + if (OB_SUCC(ret)) { + need_update_tx_data = ctx_tx_data_.get_max_replayed_rollback_scn() > replay_scn; + if (need_update_tx_data && OB_FAIL(ctx_tx_data_.deep_copy_tx_data_out(update_tx_data_guard))) { + TRANS_LOG(WARN, "deep copy tx data failed", KR(ret), K(*this)); + } + } + // prepare end_scn for tx-data items + if (OB_SUCC(ret)) { + tmp_tx_data_guard.tx_data()->end_scn_ = replay_scn; + if (need_update_tx_data) { + // if the tx-data will be inserted into frozen tx-data-memtable, and it may be not the one with largest end_scn + // we must delete others in order to ensure ourself is the valid one with largest end_scn + tmp_tx_data_guard.tx_data()->exclusive_flag_ = ObTxData::ExclusiveType::EXCLUSIVE; + // for update tx-data, use the same end_scn_ + update_tx_data_guard.tx_data()->end_scn_ = ctx_tx_data_.get_max_replayed_rollback_scn(); + update_tx_data_guard.tx_data()->exclusive_flag_ = ObTxData::ExclusiveType::EXCLUSIVE; + } + } + // prepare done, do the final step to insert tx-data-table, this should not fail + if (OB_SUCC(ret)) { + if (OB_FAIL(ctx_tx_data_.insert_tmp_tx_data(tmp_tx_data_guard.tx_data()))) { + TRANS_LOG(WARN, "insert to tx table failed", KR(ret), K(*this)); + } else if (need_update_tx_data && OB_FAIL(ctx_tx_data_.insert_tmp_tx_data(update_tx_data_guard.tx_data()))) { + TRANS_LOG(WARN, "insert to tx table failed", KR(ret), K(*this)); + } + } + // if this is the largest scn replayed, remember it + if (OB_SUCC(ret) && !need_update_tx_data) { + ctx_tx_data_.set_max_replayed_rollback_scn(replay_scn); } } else if (OB_UNLIKELY(exec_info_.max_submitted_seq_no_ > to_scn)) { /* Leader */ ObUndoAction undo_action(from_scn, to_scn); diff --git a/src/storage/tx/ob_trans_part_ctx.h b/src/storage/tx/ob_trans_part_ctx.h index b5939c64d2..34427b3331 100644 --- a/src/storage/tx/ob_trans_part_ctx.h +++ b/src/storage/tx/ob_trans_part_ctx.h @@ -459,6 +459,7 @@ public: const bool try_lock, const ObRegisterMdsFlag ®ister_flag); + int dup_table_tx_redo_sync(const bool need_retry_by_task); const share::SCN get_start_log_ts() { return ctx_tx_data_.get_start_log_ts(); @@ -548,7 +549,7 @@ private: int fix_redo_lsns_(const ObTxLogCb *log_cb); int search_unsubmitted_dup_table_redo_() __attribute__((__noinline__)); - int dup_table_tx_redo_sync_(); + int dup_table_tx_redo_sync_(const bool need_retry_by_task = true); int check_dup_trx_with_submitting_all_redo(ObTxLogBlock &log_block, memtable::ObRedoLogSubmitHelper &helper); bool is_dup_table_redo_sync_completed_(); diff --git a/src/storage/tx/ob_trans_service.cpp b/src/storage/tx/ob_trans_service.cpp index 03c502b855..af60de9954 100644 --- a/src/storage/tx/ob_trans_service.cpp +++ b/src/storage/tx/ob_trans_service.cpp @@ -531,6 +531,12 @@ void ObTransService::handle(void *task) mtl_free(standby_cleanup_task); standby_cleanup_task = nullptr; } + } else if(ObTransRetryTaskType::DUP_TABLE_TX_REDO_SYNC_RETRY_TASK == trans_task->get_task_type()) { + ObTxRedoSyncRetryTask *redo_sync_task = static_cast(trans_task); + redo_sync_task->clear_in_thread_pool_flag(); + if (OB_FAIL(redo_sync_task->iter_tx_retry_redo_sync())) { + TRANS_LOG(WARN, "execute redo sync task failed", K(ret)); + } } else { ret = OB_ERR_UNEXPECTED; TRANS_LOG(ERROR, "unexpected trans task type!!!", KR(ret), K(*trans_task)); @@ -649,6 +655,21 @@ int ObTransService::check_dup_table_lease_valid(const ObLSID ls_id, return ret; } +int ObTransService::retry_redo_sync_by_task(ObTransID tx_id, share::ObLSID ls_id) +{ + int ret = OB_SUCCESS; + const int64_t redo_sync_index = (ls_id.hash() + tx_id.hash()) % MAX_REDO_SYNC_TASK_COUNT; + + if (OB_FAIL(redo_sync_task_array_[redo_sync_index].push_back_redo_sync_object(tx_id, ls_id))) { + TRANS_LOG(WARN, "retry redo sync task failed", K(ret), K(tx_id), K(ls_id), K(redo_sync_index)); + } else { + TRANS_LOG(DEBUG, "start to retry redo sync task successfully", K(ret), K(tx_id), K(ls_id), + K(redo_sync_index)); + } + + return ret; +} + int ObTransService::handle_redo_sync_task_(ObDupTableRedoSyncTask *task, bool &need_release_task) { UNUSED(task); diff --git a/src/storage/tx/ob_trans_service.h b/src/storage/tx/ob_trans_service.h index 6354e0b634..4c60088a73 100644 --- a/src/storage/tx/ob_trans_service.h +++ b/src/storage/tx/ob_trans_service.h @@ -47,6 +47,8 @@ #include "ob_tx_free_route.h" #include "ob_tx_free_route_msg.h" +#define MAX_REDO_SYNC_TASK_COUNT 10 + namespace oceanbase { @@ -268,6 +270,8 @@ private: public: int check_dup_table_ls_readable(); int check_dup_table_tablet_readable(); + + int retry_redo_sync_by_task(ObTransID tx_id, share::ObLSID ls_id); public: int end_1pc_trans(ObTxDesc &trans_desc, ObITxCallback *endTransCb, @@ -333,6 +337,7 @@ private: ObDupTabletScanTask dup_tablet_scan_task_; ObDupTableLoopWorker dup_table_loop_worker_; ObDupTableRpc dup_table_rpc_impl_; + ObTxRedoSyncRetryTask redo_sync_task_array_[MAX_REDO_SYNC_TASK_COUNT]; obrpc::ObSrvRpcProxy *rpc_proxy_; ObTxELRUtil elr_util_; diff --git a/src/storage/tx/ob_trans_service_v4.cpp b/src/storage/tx/ob_trans_service_v4.cpp index b2fdf6d259..e286524f03 100644 --- a/src/storage/tx/ob_trans_service_v4.cpp +++ b/src/storage/tx/ob_trans_service_v4.cpp @@ -1732,6 +1732,10 @@ int ObTransService::sync_acquire_global_snapshot_(ObTxDesc &tx, [&]() -> bool { return tx.flags_.INTERRUPTED_; }); tx.lock_.lock(); bool interrupted = tx.flags_.INTERRUPTED_; + if (interrupted) { + ret = OB_ERR_INTERRUPTED; + TRANS_LOG(WARN, "acquiring global snapshot has been interrupted", KR(ret), K(tx)); + } tx.clear_interrupt(); tx.flags_.BLOCK_ = false; if (op_sn != tx.op_sn_) { @@ -1759,33 +1763,27 @@ int ObTransService::acquire_global_snapshot__(const int64_t expire_ts, int ret = OB_SUCCESS; const MonotonicTs now0 = get_req_receive_mts_(); const MonotonicTs now = now0 - MonotonicTs(gts_ahead); - int retry_times = 0; - const int MAX_RETRY_TIMES = 2000; // 2000 * 500us = 1s - do { - int64_t n = ObClockGenerator::getClock(); - MonotonicTs rts(0); - if (n >= expire_ts) { - ret = OB_TIMEOUT; - } else if (retry_times++ > MAX_RETRY_TIMES) { + const int64_t current_time = ObClockGenerator::getClock(); + // occupy current worker thread for at most 1s + const int64_t MAX_WAIT_TIME_US = 1 * 1000 * 1000; + MonotonicTs rts(0); + + if (interrupt_checker()) { + ret = OB_ERR_INTERRUPTED; + } else if (current_time >= expire_ts) { + ret = OB_TIMEOUT; + TRANS_LOG(WARN, "get gts timeout", K(ret), K(expire_ts), K(current_time)); + } else if (OB_FAIL(ts_mgr_->get_gts_sync(tenant_id_, now, MAX_WAIT_TIME_US, snapshot, rts))) { + TRANS_LOG(WARN, "get gts fail", K(ret), K(expire_ts), K(now)); + if (OB_TIMEOUT == ret) { ret = OB_GTS_NOT_READY; - TRANS_LOG(WARN, "gts not ready", K(ret), K(retry_times)); - } else if (OB_FAIL(ts_mgr_->get_gts(tenant_id_, now, NULL, snapshot, rts))) { - if (OB_EAGAIN == ret) { - if (interrupt_checker()) { - ret = OB_ERR_INTERRUPTED; - } else { - ob_usleep(500); - } - } else { - TRANS_LOG(WARN, "get gts fail", K(now)); - } - } else if (OB_UNLIKELY(!snapshot.is_valid())) { - ret = OB_ERR_UNEXPECTED; - TRANS_LOG(WARN, "invalid snapshot from gts", K(snapshot), K(now)); - } else { - uncertain_bound = rts.mts_ + gts_ahead; } - } while (OB_EAGAIN == ret); + } else if (OB_UNLIKELY(!snapshot.is_valid())) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "invalid snapshot from gts", K(snapshot), K(now)); + } else { + uncertain_bound = rts.mts_ + gts_ahead; + } if (OB_FAIL(ret)) { TRANS_LOG(WARN, "acquire global snapshot fail", K(ret), diff --git a/src/storage/tx/ob_ts_mgr.cpp b/src/storage/tx/ob_ts_mgr.cpp index 2dddb0de77..d6f06e44e3 100644 --- a/src/storage/tx/ob_ts_mgr.cpp +++ b/src/storage/tx/ob_ts_mgr.cpp @@ -107,6 +107,214 @@ ObTsSourceInfoGuard::~ObTsSourceInfoGuard() } } +int ObTsSyncGetTsCbTask::init(uint64_t task_id) +{ + int ret = OB_SUCCESS; + + if (is_inited_) { + ret = OB_INIT_TWICE; + TRANS_LOG(WARN, "ObTsSyncGetTsCbTask inited twice", KR(ret)); + } else if (OB_FAIL(cond_.init(ObWaitEventIds::SYNC_GET_GTS_WAIT))) { + TRANS_LOG(WARN, "ObTsSyncGetTsCbTask cond init failed", K(ret)); + } else { + task_id_ = task_id; + is_inited_ = true; + } + + return ret; +} + +int ObTsSyncGetTsCbTask::gts_callback_interrupted(const int errcode) +{ + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + TRANS_LOG(WARN, "not init", K(ret)); + } else { + ObThreadCondGuard cond_guard(cond_); + if (is_early_exit_) { + ObTsSyncGetTsCbTaskPool::get_instance().recycle_task(this); + } else { + errcode_ = errcode; + is_finished_ = true; + cond_.signal(); + } + } + + return ret; +} + +int ObTsSyncGetTsCbTask::get_gts_callback(const MonotonicTs srr, const share::SCN >s, + const MonotonicTs receive_gts_ts) +{ + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + TRANS_LOG(WARN, "not init", K(ret)); + } else if (srr < get_stc()) { + ret = OB_EAGAIN; + } else { + ObThreadCondGuard cond_guard(cond_); + if (is_early_exit_) { + ObTsSyncGetTsCbTaskPool::get_instance().recycle_task(this); + } else { + gts_result_ = gts; + is_finished_ = true; + cond_.signal(); + } + } + + return ret; +} + +int ObTsSyncGetTsCbTask::gts_elapse_callback(const MonotonicTs srr, const share::SCN >s) +{ + int ret = OB_NOT_SUPPORTED; + return ret; +} + +MonotonicTs ObTsSyncGetTsCbTask::get_stc() const +{ + return stc_; +} + +uint64_t ObTsSyncGetTsCbTask::hash() const +{ + return task_id_; +} + +uint64_t ObTsSyncGetTsCbTask::get_tenant_id() const +{ + return tenant_id_; +} + +int ObTsSyncGetTsCbTask::wait(const int64_t timeout_us, share::SCN &scn, bool &need_recycle_task) +{ + int ret = OB_SUCCESS; + bool need_recycle = true; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + TRANS_LOG(WARN, "not init", K(ret)); + } else { + ObThreadCondGuard cond_guard(cond_); + // wait the condition in multiple rounds, so we can check the interrupt status every round + if (!is_finished_) { + if (OB_FAIL(cond_.wait_us(timeout_us))) { + is_early_exit_ = true; + need_recycle = false; + TRANS_LOG(WARN, "ObTsSyncGetTsCbTask cond wait failed", K(ret)); + } + } + if (errcode_ != OB_SUCCESS) { + ret = errcode_; + TRANS_LOG(WARN, "ObTsSyncGetTsCbTask errcode", K(ret)); + } else { + scn = gts_result_; + } + } + + need_recycle_task = need_recycle; + return ret; +} + +int ObTsSyncGetTsCbTask::config(MonotonicTs stc, uint64_t tenant_id) { + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + TRANS_LOG(WARN, "not init", K(ret)); + } else { + is_finished_ = false; + is_early_exit_ = false; + gts_result_.reset(); + errcode_ = OB_SUCCESS; + stc_ = stc; + tenant_id_ = tenant_id; + } + + return ret; +} + +int ObTsSyncGetTsCbTaskPool::init() { + int ret = OB_SUCCESS; + + if (is_inited_) { + ret = OB_INIT_TWICE; + TRANS_LOG(WARN, "ObTsSyncGetTsCbTaskPool inited twice", KR(ret)); + } else { + for (uint64_t i = 0; i < POOL_SIZE; i++) { + if (OB_FAIL(tasks_[i].init(i))) { + TRANS_LOG(WARN, "ObTsSyncGetTsCbTaskPool init failed", KR(ret)); + break; + } + } + + if (OB_SUCC(ret)) { + is_inited_ = true; + } + } + + return ret; +} + +int ObTsSyncGetTsCbTaskPool::get_task(MonotonicTs stc, uint64_t tenant_id, + ObTsSyncGetTsCbTask *&task) { + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + TRANS_LOG(WARN, "not init", K(ret)); + } else { + // try to use thread_id to find a free cbtask + const int64_t thread_id = get_itid(); + int64_t index = thread_id % POOL_SIZE; + int iter_count = 0; + const int ITER_LIMIT = 8; + + while (iter_count < ITER_LIMIT) { + ObTsSyncGetTsCbTask *iter_task = &tasks_[index]; + if (ATOMIC_BCAS(&iter_task->is_occupied_, false, true)) { + break; + } + iter_count++; + index = (index + iter_count) % POOL_SIZE; + } + if (iter_count == ITER_LIMIT) { + ret = OB_EAGAIN; + TRANS_LOG(WARN, "ObTsSyncGetTsCbTaskPool failed to get task", K(ret), K(thread_id)); + } else { + task = &tasks_[index]; + if (OB_FAIL(task->config(stc, tenant_id))) { + TRANS_LOG(WARN, "failed to config ObTsSyncGetTsCbTask", K(ret), K(index)); + } + } + } + + return ret; +} + +int ObTsSyncGetTsCbTaskPool::recycle_task(ObTsSyncGetTsCbTask *task) { + int ret = OB_SUCCESS; + + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + TRANS_LOG(WARN, "not init", K(ret)); + } else if (OB_ISNULL(task)) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "ObTsSyncGetTsCbTask is NULL", KR(ret)); + } else { + if (!ATOMIC_BCAS(&task->is_occupied_, true, false)) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "ObTsSyncGetTsCbTask has been recycled", KR(ret)); + } + } + + return ret; +} + ////////////////////////ObTsMgr实现/////////////////////////////////// int ObTsMgr::init(const ObAddr &server, @@ -140,6 +348,8 @@ int ObTsMgr::init(const ObAddr &server, TRANS_LOG(WARN, "response rpc init failed", KR(ret), K(server)); } else if (OB_FAIL(lock_.init(lib::ObMemAttr(OB_SERVER_TENANT_ID, "TsMgr")))) { TRANS_LOG(WARN, "ObQSyncLock init failed", KR(ret), K(OB_SERVER_TENANT_ID)); + } else if (OB_FAIL(ObTsSyncGetTsCbTaskPool::get_instance().init())) { + TRANS_LOG(WARN, "ObTsSyncGetTsCbTaskPool init failed", KR(ret)); } else { server_ = server; location_adapter_ = &location_adapter_def_; @@ -685,6 +895,85 @@ int ObTsMgr::get_ts_sync(const uint64_t tenant_id, const int64_t timeout_us, sha return get_ts_sync(tenant_id, timeout_us, scn, unused_is_external_consistent); } +int ObTsMgr::get_gts_sync(const uint64_t tenant_id, + const MonotonicTs stc, + const int64_t timeout_us, + share::SCN &scn, + MonotonicTs &receive_gts_ts) +{ + int ret = OB_SUCCESS; + + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + TRANS_LOG(WARN, "ObTsMgr is not inited", K(ret)); + } else if (OB_UNLIKELY(!is_running_)) { + ret = OB_NOT_RUNNING; + TRANS_LOG(WARN, "ObTsMgr is not running", K(ret)); + } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id)) || OB_UNLIKELY(!stc.is_valid()) + || OB_UNLIKELY(timeout_us < 0)) { + ret = OB_INVALID_ARGUMENT; + TRANS_LOG(WARN, "invalid argument", K(ret), K(tenant_id), K(stc), K(timeout_us)); + } else { + ObTsSourceInfo *ts_source_info = NULL; + ObGtsSource *ts_source = NULL; + ObTsSourceInfoGuard info_guard; + ObTsSyncGetTsCbTask *task = NULL; + int64_t gts_result = 0; + bool fall_back_to_sleep = false; + if (OB_FAIL(get_ts_source_info_opt_(tenant_id, info_guard, true, true))) { + TRANS_LOG(WARN, "get ts source info failed", K(ret), K(tenant_id)); + } else if (OB_ISNULL(ts_source_info = info_guard.get_ts_source_info())) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "ts source info is NULL", K(ret), K(tenant_id)); + } else if (OB_ISNULL(ts_source = ts_source_info->get_gts_source())) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "ts source is NULL", K(ret)); + } else if (OB_SUCC(ObTsSyncGetTsCbTaskPool::get_instance().get_task(stc, tenant_id, task))) { + bool need_recycle_task = true; + if (OB_FAIL(ts_source->get_gts(stc, task, gts_result, receive_gts_ts))) { + if (OB_EAGAIN != ret) { + TRANS_LOG(WARN, "get gts error", K(ret), K(tenant_id), K(stc)); + } else if (OB_FAIL(task->wait(timeout_us, scn, need_recycle_task))) { + if (OB_TIMEOUT != ret) { + fall_back_to_sleep = true; + } + TRANS_LOG(WARN, "failed to wait ObTsSyncGetTsCbTask", K(ret), K(tenant_id), K(timeout_us)); + } + } else { + scn.convert_for_gts(gts_result); + } + if (need_recycle_task) { + ObTsSyncGetTsCbTaskPool::get_instance().recycle_task(task); + } + } else { + fall_back_to_sleep = true; + } + if (fall_back_to_sleep) { + TRANS_LOG(WARN, "failed to get ObTsSyncGetTsCbTask, fall back to sleep", K(ret)); + + int64_t expire_ts = ObClockGenerator::getClock() + timeout_us; + int retry_times = 0; + const int64_t SLEEP_TIME_US = 500; + do { + const int64_t now = ObClockGenerator::getClock(); + if (now >= expire_ts) { + ret = OB_TIMEOUT; + } else if (OB_FAIL(ts_source->get_gts(stc, NULL, gts_result, receive_gts_ts))) { + if (OB_EAGAIN == ret) { + ob_usleep(SLEEP_TIME_US); + } else { + TRANS_LOG(WARN, "get gts fail", K(ret), K(now)); + } + } else { + scn.convert_for_gts(gts_result); + } + } while (OB_EAGAIN == ret); + } + } + + return ret; +} + int ObTsMgr::get_ts_sync(const uint64_t tenant_id, const int64_t timeout_us, SCN &scn, diff --git a/src/storage/tx/ob_ts_mgr.h b/src/storage/tx/ob_ts_mgr.h index 6a1247d366..2e2e437a6f 100644 --- a/src/storage/tx/ob_ts_mgr.h +++ b/src/storage/tx/ob_ts_mgr.h @@ -86,6 +86,12 @@ public: ObTsCbTask *task, share::SCN &scn, MonotonicTs &receive_gts_ts) = 0; + virtual int get_gts_sync(const uint64_t tenant_id, + const MonotonicTs stc, + const int64_t timeout_us, + share::SCN &scn, + MonotonicTs &receive_gts_ts) = 0; + virtual int get_gts(const uint64_t tenant_id, ObTsCbTask *task, share::SCN &scn) = 0; virtual int get_ts_sync(const uint64_t tenant_id, const int64_t timeout_ts, share::SCN &scn, bool &is_external_consistent) = 0; @@ -285,6 +291,64 @@ private: bool need_revert_; }; +class ObTsSyncGetTsCbTask : public ObTsCbTask +{ +public: + friend class ObTsSyncGetTsCbTaskPool; + ObTsSyncGetTsCbTask() + :is_inited_(false), task_id_(0), is_occupied_(false), is_finished_(false), + is_early_exit_(false), stc_(0), tenant_id_(0), errcode_(OB_SUCCESS) {} + ~ObTsSyncGetTsCbTask() {} + int init(uint64_t task_id); + int config(MonotonicTs stc, uint64_t tenant_id); + int gts_callback_interrupted(const int errcode) override; + int get_gts_callback(const MonotonicTs srr, const share::SCN >s, + const MonotonicTs receive_gts_ts) override; + int gts_elapse_callback(const MonotonicTs srr, const share::SCN >s) override; + MonotonicTs get_stc() const override; + uint64_t hash() const override; + uint64_t get_tenant_id() const override; + int wait(const int64_t timeout_us, share::SCN &scn, bool &need_recycle_task); +private: + bool is_inited_; + uint64_t task_id_; + // whether this callback task is being used + bool is_occupied_ __attribute__((aligned(8))); + // whether the callback has been invoked + bool is_finished_; + // whether the caller exits (due to timeout) before the callback is invoked + bool is_early_exit_; + share::SCN gts_result_; + ObThreadCond cond_; + MonotonicTs stc_; + uint64_t tenant_id_; + int errcode_; +}; + +STATIC_ASSERT(sizeof(ObTsSyncGetTsCbTask) <= 256, "ObTsSyncGetTsCbTask is too large"); +/** + * The resource pool of ObTsSyncGetTsCbTask. The pool has a fixed size of cbtasks, and the cbtasks + * can be reused. + */ +class ObTsSyncGetTsCbTaskPool +{ +public: + static constexpr int64_t POOL_SIZE = 8000; + ObTsSyncGetTsCbTaskPool() {} + ~ObTsSyncGetTsCbTaskPool() {} + static ObTsSyncGetTsCbTaskPool& get_instance() + { + static ObTsSyncGetTsCbTaskPool pool; + return pool; + } + int init(); + int get_task(MonotonicTs stc, uint64_t tenant_id, ObTsSyncGetTsCbTask *&task); + int recycle_task(ObTsSyncGetTsCbTask *task); +private: + bool is_inited_; + ObTsSyncGetTsCbTask tasks_[POOL_SIZE]; +}; + typedef common::ObLinkHashMap ObTsSourceInfoMap; class ObTsMgr : public share::ObThreadPool, public ObITsMgr { @@ -315,6 +379,20 @@ public: ObTsCbTask *task, share::SCN &scn, MonotonicTs &receive_gts_ts); + /** + * 与`get_gts`相对应的同步接口，用于同步获取合适的GTS时间戳，可传入超时时间以避免长时间等待。 + * 相较于原有同步接口`get_ts_sync`，本接口的性能更好 + * @param[in] tenant_id + * @param[in] stc: 需要获取GTS的时间点，一般取current time + * @param[in] timeout_us: 超时时长，单位us + * @param[out] scn: 获取到的GTS时间戳结果 + * @param[out] receive_gts_ts: 收到GTS response的时间点 + */ + int get_gts_sync(const uint64_t tenant_id, + const MonotonicTs stc, + const int64_t timeout_us, + share::SCN &scn, + MonotonicTs &receive_gts_ts); //仅仅获取本地gts cache的最新值，但可能会失败，失败之后处理逻辑如下: //1. 如果task == NULL，说明调用者不需要异步回调，直接返回报错，由调用者处理 //2. 如果task != NULL，需要注册异步回调任务 diff --git a/src/storage/tx/ob_tx_api.cpp b/src/storage/tx/ob_tx_api.cpp index cc70532936..6bde477a7e 100644 --- a/src/storage/tx/ob_tx_api.cpp +++ b/src/storage/tx/ob_tx_api.cpp @@ -648,6 +648,8 @@ int ObTransService::get_read_snapshot(ObTxDesc &tx, snapshot.valid_ = true; } ObTransTraceLog &tlog = tx.get_tlog(); + common::ObTraceIdAdaptor trace_id; + trace_id.set(ObCurTraceId::get()); REC_TRANS_TRACE_EXT(&tlog, get_read_snapshot, OB_Y(ret), OB_Y(expire_ts), OB_ID(txid), tx.tx_id_, OB_ID(isolation_level), (int)isolation, @@ -655,7 +657,7 @@ int ObTransService::get_read_snapshot(ObTxDesc &tx, OB_ID(snapshot_version), snapshot.core_.version_, OB_ID(snapshot_txid), snapshot.core_.tx_id_.get_id(), OB_ID(snapshot_scn), snapshot.core_.scn_.cast_to_int(), - OB_ID(trace_id), ObCurTraceId::get_trace_id_str(), + OB_ID(trace_id), trace_id, OB_ID(ref), tx.get_ref(), OB_ID(thread_id), GETTID()); return ret; @@ -718,6 +720,8 @@ int ObTransService::get_ls_read_snapshot(ObTxDesc &tx, ret = get_read_snapshot(tx, isolation, expire_ts, snapshot); } ObTransTraceLog &tlog = tx.get_tlog(); + common::ObTraceIdAdaptor trace_id; + trace_id.set(ObCurTraceId::get()); REC_TRANS_TRACE_EXT(&tlog, get_ls_read_snapshot, OB_Y(ret), OB_Y(expire_ts), OB_ID(ls_id), lsid.id(), OB_ID(isolation_level), (int)isolation, @@ -725,7 +729,7 @@ int ObTransService::get_ls_read_snapshot(ObTxDesc &tx, OB_ID(snapshot_version), snapshot.core_.version_, OB_ID(snapshot_txid), snapshot.core_.tx_id_.get_id(), OB_ID(snapshot_scn), snapshot.core_.scn_.cast_to_int(), - OB_ID(trace_id), ObCurTraceId::get_trace_id_str(), + OB_ID(trace_id), trace_id, OB_ID(ref), tx.get_ref(), OB_ID(thread_id), GETTID()); return ret; @@ -1389,7 +1393,6 @@ int ObTransService::rollback_to_explicit_savepoint(ObTxDesc &tx, } } } - tx.state_change_flags_.EXTRA_CHANGED_ = true; int64_t elapsed_us = ObTimeUtility::current_time() - start_ts; ObTransTraceLog &tlog = tx.get_tlog(); REC_TRANS_TRACE_EXT(&tlog, rollback_explicit_savepoint, OB_Y(ret), @@ -1434,7 +1437,6 @@ int ObTransService::release_explicit_savepoint(ObTxDesc &tx, const ObString &sav TRANS_LOG(TRACE, "release savepoint", K(savepoint), K(sp_id), K(session_id), K(tx)); } } - tx.state_change_flags_.EXTRA_CHANGED_ = true; ObTransTraceLog &tlog = tx.get_tlog(); REC_TRANS_TRACE_EXT(&tlog, release_explicit_savepoint, OB_Y(ret), OB_ID(savepoint), savepoint, @@ -1533,7 +1535,7 @@ int ObTransService::rollback_savepoint_(ObTxDesc &tx, if (OB_TIMEOUT == ret && ObTimeUtility::current_time() >= tx.get_expire_ts()) { ret = OB_TRANS_TIMEOUT; } - if (OB_SUCC(ret)) { + if (OB_SUCC(ret) && parts.count() > 0) { tx.post_rb_savepoint_(parts, savepoint); } return ret; diff --git a/src/storage/tx/ob_tx_data_define.h b/src/storage/tx/ob_tx_data_define.h index 6059353b7c..0ceefb22ab 100644 --- a/src/storage/tx/ob_tx_data_define.h +++ b/src/storage/tx/ob_tx_data_define.h @@ -235,10 +235,22 @@ public: // DONT : Modify this definition class ObTxData : public ObTxCommitData, public ObTxDataLink { +public: + enum ExclusiveType { + NORMAL = 0, + EXCLUSIVE, + DELETED + }; private: const static int64_t UNIS_VERSION = 1; public: - ObTxData() : ObTxCommitData(), ObTxDataLink(), tx_data_allocator_(nullptr), ref_cnt_(0), undo_status_list_(), flag_(0) {} + ObTxData() + : ObTxCommitData(), + ObTxDataLink(), + tx_data_allocator_(nullptr), + ref_cnt_(0), + undo_status_list_(), + exclusive_flag_(ExclusiveType::NORMAL) {} ObTxData(const ObTxData &rhs); ObTxData &operator=(const ObTxData &rhs); ObTxData &operator=(const ObTxCommitData &rhs); @@ -328,7 +340,7 @@ public: share::ObTenantTxDataAllocator *tx_data_allocator_; int64_t ref_cnt_; ObUndoStatusList undo_status_list_; - int64_t flag_; + ExclusiveType exclusive_flag_; }; class ObTxDataGuard diff --git a/src/storage/tx/ob_tx_stat.h b/src/storage/tx/ob_tx_stat.h index dc25c954b9..4d7729c534 100644 --- a/src/storage/tx/ob_tx_stat.h +++ b/src/storage/tx/ob_tx_stat.h @@ -25,21 +25,21 @@ struct ObTxCallbackListStat { int id_; share::SCN sync_scn_; - share::SCN checksum_scn_; int length_; int logged_; int removed_; + int branch_removed_; void reset() {} DECLARE_TO_STRING { int64_t pos = 0; - BUF_PRINTF("(%d,%d,%d,%d,%ld,%ld)", + BUF_PRINTF("[%d,%d,%d,%d,%d,%ld]", id_, length_, logged_, removed_, - (sync_scn_.is_valid() ? sync_scn_.get_val_for_inner_table_field() : -1), - (checksum_scn_.is_valid() ? checksum_scn_.get_val_for_inner_table_field() : -1)); + branch_removed_, + (sync_scn_.is_valid() ? sync_scn_.get_val_for_inner_table_field() : 0)); return pos; } }; @@ -127,8 +127,15 @@ public: { int64_t pos = 0; if (stats_.count() > 0) { - BUF_PRINTF("[id, length, logged, removed, sync_scn, checksum_scn]"); - BUF_PRINTO(stats_); + J_ARRAY_START(); + BUF_PRINTF("\"id, length, logged, removed, branch_removed, sync_scn\""); + for (int i =0; i < stats_.count(); i++) { + if (stats_.at(i).id_ >= 0) { + J_COMMA(); + BUF_PRINTO(stats_.at(i)); + } + } + J_ARRAY_END(); } return pos; } diff --git a/src/storage/tx_storage/ob_tenant_freezer.cpp b/src/storage/tx_storage/ob_tenant_freezer.cpp index 2f23472c9a..c0214509b3 100644 --- a/src/storage/tx_storage/ob_tenant_freezer.cpp +++ b/src/storage/tx_storage/ob_tenant_freezer.cpp @@ -172,7 +172,13 @@ bool ObTenantFreezer::exist_ls_freezing() int ls_cnt = 0; int exist_ls_freezing = false; for (; OB_SUCC(iter->get_next(ls)); ++ls_cnt) { - if (ls->get_freezer()->is_freeze()) { + int tmp_ret = OB_SUCCESS; + ObRole role; + if (OB_TMP_FAIL(ls->get_ls_role(role))) { + LOG_WARN("get ls role failed", KR(tmp_ret), K(ls->get_ls_id())); + } else if (common::is_strong_leader(role)) { + // skip check leader logstream + } else if (ls->get_freezer()->is_freeze()) { exist_ls_freezing = true; } } diff --git a/src/storage/tx_table/ob_tx_data_hash_map.cpp b/src/storage/tx_table/ob_tx_data_hash_map.cpp index d2d38107d4..1d3b76fdb2 100644 --- a/src/storage/tx_table/ob_tx_data_hash_map.cpp +++ b/src/storage/tx_table/ob_tx_data_hash_map.cpp @@ -64,6 +64,20 @@ int ObTxDataHashMap::insert(const transaction::ObTransID &key, ObTxData *value) } else { int64_t pos = get_pos(key); + if (OB_UNLIKELY(ObTxData::ExclusiveType::NORMAL != value->exclusive_flag_)) { + if (ObTxData::ExclusiveType::EXCLUSIVE != value->exclusive_flag_) { + STORAGE_LOG(ERROR, "invalid exclusive flag", KPC(value)); + } else { + ObTxData *iter = buckets_[pos].next_; + while (OB_NOT_NULL(iter)) { + if (iter->contain(key)) { + iter->exclusive_flag_ = ObTxData::ExclusiveType::DELETED; + } + iter = iter->hash_node_.next_; + } + } + } + // atomic insert this value while (true) { ObTxData *next_value = ATOMIC_LOAD(&buckets_[pos].next_); @@ -133,7 +147,7 @@ int ObTxDataHashMap::Iterator::get_next(ObTxDataGuard &guard) while (++bucket_idx_ < tx_data_map_.BUCKETS_CNT) { val_ = tx_data_map_.buckets_[bucket_idx_].next_; - if (OB_NOT_NULL(val_)) { + if (OB_NOT_NULL(val_) && (ObTxData::ExclusiveType::DELETED != val_->exclusive_flag_)) { break; } } diff --git a/tools/deploy/init_for_ce.sql b/tools/deploy/init_for_ce.sql index 547118a766..8bc8708c90 100644 --- a/tools/deploy/init_for_ce.sql +++ b/tools/deploy/init_for_ce.sql @@ -28,6 +28,8 @@ alter tenant sys set variables ob_enable_truncate_flashback = 'on'; alter tenant mysql set variables ob_tcp_invited_nodes='%'; alter tenant mysql set variables recyclebin = 'on'; alter tenant mysql set variables ob_enable_truncate_flashback = 'on'; +alter tenant sys set variables _nlj_batching_enabled = true; +alter tenant mysql set variables _nlj_batching_enabled = true; alter system set ob_compaction_schedule_interval = '10s' tenant sys; alter system set ob_compaction_schedule_interval = '10s' tenant all_user; alter system set ob_compaction_schedule_interval = '10s' tenant all_meta; diff --git a/tools/deploy/mysql_test/r/mysql/information_schema.result b/tools/deploy/mysql_test/r/mysql/information_schema.result index 055ab98527..b50bbc6c49 100644 --- a/tools/deploy/mysql_test/r/mysql/information_schema.result +++ b/tools/deploy/mysql_test/r/mysql/information_schema.result @@ -234,6 +234,14 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | DBA_IND_PARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_IND_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_IND_SUBPARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVIEWS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVIEW_LOGS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_CHANGE_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_RUN_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_STATS_PARAMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_STATS_SYS_DEFAULTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_STMT_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OBJECTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_ACCESS_POINT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_ARCHIVELOG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | @@ -1591,6 +1599,14 @@ select * from information_schema.tables where table_schema in ('oceanbase', 'mys | def | oceanbase | DBA_IND_PARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_IND_STATISTICS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_IND_SUBPARTITIONS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVIEWS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVIEW_LOGS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_CHANGE_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_RUN_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_STATS_PARAMS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_STATS_SYS_DEFAULTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | +| def | oceanbase | DBA_MVREF_STMT_STATS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OBJECTS | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_ACCESS_POINT | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | | def | oceanbase | DBA_OB_ARCHIVELOG | SYSTEM VIEW | MEMORY | NULL | DYNAMIC | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | utf8mb4_general_ci | NULL | NULL | | diff --git a/tools/deploy/mysql_test/r/mysql/topk.result b/tools/deploy/mysql_test/r/mysql/topk.result index fd2e746468..c3737b3d7d 100644 --- a/tools/deploy/mysql_test/r/mysql/topk.result +++ b/tools/deploy/mysql_test/r/mysql/topk.result @@ -866,7 +866,7 @@ Outputs & filters: 0 - output([t1.c3], [cast(cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), DECIMAL(20, 0)), DECIMAL(15, 4))], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1)) + T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [cast(T_FUN_SUM(T_FUN_SUM(t1.c2)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), DECIMAL(20, 0))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, - DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(-1, -1))]), filter(nil), rowset=256 + DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(33, 0))]), filter(nil), rowset=256 limit(1), offset(nil) 1 - output([t1.c3], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1))))]), filter(nil), rowset=256 @@ -921,7 +921,7 @@ Outputs & filters: 0 - output([t1.c3], [cast(cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), DECIMAL(20, 0)), DECIMAL(15, 4))], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1)) + T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [cast(T_FUN_SUM(T_FUN_SUM(t1.c2)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), DECIMAL(20, 0))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, - DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(-1, -1))]), filter(nil), rowset=256 + DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(33, 0))]), filter(nil), rowset=256 limit(10), offset(nil) 1 - output([t1.c3], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1))))]), filter(nil), rowset=256 @@ -976,7 +976,7 @@ Outputs & filters: 0 - output([t1.c3], [cast(cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), DECIMAL(20, 0)), DECIMAL(15, 4))], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1)) + T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [cast(T_FUN_SUM(T_FUN_SUM(t1.c2)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), DECIMAL(20, 0))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, - DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(-1, -1))]), filter(nil), rowset=256 + DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(33, 0))]), filter(nil), rowset=256 limit(1), offset(nil) 1 - output([t1.c3], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1))))]), filter(nil), rowset=256 @@ -1031,7 +1031,7 @@ Outputs & filters: 0 - output([t1.c3], [cast(cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), DECIMAL(20, 0)), DECIMAL(15, 4))], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1)) + T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [cast(T_FUN_SUM(T_FUN_SUM(t1.c2)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), DECIMAL(20, 0))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, - DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(-1, -1))]), filter(nil), rowset=256 + DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(33, 0))]), filter(nil), rowset=256 limit(1), offset(4) 1 - output([t1.c3], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1))))]), filter(nil), rowset=256 @@ -1086,7 +1086,7 @@ Outputs & filters: 0 - output([t1.c3], [cast(cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), DECIMAL(20, 0)), DECIMAL(15, 4))], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1)) + T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [cast(T_FUN_SUM(T_FUN_SUM(t1.c2)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), DECIMAL(20, 0))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, - DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(-1, -1))]), filter(nil), startup_filter([0]), rowset=256 + DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(33, 0))]), filter(nil), startup_filter([0]), rowset=256 limit(0), offset(nil) 1 - output([t1.c3], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1))))]), filter(nil), rowset=256 @@ -1141,7 +1141,7 @@ Outputs & filters: 0 - output([t1.c2], [cast(cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), DECIMAL(20, 0)), DECIMAL(15, 4))], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1)) + T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [cast(T_FUN_SUM(T_FUN_SUM(t1.c2)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), DECIMAL(20, 0))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, - DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(-1, -1))]), filter(nil), rowset=256 + DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(33, 0))]), filter(nil), rowset=256 limit(5), offset(nil) 1 - output([t1.c2], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1))))]), filter(nil), rowset=256 @@ -1207,7 +1207,7 @@ Outputs & filters: 0 - output([t1.c1], [cast(cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), DECIMAL(20, 0)), DECIMAL(15, 4))], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1)) + T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [cast(T_FUN_SUM(T_FUN_SUM(t1.c2)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), DECIMAL(20, 0))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, - DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(-1, -1))]), filter(nil), rowset=256 + DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(33, 0))]), filter(nil), rowset=256 limit(5), offset(nil) 1 - output([t1.c1], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1))))]), filter(nil), rowset=256 @@ -1273,7 +1273,7 @@ Outputs & filters: 0 - output([t1.c2], [cast(cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), DECIMAL(20, 0)), DECIMAL(15, 4))], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1)) + T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [cast(T_FUN_SUM(T_FUN_SUM(t1.c2)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), DECIMAL(20, 0))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, - DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(-1, -1))]), filter(nil), rowset=256 + DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(33, 0))]), filter(nil), rowset=256 limit(5), offset(nil) 1 - output([t1.c2], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1))))]), filter(nil), rowset=256 @@ -1346,7 +1346,7 @@ Outputs & filters: 0 - output([cast(cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), DECIMAL(20, 0)), DECIMAL(15, 4))], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1)) + T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [cast(T_FUN_SUM(T_FUN_SUM(t1.c2)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), DECIMAL(20, 0))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, - -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(-1, -1))]), filter(nil), rowset=256 + -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(33, 0))]), filter(nil), rowset=256 limit(5), offset(nil) 1 - output([T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1))], [T_FUN_SUM(T_FUN_SUM(t1.c1)) + T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1))))]), filter(nil), rowset=256 @@ -1407,7 +1407,7 @@ Outputs & filters: ------------------------------------- 0 - output([t1.c2], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1)) + T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [cast(T_FUN_SUM(T_FUN_SUM(t1.c2)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), - DECIMAL(20, 0))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(-1, -1))]), filter(nil), rowset=256 + DECIMAL(20, 0))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(33, 0))]), filter(nil), rowset=256 limit(5), offset(nil) 1 - output([t1.c2], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1)) + T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [T_FUN_SUM(T_FUN_SUM(t1.c1))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1))))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1))]), filter(nil), rowset=256 @@ -1464,7 +1464,7 @@ Outputs & filters: ------------------------------------- 0 - output([t1.c2], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1)) + T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [cast(T_FUN_SUM(T_FUN_SUM(t1.c2)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), - DECIMAL(20, 0))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(-1, -1))]), filter(nil), rowset=256 + DECIMAL(20, 0))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(33, 0))]), filter(nil), rowset=256 limit(5), offset(nil) 1 - output([t1.c2], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1)) + T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [T_FUN_SUM(T_FUN_SUM(t1.c1))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1))))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1))]), filter(nil), rowset=256 @@ -1526,7 +1526,7 @@ Outputs & filters: ------------------------------------- 0 - output([t1.c2], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1)) + T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [cast(T_FUN_SUM(T_FUN_SUM(t1.c2)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), - DECIMAL(20, 0))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(-1, -1))]), filter(nil), rowset=256 + DECIMAL(20, 0))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(33, 0))]), filter(nil), rowset=256 limit(5), offset(nil) 1 - output([t1.c2], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1)) + T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [T_FUN_SUM(T_FUN_SUM(t1.c1))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1))))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1))]), filter(nil), rowset=256 @@ -1588,7 +1588,7 @@ Outputs & filters: ------------------------------------- 0 - output([t1.c2], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1)) + T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [cast(T_FUN_SUM(T_FUN_SUM(t1.c2)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), - DECIMAL(20, 0))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(-1, -1))]), filter(nil), rowset=256 + DECIMAL(20, 0))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(33, 0))]), filter(nil), rowset=256 limit(5), offset(nil) 1 - output([t1.c2], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1)) + T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [T_FUN_SUM(T_FUN_SUM(t1.c1))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1))))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1))]), filter(nil), rowset=256 @@ -1650,7 +1650,7 @@ Outputs & filters: ------------------------------------- 0 - output([t1.c2], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1)) + T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [cast(T_FUN_SUM(T_FUN_SUM(t1.c2)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), - DECIMAL(20, 0))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(-1, -1))]), filter(nil), rowset=256 + DECIMAL(20, 0))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(33, 0))]), filter(nil), rowset=256 limit(5), offset(nil) 1 - output([t1.c2], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1)) + T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [T_FUN_SUM(T_FUN_SUM(t1.c1))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1))))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1))]), filter(nil), rowset=256 @@ -1991,7 +1991,7 @@ Outputs & filters: 13 - output([t1.c3], [t1.c1], [cast(cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), DECIMAL(20, 0)), DECIMAL(15, 4))], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1)) + T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [cast(T_FUN_SUM(T_FUN_SUM(t1.c2)), DECIMAL(33, 0)) / cast(T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1)), DECIMAL(20, 0))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, - DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(-1, -1))]), filter(nil), rowset=256 + DOUBLE(-1, -1)))) / cast(T_FUN_SUM(T_FUN_SUM(t1.c1)), DOUBLE(33, 0))]), filter(nil), rowset=256 limit(7), offset(nil) 14 - output([t1.c3], [t1.c1], [T_FUN_SUM(T_FUN_SUM(t1.c2))], [T_FUN_SUM(T_FUN_SUM(t1.c1))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c3))], [T_FUN_MIN(T_FUN_MIN(t1.c3))], [T_FUN_MAX(T_FUN_MAX(t1.c1))], [T_FUN_COUNT_SUM(T_FUN_COUNT(t1.c1))], [T_FUN_SUM(T_FUN_SUM(cast(t1.c3, DOUBLE(-1, -1))))]), filter(nil), rowset=256 diff --git a/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_bugfix_mysql.result b/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_bugfix_mysql.result index d3935060a9..ec6aaef568 100644 --- a/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_bugfix_mysql.result +++ b/tools/deploy/mysql_test/test_suite/geometry/r/mysql/geometry_bugfix_mysql.result @@ -507,7 +507,7 @@ Outputs & filters: access([a.g]), partitions(p0) is_index_back=false, is_global_index=false, range_key([a.__pk_increment]), range(MIN ; MAX)always true - 2 - output([b.g]), filter([_st_dwithin(:0, b.g, cast(0.1, DOUBLE(-1, -1)))]) + 2 - output([b.g]), filter([_st_dwithin(:0, b.g, cast(0.1, DOUBLE(2, 1)))]) access([b.__pk_increment], [b.g]), partitions(p0) is_index_back=true, is_global_index=false, filter_before_indexback[false], range_key([b.__cellid_16], [b.__mbr_16], [b.__pk_increment]), range(MIN ; MAX) diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result index dbe827b678..9f3b17a88d 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_mysql.result @@ -5793,3 +5793,161 @@ TSNAP_META_EXISTED varchar(3) NO select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.V$OB_LS_SNAPSHOTS limit 1); cnt 1 +desc oceanbase.DBA_MVIEW_LOGS; +Field Type Null Key Default Extra +LOG_OWNER varchar(128) NO +MASTER varchar(128) NO +LOG_TABLE varchar(128) NO +LOG_TRIGGER varchar(128) NO +ROWIDS varchar(3) NO +PRIMARY_KEY varchar(3) NO +OBJECT_ID varchar(3) NO +FILTER_COLUMNS varchar(3) NO +SEQUENCE varchar(3) NO +INCLUDE_NEW_VALUES varchar(3) NO +PURGE_ASYNCHRONOUS varchar(3) NO +PURGE_DEFERRED varchar(3) NO +PURGE_START datetime YES NULL +PURGE_INTERVAL varchar(200) YES NULL +LAST_PURGE_DATE datetime YES NULL +LAST_PURGE_STATUS bigint(1) NO +NUM_ROWS_PURGED bigint(20) YES NULL +COMMIT_SCN_BASED varchar(3) NO +STAGING_LOG varchar(3) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVIEW_LOGS limit 1); +cnt +1 +desc oceanbase.DBA_MVIEWS; +Field Type Null Key Default Extra +OWNER varchar(128) NO +MVIEW_NAME varchar(128) NO +CONTAINER_NAME varchar(128) NO +QUERY longtext NO NULL +QUERY_LEN bigint(20) NO +UPDATABLE varchar(1) NO +UPDATE_LOG varchar(128) NO +MASTER_ROLLBACK_SEG varchar(128) NO +MASTER_LINK varchar(128) NO +REWRITE_ENABLED varchar(1) NO +REWRITE_CAPABILITY varchar(9) NO +REFRESH_MODE varchar(6) NO +REFRESH_METHOD varchar(8) NO +BUILD_MODE varchar(9) NO +FAST_REFRESHABLE varchar(18) NO +LAST_REFRESH_TYPE varchar(8) NO +LAST_REFRESH_DATE datetime YES NULL +LAST_REFRESH_END_TIME datetime NO +STALENESS varchar(19) NO +AFTER_FAST_REFRESH varchar(19) NO +UNKNOWN_PREBUILT varchar(1) NO +UNKNOWN_PLSQL_FUNC varchar(1) NO +UNKNOWN_EXTERNAL_TABLE varchar(1) NO +UNKNOWN_CONSIDER_FRESH varchar(1) NO +UNKNOWN_IMPORT varchar(1) NO +UNKNOWN_TRUSTED_FD varchar(1) NO +COMPILE_STATE varchar(19) NO +USE_NO_INDEX varchar(1) NO +STALE_SINCE datetime NO +NUM_PCT_TABLES bigint(0) NO +NUM_FRESH_PCT_REGIONS bigint(0) NO +NUM_STALE_PCT_REGIONS bigint(0) NO +SEGMENT_CREATED varchar(3) NO +EVALUATION_EDITION varchar(128) NO +UNUSABLE_BEFORE varchar(128) NO +UNUSABLE_BEGINNING varchar(128) NO +DEFAULT_COLLATION varchar(100) NO +ON_QUERY_COMPUTATION varchar(1) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVIEWS limit 1); +cnt +1 +desc oceanbase.DBA_MVREF_STATS_SYS_DEFAULTS; +Field Type Null Key Default Extra +PARAMETER_NAME varchar(16) NO +VALUE varchar(40) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_STATS_SYS_DEFAULTS limit 1); +cnt +1 +desc oceanbase.DBA_MVREF_STATS_PARAMS; +Field Type Null Key Default Extra +MV_OWNER varchar(128) NO +MV_NAME varchar(128) NO +COLLECTION_LEVEL varchar(8) NO +RETENTION_PERIOD bigint(20) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_STATS_PARAMS limit 1); +cnt +1 +desc oceanbase.DBA_MVREF_RUN_STATS; +Field Type Null Key Default Extra +RUN_OWNER varchar(128) NO NULL +REFRESH_ID bigint(20) NO NULL +NUM_MVS bigint(20) NO NULL +MVIEWS text NO NULL +BASE_TABLES text YES NULL +METHOD text YES NULL +ROLLBACK_SEG text YES NULL +PUSH_DEFERRED_RPC varchar(1) NO +REFRESH_AFTER_ERRORS varchar(1) NO +PURGE_OPTION bigint(20) NO NULL +PARALLELISM bigint(20) NO NULL +HEAP_SIZE bigint(20) NO NULL +ATOMIC_REFRESH varchar(1) NO +NESTED varchar(1) NO +OUT_OF_PLACE varchar(1) NO +NUMBER_OF_FAILURES bigint(20) NO NULL +START_TIME datetime NO NULL +END_TIME datetime NO NULL +ELAPSED_TIME bigint(20) NO NULL +LOG_SETUP_TIME bigint(1) NO +LOG_PURGE_TIME bigint(20) NO NULL +COMPLETE_STATS_AVAILABLE varchar(1) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_RUN_STATS limit 1); +cnt +1 +desc oceanbase.DBA_MVREF_STATS; +Field Type Null Key Default Extra +MV_OWNER varchar(128) NO +MV_NAME varchar(128) NO +REFRESH_ID bigint(20) NO NULL +REFRESH_METHOD varchar(30) NO +REFRESH_OPTIMIZATIONS text NO +ADDITIONAL_EXECUTIONS text NO +START_TIME datetime NO NULL +END_TIME datetime NO NULL +ELAPSED_TIME bigint(20) NO NULL +LOG_SETUP_TIME bigint(1) NO +LOG_PURGE_TIME bigint(20) NO NULL +INITIAL_NUM_ROWS bigint(20) NO NULL +FINAL_NUM_ROWS bigint(20) NO NULL +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_STATS limit 1); +cnt +1 +desc oceanbase.DBA_MVREF_CHANGE_STATS; +Field Type Null Key Default Extra +TBL_OWNER varchar(128) NO +TBL_NAME varchar(128) NO +MV_OWNER varchar(128) NO +MV_NAME varchar(128) NO +REFRESH_ID bigint(20) NO NULL +NUM_ROWS_INS bigint(20) YES NULL +NUM_ROWS_UPD bigint(20) YES NULL +NUM_ROWS_DEL bigint(20) YES NULL +NUM_ROWS_DL_INS bigint(1) NO +PMOPS_OCCURRED varchar(1) NO +PMOP_DETAILS text NO +NUM_ROWS bigint(20) YES NULL +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_CHANGE_STATS limit 1); +cnt +1 +desc oceanbase.DBA_MVREF_STMT_STATS; +Field Type Null Key Default Extra +MV_OWNER varchar(128) NO +MV_NAME varchar(128) NO +REFRESH_ID bigint(20) NO NULL +STEP bigint(20) NO NULL +SQLID varchar(32) YES NULL +STMT longtext NO NULL +EXECUTION_TIME bigint(20) NO NULL +EXECUTION_PLAN longtext YES NULL +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_STMT_STATS limit 1); +cnt +1 diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result index 97d7e74159..e4a53ff815 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_sys_views_in_sys.result @@ -8454,3 +8454,327 @@ ERROR_MESSAGE varchar(512) YES NULL select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_OB_CLONE_HISTORY limit 1); cnt 1 +desc oceanbase.CDB_MVIEW_LOGS; +Field Type Null Key Default Extra +TENANT_ID bigint(20) NO NULL +LOG_OWNER varchar(128) NO +MASTER varchar(128) NO +LOG_TABLE varchar(128) NO +LOG_TRIGGER varchar(128) NO +ROWIDS varchar(3) NO +PRIMARY_KEY varchar(3) NO +OBJECT_ID varchar(3) NO +FILTER_COLUMNS varchar(3) NO +SEQUENCE varchar(3) NO +INCLUDE_NEW_VALUES varchar(3) NO +PURGE_ASYNCHRONOUS varchar(3) NO +PURGE_DEFERRED varchar(3) NO +PURGE_START datetime YES NULL +PURGE_INTERVAL varchar(200) YES NULL +LAST_PURGE_DATE datetime YES NULL +LAST_PURGE_STATUS bigint(1) NO +NUM_ROWS_PURGED bigint(20) YES NULL +COMMIT_SCN_BASED varchar(3) NO +STAGING_LOG varchar(3) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.CDB_MVIEW_LOGS limit 1); +cnt +1 +desc oceanbase.DBA_MVIEW_LOGS; +Field Type Null Key Default Extra +LOG_OWNER varchar(128) NO +MASTER varchar(128) NO +LOG_TABLE varchar(128) NO +LOG_TRIGGER varchar(128) NO +ROWIDS varchar(3) NO +PRIMARY_KEY varchar(3) NO +OBJECT_ID varchar(3) NO +FILTER_COLUMNS varchar(3) NO +SEQUENCE varchar(3) NO +INCLUDE_NEW_VALUES varchar(3) NO +PURGE_ASYNCHRONOUS varchar(3) NO +PURGE_DEFERRED varchar(3) NO +PURGE_START datetime YES NULL +PURGE_INTERVAL varchar(200) YES NULL +LAST_PURGE_DATE datetime YES NULL +LAST_PURGE_STATUS bigint(1) NO +NUM_ROWS_PURGED bigint(20) YES NULL +COMMIT_SCN_BASED varchar(3) NO +STAGING_LOG varchar(3) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVIEW_LOGS limit 1); +cnt +1 +desc oceanbase.CDB_MVIEWS; +Field Type Null Key Default Extra +TENANT_ID bigint(20) NO NULL +OWNER varchar(128) NO +MVIEW_NAME varchar(128) NO +CONTAINER_NAME varchar(128) NO +QUERY longtext NO NULL +QUERY_LEN bigint(20) NO +UPDATABLE varchar(1) NO +UPDATE_LOG varchar(128) NO +MASTER_ROLLBACK_SEG varchar(128) NO +MASTER_LINK varchar(128) NO +REWRITE_ENABLED varchar(1) NO +REWRITE_CAPABILITY varchar(9) NO +REFRESH_MODE varchar(6) NO +REFRESH_METHOD varchar(8) NO +BUILD_MODE varchar(9) NO +FAST_REFRESHABLE varchar(18) NO +LAST_REFRESH_TYPE varchar(8) NO +LAST_REFRESH_DATE datetime YES NULL +LAST_REFRESH_END_TIME datetime NO +STALENESS varchar(19) NO +AFTER_FAST_REFRESH varchar(19) NO +UNKNOWN_PREBUILT varchar(1) NO +UNKNOWN_PLSQL_FUNC varchar(1) NO +UNKNOWN_EXTERNAL_TABLE varchar(1) NO +UNKNOWN_CONSIDER_FRESH varchar(1) NO +UNKNOWN_IMPORT varchar(1) NO +UNKNOWN_TRUSTED_FD varchar(1) NO +COMPILE_STATE varchar(19) NO +USE_NO_INDEX varchar(1) NO +STALE_SINCE datetime NO +NUM_PCT_TABLES bigint(0) NO +NUM_FRESH_PCT_REGIONS bigint(0) NO +NUM_STALE_PCT_REGIONS bigint(0) NO +SEGMENT_CREATED varchar(3) NO +EVALUATION_EDITION varchar(128) NO +UNUSABLE_BEFORE varchar(128) NO +UNUSABLE_BEGINNING varchar(128) NO +DEFAULT_COLLATION varchar(100) NO +ON_QUERY_COMPUTATION varchar(1) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.CDB_MVIEWS limit 1); +cnt +1 +desc oceanbase.DBA_MVIEWS; +Field Type Null Key Default Extra +OWNER varchar(128) NO +MVIEW_NAME varchar(128) NO +CONTAINER_NAME varchar(128) NO +QUERY longtext NO NULL +QUERY_LEN bigint(20) NO +UPDATABLE varchar(1) NO +UPDATE_LOG varchar(128) NO +MASTER_ROLLBACK_SEG varchar(128) NO +MASTER_LINK varchar(128) NO +REWRITE_ENABLED varchar(1) NO +REWRITE_CAPABILITY varchar(9) NO +REFRESH_MODE varchar(6) NO +REFRESH_METHOD varchar(8) NO +BUILD_MODE varchar(9) NO +FAST_REFRESHABLE varchar(18) NO +LAST_REFRESH_TYPE varchar(8) NO +LAST_REFRESH_DATE datetime YES NULL +LAST_REFRESH_END_TIME datetime NO +STALENESS varchar(19) NO +AFTER_FAST_REFRESH varchar(19) NO +UNKNOWN_PREBUILT varchar(1) NO +UNKNOWN_PLSQL_FUNC varchar(1) NO +UNKNOWN_EXTERNAL_TABLE varchar(1) NO +UNKNOWN_CONSIDER_FRESH varchar(1) NO +UNKNOWN_IMPORT varchar(1) NO +UNKNOWN_TRUSTED_FD varchar(1) NO +COMPILE_STATE varchar(19) NO +USE_NO_INDEX varchar(1) NO +STALE_SINCE datetime NO +NUM_PCT_TABLES bigint(0) NO +NUM_FRESH_PCT_REGIONS bigint(0) NO +NUM_STALE_PCT_REGIONS bigint(0) NO +SEGMENT_CREATED varchar(3) NO +EVALUATION_EDITION varchar(128) NO +UNUSABLE_BEFORE varchar(128) NO +UNUSABLE_BEGINNING varchar(128) NO +DEFAULT_COLLATION varchar(100) NO +ON_QUERY_COMPUTATION varchar(1) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVIEWS limit 1); +cnt +1 +desc oceanbase.CDB_MVREF_STATS_SYS_DEFAULTS; +Field Type Null Key Default Extra +TENANT_ID bigint(20) NO +PARAMETER_NAME varchar(16) NO +VALUE varchar(40) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.CDB_MVREF_STATS_SYS_DEFAULTS limit 1); +cnt +1 +desc oceanbase.DBA_MVREF_STATS_SYS_DEFAULTS; +Field Type Null Key Default Extra +PARAMETER_NAME varchar(16) NO +VALUE varchar(40) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_STATS_SYS_DEFAULTS limit 1); +cnt +1 +desc oceanbase.CDB_MVREF_STATS_PARAMS; +Field Type Null Key Default Extra +TENANT_ID bigint(20) NO +MV_OWNER varchar(128) NO +MV_NAME varchar(128) NO +COLLECTION_LEVEL varchar(8) NO +RETENTION_PERIOD bigint(20) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.CDB_MVREF_STATS_PARAMS limit 1); +cnt +1 +desc oceanbase.DBA_MVREF_STATS_PARAMS; +Field Type Null Key Default Extra +MV_OWNER varchar(128) NO +MV_NAME varchar(128) NO +COLLECTION_LEVEL varchar(8) NO +RETENTION_PERIOD bigint(20) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_STATS_PARAMS limit 1); +cnt +1 +desc oceanbase.CDB_MVREF_RUN_STATS; +Field Type Null Key Default Extra +TENANT_ID bigint(20) NO NULL +RUN_OWNER varchar(128) NO NULL +REFRESH_ID bigint(20) NO NULL +NUM_MVS bigint(20) NO NULL +MVIEWS text NO NULL +BASE_TABLES text YES NULL +METHOD text YES NULL +ROLLBACK_SEG text YES NULL +PUSH_DEFERRED_RPC varchar(1) NO +REFRESH_AFTER_ERRORS varchar(1) NO +PURGE_OPTION bigint(20) NO NULL +PARALLELISM bigint(20) NO NULL +HEAP_SIZE bigint(20) NO NULL +ATOMIC_REFRESH varchar(1) NO +NESTED varchar(1) NO +OUT_OF_PLACE varchar(1) NO +NUMBER_OF_FAILURES bigint(20) NO NULL +START_TIME datetime NO NULL +END_TIME datetime NO NULL +ELAPSED_TIME bigint(20) NO NULL +LOG_SETUP_TIME bigint(1) NO +LOG_PURGE_TIME bigint(20) NO NULL +COMPLETE_STATS_AVAILABLE varchar(1) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.CDB_MVREF_RUN_STATS limit 1); +cnt +1 +desc oceanbase.DBA_MVREF_RUN_STATS; +Field Type Null Key Default Extra +RUN_OWNER varchar(128) NO NULL +REFRESH_ID bigint(20) NO NULL +NUM_MVS bigint(20) NO NULL +MVIEWS text NO NULL +BASE_TABLES text YES NULL +METHOD text YES NULL +ROLLBACK_SEG text YES NULL +PUSH_DEFERRED_RPC varchar(1) NO +REFRESH_AFTER_ERRORS varchar(1) NO +PURGE_OPTION bigint(20) NO NULL +PARALLELISM bigint(20) NO NULL +HEAP_SIZE bigint(20) NO NULL +ATOMIC_REFRESH varchar(1) NO +NESTED varchar(1) NO +OUT_OF_PLACE varchar(1) NO +NUMBER_OF_FAILURES bigint(20) NO NULL +START_TIME datetime NO NULL +END_TIME datetime NO NULL +ELAPSED_TIME bigint(20) NO NULL +LOG_SETUP_TIME bigint(1) NO +LOG_PURGE_TIME bigint(20) NO NULL +COMPLETE_STATS_AVAILABLE varchar(1) NO +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_RUN_STATS limit 1); +cnt +1 +desc oceanbase.CDB_MVREF_STATS; +Field Type Null Key Default Extra +TENANT_ID bigint(20) NO NULL +MV_OWNER varchar(128) NO +MV_NAME varchar(128) NO +REFRESH_ID bigint(20) NO NULL +REFRESH_METHOD varchar(30) NO +REFRESH_OPTIMIZATIONS text NO +ADDITIONAL_EXECUTIONS text NO +START_TIME datetime NO NULL +END_TIME datetime NO NULL +ELAPSED_TIME bigint(20) NO NULL +LOG_SETUP_TIME bigint(1) NO +LOG_PURGE_TIME bigint(20) NO NULL +INITIAL_NUM_ROWS bigint(20) NO NULL +FINAL_NUM_ROWS bigint(20) NO NULL +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.CDB_MVREF_STATS limit 1); +cnt +1 +desc oceanbase.DBA_MVREF_STATS; +Field Type Null Key Default Extra +MV_OWNER varchar(128) NO +MV_NAME varchar(128) NO +REFRESH_ID bigint(20) NO NULL +REFRESH_METHOD varchar(30) NO +REFRESH_OPTIMIZATIONS text NO +ADDITIONAL_EXECUTIONS text NO +START_TIME datetime NO NULL +END_TIME datetime NO NULL +ELAPSED_TIME bigint(20) NO NULL +LOG_SETUP_TIME bigint(1) NO +LOG_PURGE_TIME bigint(20) NO NULL +INITIAL_NUM_ROWS bigint(20) NO NULL +FINAL_NUM_ROWS bigint(20) NO NULL +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_STATS limit 1); +cnt +1 +desc oceanbase.CDB_MVREF_CHANGE_STATS; +Field Type Null Key Default Extra +TENANT_ID bigint(20) NO NULL +TBL_OWNER varchar(128) NO +TBL_NAME varchar(128) NO +MV_OWNER varchar(128) NO +MV_NAME varchar(128) NO +REFRESH_ID bigint(20) NO NULL +NUM_ROWS_INS bigint(20) YES NULL +NUM_ROWS_UPD bigint(20) YES NULL +NUM_ROWS_DEL bigint(20) YES NULL +NUM_ROWS_DL_INS bigint(1) NO +PMOPS_OCCURRED varchar(1) NO +PMOP_DETAILS text NO +NUM_ROWS bigint(20) YES NULL +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.CDB_MVREF_CHANGE_STATS limit 1); +cnt +1 +desc oceanbase.DBA_MVREF_CHANGE_STATS; +Field Type Null Key Default Extra +TBL_OWNER varchar(128) NO +TBL_NAME varchar(128) NO +MV_OWNER varchar(128) NO +MV_NAME varchar(128) NO +REFRESH_ID bigint(20) NO NULL +NUM_ROWS_INS bigint(20) YES NULL +NUM_ROWS_UPD bigint(20) YES NULL +NUM_ROWS_DEL bigint(20) YES NULL +NUM_ROWS_DL_INS bigint(1) NO +PMOPS_OCCURRED varchar(1) NO +PMOP_DETAILS text NO +NUM_ROWS bigint(20) YES NULL +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_CHANGE_STATS limit 1); +cnt +1 +desc oceanbase.CDB_MVREF_STMT_STATS; +Field Type Null Key Default Extra +TENANT_ID bigint(20) NO NULL +MV_OWNER varchar(128) NO +MV_NAME varchar(128) NO +REFRESH_ID bigint(20) NO NULL +STEP bigint(20) NO NULL +SQLID varchar(32) YES NULL +STMT longtext NO NULL +EXECUTION_TIME bigint(20) NO NULL +EXECUTION_PLAN longtext YES NULL +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.CDB_MVREF_STMT_STATS limit 1); +cnt +1 +desc oceanbase.DBA_MVREF_STMT_STATS; +Field Type Null Key Default Extra +MV_OWNER varchar(128) NO +MV_NAME varchar(128) NO +REFRESH_ID bigint(20) NO NULL +STEP bigint(20) NO NULL +SQLID varchar(32) YES NULL +STMT longtext NO NULL +EXECUTION_TIME bigint(20) NO NULL +EXECUTION_PLAN longtext YES NULL +select /*+QUERY_TIMEOUT(60000000)*/ count(*) as cnt from (select * from oceanbase.DBA_MVREF_STMT_STATS limit 1); +cnt +1 diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result index e85e03f198..8d08a3a9da 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/desc_virtual_table_in_sys.result @@ -4835,6 +4835,7 @@ execution_id bigint(20) NO 0 ddl_stmt_str longtext YES NULL ret_code bigint(20) NO 0 message longtext YES NULL +consensus_schema_version bigint(20) NO -1 select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_ddl_task_status; IF(count(*) >= 0, 1, 0) 1 @@ -5900,6 +5901,7 @@ user_message longtext YES NULL dba_message varchar(512) YES NULL parent_task_id bigint(20) NO 0 trace_id varchar(64) YES NULL +consensus_schema_version bigint(20) NO -1 select /*+QUERY_TIMEOUT(60000000)*/ IF(count(*) >= 0, 1, 0) from oceanbase.__all_virtual_ddl_error_message; IF(count(*) >= 0, 1, 0) 1 diff --git a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result index fb885beb62..30a6eca9c0 100644 --- a/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result +++ b/tools/deploy/mysql_test/test_suite/inner_table/r/mysql/inner_table_overall.result @@ -1101,6 +1101,22 @@ select 0xffffffffff & table_id, table_name, table_type, database_id, part_num fr 21517 GV$OB_LS_SNAPSHOTS 1 201001 1 21518 V$OB_LS_SNAPSHOTS 1 201001 1 21519 DBA_OB_CLONE_HISTORY 1 201001 1 +21525 CDB_MVIEW_LOGS 1 201001 1 +21526 DBA_MVIEW_LOGS 1 201001 1 +21527 CDB_MVIEWS 1 201001 1 +21528 DBA_MVIEWS 1 201001 1 +21529 CDB_MVREF_STATS_SYS_DEFAULTS 1 201001 1 +21530 DBA_MVREF_STATS_SYS_DEFAULTS 1 201001 1 +21531 CDB_MVREF_STATS_PARAMS 1 201001 1 +21532 DBA_MVREF_STATS_PARAMS 1 201001 1 +21533 CDB_MVREF_RUN_STATS 1 201001 1 +21534 DBA_MVREF_RUN_STATS 1 201001 1 +21535 CDB_MVREF_STATS 1 201001 1 +21536 DBA_MVREF_STATS 1 201001 1 +21537 CDB_MVREF_CHANGE_STATS 1 201001 1 +21538 DBA_MVREF_CHANGE_STATS 1 201001 1 +21539 CDB_MVREF_STMT_STATS 1 201001 1 +21540 DBA_MVREF_STMT_STATS 1 201001 1 check sys table count and table_id range success check count and table_id range for virtual table success select * from information_schema.CHARACTER_SETS limit 1; diff --git a/tools/deploy/mysql_test/test_suite/msdt/t/lob.test b/tools/deploy/mysql_test/test_suite/msdt/t/lob.test index f11f073c20..25497e11ef 100644 --- a/tools/deploy/mysql_test/test_suite/msdt/t/lob.test +++ b/tools/deploy/mysql_test/test_suite/msdt/t/lob.test @@ -21,6 +21,20 @@ let $report_cnt1 = query_get_value(select count(*) as cnt from oceanbase.__all_v let $unit_cnt1 = query_get_value(select count(*) as cnt from oceanbase.__all_virtual_unit where tenant_id = $tenant_id1, cnt, 1); --disable_query_log + +let $__i__= 120; +while($__i__ > 0) +{ + sleep 1; + dec $__i__; + let $report_cnt1 = query_get_value(select count(*) as cnt from oceanbase.__all_virtual_tablet_meta_table where tenant_id = $tenant_id1 and ls_id = $ls_id1 and tablet_id = $tablet_id1, cnt, 1); + if($report_cnt1 == $unit_cnt1) + { + let $__i__ = -1; + } +} + + eval select $report_cnt1 = $unit_cnt1 as result; --enable_query_log diff --git a/tools/deploy/mysql_test/test_suite/skyline/r/mysql/skyline_complicate_mysql.result b/tools/deploy/mysql_test/test_suite/skyline/r/mysql/skyline_complicate_mysql.result index 3582b5ab78..c7d64f5483 100644 --- a/tools/deploy/mysql_test/test_suite/skyline/r/mysql/skyline_complicate_mysql.result +++ b/tools/deploy/mysql_test/test_suite/skyline/r/mysql/skyline_complicate_mysql.result @@ -739,7 +739,7 @@ Outputs & filters: 0 - output([T_FUN_MAX(skyline_int.v1)], [skyline_int.tenant_id], [skyline_int.v6]), filter(nil), rowset=16 sort_keys([skyline_int.v3, ASC], [skyline_int.v4, ASC], [skyline_int.v5, ASC]) 1 - output([skyline_int.v3], [skyline_int.v4], [skyline_int.v5], [T_FUN_MAX(skyline_int.v1)], [skyline_int.tenant_id], [skyline_int.v6]), filter(nil), rowset=16 - group([skyline_int.v3], [skyline_int.v5], [skyline_int.v4]), agg_func([T_FUN_MAX(skyline_int.v1)]) + group([skyline_int.v5], [skyline_int.v4], [skyline_int.v3]), agg_func([T_FUN_MAX(skyline_int.v1)]) 2 - output([skyline_int.v3], [skyline_int.v4], [skyline_int.v5], [skyline_int.tenant_id], [skyline_int.v6], [skyline_int.v1]), filter(nil), rowset=16 equal_conds([skyline_int.v3 = other.c1]), other_conds(nil) merge_directions([ASC]) @@ -852,7 +852,7 @@ Outputs & filters: 0 - output([skyline_int.v5], [skyline_int.v4], [skyline_int.v3]), filter(nil), rowset=16 sort_keys([skyline_int.v3, ASC]) 1 - output([skyline_int.v3], [skyline_int.v5], [skyline_int.v4]), filter(nil), rowset=16 - distinct([skyline_int.v4], [skyline_int.v5], [skyline_int.v3]) + distinct([skyline_int.v5], [skyline_int.v4], [skyline_int.v3]) 2 - output([skyline_int.v3], [skyline_int.v5], [skyline_int.v4]), filter(nil), rowset=16 equal_conds([skyline_int.v4 = other.c1]), other_conds(nil) merge_directions([ASC]) diff --git a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_left.result b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_left.result index 585b678b39..bfd8c8f333 100644 --- a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_left.result +++ b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_left.result @@ -652,7 +652,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c1], [t1.c17], [left(cast(t1.c1, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, 0)))], [right(cast(t1.c1, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, + 0 - output([t1.c1], [t1.c17], [left(cast(t1.c1, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))], [right(cast(t1.c1, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c1], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, @@ -1123,7 +1123,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c2], [t1.c17], [left(cast(t1.c2, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, 0)))], [right(cast(t1.c2, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, + 0 - output([t1.c2], [t1.c17], [left(cast(t1.c2, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))], [right(cast(t1.c2, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c2], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, @@ -1594,7 +1594,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c3], [t1.c17], [left(cast(t1.c3, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, 0)))], [right(cast(t1.c3, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, + 0 - output([t1.c3], [t1.c17], [left(cast(t1.c3, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))], [right(cast(t1.c3, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c3], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, @@ -2065,7 +2065,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c4], [t1.c17], [left(cast(t1.c4, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, 0)))], [right(cast(t1.c4, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, + 0 - output([t1.c4], [t1.c17], [left(cast(t1.c4, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))], [right(cast(t1.c4, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c4], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, @@ -2536,7 +2536,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c5], [t1.c17], [left(cast(t1.c5, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, 0)))], [right(cast(t1.c5, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, + 0 - output([t1.c5], [t1.c17], [left(cast(t1.c5, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))], [right(cast(t1.c5, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c5], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, @@ -3007,7 +3007,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c6], [t1.c17], [left(cast(t1.c6, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, 0)))], [right(cast(t1.c6, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, + 0 - output([t1.c6], [t1.c17], [left(cast(t1.c6, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))], [right(cast(t1.c6, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c6], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, @@ -3478,7 +3478,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c7], [t1.c17], [left(cast(t1.c7, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, 0)))], [right(cast(t1.c7, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, + 0 - output([t1.c7], [t1.c17], [left(cast(t1.c7, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))], [right(cast(t1.c7, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c7], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, @@ -3949,7 +3949,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c8], [t1.c17], [left(cast(t1.c8, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, 0)))], [right(cast(t1.c8, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, + 0 - output([t1.c8], [t1.c17], [left(cast(t1.c8, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))], [right(cast(t1.c8, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c8], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, @@ -4420,7 +4420,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c9], [t1.c17], [left(cast(t1.c9, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, 0)))], [right(cast(t1.c9, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, + 0 - output([t1.c9], [t1.c17], [left(cast(t1.c9, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))], [right(cast(t1.c9, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c9], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, @@ -4891,8 +4891,8 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c10], [t1.c17], [left(cast(t1.c10, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, 0)))], [right(cast(t1.c10, VARCHAR(1048576)), cast(t1.c17, - BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c10], [t1.c17], [left(cast(t1.c10, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))], [right(cast(t1.c10, VARCHAR(1048576)), cast(t1.c17, + BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c10], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5362,8 +5362,8 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c11], [t1.c17], [left(cast(t1.c11, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, 0)))], [right(cast(t1.c11, VARCHAR(1048576)), cast(t1.c17, - BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c11], [t1.c17], [left(cast(t1.c11, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))], [right(cast(t1.c11, VARCHAR(1048576)), cast(t1.c17, + BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c11], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5833,8 +5833,8 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c12], [t1.c17], [left(cast(t1.c12, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, 0)))], [right(cast(t1.c12, VARCHAR(1048576)), cast(t1.c17, - BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c12], [t1.c17], [left(cast(t1.c12, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))], [right(cast(t1.c12, VARCHAR(1048576)), cast(t1.c17, + BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c12], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6304,8 +6304,8 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c13], [t1.c17], [left(cast(t1.c13, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, 0)))], [right(cast(t1.c13, VARCHAR(1048576)), cast(t1.c17, - BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c13], [t1.c17], [left(cast(t1.c13, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))], [right(cast(t1.c13, VARCHAR(1048576)), cast(t1.c17, + BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c13], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6775,8 +6775,8 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c14], [t1.c17], [left(cast(t1.c14, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, 0)))], [right(cast(t1.c14, VARCHAR(1048576)), cast(t1.c17, - BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c14], [t1.c17], [left(cast(t1.c14, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))], [right(cast(t1.c14, VARCHAR(1048576)), cast(t1.c17, + BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c14], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7246,8 +7246,8 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c15], [t1.c17], [left(cast(t1.c15, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, 0)))], [right(cast(t1.c15, VARCHAR(1048576)), cast(t1.c17, - BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c15], [t1.c17], [left(cast(t1.c15, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))], [right(cast(t1.c15, VARCHAR(1048576)), cast(t1.c17, + BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c15], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7717,8 +7717,8 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c16], [t1.c17], [left(cast(t1.c16, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, 0)))], [right(cast(t1.c16, VARCHAR(1048576)), cast(t1.c17, - BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c16], [t1.c17], [left(cast(t1.c16, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))], [right(cast(t1.c16, VARCHAR(1048576)), cast(t1.c17, + BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c16], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8188,8 +8188,8 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c17], [t1.c17], [left(cast(t1.c17, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, 0)))], [right(cast(t1.c17, VARCHAR(1048576)), cast(t1.c17, - BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c17], [t1.c17], [left(cast(t1.c17, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))], [right(cast(t1.c17, VARCHAR(1048576)), cast(t1.c17, + BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -8659,8 +8659,8 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c18], [t1.c17], [left(cast(t1.c18, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, 0)))], [right(cast(t1.c18, VARCHAR(1048576)), cast(t1.c17, - BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c18], [t1.c17], [left(cast(t1.c18, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))], [right(cast(t1.c18, VARCHAR(1048576)), cast(t1.c17, + BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c18], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9130,8 +9130,8 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c19], [t1.c17], [left(cast(t1.c19, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, 0)))], [right(cast(t1.c19, VARCHAR(1048576)), cast(t1.c17, - BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c19], [t1.c17], [left(cast(t1.c19, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))], [right(cast(t1.c19, VARCHAR(1048576)), cast(t1.c17, + BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c19], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9601,8 +9601,8 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c20], [t1.c17], [left(cast(t1.c20, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, 0)))], [right(cast(t1.c20, VARCHAR(1048576)), cast(t1.c17, - BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c20], [t1.c17], [left(cast(t1.c20, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))], [right(cast(t1.c20, VARCHAR(1048576)), cast(t1.c17, + BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c20], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10057,7 +10057,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c21], [t1.c17], [left(t1.c21, cast(t1.c17, BIGINT(-1, 0)))], [right(t1.c21, cast(t1.c17, BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c21], [t1.c17], [left(t1.c21, cast(t1.c17, BIGINT(10, 0)))], [right(t1.c21, cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c21], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10522,8 +10522,8 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c22], [t1.c17], [left(cast(t1.c22, VARCHAR(1048576)), cast(t1.c17, BIGINT(-1, 0)))], [right(cast(t1.c22, VARCHAR(1048576)), cast(t1.c17, - BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c22], [t1.c17], [left(cast(t1.c22, VARCHAR(1048576)), cast(t1.c17, BIGINT(10, 0)))], [right(cast(t1.c22, VARCHAR(1048576)), cast(t1.c17, + BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c22], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true diff --git a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_substr.result b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_substr.result index ebc4b6991a..70ca736d90 100644 --- a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_substr.result +++ b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_substr.result @@ -117,8 +117,8 @@ Query Plan ========================================== Outputs & filters: ------------------------------------- - 0 - output([substr(cast(3.14159, VARCHAR(1048576)), cast('2', BIGINT(-1, 0)), cast('3', BIGINT(-1, 0)))]), filter(nil) - values({substr(cast(3.14159, VARCHAR(1048576)), cast('2', BIGINT(-1, 0)), cast('3', BIGINT(-1, 0)))}) + 0 - output([substr(cast(3.14159, VARCHAR(4)), cast('2', BIGINT(-1, 0)), cast('3', BIGINT(-1, 0)))]), filter(nil) + values({substr(cast(3.14159, VARCHAR(4)), cast('2', BIGINT(-1, 0)), cast('3', BIGINT(-1, 0)))}) select substr(3.14159, '2', '2.1') from dual; +-----------------------------+ | substr(3.14159, '2', '2.1') | @@ -173,3 +173,37 @@ select substr(3.14159, 2.1, 2.1) from dual; +---------------------------+ | .1 | +---------------------------+ +select substr(1, 1, 9223372036854775807); ++-----------------------------------+ +| substr(1, 1, 9223372036854775807) | ++-----------------------------------+ +| 1 | ++-----------------------------------+ +select substr(123, 2, 9223372036854775806); ++-------------------------------------+ +| substr(123, 2, 9223372036854775806) | ++-------------------------------------+ +| 23 | ++-------------------------------------+ +drop view if exists v1; +create view v1 as select substr(123, 2, 9223372036854775807); +desc v1; ++-------------------------------------+------------+------+-----+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++-------------------------------------+------------+------+-----+---------+-------+ +| substr(123, 2, 9223372036854775807) | varchar(2) | NO | | | | ++-------------------------------------+------------+------+-----+---------+-------+ +select * from v1; ++-------------------------------------+ +| substr(123, 2, 9223372036854775807) | ++-------------------------------------+ +| 23 | ++-------------------------------------+ +drop view v1; +SELECT NULL ORDER BY SUBSTR ( 9223372036854775807 FROM @@ob_query_timeout FOR 9223372036854775807 ); ++------+ +| NULL | ++------+ +| NULL | ++------+ + diff --git a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_timestampadd.result b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_timestampadd.result index ccf63da29e..e5eadaf2ba 100644 --- a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_timestampadd.result +++ b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_timestampadd.result @@ -20473,7 +20473,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(1, cast(t1.c17, BIGINT(-1, 0)), t1.c1)]), filter(nil), rowset=16 + 0 - output([timestampadd(1, cast(t1.c17, BIGINT(10, 0)), t1.c1)]), filter(nil), rowset=16 access([t1.c17], [t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -20492,7 +20492,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(8, cast(t1.c17, BIGINT(-1, 0)), t1.c1)]), filter(nil), rowset=16 + 0 - output([timestampadd(8, cast(t1.c17, BIGINT(10, 0)), t1.c1)]), filter(nil), rowset=16 access([t1.c17], [t1.c1]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -20531,7 +20531,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(1, cast(t1.c17, BIGINT(-1, 0)), t1.c2)]), filter(nil), rowset=16 + 0 - output([timestampadd(1, cast(t1.c17, BIGINT(10, 0)), t1.c2)]), filter(nil), rowset=16 access([t1.c17], [t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -20550,7 +20550,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(8, cast(t1.c17, BIGINT(-1, 0)), t1.c2)]), filter(nil), rowset=16 + 0 - output([timestampadd(8, cast(t1.c17, BIGINT(10, 0)), t1.c2)]), filter(nil), rowset=16 access([t1.c17], [t1.c2]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -20589,7 +20589,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(1, cast(t1.c17, BIGINT(-1, 0)), t1.c3)]), filter(nil), rowset=16 + 0 - output([timestampadd(1, cast(t1.c17, BIGINT(10, 0)), t1.c3)]), filter(nil), rowset=16 access([t1.c17], [t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -20608,7 +20608,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(8, cast(t1.c17, BIGINT(-1, 0)), t1.c3)]), filter(nil), rowset=16 + 0 - output([timestampadd(8, cast(t1.c17, BIGINT(10, 0)), t1.c3)]), filter(nil), rowset=16 access([t1.c17], [t1.c3]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -20647,7 +20647,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(1, cast(t1.c17, BIGINT(-1, 0)), t1.c4)]), filter(nil), rowset=16 + 0 - output([timestampadd(1, cast(t1.c17, BIGINT(10, 0)), t1.c4)]), filter(nil), rowset=16 access([t1.c17], [t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -20666,7 +20666,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(8, cast(t1.c17, BIGINT(-1, 0)), t1.c4)]), filter(nil), rowset=16 + 0 - output([timestampadd(8, cast(t1.c17, BIGINT(10, 0)), t1.c4)]), filter(nil), rowset=16 access([t1.c17], [t1.c4]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -20705,7 +20705,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(1, cast(t1.c17, BIGINT(-1, 0)), t1.c5)]), filter(nil), rowset=16 + 0 - output([timestampadd(1, cast(t1.c17, BIGINT(10, 0)), t1.c5)]), filter(nil), rowset=16 access([t1.c17], [t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -20724,7 +20724,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(8, cast(t1.c17, BIGINT(-1, 0)), t1.c5)]), filter(nil), rowset=16 + 0 - output([timestampadd(8, cast(t1.c17, BIGINT(10, 0)), t1.c5)]), filter(nil), rowset=16 access([t1.c17], [t1.c5]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -20763,7 +20763,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(1, cast(t1.c17, BIGINT(-1, 0)), t1.c6)]), filter(nil), rowset=16 + 0 - output([timestampadd(1, cast(t1.c17, BIGINT(10, 0)), t1.c6)]), filter(nil), rowset=16 access([t1.c17], [t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -20782,7 +20782,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(8, cast(t1.c17, BIGINT(-1, 0)), t1.c6)]), filter(nil), rowset=16 + 0 - output([timestampadd(8, cast(t1.c17, BIGINT(10, 0)), t1.c6)]), filter(nil), rowset=16 access([t1.c17], [t1.c6]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -20821,7 +20821,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(1, cast(t1.c17, BIGINT(-1, 0)), t1.c7)]), filter(nil), rowset=16 + 0 - output([timestampadd(1, cast(t1.c17, BIGINT(10, 0)), t1.c7)]), filter(nil), rowset=16 access([t1.c17], [t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -20840,7 +20840,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(8, cast(t1.c17, BIGINT(-1, 0)), t1.c7)]), filter(nil), rowset=16 + 0 - output([timestampadd(8, cast(t1.c17, BIGINT(10, 0)), t1.c7)]), filter(nil), rowset=16 access([t1.c17], [t1.c7]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -20879,7 +20879,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(1, cast(t1.c17, BIGINT(-1, 0)), t1.c8)]), filter(nil), rowset=16 + 0 - output([timestampadd(1, cast(t1.c17, BIGINT(10, 0)), t1.c8)]), filter(nil), rowset=16 access([t1.c17], [t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -20898,7 +20898,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(8, cast(t1.c17, BIGINT(-1, 0)), t1.c8)]), filter(nil), rowset=16 + 0 - output([timestampadd(8, cast(t1.c17, BIGINT(10, 0)), t1.c8)]), filter(nil), rowset=16 access([t1.c17], [t1.c8]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -20937,7 +20937,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(1, cast(t1.c17, BIGINT(-1, 0)), t1.c9)]), filter(nil), rowset=16 + 0 - output([timestampadd(1, cast(t1.c17, BIGINT(10, 0)), t1.c9)]), filter(nil), rowset=16 access([t1.c17], [t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -20956,7 +20956,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(8, cast(t1.c17, BIGINT(-1, 0)), t1.c9)]), filter(nil), rowset=16 + 0 - output([timestampadd(8, cast(t1.c17, BIGINT(10, 0)), t1.c9)]), filter(nil), rowset=16 access([t1.c17], [t1.c9]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -20995,7 +20995,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(1, cast(t1.c17, BIGINT(-1, 0)), t1.c10)]), filter(nil), rowset=16 + 0 - output([timestampadd(1, cast(t1.c17, BIGINT(10, 0)), t1.c10)]), filter(nil), rowset=16 access([t1.c17], [t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21014,7 +21014,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(8, cast(t1.c17, BIGINT(-1, 0)), t1.c10)]), filter(nil), rowset=16 + 0 - output([timestampadd(8, cast(t1.c17, BIGINT(10, 0)), t1.c10)]), filter(nil), rowset=16 access([t1.c17], [t1.c10]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21053,7 +21053,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(1, cast(t1.c17, BIGINT(-1, 0)), t1.c11)]), filter(nil), rowset=16 + 0 - output([timestampadd(1, cast(t1.c17, BIGINT(10, 0)), t1.c11)]), filter(nil), rowset=16 access([t1.c17], [t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21072,7 +21072,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(8, cast(t1.c17, BIGINT(-1, 0)), t1.c11)]), filter(nil), rowset=16 + 0 - output([timestampadd(8, cast(t1.c17, BIGINT(10, 0)), t1.c11)]), filter(nil), rowset=16 access([t1.c17], [t1.c11]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21111,7 +21111,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(1, cast(t1.c17, BIGINT(-1, 0)), t1.c12)]), filter(nil), rowset=16 + 0 - output([timestampadd(1, cast(t1.c17, BIGINT(10, 0)), t1.c12)]), filter(nil), rowset=16 access([t1.c17], [t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21130,7 +21130,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(8, cast(t1.c17, BIGINT(-1, 0)), t1.c12)]), filter(nil), rowset=16 + 0 - output([timestampadd(8, cast(t1.c17, BIGINT(10, 0)), t1.c12)]), filter(nil), rowset=16 access([t1.c17], [t1.c12]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21169,7 +21169,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(1, cast(t1.c17, BIGINT(-1, 0)), t1.c13)]), filter(nil), rowset=16 + 0 - output([timestampadd(1, cast(t1.c17, BIGINT(10, 0)), t1.c13)]), filter(nil), rowset=16 access([t1.c17], [t1.c13]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21188,7 +21188,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(8, cast(t1.c17, BIGINT(-1, 0)), t1.c13)]), filter(nil), rowset=16 + 0 - output([timestampadd(8, cast(t1.c17, BIGINT(10, 0)), t1.c13)]), filter(nil), rowset=16 access([t1.c17], [t1.c13]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21227,7 +21227,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(1, cast(t1.c17, BIGINT(-1, 0)), t1.c14)]), filter(nil), rowset=16 + 0 - output([timestampadd(1, cast(t1.c17, BIGINT(10, 0)), t1.c14)]), filter(nil), rowset=16 access([t1.c17], [t1.c14]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21246,7 +21246,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(8, cast(t1.c17, BIGINT(-1, 0)), t1.c14)]), filter(nil), rowset=16 + 0 - output([timestampadd(8, cast(t1.c17, BIGINT(10, 0)), t1.c14)]), filter(nil), rowset=16 access([t1.c17], [t1.c14]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21285,7 +21285,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(1, cast(t1.c17, BIGINT(-1, 0)), t1.c15)]), filter(nil), rowset=16 + 0 - output([timestampadd(1, cast(t1.c17, BIGINT(10, 0)), t1.c15)]), filter(nil), rowset=16 access([t1.c17], [t1.c15]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21304,7 +21304,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(8, cast(t1.c17, BIGINT(-1, 0)), t1.c15)]), filter(nil), rowset=16 + 0 - output([timestampadd(8, cast(t1.c17, BIGINT(10, 0)), t1.c15)]), filter(nil), rowset=16 access([t1.c17], [t1.c15]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21343,7 +21343,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(1, cast(t1.c17, BIGINT(-1, 0)), t1.c16)]), filter(nil), rowset=16 + 0 - output([timestampadd(1, cast(t1.c17, BIGINT(10, 0)), t1.c16)]), filter(nil), rowset=16 access([t1.c17], [t1.c16]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21362,7 +21362,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(8, cast(t1.c17, BIGINT(-1, 0)), t1.c16)]), filter(nil), rowset=16 + 0 - output([timestampadd(8, cast(t1.c17, BIGINT(10, 0)), t1.c16)]), filter(nil), rowset=16 access([t1.c17], [t1.c16]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21401,7 +21401,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(1, cast(t1.c17, BIGINT(-1, 0)), t1.c17)]), filter(nil), rowset=16 + 0 - output([timestampadd(1, cast(t1.c17, BIGINT(10, 0)), t1.c17)]), filter(nil), rowset=16 access([t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21420,7 +21420,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(8, cast(t1.c17, BIGINT(-1, 0)), t1.c17)]), filter(nil), rowset=16 + 0 - output([timestampadd(8, cast(t1.c17, BIGINT(10, 0)), t1.c17)]), filter(nil), rowset=16 access([t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21459,7 +21459,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(1, cast(t1.c17, BIGINT(-1, 0)), t1.c18)]), filter(nil), rowset=16 + 0 - output([timestampadd(1, cast(t1.c17, BIGINT(10, 0)), t1.c18)]), filter(nil), rowset=16 access([t1.c17], [t1.c18]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21478,7 +21478,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(8, cast(t1.c17, BIGINT(-1, 0)), t1.c18)]), filter(nil), rowset=16 + 0 - output([timestampadd(8, cast(t1.c17, BIGINT(10, 0)), t1.c18)]), filter(nil), rowset=16 access([t1.c17], [t1.c18]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21517,7 +21517,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(1, cast(t1.c17, BIGINT(-1, 0)), t1.c19)]), filter(nil), rowset=16 + 0 - output([timestampadd(1, cast(t1.c17, BIGINT(10, 0)), t1.c19)]), filter(nil), rowset=16 access([t1.c17], [t1.c19]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21536,7 +21536,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(8, cast(t1.c17, BIGINT(-1, 0)), t1.c19)]), filter(nil), rowset=16 + 0 - output([timestampadd(8, cast(t1.c17, BIGINT(10, 0)), t1.c19)]), filter(nil), rowset=16 access([t1.c17], [t1.c19]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21575,7 +21575,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(1, cast(t1.c17, BIGINT(-1, 0)), t1.c20)]), filter(nil), rowset=16 + 0 - output([timestampadd(1, cast(t1.c17, BIGINT(10, 0)), t1.c20)]), filter(nil), rowset=16 access([t1.c17], [t1.c20]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21594,7 +21594,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(8, cast(t1.c17, BIGINT(-1, 0)), t1.c20)]), filter(nil), rowset=16 + 0 - output([timestampadd(8, cast(t1.c17, BIGINT(10, 0)), t1.c20)]), filter(nil), rowset=16 access([t1.c17], [t1.c20]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21633,7 +21633,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(1, cast(t1.c17, BIGINT(-1, 0)), t1.c21)]), filter(nil), rowset=16 + 0 - output([timestampadd(1, cast(t1.c17, BIGINT(10, 0)), t1.c21)]), filter(nil), rowset=16 access([t1.c17], [t1.c21]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21652,7 +21652,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(8, cast(t1.c17, BIGINT(-1, 0)), t1.c21)]), filter(nil), rowset=16 + 0 - output([timestampadd(8, cast(t1.c17, BIGINT(10, 0)), t1.c21)]), filter(nil), rowset=16 access([t1.c17], [t1.c21]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21691,7 +21691,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(1, cast(t1.c17, BIGINT(-1, 0)), t1.c22)]), filter(nil), rowset=16 + 0 - output([timestampadd(1, cast(t1.c17, BIGINT(10, 0)), t1.c22)]), filter(nil), rowset=16 access([t1.c17], [t1.c22]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -21710,7 +21710,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([timestampadd(8, cast(t1.c17, BIGINT(-1, 0)), t1.c22)]), filter(nil), rowset=16 + 0 - output([timestampadd(8, cast(t1.c17, BIGINT(10, 0)), t1.c22)]), filter(nil), rowset=16 access([t1.c17], [t1.c22]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true diff --git a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_trunc.result b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_trunc.result index 519c95a89f..ba290307f3 100644 --- a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_trunc.result +++ b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/expr_trunc.result @@ -1101,7 +1101,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c1], [t1.c17], [truncate(cast(t1.c1, DOUBLE(-1, -1)), cast(t1.c17, BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c1], [t1.c17], [truncate(cast(t1.c1, DOUBLE(-1, -1)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c1], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -2421,7 +2421,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c2], [t1.c17], [truncate(cast(t1.c2, DOUBLE UNSIGNED(-1, -1)), cast(t1.c17, BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c2], [t1.c17], [truncate(cast(t1.c2, DOUBLE UNSIGNED(-1, -1)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c2], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -3741,7 +3741,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c3], [t1.c17], [truncate(cast(t1.c3, DOUBLE(-1, -1)), cast(t1.c17, BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c3], [t1.c17], [truncate(cast(t1.c3, DOUBLE(-1, -1)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c3], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -5061,7 +5061,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c4], [t1.c17], [truncate(cast(t1.c4, DOUBLE UNSIGNED(-1, -1)), cast(t1.c17, BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c4], [t1.c17], [truncate(cast(t1.c4, DOUBLE UNSIGNED(-1, -1)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c4], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -6381,7 +6381,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c5], [t1.c17], [truncate(cast(t1.c5, DOUBLE(-1, -1)), cast(t1.c17, BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c5], [t1.c17], [truncate(cast(t1.c5, DOUBLE(-1, -1)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c5], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -7701,7 +7701,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c6], [t1.c17], [truncate(cast(t1.c6, DOUBLE UNSIGNED(-1, -1)), cast(t1.c17, BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c6], [t1.c17], [truncate(cast(t1.c6, DOUBLE UNSIGNED(-1, -1)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c6], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -9021,7 +9021,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c7], [t1.c17], [truncate(cast(t1.c7, DOUBLE(-1, -1)), cast(t1.c17, BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c7], [t1.c17], [truncate(cast(t1.c7, DOUBLE(-1, -1)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c7], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -10341,7 +10341,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c8], [t1.c17], [truncate(cast(t1.c8, DOUBLE UNSIGNED(-1, -1)), cast(t1.c17, BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c8], [t1.c17], [truncate(cast(t1.c8, DOUBLE UNSIGNED(-1, -1)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c8], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -11661,7 +11661,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c9], [t1.c17], [truncate(cast(t1.c9, DOUBLE(-1, -1)), cast(t1.c17, BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c9], [t1.c17], [truncate(cast(t1.c9, DOUBLE(-1, -1)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c9], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -12981,7 +12981,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c10], [t1.c17], [truncate(cast(t1.c10, DOUBLE UNSIGNED(-1, -1)), cast(t1.c17, BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c10], [t1.c17], [truncate(cast(t1.c10, DOUBLE UNSIGNED(-1, -1)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c10], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -14285,7 +14285,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c11], [t1.c17], [truncate(cast(t1.c11, DOUBLE(-1, -1)), cast(t1.c17, BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c11], [t1.c17], [truncate(cast(t1.c11, DOUBLE(-1, -1)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c11], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -15583,7 +15583,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c12], [t1.c17], [truncate(cast(t1.c12, DOUBLE UNSIGNED(-1, -1)), cast(t1.c17, BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c12], [t1.c17], [truncate(cast(t1.c12, DOUBLE UNSIGNED(-1, -1)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c12], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -16897,7 +16897,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c13], [t1.c17], [truncate(cast(t1.c13, DOUBLE(-1, -1)), cast(t1.c17, BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c13], [t1.c17], [truncate(cast(t1.c13, DOUBLE(-1, -1)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c13], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -18217,7 +18217,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c14], [t1.c17], [truncate(cast(t1.c14, DOUBLE UNSIGNED(-1, -1)), cast(t1.c17, BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c14], [t1.c17], [truncate(cast(t1.c14, DOUBLE UNSIGNED(-1, -1)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c14], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -19521,7 +19521,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c15], [t1.c17], [truncate(t1.c15, cast(t1.c17, BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c15], [t1.c17], [truncate(t1.c15, cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c15], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -20819,7 +20819,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c16], [t1.c17], [truncate(t1.c16, cast(t1.c17, BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c16], [t1.c17], [truncate(t1.c16, cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c16], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -22117,7 +22117,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c17], [t1.c17], [truncate(t1.c17, cast(t1.c17, BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c17], [t1.c17], [truncate(t1.c17, cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -23415,7 +23415,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c18], [t1.c17], [truncate(t1.c18, cast(t1.c17, BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c18], [t1.c17], [truncate(t1.c18, cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c18], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -24729,7 +24729,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c19], [t1.c17], [truncate(cast(t1.c19, DOUBLE(-1, -1)), cast(t1.c17, BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c19], [t1.c17], [truncate(cast(t1.c19, DOUBLE(-1, -1)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c19], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -26049,7 +26049,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c20], [t1.c17], [truncate(cast(t1.c20, DOUBLE(-1, -1)), cast(t1.c17, BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c20], [t1.c17], [truncate(cast(t1.c20, DOUBLE(-1, -1)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c20], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -27369,7 +27369,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c21], [t1.c17], [truncate(cast(t1.c21, DOUBLE(-1, -1)), cast(t1.c17, BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c21], [t1.c17], [truncate(cast(t1.c21, DOUBLE(-1, -1)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c21], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true @@ -28689,7 +28689,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([t1.c22], [t1.c17], [truncate(cast(t1.c22, DOUBLE(-1, -1)), cast(t1.c17, BIGINT(-1, 0)))]), filter(nil), rowset=16 + 0 - output([t1.c22], [t1.c17], [truncate(cast(t1.c22, DOUBLE(-1, -1)), cast(t1.c17, BIGINT(10, 0)))]), filter(nil), rowset=16 access([t1.c22], [t1.c17]), partitions(p0) is_index_back=false, is_global_index=false, range_key([t1.__pk_increment]), range(MIN ; MAX)always true diff --git a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/hash_distinct.result b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/hash_distinct.result index d7930958bf..20c53d4d15 100644 --- a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/hash_distinct.result +++ b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/hash_distinct.result @@ -277,15 +277,17 @@ Query Plan ============================================ Outputs & filters: ------------------------------------- - 0 - output([T_FUN_SUM(.c0)], [T_FUN_SUM(length(.c1))], [T_FUN_COUNT(.c2)], [T_FUN_COUNT(.c3)], [T_FUN_COUNT(.c4)], [T_FUN_COUNT(.c5)], [T_FUN_COUNT(.c6)], - [T_FUN_COUNT(.c7)]), filter(nil) + 0 - output([T_FUN_SUM(ANONYMOUS_VIEW1.c0)], [T_FUN_SUM(length(ANONYMOUS_VIEW1.c1))], [T_FUN_COUNT(ANONYMOUS_VIEW1.c2)], [T_FUN_COUNT(ANONYMOUS_VIEW1.c3)], + [T_FUN_COUNT(ANONYMOUS_VIEW1.c4)], [T_FUN_COUNT(ANONYMOUS_VIEW1.c5)], [T_FUN_COUNT(ANONYMOUS_VIEW1.c6)], [T_FUN_COUNT(ANONYMOUS_VIEW1.c7)]), filter(nil) limit(1), offset(nil) - 1 - output([T_FUN_SUM(.c0)], [T_FUN_SUM(length(.c1))], [T_FUN_COUNT(.c2)], [T_FUN_COUNT(.c3)], [T_FUN_COUNT(.c4)], [T_FUN_COUNT(.c5)], [T_FUN_COUNT(.c6)], - [T_FUN_COUNT(.c7)]), filter(nil) - group(nil), agg_func([T_FUN_SUM(.c0)], [T_FUN_SUM(length(.c1))], [T_FUN_COUNT(.c2)], [T_FUN_COUNT(.c3)], [T_FUN_COUNT(.c4)], [T_FUN_COUNT(.c5)], [T_FUN_COUNT(.c6)], - [T_FUN_COUNT(.c7)]) - 2 - output([.c0], [.c1], [.c2], [.c3], [.c4], [.c5], [.c6], [.c7]), filter(nil) - access([.c0], [.c1], [.c2], [.c3], [.c4], [.c5], [.c6], [.c7]) + 1 - output([T_FUN_SUM(ANONYMOUS_VIEW1.c0)], [T_FUN_SUM(length(ANONYMOUS_VIEW1.c1))], [T_FUN_COUNT(ANONYMOUS_VIEW1.c2)], [T_FUN_COUNT(ANONYMOUS_VIEW1.c3)], + [T_FUN_COUNT(ANONYMOUS_VIEW1.c4)], [T_FUN_COUNT(ANONYMOUS_VIEW1.c5)], [T_FUN_COUNT(ANONYMOUS_VIEW1.c6)], [T_FUN_COUNT(ANONYMOUS_VIEW1.c7)]), filter(nil) + group(nil), agg_func([T_FUN_SUM(ANONYMOUS_VIEW1.c0)], [T_FUN_SUM(length(ANONYMOUS_VIEW1.c1))], [T_FUN_COUNT(ANONYMOUS_VIEW1.c2)], [T_FUN_COUNT(ANONYMOUS_VIEW1.c3)], + [T_FUN_COUNT(ANONYMOUS_VIEW1.c4)], [T_FUN_COUNT(ANONYMOUS_VIEW1.c5)], [T_FUN_COUNT(ANONYMOUS_VIEW1.c6)], [T_FUN_COUNT(ANONYMOUS_VIEW1.c7)]) + 2 - output([ANONYMOUS_VIEW1.c0], [ANONYMOUS_VIEW1.c1], [ANONYMOUS_VIEW1.c2], [ANONYMOUS_VIEW1.c3], [ANONYMOUS_VIEW1.c4], [ANONYMOUS_VIEW1.c5], [ANONYMOUS_VIEW1.c6], + [ANONYMOUS_VIEW1.c7]), filter(nil) + access([ANONYMOUS_VIEW1.c0], [ANONYMOUS_VIEW1.c1], [ANONYMOUS_VIEW1.c2], [ANONYMOUS_VIEW1.c3], [ANONYMOUS_VIEW1.c4], [ANONYMOUS_VIEW1.c5], [ANONYMOUS_VIEW1.c6], + [ANONYMOUS_VIEW1.c7]) 3 - output([t111_var.c0], [t111_var.c1], [t111_var.c2], [t111_var.c3], [t111_var.c4], [t111_var.c5], [t111_var.c6], [t111_var.c7]), filter(nil) distinct([t111_var.c0], [t111_var.c1], [t111_var.c2], [t111_var.c3], [t111_var.c4], [t111_var.c5], [t111_var.c6], [t111_var.c7]) 4 - output([t111_var.c0], [t111_var.c1], [t111_var.c2], [t111_var.c3], [t111_var.c4], [t111_var.c5], [t111_var.c6], [t111_var.c7]), filter(nil) diff --git a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/static_engine_cmp_null.result b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/static_engine_cmp_null.result index 0caf57d02f..8133e6558e 100644 --- a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/static_engine_cmp_null.result +++ b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/static_engine_cmp_null.result @@ -238,7 +238,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([NULL = t.float_t]), filter(nil), rowset=16 + 0 - output([cast(NULL, FLOAT(-1, -1)) = t.float_t]), filter(nil), rowset=16 access([t.float_t]), partitions(p0) limit(1), offset(nil), is_index_back=false, is_global_index=false, range_key([t.__pk_increment]), range(MIN ; MAX)always true @@ -257,7 +257,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([NULL = t.ufloat_t]), filter(nil), rowset=16 + 0 - output([cast(NULL, FLOAT UNSIGNED(-1, -1)) = t.ufloat_t]), filter(nil), rowset=16 access([t.ufloat_t]), partitions(p0) limit(1), offset(nil), is_index_back=false, is_global_index=false, range_key([t.__pk_increment]), range(MIN ; MAX)always true @@ -276,7 +276,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([NULL = t.double_t]), filter(nil), rowset=16 + 0 - output([cast(NULL, DOUBLE(-1, -1)) = t.double_t]), filter(nil), rowset=16 access([t.double_t]), partitions(p0) limit(1), offset(nil), is_index_back=false, is_global_index=false, range_key([t.__pk_increment]), range(MIN ; MAX)always true @@ -295,7 +295,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([NULL = t.udouble_t]), filter(nil), rowset=16 + 0 - output([cast(NULL, DOUBLE UNSIGNED(-1, -1)) = t.udouble_t]), filter(nil), rowset=16 access([t.udouble_t]), partitions(p0) limit(1), offset(nil), is_index_back=false, is_global_index=false, range_key([t.__pk_increment]), range(MIN ; MAX)always true @@ -447,7 +447,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([NULL = t.varchar_t]), filter(nil), rowset=16 + 0 - output([cast(NULL, VARCHAR(1048576)) = t.varchar_t]), filter(nil), rowset=16 access([t.varchar_t]), partitions(p0) limit(1), offset(nil), is_index_back=false, is_global_index=false, range_key([t.__pk_increment]), range(MIN ; MAX)always true @@ -466,7 +466,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([NULL = t.char_t]), filter(nil), rowset=16 + 0 - output([cast(NULL, CHAR(1048576)) = t.char_t]), filter(nil), rowset=16 access([t.char_t]), partitions(p0) limit(1), offset(nil), is_index_back=false, is_global_index=false, range_key([t.__pk_increment]), range(MIN ; MAX)always true @@ -485,7 +485,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([NULL = t.tinytext_t]), filter(nil), rowset=16 + 0 - output([cast(NULL, TINYTEXT(256)) = t.tinytext_t]), filter(nil), rowset=16 access([t.tinytext_t]), partitions(p0) limit(1), offset(nil), is_index_back=false, is_global_index=false, range_key([t.__pk_increment]), range(MIN ; MAX)always true @@ -504,7 +504,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([NULL = t.mediumtext_t]), filter(nil) + 0 - output([cast(NULL, MEDIUMTEXT(16777216)) = t.mediumtext_t]), filter(nil) access([t.mediumtext_t]), partitions(p0) limit(1), offset(nil), is_index_back=false, is_global_index=false, range_key([t.__pk_increment]), range(MIN ; MAX)always true @@ -523,7 +523,7 @@ Query Plan =============================================== Outputs & filters: ------------------------------------- - 0 - output([NULL = t.longtext_t]), filter(nil) + 0 - output([cast(NULL, LONGTEXT(536870911)) = t.longtext_t]), filter(nil) access([t.longtext_t]), partitions(p0) limit(1), offset(nil), is_index_back=false, is_global_index=false, range_key([t.__pk_increment]), range(MIN ; MAX)always true diff --git a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/subplan_filter.result b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/subplan_filter.result index 5f4a257830..c2a850d1f5 100644 --- a/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/subplan_filter.result +++ b/tools/deploy/mysql_test/test_suite/static_engine/r/mysql/subplan_filter.result @@ -1757,6 +1757,7 @@ Outline Data: ------------------------------------- /*+ BEGIN_OUTLINE_DATA + PQ_SUBQUERY(@"SEL$1" ("SEL$2") NONE ALL) FULL(@"SEL$1" "test"."t1"@"SEL$1") FULL(@"SEL$2" "test"."t2"@"SEL$2") USE_DAS(@"SEL$2" "test"."t2"@"SEL$2") @@ -1858,6 +1859,7 @@ Outline Data: ------------------------------------- /*+ BEGIN_OUTLINE_DATA + PQ_SUBQUERY(@"SEL$1" ("SEL$2") NONE ALL) FULL(@"SEL$1" "test"."t1"@"SEL$1") FULL(@"SEL$2" "test"."t2"@"SEL$2") USE_DAS(@"SEL$2" "test"."t2"@"SEL$2") @@ -1959,6 +1961,7 @@ Outline Data: ------------------------------------- /*+ BEGIN_OUTLINE_DATA + PQ_SUBQUERY(@"SEL$1" ("SEL$2") NONE ALL) FULL(@"SEL$1" "test"."t1"@"SEL$1") FULL(@"SEL$2" "test"."t2"@"SEL$2") USE_DAS(@"SEL$2" "test"."t2"@"SEL$2") @@ -2067,6 +2070,7 @@ Outline Data: ------------------------------------- /*+ BEGIN_OUTLINE_DATA + PQ_SUBQUERY(@"SEL$1" ("SEL$2" "SEL$3") LOCAL LOCAL) FULL(@"SEL$1" "test"."t1"@"SEL$1") FULL(@"SEL$2" "test"."t2"@"SEL$2") FULL(@"SEL$3" "t2"@"SEL$3") @@ -2187,6 +2191,7 @@ Outline Data: ------------------------------------- /*+ BEGIN_OUTLINE_DATA + PQ_SUBQUERY(@"SEL$1" ("SEL$2" "SEL$3") LOCAL LOCAL) FULL(@"SEL$1" "test"."t1"@"SEL$1") FULL(@"SEL$2" "test"."t2"@"SEL$2") FULL(@"SEL$3" "t2"@"SEL$3") @@ -2307,6 +2312,7 @@ Outline Data: ------------------------------------- /*+ BEGIN_OUTLINE_DATA + PQ_SUBQUERY(@"SEL$1" ("SEL$2" "SEL$3") LOCAL LOCAL) FULL(@"SEL$1" "test"."t1"@"SEL$1") FULL(@"SEL$2" "test"."t2"@"SEL$2") FULL(@"SEL$3" "t2"@"SEL$3") @@ -2427,6 +2433,7 @@ Outline Data: ------------------------------------- /*+ BEGIN_OUTLINE_DATA + PQ_SUBQUERY(@"SEL$1" ("SEL$2" "SEL$3") LOCAL LOCAL) FULL(@"SEL$1" "test"."t1"@"SEL$1") FULL(@"SEL$2" "test"."t2"@"SEL$2") FULL(@"SEL$3" "t2"@"SEL$3") @@ -2554,6 +2561,7 @@ Outline Data: ------------------------------------- /*+ BEGIN_OUTLINE_DATA + PQ_SUBQUERY(@"SEL$1" ("SEL$2" "SEL$3" "SEL$4") LOCAL LOCAL) FULL(@"SEL$1" "test"."t1"@"SEL$1") FULL(@"SEL$2" "test"."t2"@"SEL$2") FULL(@"SEL$3" "t2"@"SEL$3") @@ -2727,6 +2735,7 @@ Outline Data: ------------------------------------- /*+ BEGIN_OUTLINE_DATA + PQ_SUBQUERY(@"SEL$1" ("SEL$2" "SEL$3" "SEL$4" "SEL$5" "SEL$6" "SEL$7" "SEL$8") NONE ALL) FULL(@"SEL$1" "test"."t1"@"SEL$1") FULL(@"SEL$2" "test"."t2"@"SEL$2") USE_DAS(@"SEL$2" "test"."t2"@"SEL$2") @@ -2953,6 +2962,7 @@ Outline Data: ------------------------------------- /*+ BEGIN_OUTLINE_DATA + PQ_SUBQUERY(@"SEL$1" ("SEL$2" "SEL$3" "SEL$4" "SEL$5" "SEL$6" "SEL$7" "SEL$8") NONE ALL) FULL(@"SEL$1" "test"."t1"@"SEL$1") FULL(@"SEL$2" "test"."t2"@"SEL$2") USE_DAS(@"SEL$2" "test"."t2"@"SEL$2") @@ -3179,6 +3189,7 @@ Outline Data: ------------------------------------- /*+ BEGIN_OUTLINE_DATA + PQ_SUBQUERY(@"SEL$1" ("SEL$2" "SEL$3" "SEL$4" "SEL$5" "SEL$6" "SEL$7" "SEL$8") NONE ALL) FULL(@"SEL$1" "test"."t1"@"SEL$1") FULL(@"SEL$2" "test"."t2"@"SEL$2") USE_DAS(@"SEL$2" "test"."t2"@"SEL$2") diff --git a/tools/deploy/mysql_test/test_suite/static_engine/t/expr_substr.test b/tools/deploy/mysql_test/test_suite/static_engine/t/expr_substr.test index 72afb281c6..95dbf60406 100644 --- a/tools/deploy/mysql_test/test_suite/static_engine/t/expr_substr.test +++ b/tools/deploy/mysql_test/test_suite/static_engine/t/expr_substr.test @@ -47,5 +47,17 @@ select substr(3.14159, 2.9, 2.9) from dual; select substr(3.14159, 2.1, 2.1) from dual; --enable_warnings +# bug: +select substr(1, 1, 9223372036854775807); +select substr(123, 2, 9223372036854775806); +--disable_warnings +drop view if exists v1; +--enable_warnings +create view v1 as select substr(123, 2, 9223372036854775807); +desc v1; +select * from v1; +drop view v1; +SELECT NULL ORDER BY SUBSTR ( 9223372036854775807 FROM @@ob_query_timeout FOR 9223372036854775807 ); + connection syscon; --sleep 2 diff --git a/tools/deploy/mysql_test/test_suite/update/r/mysql/update_ignore.result b/tools/deploy/mysql_test/test_suite/update/r/mysql/update_ignore.result index 964af576ea..b58b3b1224 100644 Binary files a/tools/deploy/mysql_test/test_suite/update/r/mysql/update_ignore.result and b/tools/deploy/mysql_test/test_suite/update/r/mysql/update_ignore.result differ diff --git a/tools/deploy/mysql_test/test_suite/update/t/update_ignore.test b/tools/deploy/mysql_test/test_suite/update/t/update_ignore.test index 9b8f079292..167b786d20 100644 --- a/tools/deploy/mysql_test/test_suite/update/t/update_ignore.test +++ b/tools/deploy/mysql_test/test_suite/update/t/update_ignore.test @@ -154,3 +154,97 @@ select * from z0case; UPDATE Z0CASE SET T1='AbC' WHERE T2='11'; select * from z0case; drop table z0case; + +--disable_warnings +drop table if exists table0_innodb_key_pk_parts_2; + +CREATE TABLE table0_innodb_key_pk_parts_2 ( +col_bit bit, +col_blob blob, +col_char_key char (1), +col_blob_not_null blob not null, +col_int_not_null_key int not null, +col_float_not_null float not null, +col_bit_not_null bit not null, +col_decimal_not_null_key decimal not null, +col_decimal decimal, +col_set_not_null set ('a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z') not null, +col_bit_not_null_key bit not null, +col_int_key int, +col_char_not_null_key char (1) not null, +col_int_unsigned_not_null_key int unsigned not null, +col_decimal_not_null decimal not null, +col_bit_key bit, +col_set_not_null_key set ('a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z') not null, +col_set_key set ('a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z'), +col_enum_not_null enum ('a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z') not null, +col_float_unsigned_not_null_ke float unsigned not null, +col_int_unsigned_key int unsigned, +col_int_unsigned int unsigned, +col_int_not_null int not null, +col_float float, +col_char char (1), +col_decimal_unsigned decimal unsigned, +col_enum_not_null_key enum ('a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z') not null, +col_set set ('a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z'), +col_decimal_unsigned_key decimal unsigned, +col_char_not_null char (1) not null, +col_float_unsigned_not_null float unsigned not null, +pk int, +col_decimal_unsigned_not_null decimal unsigned not null, +col_decimal_unsigned_not_null_ decimal unsigned not null, +col_float_key float, +col_float_not_null_key float not null, +col_blob_not_null_key blob not null, +col_decimal_key decimal, +col_int int, +col_enum_key enum ('a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z'), +col_blob_key blob, +col_enum enum ('a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z'), +col_float_unsigned float unsigned, +col_float_unsigned_key float unsigned, +col_int_unsigned_not_null int unsigned not null, +/*Indices*/ +key idx13(col_char_key ), +key idx9(col_int_not_null_key ), +key idx41(col_decimal_not_null_key ), +key idx3(col_bit_not_null_key ), +key idx5(col_int_key ), +key idx15(col_char_not_null_key ), +key idx11(col_int_unsigned_not_null_key ), +key idx1(col_bit_key ), +key idx23(col_set_not_null_key ), +key idx21(col_set_key ), +key idx35(col_float_unsigned_not_null_ke ), +key idx7(col_int_unsigned_key ), +key idx19(col_enum_not_null_key ), +key idx39(col_decimal_unsigned_key ), +key idx43(col_decimal_unsigned_not_null_ ), +key idx29(col_float_key ), +key idx33(col_float_not_null_key ), +key idx27(col_blob_not_null_key (255)), +key idx37(col_decimal_key ), +key idx17(col_enum_key ), +key idx25(col_blob_key (255)), +key idx31(col_float_unsigned_key )) ENGINE=innodb PARTITION BY key (pk) partitions 2; + +set sql_mode = ''; + +INSERT /*+ ENABLE_PARALLEL_DML PARALLEL(3) */ INTO table0_innodb_key_pk_parts_2 ( col_blob_not_null , col_float , col_float_unsigned_not_null_ke ) VALUES ( '2009-06-17 02:00:49' , '2008-05-04 09:58:22.031383' , -24420 ); +--error 1048 +INSERT /*+ ENABLE_PARALLEL_DML PARALLEL(3) */ INTO table0_innodb_key_pk_parts_2 ( col_decimal_key , col_set_not_null , col_bit_not_null_key ) VALUES ( '2002-08-21 15:50:49' , NULL , '2008-04-08 00:54:35.035855' ); +INSERT /*+ ENABLE_PARALLEL_DML PARALLEL(3) */ INTO table0_innodb_key_pk_parts_2 ( col_decimal_unsigned_not_null , col_set_key , col_enum_key ) VALUES ( 4 , -4338432 , '05:31:13' ); +INSERT /*+ ENABLE_PARALLEL_DML PARALLEL(3) */ INTO table0_innodb_key_pk_parts_2 ( col_char_not_null_key , col_enum_not_null , col_enum ) VALUES ( '2003-06-24 04:26:58.057298' , 'arzwkjqshciwulehkuyzewxtpbpfmfj' , '2003' ); +INSERT /*+ ENABLE_PARALLEL_DML PARALLEL(3) */ INTO table0_innodb_key_pk_parts_2 ( col_decimal_unsigned , col_int_key , col_float ) VALUES ( 'lq' , '2005' , '2003-06-11 10:08:30' ); +INSERT /*+ ENABLE_PARALLEL_DML PARALLEL(3) */ INTO table0_innodb_key_pk_parts_2 ( col_char_not_null , col_char , col_int_not_null_key ) VALUES ( 8 , -7474944 , 896512 ); +INSERT /*+ ENABLE_PARALLEL_DML PARALLEL(3) */ INTO table0_innodb_key_pk_parts_2 ( col_bit_not_null_key , col_blob_not_null , pk ) VALUES ( '20:58:15' , 'hrzplcvmxggypuvoppkxwfvkhchskzmiwiombucu' , 30604 ); +INSERT /*+ ENABLE_PARALLEL_DML PARALLEL(3) */ INTO table0_innodb_key_pk_parts_2 ( col_float_unsigned_not_null_ke , col_char_key , col_char ) VALUES ( '2008' , '2001' , 3 ); +--error 1048 +INSERT /*+ ENABLE_PARALLEL_DML PARALLEL(3) */ INTO table0_innodb_key_pk_parts_2 ( col_enum_not_null_key , col_decimal_unsigned_not_null_ , col_blob ) VALUES ( 26681 , NULL , 14465 ); +INSERT /*+ ENABLE_PARALLEL_DML PARALLEL(3) */ INTO table0_innodb_key_pk_parts_2 ( col_enum , col_enum_not_null , col_decimal ) VALUES ( '2006-04-09 05:33:14.045216' , '2008' , '2004-09-17 09:11:54.004359' ); +INSERT /*+ ENABLE_PARALLEL_DML PARALLEL(3) */ INTO table0_innodb_key_pk_parts_2 ( col_blob_key , col_int_unsigned_not_null , col_int_not_null_key ) VALUES ( '20:35:45' , 47682 , 1319 ); + +UPDATE /*+ ENABLE_PARALLEL_DML PARALLEL(3) */ table0_innodb_key_pk_parts_2 SET col_blob_not_null_key = NULL WHERE col_bit_not_null <> '08:59:21'; +select * from table0_innodb_key_pk_parts_2; +drop table table0_innodb_key_pk_parts_2; +--enable_warnings diff --git a/tools/deploy/obd/ocitest.yaml.template b/tools/deploy/obd/ocitest.yaml.template new file mode 100644 index 0000000000..d3a933e3c7 --- /dev/null +++ b/tools/deploy/obd/ocitest.yaml.template @@ -0,0 +1,37 @@ +oceanbase-ce: +{{%% SERVERS %%}} + tag: latest + global: + log_disk_size: '240G' + home_path: /data/{user}/{deploy_name}/observer/{server} + production_mode: false + memory_limit: '80G' + system_memory: '10G' + datafile_size: '40G' + runtime_dependencies: + - src_path: plugin_dir + target_path: plugin_dir + - src_path: admin + target_path: admin + - src_path: lib + target_path: lib + - src_path: tools + target_path: tools + - src_path: etc + target_path: etc + - src_path: wallet + target_path: wallet +obproxy-ce: +{{%% OBPORXY_SERVERS %%}} + global: + listen_port: {{%% LISTEN_PORT %%}} + prometheus_listen_port: {{%% PROMETHEUS_LISTEN_PORT %%}} + home_path: {{%% OBPORXY_HOME_PATH %%}} + enable_cluster_checkout: false + skip_proxy_sys_private_check: true + syslog_level: 'DEBUG' + enable_metadb_used: 'false' + enable_qa_mode: 'TRUE' + location_expire_period: '1' + depends: + - {{%% COMPONENT %%}} diff --git a/tools/docker/standalone/Dockerfile b/tools/docker/standalone/Dockerfile index ab3a433157..3cb3fbca9e 100644 --- a/tools/docker/standalone/Dockerfile +++ b/tools/docker/standalone/Dockerfile @@ -2,6 +2,7 @@ FROM openanolis/anolisos # docker build --build-arg VERSION={VERSION} . ARG VERSION +ARG STEP RUN yum install -y yum-utils && \ yum-config-manager --add-repo https://mirrors.aliyun.com/oceanbase/OceanBase.repo && \ @@ -10,6 +11,11 @@ RUN yum install -y yum-utils && \ rm -rf /usr/obd/mirror/remote/* && \ yum clean all +ENV STEP=$STEP +RUN if [ "$STEP" == 1 ]; then \ + yum install -y mysql; \ + fi + ENV OBVersion=$VERSION RUN if [ -z "${OBVersion}" ]; then \ echo "VERSION is empty, then build the docker with latest rpm"; \ @@ -19,15 +25,16 @@ RUN if [ -z "${OBVersion}" ]; then \ else \ echo "build the docker with VERSION ${OBVersion}"; \ fi && \ - mkdir -p /root/pkg && \ + mkdir -p /root/pkg && mkdir -p /root/store && mkdir -p /root/dest && \ cd /root/pkg && \ yum install -y --downloadonly --downloaddir=. oceanbase-ce-${OBVersion}.el7 oceanbase-ce-libs-${OBVersion}.el7 obagent ob-configserver && \ rm -rf /usr/obd/mirror/remote/* && \ yum clean all COPY boot /root/boot/ +COPY init_store_for_fast_start.py /root/boot/ ENV PATH /root/boot:$PATH -ENV LD_LIBRARY_PATH /root/ob/lib:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH /home/admin/oceanbase/lib:/root/ob/lib:$LD_LIBRARY_PATH WORKDIR /root CMD _boot diff --git a/tools/docker/standalone/README-CN.md b/tools/docker/standalone/README-CN.md index 11c7909d61..98b5372446 100644 --- a/tools/docker/standalone/README-CN.md +++ b/tools/docker/standalone/README-CN.md @@ -39,6 +39,9 @@ docker run -p 2881:2881 --name oceanbase-ce -e MODE=slim -e OB_MEMORY_LIMIT=5G - # 根据当前容器情况部署最大规格的实例 docker run -p 2881:2881 --name oceanbase-ce -e MODE=normal -d oceanbase/oceanbase-ce + +# 部署一个快速启动镜像，mode可以为任意模式 +docker run -p 2881:2881 --name oceanbase-ce -e FASTBOOT=true -d oceanbase/oceanbase-ce ``` 启动预计需要 2-5 分钟。执行以下命令，如果返回 `boot success!`，则启动成功。 @@ -75,7 +78,8 @@ mysql -uroot -h127.1 -P2881 | 变量名称 | 默认值 | 描述 | | ---------------- | ------------- | ------------------------------------------------------------ | | MODE | {mini, slim, normal} | mini或者不赋值变量表示使用mini模式部署OceanBase数据库实例，仅用来研究学习使用。不适合用于生产或性能测试。slim适用于更小的自定义配置，移除obagent，支持自定义的初始化脚本在绑定目录/root/boot/init.d，如果不绑定该目录，docker不会执行该租户的初始化sql。| -| EXIT_WHILE_ERROR | true | OceanBase 如果启动失败，是否退出容器。比如初次run镜像失败，或start容器失败，可以将此参数设置为false,那么OB启动失败，也可以进入容器，查看OceanBase的运行日志，然后进行排查。 | +| FASTBOOT | false | true表示镜像会以快速启动的方式运行。 | +| EXIT_WHILE_ERROR | true | OceanBase 如果启动失败，是否退出容器。比如初次run镜像失败，或start容器失败，可以将此参数设置为false,那么OB启动失败，也可以进入容器，查看OceanBase的运行日志，然后进行排查。 | | OB_CLUSTER_NAME | obcluster | oceanbase集群名 | | OB_TENANT_NAME | test | oceanbase mysql租户名| | OB_MEMORY_LIMIT | 6G | oceanbase启动memory_limit参数配置 | @@ -113,14 +117,14 @@ docker run -d -p 2881:2881 -v $PWD/ob:/root/ob -v $PWD/obd:/root/.obd --name oce docker -v 参数的详细说明可以参考 [docker volumn](https://docs.docker.com/storage/volumes/)。 ## 快速单机启动镜像构建 -在`tools/docker/standalone`目录下提供`fast_boot_docker_build.sh`脚本，通过该脚本可以构建快速启动镜像。在运行脚本之前，请首先修改`tools/docker/standalone/boot/_env`环境配置脚本： +在`tools/docker/standalone`目录下提供`docker_build.sh`脚本，通过该脚本可以构建快速启动镜像。在运行脚本之前，请首先修改`tools/docker/standalone/boot/_env`环境配置脚本： -- 必须：将`MODE`配置项修改为`STANDALONE` - 可选：修改其余配置项 修改完毕后，执行镜像构建脚本： -- `./fast_boot_docker_build.sh ` 例如：`./fast_boot_docker_build.sh 4.2.1.0-100000102023092807` +- 构建最新版镜像 `./docker_build.sh` +- 构建某个特别版本的oceanbase镜像 `./docker_build.sh ` 例如：`./docker_build.sh 4.2.1.0-100000102023092807` 等待构建完毕后，可使用前述相同的方式启动、测试实例。 diff --git a/tools/docker/standalone/README.md b/tools/docker/standalone/README.md index 29eaeb3971..b29a5d51fb 100644 --- a/tools/docker/standalone/README.md +++ b/tools/docker/standalone/README.md @@ -39,6 +39,9 @@ docker run -p 2881:2881 --name oceanbase-ce -e MODE=slim -e OB_MEMORY_LIMIT=5G - # deploy an instance of the largest size according to the current container docker run -p 2881:2881 --name oceanbase-ce -e MODE=normal -d oceanbase/oceanbase-ce + +# deploy a quick-start instance in any mode as desired to the current container +docker run -p 2881:2881 --name oceanbase-ce -e FASTBOOT=true -d oceanbase/oceanbase-ce ``` Two to five minutes are necessary for the boot procedure. To make sure that the boot procedure is successful, run this command: @@ -73,6 +76,7 @@ This table shows the supported environment variables of the current oceanbase-ce | Variable name | Default value | Description | | ---------------- | ------------- | ------------------------------------------------------------ | | MODE | {mini, slim, normal} | If it is mini, then the docker use mini mode to deploy OceanBase Database instance, it should be used only for research/study/evaluation. DO NOT use it for production or performance testing. If it is slim, then the docker can run in a smaller instance. It remove the obagent and can run a self tenant initial sql by yourself in the mount volume /root/boot/init.d. If you do not mount the volume path the docker does not init the tenant sql. | +| FASTBOOT | false | The container can run in a quick-start mode when FASTBOOT=true. | | EXIT_WHILE_ERROR | true | Whether quit the container while start observer failed. If start observer failed, you can not explore the logs as the container will exit. But if you set the EXIT_WHILE_ERROR=false, the container will not exit while observer starting fail and you can use docker exec to debug. | | OB_CLUSTER_NAME | obcluster | The oceanbase cluster name | | OB_TENANT_NAME | test | The oceanbase mysql tenant name | @@ -109,14 +113,14 @@ The docker image `oceanbase-ce` saves the data to /root/ob directory default. Yo You can view more information about `docker -v` at [docker volume](https://docs.docker.com/storage/volumes/). ## Fast boot image building for a standalone node -The `fast_boot_docker_build.sh` script is provided in the `tools/docker/standalone` directory, through which the fast boot image can be built. Before running the script, please first modify the `tools/docker/standalone/boot/_env` environment configuration script: +The `docker_build.sh` script is provided in the `tools/docker/standalone` directory, through which the fast boot image can be built. Before running the script, please first modify the `tools/docker/standalone/boot/_env` environment configuration script: -- Required: Modify the `MODE` configuration item to `STANDALONE` - Optional: Modify the remaining configuration items After the modification is completed, execute the image build script: -- `./fast_boot_docker_build.sh `. For example `./fast_boot_docker_build.sh 4.2.1.0-100000102023092807` +- build the image with latest oceanbase version `./docker_build.sh` +- build the image with specific oceanbase version `./docker_build.sh `. For example `./docker_build.sh 4.2.1.0-100000102023092807` After waiting for the build to be completed, you can start and test the instance in the same way as mentioned above. diff --git a/tools/docker/standalone/docker_build.sh b/tools/docker/standalone/docker_build.sh new file mode 100755 index 0000000000..cb1031a134 --- /dev/null +++ b/tools/docker/standalone/docker_build.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +if [ "$#" -gt 1 ]; then + exit 1 +elif [ "$#" -gt 1 ]; then + BUILD_ARG="--build-arg VERSION=$1" +else + BUILD_ARG="" +fi + +TMP_INIT_STORE_PY_SCRIPT="init_store_for_fast_start.tmp.py" +ACTUAL_INIT_STORE_PY_SCRIPT="init_store_for_fast_start.py" + +CWD=$(cd `dirname $0`;pwd) +cd "${CWD}" + +function fast_boot_docker_build() { + rm -rf boot + cp -r step_1_boot boot + docker build $BUILD_ARG --build-arg STEP=1 -t raw_observer . + if [ $? == 0 ]; then + echo "================== build prepare docker ok ===============" + else + echo "================== build prepare docker failed ===============" + exit -1 + fi + rm -rf boot + + cd "${CWD}" && mkdir -p ${CWD}/boot/etc + docker run -it -v ${CWD}/boot:/root/dest raw_observer + if [ $? == 0 ]; then + echo "================== prepare docker run ok ===============" + else + echo "================== prepare docker run failed ===============" + rm -rf ${CWD}/boot/etc + rm -rf ${CWD}/boot/store.tar.gz + exit -1 + fi + + cd "${CWD}" + cp -r step_2_boot/* boot + docker build $BUILD_ARG --build-arg STEP=2 -t oceanbase-ce . + if [ $? == 0 ]; then + echo "================== docker build ok ===============" + else + echo "================== docker build failed ===============" + exit -1 + fi +} + +source ./step_2_boot/_env +OS=`uname` +cp ${TMP_INIT_STORE_PY_SCRIPT} ${ACTUAL_INIT_STORE_PY_SCRIPT} + +if [ "$OS" == 'Darwin' ]; then + sed -i '' -e "s/@OB_SERVER_IP@/${OB_SERVER_IP}/g" ${ACTUAL_INIT_STORE_PY_SCRIPT} + sed -i '' -e "s/@OB_MYSQL_PORT@/${OB_MYSQL_PORT}/g" ${ACTUAL_INIT_STORE_PY_SCRIPT} + sed -i '' -e "s/@OB_RPC_PORT@/${OB_RPC_PORT}/g" ${ACTUAL_INIT_STORE_PY_SCRIPT} + sed -i '' -e "s/@OB_TENANT_NAME@/${OB_TENANT_NAME}/g" ${ACTUAL_INIT_STORE_PY_SCRIPT} + sed -i '' -e "s/@OB_TENANT_LOWER_CASE_TABLE_NAMES@/${OB_TENANT_LOWER_CASE_TABLE_NAMES}/g" ${ACTUAL_INIT_STORE_PY_SCRIPT} +else + sed -i'' -e "s/@OB_SERVER_IP@/${OB_SERVER_IP}/g" ${ACTUAL_INIT_STORE_PY_SCRIPT} + sed -i'' -e "s/@OB_MYSQL_PORT@/${OB_MYSQL_PORT}/g" ${ACTUAL_INIT_STORE_PY_SCRIPT} + sed -i'' -e "s/@OB_RPC_PORT@/${OB_RPC_PORT}/g" ${ACTUAL_INIT_STORE_PY_SCRIPT} + sed -i'' -e "s/@OB_TENANT_NAME@/${OB_TENANT_NAME}/g" ${ACTUAL_INIT_STORE_PY_SCRIPT} + sed -i'' -e "s/@OB_TENANT_LOWER_CASE_TABLE_NAMES@/${OB_TENANT_LOWER_CASE_TABLE_NAMES}/g" ${ACTUAL_INIT_STORE_PY_SCRIPT} +fi + +fast_boot_docker_build +if [ $? != 0 ]; then + echo "use local rpm build docker failed" + exit -1 +fi diff --git a/tools/docker/standalone/fast_boot_docker_build.sh b/tools/docker/standalone/fast_boot_docker_build.sh deleted file mode 100755 index ee6ed636a0..0000000000 --- a/tools/docker/standalone/fast_boot_docker_build.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -RPM_VERSION=$1 - -TMP_INIT_STORE_PY_SCRIPT="init_store_for_fast_start.tmp.py" -ACTUAL_INIT_STORE_PY_SCRIPT="./fast_boot_docker_build_prepare/init_store_for_fast_start.py" - -CWD=$(cd `dirname $0`;pwd) -cd "${CWD}" - -function fast_boot_docker_build() { - cd fast_boot_docker_build_prepare && \ - docker build --build-arg VERSION="${RPM_VERSION}" -t raw_observer . - if [ $? == 0 ]; then - echo "================== build prepare docker ok ===============" - else - echo "================== build prepare docker failed ===============" - exit -1 - fi - - cd "${CWD}" && mkdir -p ${CWD}/boot/etc - docker run -it -v ${CWD}/boot:/root/dest raw_observer - if [ $? == 0 ]; then - echo "================== prepare docker run ok ===============" - else - echo "================== prepare docker run failed ===============" - rm -rf ${CWD}/boot/etc - rm -rf ${CWD}/boot/store.tar.gz - exit -1 - fi - - cd "${CWD}" - docker build --build-arg VERSION="${RPM_VERSION}" -t oceanbase-ce . - if [ $? == 0 ]; then - echo "================== fast boot docker build ok ===============" - else - echo "================== fast boot docker build failed ===============" - exit -1 - fi -} - -source ./boot/_env -if [ "x${MODE}" != "xSTANDALONE" ]; then - echo "please set MODE to STANDALONE for building fast boot docker" - exit -1 -fi -OS=`uname` -cp ${TMP_INIT_STORE_PY_SCRIPT} ${ACTUAL_INIT_STORE_PY_SCRIPT} - -if [ "$OS" == 'Darwin' ]; then - alias sedi="sed -i '' -e" -else - alias sedi="sed -i'' -e" -fi - -sedi "s/@OB_MYSQL_PORT@/${OB_MYSQL_PORT}/g" ${ACTUAL_INIT_STORE_PY_SCRIPT} -sedi "s/@OB_RPC_PORT@/${OB_RPC_PORT}/g" ${ACTUAL_INIT_STORE_PY_SCRIPT} -sedi "s/@OB_TENANT_NAME@/${OB_TENANT_NAME}/g" ${ACTUAL_INIT_STORE_PY_SCRIPT} -sedi "s/@OB_TENANT_LOWER_CASE_TABLE_NAMES@/${OB_TENANT_LOWER_CASE_TABLE_NAMES}/g" ${ACTUAL_INIT_STORE_PY_SCRIPT} - -fast_boot_docker_build -if [ $? != 0 ]; then - echo "use local rpm build docker failed" - exit -1 -fi diff --git a/tools/docker/standalone/fast_boot_docker_build_prepare/Dockerfile b/tools/docker/standalone/fast_boot_docker_build_prepare/Dockerfile deleted file mode 100644 index fcc8584352..0000000000 --- a/tools/docker/standalone/fast_boot_docker_build_prepare/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -FROM openanolis/anolisos - -# docker build --build-arg VERSION={VERSION} . -ARG VERSION - -RUN yum install -y yum-utils && \ - yum-config-manager --add-repo https://mirrors.aliyun.com/oceanbase/OceanBase.repo && \ - sed -i 's/$releasever/7/' /etc/yum.repos.d/OceanBase.repo && \ - yum install -y libaio mysql && \ - yum clean all - -ENV OBVersion=$VERSION -RUN if [ -z "${OBVersion}" ]; then \ - echo "VERSION is empty, then build the docker with latest rpm"; \ - VersionPre=`yum info oceanbase-ce | grep Version | awk '{print $3}'`; \ - Release=`yum info oceanbase-ce | grep Release | awk '{print $3}' | awk -F. '{print $1}'`; \ - OBVersion="${VersionPre}-${Release}"; \ - else \ - echo "build the docker with VERSION ${OBVersion}"; \ - fi && \ - mkdir -p /root/pkg && mkdir -p /root/store && mkdir -p /root/dest && \ - cd /root/pkg && \ - yum install -y --downloadonly --downloaddir=. oceanbase-ce-${OBVersion}.el7 oceanbase-ce-libs-${OBVersion}.el7 && \ - rm -rf /usr/obd/mirror/remote/* && \ - yum clean all - -COPY boot /root/boot/ -COPY init_store_for_fast_start.py /root/boot/ -ENV PATH /root/boot:$PATH -ENV LD_LIBRARY_PATH /home/admin/oceanbase/lib:/root/ob/lib:$LD_LIBRARY_PATH - -WORKDIR /root -CMD _boot - -EXPOSE 2881 \ No newline at end of file diff --git a/tools/docker/standalone/init_store_for_fast_start.tmp.py b/tools/docker/standalone/init_store_for_fast_start.tmp.py index 1d19bf1e47..f1333668f8 100644 --- a/tools/docker/standalone/init_store_for_fast_start.tmp.py +++ b/tools/docker/standalone/init_store_for_fast_start.tmp.py @@ -25,6 +25,7 @@ def check_file_or_path_exist(bin_abs_path, home_abs_path, store_tar_file_path, e logging.warn("invalid store tar file path") return False if not os.path.isdir(etc_dest_dir): + logging.warn(etc_dest_dir) logging.warn("invalid etc dest dir") return False return True @@ -46,16 +47,15 @@ if __name__ == "__main__": parser.add_argument("-z", dest="zone", type=str, default="zone1") parser.add_argument("-c", dest="cluster_id", type=str, default="1") parser.add_argument("-d", dest="data_path", type=str, default="/data/store") - parser.add_argument("-i", dest="devname", type=str, default="lo") - parser.add_argument("-r", dest="rootservice", type=str, default="127.0.0.1:@OB_RPC_PORT@") - parser.add_argument("-I", dest="ip", type=str, default="127.0.0.1") + parser.add_argument("-r", dest="rootservice", type=str, default="@OB_SERVER_IP@:@OB_RPC_PORT@") + parser.add_argument("-I", dest="ip", type=str, default="@OB_SERVER_IP@") parser.add_argument("-l", dest="log_level", type=str, default="INFO") parser.add_argument("-o", dest="opt_str", type=str, default="__min_full_resource_pool_memory=2147483648,memory_limit=6G,system_memory=1G,datafile_size=256M,log_disk_size=5G,cpu_count=16") parser.add_argument("-N", dest="daemon", type=str, default="1") parser.add_argument("--tenant_name", type=str, default="@OB_TENANT_NAME@") parser.add_argument("--tenant_lower_case_table_names", type=int, default="@OB_TENANT_LOWER_CASE_TABLE_NAMES@") - parser.add_argument("--max_cpu", type=float, default=14.0) - parser.add_argument("--min_cpu", type=float, default=14.0) + parser.add_argument("--max_cpu", type=float, default=7.0) + parser.add_argument("--min_cpu", type=float, default=7.0) parser.add_argument("--memory_size", type=int, default=3221225472) parser.add_argument("--log_disk_size", type=int, default=3221225472) args = parser.parse_args() @@ -83,8 +83,8 @@ if __name__ == "__main__": # prepare observer start parameters daemon_option = "-N" if args.daemon=="1" else "" - observer_args = "-p %s -P %s -z %s -c %s -d %s -i %s -r %s -I %s -l %s -o %s %s" % (args.mysql_port, args.rpc_port, args.zone, \ - args.cluster_id, data_abs_path, args.devname, \ + observer_args = "-p %s -P %s -z %s -c %s -d %s -r %s -I %s -l %s -o %s %s" % (args.mysql_port, args.rpc_port, args.zone, \ + args.cluster_id, data_abs_path, \ args.rootservice, args.ip, args.log_level, args.opt_str, \ daemon_option) os.chdir(home_abs_path) diff --git a/tools/docker/standalone/fast_boot_docker_build_prepare/boot/_boot b/tools/docker/standalone/step_1_boot/_boot similarity index 100% rename from tools/docker/standalone/fast_boot_docker_build_prepare/boot/_boot rename to tools/docker/standalone/step_1_boot/_boot diff --git a/tools/docker/standalone/fast_boot_docker_build_prepare/boot/env.sh b/tools/docker/standalone/step_1_boot/env.sh similarity index 100% rename from tools/docker/standalone/fast_boot_docker_build_prepare/boot/env.sh rename to tools/docker/standalone/step_1_boot/env.sh diff --git a/tools/docker/standalone/boot/_boot b/tools/docker/standalone/step_2_boot/_boot similarity index 95% rename from tools/docker/standalone/boot/_boot rename to tools/docker/standalone/step_2_boot/_boot index ee5cbce7d9..f0feb805b2 100755 --- a/tools/docker/standalone/boot/_boot +++ b/tools/docker/standalone/step_2_boot/_boot @@ -56,7 +56,6 @@ function get_mode() { MODE=${MODE^^} } - function exit_while_error() { if test -z ${EXIT_WHILE_ERROR} then @@ -66,6 +65,15 @@ function exit_while_error() { return `is_true ${EXIT_WHILE_ERROR}` } +function fastboot() { + if test -z ${FASTBOOT} + then + return 0 + fi + + return `is_true ${FASTBOOT}` +} + function remove_disk_check_logic_in_obd() { # make sure obd copy the plugin code obd cluster list @@ -126,9 +134,9 @@ else # nothing here, bootstrap TMPFILE="boot.${STAMP}.yaml" get_mode - if [ "x${MODE}" == "xSTANDALONE" ]; then - echo "oceanbase-ce docker in standalone mode" - cp -f boot-mini-tmp.yaml $TMPFILE + if [ "x${MODE}" == "xNORMAL" ]; then + echo "oceanbase-ce docker in normal mode" + cp -f boot-tmp.yaml $TMPFILE elif [ "x${MODE}" == "xMINI" ]; then echo "oceanbase-ce docker in mini mode" cp -f boot-mini-tmp.yaml $TMPFILE @@ -136,14 +144,14 @@ else # nothing here, bootstrap echo "oceanbase-ce docker in slim mode" cp -f boot-mini-tmp.yaml $TMPFILE else - cp -f boot-tmp.yaml $TMPFILE + cp -f boot-mini-tmp.yaml $TMPFILE fi if [ "x${MODE}" != "xSLIM" ]; then cat obagent.yaml >> $TMPFILE fi - if [ "x${MODE}" != "xMINI" ] && [ "x${MODE}" != "xSTANDALONE" ]; then + if [ "x${MODE}" == "xSLIM" ]; then cat ob-configserver.yaml >> $TMPFILE fi @@ -171,7 +179,7 @@ else # nothing here, bootstrap print_start_phase "Ob-deploy deploy" remove_disk_check_logic_in_obd - if [ "x${MODE}" == "xSTANDALONE" ]; then + if fastboot; then obd devmode enable && obd cluster deploy "${OB_CLUSTER_NAME}" -c $TMPFILE; if [ $? -ne 0 ]; then deploy_failed @@ -195,6 +203,11 @@ else # nothing here, bootstrap print_start_phase "Ob-deploy start" obd cluster start ${OB_CLUSTER_NAME} print_end_phase + + if [ "x${MODE}" == "xSLIM" ]; then + run_custom_scripts /root/boot/init.d + fi + else print_start_phase "Ob-deploy autodeploy" obd devmode enable && obd cluster autodeploy "${OB_CLUSTER_NAME}" -c $TMPFILE; @@ -223,6 +236,7 @@ else # nothing here, bootstrap fi print_end_phase fi + if [ $? -ne 0 ]; then deploy_failed fi diff --git a/tools/docker/standalone/boot/_env b/tools/docker/standalone/step_2_boot/_env similarity index 90% rename from tools/docker/standalone/boot/_env rename to tools/docker/standalone/step_2_boot/_env index 25cf09d346..847baa91aa 100755 --- a/tools/docker/standalone/boot/_env +++ b/tools/docker/standalone/step_2_boot/_env @@ -1,7 +1,8 @@ MODE=${MODE:-MINI} +FASTBOOT=${FASTBOOT:-false} EXIT_WHILE_ERROR=${EXIT_WHILE_ERROR:-true} OB_HOME_PATH="/root/ob" -OB_SERVER_IP=$(hostname -i) +OB_SERVER_IP="127.0.0.1" OB_MYSQL_PORT="2881" OB_RPC_PORT="2882" OB_CLUSTER_NAME=${OB_CLUSTER_NAME:-obcluster} diff --git a/tools/docker/standalone/boot/boot-mini-tmp.yaml b/tools/docker/standalone/step_2_boot/boot-mini-tmp.yaml similarity index 98% rename from tools/docker/standalone/boot/boot-mini-tmp.yaml rename to tools/docker/standalone/step_2_boot/boot-mini-tmp.yaml index b4819829f5..115d7cc4e1 100644 --- a/tools/docker/standalone/boot/boot-mini-tmp.yaml +++ b/tools/docker/standalone/step_2_boot/boot-mini-tmp.yaml @@ -17,7 +17,7 @@ oceanbase-ce: datafile_size: @OB_DATAFILE_SIZE@ # Size of the data file. log_disk_size: @OB_LOG_DISK_SIZE@ # The size of disk space used by the clog files. cpu_count: 16 - production_mode: true + production_mode: false syslog_level: INFO # System log level. The default value is INFO. enable_syslog_wf: false # Print system logs whose levels are higher than WARNING to a separate log file. The default value is true. enable_syslog_recycle: true # Enable auto system log recycling or not. The default value is false. diff --git a/tools/docker/standalone/boot/boot-tmp.yaml b/tools/docker/standalone/step_2_boot/boot-tmp.yaml similarity index 92% rename from tools/docker/standalone/boot/boot-tmp.yaml rename to tools/docker/standalone/step_2_boot/boot-tmp.yaml index 74470882c9..b3ea85ae39 100644 --- a/tools/docker/standalone/boot/boot-tmp.yaml +++ b/tools/docker/standalone/step_2_boot/boot-tmp.yaml @@ -9,6 +9,7 @@ oceanbase-ce: rpc_port: @OB_RPC_PORT@ datafile_size: 20G log_disk_size: 20G + production_mode: false appname: @OB_CLUSTER_NAME@ root_password: @OB_ROOT_PASSWORD@ enable_rich_error_msg: true diff --git a/tools/docker/standalone/boot/init_tenant_user.sql b/tools/docker/standalone/step_2_boot/init_tenant_user.sql similarity index 100% rename from tools/docker/standalone/boot/init_tenant_user.sql rename to tools/docker/standalone/step_2_boot/init_tenant_user.sql diff --git a/tools/docker/standalone/boot/ob-configserver.yaml b/tools/docker/standalone/step_2_boot/ob-configserver.yaml similarity index 100% rename from tools/docker/standalone/boot/ob-configserver.yaml rename to tools/docker/standalone/step_2_boot/ob-configserver.yaml diff --git a/tools/docker/standalone/boot/ob-mysql b/tools/docker/standalone/step_2_boot/ob-mysql similarity index 100% rename from tools/docker/standalone/boot/ob-mysql rename to tools/docker/standalone/step_2_boot/ob-mysql diff --git a/tools/docker/standalone/boot/obagent.yaml b/tools/docker/standalone/step_2_boot/obagent.yaml similarity index 100% rename from tools/docker/standalone/boot/obagent.yaml rename to tools/docker/standalone/step_2_boot/obagent.yaml diff --git a/tools/upgrade/actions.py b/tools/upgrade/actions.py index 5a58c7fb33..727b097b90 100755 --- a/tools/upgrade/actions.py +++ b/tools/upgrade/actions.py @@ -127,16 +127,45 @@ def set_parameter(cur, parameter, value, timeout = 0): cur.execute(sql) wait_parameter_sync(cur, False, parameter, value, timeout) +def set_session_timeout(cur, seconds): + sql = "set @@session.ob_query_timeout = {0}".format(seconds * 1000 * 1000) + logging.info(sql) + cur.execute(sql) + +def set_default_timeout_by_tenant(cur, timeout, timeout_per_tenant, min_timeout): + if timeout > 0: + logging.info("use timeout from opt, timeout(s):{0}".format(timeout)) + else: + query_cur = QueryCursor(cur) + tenant_id_list = fetch_tenant_ids(query_cur) + cal_timeout = len(tenant_id_list) * timeout_per_tenant + timeout = (cal_timeout if cal_timeout > min_timeout else min_timeout) + logging.info("use default timeout caculated by tenants, " + "timeout(s):{0}, tenant_count:{1}, " + "timeout_per_tenant(s):{2}, min_timeout(s):{3}" + .format(timeout, len(tenant_id_list), timeout_per_tenant, min_timeout)) + + return timeout + def set_tenant_parameter(cur, parameter, value, timeout = 0): + tenants_list = [] if get_min_cluster_version(cur) < get_version("4.2.1.0"): tenants_list = ['all'] else: tenants_list = ['sys', 'all_user', 'all_meta'] + + query_timeout = set_default_timeout_by_tenant(cur, timeout, 10, 60) + + set_session_timeout(cur, query_timeout) + for tenants in tenants_list: sql = """alter system set {0} = '{1}' tenant = '{2}'""".format(parameter, value, tenants) logging.info(sql) cur.execute(sql) + + set_session_timeout(cur, 10) + wait_parameter_sync(cur, True, parameter, value, timeout) def get_ori_enable_ddl(cur, timeout): @@ -225,7 +254,20 @@ def wait_parameter_sync(cur, is_tenant_config, key, value, timeout): table_name = "GV$OB_PARAMETERS" if not is_tenant_config else "__all_virtual_tenant_parameter_info" sql = """select count(*) as cnt from oceanbase.{0} where name = '{1}' and value != '{2}'""".format(table_name, key, value) - times = (timeout if timeout > 0 else 60) / 5 + + wait_timeout = 0 + query_timeout = 0 + if not is_tenant_config or timeout > 0: + wait_timeout = (timeout if timeout > 0 else 60) + query_timeout = wait_timeout + else: + # is_tenant_config & timeout not set + wait_timeout = set_default_timeout_by_tenant(cur, timeout, 10, 60) + query_timeout = set_default_timeout_by_tenant(cur, timeout, 2, 60) + + set_session_timeout(cur, query_timeout) + + times = wait_timeout / 5 while times >= 0: logging.info(sql) cur.execute(sql) @@ -245,6 +287,8 @@ def wait_parameter_sync(cur, is_tenant_config, key, value, timeout): raise e time.sleep(5) + set_session_timeout(cur, 10) + def do_begin_upgrade(cur, timeout): if not check_parameter(cur, False, "enable_upgrade_mode", "True"): @@ -316,24 +360,38 @@ def do_suspend_merge(cur, timeout): tenants_list = ['all'] else: tenants_list = ['sys', 'all_user', 'all_meta'] + + query_timeout = set_default_timeout_by_tenant(cur, timeout, 10, 60) + + set_session_timeout(cur, query_timeout) + for tenants in tenants_list: action_sql = "alter system suspend merge tenant = {0}".format(tenants) rollback_sql = "alter system resume merge tenant = {0}".format(tenants) logging.info(action_sql) cur.execute(action_sql) + set_session_timeout(cur, 10) + def do_resume_merge(cur, timeout): tenants_list = [] if get_min_cluster_version(cur) < get_version("4.2.1.0"): tenants_list = ['all'] else: tenants_list = ['sys', 'all_user', 'all_meta'] + + query_timeout = set_default_timeout_by_tenant(cur, timeout, 10, 60) + + set_session_timeout(cur, query_timeout) + for tenants in tenants_list: action_sql = "alter system resume merge tenant = {0}".format(tenants) rollback_sql = "alter system suspend merge tenant = {0}".format(tenants) logging.info(action_sql) cur.execute(action_sql) + set_session_timeout(cur, 10) + class Cursor: __cursor = None def __init__(self, cursor): diff --git a/tools/upgrade/tenant_upgrade_action.py b/tools/upgrade/tenant_upgrade_action.py index fac3d7d281..77b53ccd89 100755 --- a/tools/upgrade/tenant_upgrade_action.py +++ b/tools/upgrade/tenant_upgrade_action.py @@ -21,17 +21,7 @@ def do_upgrade(conn, cur, timeout, user, pwd): else: run_upgrade_job(conn, cur, "UPGRADE_VIRTUAL_SCHEMA", timeout) - # just to make __all_virtual_upgrade_inspection avaliable - timeout_ts = (timeout if timeout > 0 else 600) * 1000 * 1000 - sql = "set @@session.ob_query_timeout = {0}".format(timeout_ts) - logging.info(sql) - cur.execute(sql) - sql = "alter system run job 'root_inspection'" - logging.info(sql) - cur.execute(sql) - sql = "set @@session.ob_query_timeout = 10000000" - logging.info(sql) - cur.execute(sql) + run_root_inspection(cur, timeout) ####========******####======== actions begin ========####******========#### upgrade_syslog_level(conn, cur) return @@ -47,7 +37,6 @@ def upgrade_syslog_level(conn, cur): info_cnt = result[0][0] if info_cnt > 0: actions.set_parameter(cur, "syslog_level", "WDIAG") - except Exception, e: logging.warn("upgrade syslog level failed!") raise e @@ -62,6 +51,18 @@ def query(cur, sql): def get_tenant_ids(cur): return [_[0] for _ in query(cur, 'select tenant_id from oceanbase.__all_tenant')] +def run_root_inspection(cur, timeout): + + query_timeout = actions.set_default_timeout_by_tenant(cur, timeout, 10, 600) + + actions.set_session_timeout(cur, query_timeout) + + sql = "alter system run job 'root_inspection'" + logging.info(sql) + cur.execute(sql) + + actions.set_session_timeout(cur, 10) + def upgrade_across_version(cur): current_data_version = actions.get_current_data_version() int_current_data_version = actions.get_version(current_data_version) @@ -167,7 +168,9 @@ def check_can_run_upgrade_job(cur, job_name): def check_upgrade_job_result(cur, job_name, timeout, max_used_job_id): try: - times = (timeout if timeout > 0 else 3600) / 10 + wait_timeout = actions.set_default_timeout_by_tenant(cur, timeout, 100, 3600) + + times = wait_timeout / 10 while (times >= 0): sql = """select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job where job_type = '{0}' and job_id > {1} order by job_id desc limit 1 diff --git a/tools/upgrade/upgrade_health_checker.py b/tools/upgrade/upgrade_health_checker.py index 5f8eeb47ec..255ed1dcf2 100755 --- a/tools/upgrade/upgrade_health_checker.py +++ b/tools/upgrade/upgrade_health_checker.py @@ -73,7 +73,7 @@ sys.argv[0] + """ [OPTIONS]""" +\ ' that all modules should be run. They are splitted by ",".\n' +\ ' For example: -m all, or --module=ddl,normal_dml,special_action\n' +\ '-l, --log-file=name Log file path. If log file path is not given it\'s ' + os.path.splitext(sys.argv[0])[0] + '.log\n' +\ -'-t, --timeout=name check timeout, default: 600(s).\n' + \ +'-t, --timeout=name check timeout.\n' + \ '-z, --zone=name If zone is not specified, check all servers status in cluster. \n' +\ ' Otherwise, only check servers status in specified zone. \n' + \ '\n\n' +\ @@ -135,8 +135,7 @@ Option('p', 'password', True, False, ''),\ Option('m', 'module', True, False, 'all'),\ # 日志文件路径，不同脚本的main函数中中会改成不同的默认值 Option('l', 'log-file', True, False),\ -# 一些检查的超时时间，默认是600s -Option('t', 'timeout', True, False, '600'),\ +Option('t', 'timeout', True, False, 0),\ Option('z', 'zone', True, False, ''),\ ]\ @@ -288,13 +287,38 @@ def check_zone_valid(query_cur, zone): else: logging.info("zone is empty, check all servers in cluster") +def fetch_tenant_ids(query_cur): + try: + tenant_id_list = [] + (desc, results) = query_cur.exec_query("""select distinct tenant_id from oceanbase.__all_tenant order by tenant_id desc""") + for r in results: + tenant_id_list.append(r[0]) + return tenant_id_list + except Exception, e: + logging.exception('fail to fetch distinct tenant ids') + raise e + +def set_default_timeout_by_tenant(query_cur, timeout, timeout_per_tenant, min_timeout): + if timeout > 0: + logging.info("use timeout from opt, timeout(s):{0}".format(timeout)) + else: + tenant_id_list = fetch_tenant_ids(query_cur) + cal_timeout = len(tenant_id_list) * timeout_per_tenant + timeout = (cal_timeout if cal_timeout > min_timeout else min_timeout) + logging.info("use default timeout caculated by tenants, " + "timeout(s):{0}, tenant_count:{1}, " + "timeout_per_tenant(s):{2}, min_timeout(s):{3}" + .format(timeout, len(tenant_id_list), timeout_per_tenant, min_timeout)) + + return timeout + #### START #### # 0. 检查server版本是否严格一致 def check_server_version_by_zone(query_cur, zone): if zone == '': logging.info("skip check server version by cluster") else: - sql = """select distinct(substring_index(build_version, '_', 1)) from __all_server where zone = '{0}'""".format(zone); + sql = """select distinct(substring_index(build_version, '_', 1)) from oceanbase.__all_server where zone = '{0}'""".format(zone); (desc, results) = query_cur.exec_query(sql); if len(results) != 1: raise MyError("servers build_version not match") @@ -304,8 +328,9 @@ def check_server_version_by_zone(query_cur, zone): # 1. 检查paxos副本是否同步, paxos副本是否缺失 def check_paxos_replica(query_cur, timeout): # 1.1 检查paxos副本是否同步 - sql = """select count(*) from GV$OB_LOG_STAT where in_sync = 'NO'""" - check_until_timeout(query_cur, sql, 0, timeout) + sql = """select count(*) from oceanbase.GV$OB_LOG_STAT where in_sync = 'NO'""" + wait_timeout = set_default_timeout_by_tenant(query_cur, timeout, 10, 600) + check_until_timeout(query_cur, sql, 0, wait_timeout) # 1.2 检查paxos副本是否有缺失 TODO logging.info('check paxos replica success') @@ -315,26 +340,29 @@ def check_observer_status(query_cur, zone, timeout): sql = """select count(*) from oceanbase.__all_server where (start_service_time <= 0 or status='inactive')""" if zone != '': sql += """ and zone = '{0}'""".format(zone) - check_until_timeout(query_cur, sql, 0, timeout) + wait_timeout = set_default_timeout_by_tenant(query_cur, timeout, 10, 600) + check_until_timeout(query_cur, sql, 0, wait_timeout) # 3. 检查schema是否刷新成功 def check_schema_status(query_cur, timeout): sql = """select if (a.cnt = b.cnt, 1, 0) as passed from (select count(*) as cnt from oceanbase.__all_virtual_server_schema_info where refreshed_schema_version > 1 and refreshed_schema_version % 8 = 0) as a join (select count(*) as cnt from oceanbase.__all_server join oceanbase.__all_tenant) as b""" - check_until_timeout(query_cur, sql, 1, timeout) + wait_timeout = set_default_timeout_by_tenant(query_cur, timeout, 30, 600) + check_until_timeout(query_cur, sql, 1, wait_timeout) # 4. check major finish def check_major_merge(query_cur, timeout): need_check = 0 - (desc, results) = query_cur.exec_query("""select distinct value from GV$OB_PARAMETERs where name = 'enable_major_freeze';""") + (desc, results) = query_cur.exec_query("""select distinct value from oceanbase.GV$OB_PARAMETERS where name = 'enable_major_freeze';""") if len(results) != 1: need_check = 1 elif results[0][0] != 'True': need_check = 1 if need_check == 1: - sql = """select count(1) from CDB_OB_MAJOR_COMPACTION where (GLOBAL_BROADCAST_SCN > LAST_SCN or STATUS != 'IDLE')""" - check_until_timeout(query_cur, sql, 0, timeout) - sql2 = """select /*+ query_timeout(1000000000) */ count(1) from __all_virtual_tablet_compaction_info where max_received_scn > finished_scn and max_received_scn > 0""" - check_until_timeout(query_cur, sql2, 0, timeout) + wait_timeout = set_default_timeout_by_tenant(query_cur, timeout, 30, 600) + sql = """select count(1) from oceanbase.CDB_OB_MAJOR_COMPACTION where (GLOBAL_BROADCAST_SCN > LAST_SCN or STATUS != 'IDLE')""" + check_until_timeout(query_cur, sql, 0, wait_timeout) + sql2 = """select /*+ query_timeout(1000000000) */ count(1) from oceanbase.__all_virtual_tablet_compaction_info where max_received_scn > finished_scn and max_received_scn > 0""" + check_until_timeout(query_cur, sql2, 0, wait_timeout) def check_until_timeout(query_cur, sql, value, timeout): times = timeout / 10 @@ -366,7 +394,6 @@ def do_check(my_host, my_port, my_user, my_passwd, upgrade_params, timeout, need raise_on_warnings = True) conn.autocommit = True cur = conn.cursor(buffered=True) - timeout = timeout if timeout > 0 else 600 try: query_cur = QueryCursor(cur) check_zone_valid(query_cur, zone) diff --git a/tools/upgrade/upgrade_post.py b/tools/upgrade/upgrade_post.py index ca38333d3c..cc10931137 100755 --- a/tools/upgrade/upgrade_post.py +++ b/tools/upgrade/upgrade_post.py @@ -135,16 +135,45 @@ # cur.execute(sql) # wait_parameter_sync(cur, False, parameter, value, timeout) # +#def set_session_timeout(cur, seconds): +# sql = "set @@session.ob_query_timeout = {0}".format(seconds * 1000 * 1000) +# logging.info(sql) +# cur.execute(sql) +# +#def set_default_timeout_by_tenant(cur, timeout, timeout_per_tenant, min_timeout): +# if timeout > 0: +# logging.info("use timeout from opt, timeout(s):{0}".format(timeout)) +# else: +# query_cur = QueryCursor(cur) +# tenant_id_list = fetch_tenant_ids(query_cur) +# cal_timeout = len(tenant_id_list) * timeout_per_tenant +# timeout = (cal_timeout if cal_timeout > min_timeout else min_timeout) +# logging.info("use default timeout caculated by tenants, " +# "timeout(s):{0}, tenant_count:{1}, " +# "timeout_per_tenant(s):{2}, min_timeout(s):{3}" +# .format(timeout, len(tenant_id_list), timeout_per_tenant, min_timeout)) +# +# return timeout +# #def set_tenant_parameter(cur, parameter, value, timeout = 0): +# # tenants_list = [] # if get_min_cluster_version(cur) < get_version("4.2.1.0"): # tenants_list = ['all'] # else: # tenants_list = ['sys', 'all_user', 'all_meta'] +# +# query_timeout = set_default_timeout_by_tenant(cur, timeout, 10, 60) +# +# set_session_timeout(cur, query_timeout) +# # for tenants in tenants_list: # sql = """alter system set {0} = '{1}' tenant = '{2}'""".format(parameter, value, tenants) # logging.info(sql) # cur.execute(sql) +# +# set_session_timeout(cur, 10) +# # wait_parameter_sync(cur, True, parameter, value, timeout) # #def get_ori_enable_ddl(cur, timeout): @@ -233,7 +262,20 @@ # table_name = "GV$OB_PARAMETERS" if not is_tenant_config else "__all_virtual_tenant_parameter_info" # sql = """select count(*) as cnt from oceanbase.{0} # where name = '{1}' and value != '{2}'""".format(table_name, key, value) -# times = (timeout if timeout > 0 else 60) / 5 +# +# wait_timeout = 0 +# query_timeout = 0 +# if not is_tenant_config or timeout > 0: +# wait_timeout = (timeout if timeout > 0 else 60) +# query_timeout = wait_timeout +# else: +# # is_tenant_config & timeout not set +# wait_timeout = set_default_timeout_by_tenant(cur, timeout, 10, 60) +# query_timeout = set_default_timeout_by_tenant(cur, timeout, 2, 60) +# +# set_session_timeout(cur, query_timeout) +# +# times = wait_timeout / 5 # while times >= 0: # logging.info(sql) # cur.execute(sql) @@ -253,6 +295,8 @@ # raise e # time.sleep(5) # +# set_session_timeout(cur, 10) +# #def do_begin_upgrade(cur, timeout): # # if not check_parameter(cur, False, "enable_upgrade_mode", "True"): @@ -324,24 +368,38 @@ # tenants_list = ['all'] # else: # tenants_list = ['sys', 'all_user', 'all_meta'] +# +# query_timeout = set_default_timeout_by_tenant(cur, timeout, 10, 60) +# +# set_session_timeout(cur, query_timeout) +# # for tenants in tenants_list: # action_sql = "alter system suspend merge tenant = {0}".format(tenants) # rollback_sql = "alter system resume merge tenant = {0}".format(tenants) # logging.info(action_sql) # cur.execute(action_sql) # +# set_session_timeout(cur, 10) +# #def do_resume_merge(cur, timeout): # tenants_list = [] # if get_min_cluster_version(cur) < get_version("4.2.1.0"): # tenants_list = ['all'] # else: # tenants_list = ['sys', 'all_user', 'all_meta'] +# +# query_timeout = set_default_timeout_by_tenant(cur, timeout, 10, 60) +# +# set_session_timeout(cur, query_timeout) +# # for tenants in tenants_list: # action_sql = "alter system resume merge tenant = {0}".format(tenants) # rollback_sql = "alter system suspend merge tenant = {0}".format(tenants) # logging.info(action_sql) # cur.execute(action_sql) # +# set_session_timeout(cur, 10) +# #class Cursor: # __cursor = None # def __init__(self, cursor): @@ -1334,17 +1392,7 @@ # else: # run_upgrade_job(conn, cur, "UPGRADE_VIRTUAL_SCHEMA", timeout) # -# # just to make __all_virtual_upgrade_inspection avaliable -# timeout_ts = (timeout if timeout > 0 else 600) * 1000 * 1000 -# sql = "set @@session.ob_query_timeout = {0}".format(timeout_ts) -# logging.info(sql) -# cur.execute(sql) -# sql = "alter system run job 'root_inspection'" -# logging.info(sql) -# cur.execute(sql) -# sql = "set @@session.ob_query_timeout = 10000000" -# logging.info(sql) -# cur.execute(sql) +# run_root_inspection(cur, timeout) #####========******####======== actions begin ========####******========#### # upgrade_syslog_level(conn, cur) # return @@ -1360,7 +1408,6 @@ # info_cnt = result[0][0] # if info_cnt > 0: # actions.set_parameter(cur, "syslog_level", "WDIAG") -# # except Exception, e: # logging.warn("upgrade syslog level failed!") # raise e @@ -1375,6 +1422,18 @@ #def get_tenant_ids(cur): # return [_[0] for _ in query(cur, 'select tenant_id from oceanbase.__all_tenant')] # +#def run_root_inspection(cur, timeout): +# +# query_timeout = actions.set_default_timeout_by_tenant(cur, timeout, 10, 600) +# +# actions.set_session_timeout(cur, query_timeout) +# +# sql = "alter system run job 'root_inspection'" +# logging.info(sql) +# cur.execute(sql) +# +# actions.set_session_timeout(cur, 10) +# #def upgrade_across_version(cur): # current_data_version = actions.get_current_data_version() # int_current_data_version = actions.get_version(current_data_version) @@ -1480,7 +1539,9 @@ # #def check_upgrade_job_result(cur, job_name, timeout, max_used_job_id): # try: -# times = (timeout if timeout > 0 else 3600) / 10 +# wait_timeout = actions.set_default_timeout_by_tenant(cur, timeout, 100, 3600) +# +# times = wait_timeout / 10 # while (times >= 0): # sql = """select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job # where job_type = '{0}' and job_id > {1} order by job_id desc limit 1 @@ -2217,7 +2278,8 @@ # fail_list.append("""still has tenant snapshot task, upgrade is not allowed temporarily""") # else: # logging.info('check tenant snapshot task success') -## 19. 检查是否有租户在升到4.3.0版本之前已将binlog_row_image设为MINIMAL +# +## 17. 检查是否有租户在升到4.3.0版本之前已将binlog_row_image设为MINIMAL #def check_variable_binlog_row_image(query_cur): ## 4.3.0.0之前的版本,MINIMAL模式生成的日志CDC无法正常消费(DELETE日志). ## 4.3.0版本开始,MINIMAL模式做了改进,支持CDC消费,需要在升级到4.3.0.0之后再打开. @@ -2405,7 +2467,7 @@ #' that all modules should be run. They are splitted by ",".\n' +\ #' For example: -m all, or --module=ddl,normal_dml,special_action\n' +\ #'-l, --log-file=name Log file path. If log file path is not given it\'s ' + os.path.splitext(sys.argv[0])[0] + '.log\n' +\ -#'-t, --timeout=name check timeout, default: 600(s).\n' + \ +#'-t, --timeout=name check timeout.\n' + \ #'-z, --zone=name If zone is not specified, check all servers status in cluster. \n' +\ #' Otherwise, only check servers status in specified zone. \n' + \ #'\n\n' +\ @@ -2467,8 +2529,7 @@ #Option('m', 'module', True, False, 'all'),\ ## 日志文件路径，不同脚本的main函数中中会改成不同的默认值 #Option('l', 'log-file', True, False),\ -## 一些检查的超时时间，默认是600s -#Option('t', 'timeout', True, False, '600'),\ +#Option('t', 'timeout', True, False, 0),\ #Option('z', 'zone', True, False, ''),\ #]\ # @@ -2620,13 +2681,38 @@ # else: # logging.info("zone is empty, check all servers in cluster") # +#def fetch_tenant_ids(query_cur): +# try: +# tenant_id_list = [] +# (desc, results) = query_cur.exec_query("""select distinct tenant_id from oceanbase.__all_tenant order by tenant_id desc""") +# for r in results: +# tenant_id_list.append(r[0]) +# return tenant_id_list +# except Exception, e: +# logging.exception('fail to fetch distinct tenant ids') +# raise e +# +#def set_default_timeout_by_tenant(query_cur, timeout, timeout_per_tenant, min_timeout): +# if timeout > 0: +# logging.info("use timeout from opt, timeout(s):{0}".format(timeout)) +# else: +# tenant_id_list = fetch_tenant_ids(query_cur) +# cal_timeout = len(tenant_id_list) * timeout_per_tenant +# timeout = (cal_timeout if cal_timeout > min_timeout else min_timeout) +# logging.info("use default timeout caculated by tenants, " +# "timeout(s):{0}, tenant_count:{1}, " +# "timeout_per_tenant(s):{2}, min_timeout(s):{3}" +# .format(timeout, len(tenant_id_list), timeout_per_tenant, min_timeout)) +# +# return timeout +# ##### START #### ## 0. 检查server版本是否严格一致 #def check_server_version_by_zone(query_cur, zone): # if zone == '': # logging.info("skip check server version by cluster") # else: -# sql = """select distinct(substring_index(build_version, '_', 1)) from __all_server where zone = '{0}'""".format(zone); +# sql = """select distinct(substring_index(build_version, '_', 1)) from oceanbase.__all_server where zone = '{0}'""".format(zone); # (desc, results) = query_cur.exec_query(sql); # if len(results) != 1: # raise MyError("servers build_version not match") @@ -2636,8 +2722,9 @@ ## 1. 检查paxos副本是否同步, paxos副本是否缺失 #def check_paxos_replica(query_cur, timeout): # # 1.1 检查paxos副本是否同步 -# sql = """select count(*) from GV$OB_LOG_STAT where in_sync = 'NO'""" -# check_until_timeout(query_cur, sql, 0, timeout) +# sql = """select count(*) from oceanbase.GV$OB_LOG_STAT where in_sync = 'NO'""" +# wait_timeout = set_default_timeout_by_tenant(query_cur, timeout, 10, 600) +# check_until_timeout(query_cur, sql, 0, wait_timeout) # # # 1.2 检查paxos副本是否有缺失 TODO # logging.info('check paxos replica success') @@ -2647,26 +2734,29 @@ # sql = """select count(*) from oceanbase.__all_server where (start_service_time <= 0 or status='inactive')""" # if zone != '': # sql += """ and zone = '{0}'""".format(zone) -# check_until_timeout(query_cur, sql, 0, timeout) +# wait_timeout = set_default_timeout_by_tenant(query_cur, timeout, 10, 600) +# check_until_timeout(query_cur, sql, 0, wait_timeout) # ## 3. 检查schema是否刷新成功 #def check_schema_status(query_cur, timeout): # sql = """select if (a.cnt = b.cnt, 1, 0) as passed from (select count(*) as cnt from oceanbase.__all_virtual_server_schema_info where refreshed_schema_version > 1 and refreshed_schema_version % 8 = 0) as a join (select count(*) as cnt from oceanbase.__all_server join oceanbase.__all_tenant) as b""" -# check_until_timeout(query_cur, sql, 1, timeout) +# wait_timeout = set_default_timeout_by_tenant(query_cur, timeout, 30, 600) +# check_until_timeout(query_cur, sql, 1, wait_timeout) # ## 4. check major finish #def check_major_merge(query_cur, timeout): # need_check = 0 -# (desc, results) = query_cur.exec_query("""select distinct value from GV$OB_PARAMETERs where name = 'enable_major_freeze';""") +# (desc, results) = query_cur.exec_query("""select distinct value from oceanbase.GV$OB_PARAMETERS where name = 'enable_major_freeze';""") # if len(results) != 1: # need_check = 1 # elif results[0][0] != 'True': # need_check = 1 # if need_check == 1: -# sql = """select count(1) from CDB_OB_MAJOR_COMPACTION where (GLOBAL_BROADCAST_SCN > LAST_SCN or STATUS != 'IDLE')""" -# check_until_timeout(query_cur, sql, 0, timeout) -# sql2 = """select /*+ query_timeout(1000000000) */ count(1) from __all_virtual_tablet_compaction_info where max_received_scn > finished_scn and max_received_scn > 0""" -# check_until_timeout(query_cur, sql2, 0, timeout) +# wait_timeout = set_default_timeout_by_tenant(query_cur, timeout, 30, 600) +# sql = """select count(1) from oceanbase.CDB_OB_MAJOR_COMPACTION where (GLOBAL_BROADCAST_SCN > LAST_SCN or STATUS != 'IDLE')""" +# check_until_timeout(query_cur, sql, 0, wait_timeout) +# sql2 = """select /*+ query_timeout(1000000000) */ count(1) from oceanbase.__all_virtual_tablet_compaction_info where max_received_scn > finished_scn and max_received_scn > 0""" +# check_until_timeout(query_cur, sql2, 0, wait_timeout) # #def check_until_timeout(query_cur, sql, value, timeout): # times = timeout / 10 @@ -2698,7 +2788,6 @@ # raise_on_warnings = True) # conn.autocommit = True # cur = conn.cursor(buffered=True) -# timeout = timeout if timeout > 0 else 600 # try: # query_cur = QueryCursor(cur) # check_zone_valid(query_cur, zone) @@ -2795,8 +2884,14 @@ # # check compatible sync # parameter_count = int(server_count) * int(tenant_count) # current_data_version = actions.get_current_data_version() +# +# query_timeout = actions.set_default_timeout_by_tenant(cur, timeout, 2, 60) +# actions.set_session_timeout(cur, query_timeout) +# # sql = """select count(*) as cnt from oceanbase.__all_virtual_tenant_parameter_info where name = 'compatible' and value = '{0}' and tenant_id in ({1})""".format(current_data_version, tenant_ids_str) -# times = (timeout if timeout > 0 else 60) / 5 +# +# wait_timeout = actions.set_default_timeout_by_tenant(cur, timeout, 10, 60) +# times = wait_timeout / 5 # while times >= 0: # logging.info(sql) # cur.execute(sql) @@ -2816,6 +2911,8 @@ # raise e # time.sleep(5) # +# actions.set_session_timeout(cur, 10) +# # # check target_data_version/current_data_version from __all_core_table # int_current_data_version = actions.get_version(current_data_version) # sql = "select count(*) from __all_virtual_core_table where column_name in ('target_data_version', 'current_data_version') and column_value = {0} and tenant_id in ({1})".format(int_current_data_version, tenant_ids_str) @@ -2830,16 +2927,20 @@ # logging.info("all tenant's target_data_version/current_data_version are match with {0}".format(current_data_version)) # ## 3 检查内部表自检是否成功 -#def check_root_inspection(query_cur, timeout): +#def check_root_inspection(cur, query_cur, timeout): # sql = "select count(*) from oceanbase.__all_virtual_upgrade_inspection where info != 'succeed'" -# times = timeout if timeout > 0 else 180 -# while times > 0 : +# +# wait_timeout = actions.set_default_timeout_by_tenant(cur, timeout, 10, 600) +# +# times = wait_timeout / 10 +# while times >= 0 : # (desc, results) = query_cur.exec_query(sql) # if results[0][0] == 0: # break # time.sleep(10) # times -= 1 -# if times == 0: +# +# if times == -1: # logging.warn('check root inspection failed!') # raise e # logging.info('check root inspection success') @@ -2867,7 +2968,7 @@ # try: # check_cluster_version(cur, timeout) # check_data_version(cur, query_cur, timeout) -# check_root_inspection(query_cur, timeout) +# check_root_inspection(cur, query_cur, timeout) # enable_ddl(cur, timeout) # enable_rebalance(cur, timeout) # enable_rereplication(cur, timeout) diff --git a/tools/upgrade/upgrade_post_checker.py b/tools/upgrade/upgrade_post_checker.py index e49ddbb5ee..2a6e56b871 100755 --- a/tools/upgrade/upgrade_post_checker.py +++ b/tools/upgrade/upgrade_post_checker.py @@ -41,8 +41,14 @@ def check_data_version(cur, query_cur, timeout): # check compatible sync parameter_count = int(server_count) * int(tenant_count) current_data_version = actions.get_current_data_version() + + query_timeout = actions.set_default_timeout_by_tenant(cur, timeout, 2, 60) + actions.set_session_timeout(cur, query_timeout) + sql = """select count(*) as cnt from oceanbase.__all_virtual_tenant_parameter_info where name = 'compatible' and value = '{0}' and tenant_id in ({1})""".format(current_data_version, tenant_ids_str) - times = (timeout if timeout > 0 else 60) / 5 + + wait_timeout = actions.set_default_timeout_by_tenant(cur, timeout, 10, 60) + times = wait_timeout / 5 while times >= 0: logging.info(sql) cur.execute(sql) @@ -62,6 +68,8 @@ def check_data_version(cur, query_cur, timeout): raise e time.sleep(5) + actions.set_session_timeout(cur, 10) + # check target_data_version/current_data_version from __all_core_table int_current_data_version = actions.get_version(current_data_version) sql = "select count(*) from __all_virtual_core_table where column_name in ('target_data_version', 'current_data_version') and column_value = {0} and tenant_id in ({1})".format(int_current_data_version, tenant_ids_str) @@ -76,16 +84,20 @@ def check_data_version(cur, query_cur, timeout): logging.info("all tenant's target_data_version/current_data_version are match with {0}".format(current_data_version)) # 3 检查内部表自检是否成功 -def check_root_inspection(query_cur, timeout): +def check_root_inspection(cur, query_cur, timeout): sql = "select count(*) from oceanbase.__all_virtual_upgrade_inspection where info != 'succeed'" - times = timeout if timeout > 0 else 180 - while times > 0 : + + wait_timeout = actions.set_default_timeout_by_tenant(cur, timeout, 10, 600) + + times = wait_timeout / 10 + while times >= 0 : (desc, results) = query_cur.exec_query(sql) if results[0][0] == 0: break time.sleep(10) times -= 1 - if times == 0: + + if times == -1: logging.warn('check root inspection failed!') raise e logging.info('check root inspection success') @@ -113,7 +125,7 @@ def do_check(conn, cur, query_cur, timeout): try: check_cluster_version(cur, timeout) check_data_version(cur, query_cur, timeout) - check_root_inspection(query_cur, timeout) + check_root_inspection(cur, query_cur, timeout) enable_ddl(cur, timeout) enable_rebalance(cur, timeout) enable_rereplication(cur, timeout) diff --git a/tools/upgrade/upgrade_pre.py b/tools/upgrade/upgrade_pre.py index 6ce6108cd7..181d3e1978 100755 --- a/tools/upgrade/upgrade_pre.py +++ b/tools/upgrade/upgrade_pre.py @@ -135,16 +135,45 @@ # cur.execute(sql) # wait_parameter_sync(cur, False, parameter, value, timeout) # +#def set_session_timeout(cur, seconds): +# sql = "set @@session.ob_query_timeout = {0}".format(seconds * 1000 * 1000) +# logging.info(sql) +# cur.execute(sql) +# +#def set_default_timeout_by_tenant(cur, timeout, timeout_per_tenant, min_timeout): +# if timeout > 0: +# logging.info("use timeout from opt, timeout(s):{0}".format(timeout)) +# else: +# query_cur = QueryCursor(cur) +# tenant_id_list = fetch_tenant_ids(query_cur) +# cal_timeout = len(tenant_id_list) * timeout_per_tenant +# timeout = (cal_timeout if cal_timeout > min_timeout else min_timeout) +# logging.info("use default timeout caculated by tenants, " +# "timeout(s):{0}, tenant_count:{1}, " +# "timeout_per_tenant(s):{2}, min_timeout(s):{3}" +# .format(timeout, len(tenant_id_list), timeout_per_tenant, min_timeout)) +# +# return timeout +# #def set_tenant_parameter(cur, parameter, value, timeout = 0): +# # tenants_list = [] # if get_min_cluster_version(cur) < get_version("4.2.1.0"): # tenants_list = ['all'] # else: # tenants_list = ['sys', 'all_user', 'all_meta'] +# +# query_timeout = set_default_timeout_by_tenant(cur, timeout, 10, 60) +# +# set_session_timeout(cur, query_timeout) +# # for tenants in tenants_list: # sql = """alter system set {0} = '{1}' tenant = '{2}'""".format(parameter, value, tenants) # logging.info(sql) # cur.execute(sql) +# +# set_session_timeout(cur, 10) +# # wait_parameter_sync(cur, True, parameter, value, timeout) # #def get_ori_enable_ddl(cur, timeout): @@ -233,7 +262,20 @@ # table_name = "GV$OB_PARAMETERS" if not is_tenant_config else "__all_virtual_tenant_parameter_info" # sql = """select count(*) as cnt from oceanbase.{0} # where name = '{1}' and value != '{2}'""".format(table_name, key, value) -# times = (timeout if timeout > 0 else 60) / 5 +# +# wait_timeout = 0 +# query_timeout = 0 +# if not is_tenant_config or timeout > 0: +# wait_timeout = (timeout if timeout > 0 else 60) +# query_timeout = wait_timeout +# else: +# # is_tenant_config & timeout not set +# wait_timeout = set_default_timeout_by_tenant(cur, timeout, 10, 60) +# query_timeout = set_default_timeout_by_tenant(cur, timeout, 2, 60) +# +# set_session_timeout(cur, query_timeout) +# +# times = wait_timeout / 5 # while times >= 0: # logging.info(sql) # cur.execute(sql) @@ -253,6 +295,8 @@ # raise e # time.sleep(5) # +# set_session_timeout(cur, 10) +# #def do_begin_upgrade(cur, timeout): # # if not check_parameter(cur, False, "enable_upgrade_mode", "True"): @@ -324,24 +368,38 @@ # tenants_list = ['all'] # else: # tenants_list = ['sys', 'all_user', 'all_meta'] +# +# query_timeout = set_default_timeout_by_tenant(cur, timeout, 10, 60) +# +# set_session_timeout(cur, query_timeout) +# # for tenants in tenants_list: # action_sql = "alter system suspend merge tenant = {0}".format(tenants) # rollback_sql = "alter system resume merge tenant = {0}".format(tenants) # logging.info(action_sql) # cur.execute(action_sql) # +# set_session_timeout(cur, 10) +# #def do_resume_merge(cur, timeout): # tenants_list = [] # if get_min_cluster_version(cur) < get_version("4.2.1.0"): # tenants_list = ['all'] # else: # tenants_list = ['sys', 'all_user', 'all_meta'] +# +# query_timeout = set_default_timeout_by_tenant(cur, timeout, 10, 60) +# +# set_session_timeout(cur, query_timeout) +# # for tenants in tenants_list: # action_sql = "alter system resume merge tenant = {0}".format(tenants) # rollback_sql = "alter system suspend merge tenant = {0}".format(tenants) # logging.info(action_sql) # cur.execute(action_sql) # +# set_session_timeout(cur, 10) +# #class Cursor: # __cursor = None # def __init__(self, cursor): @@ -1334,17 +1392,7 @@ # else: # run_upgrade_job(conn, cur, "UPGRADE_VIRTUAL_SCHEMA", timeout) # -# # just to make __all_virtual_upgrade_inspection avaliable -# timeout_ts = (timeout if timeout > 0 else 600) * 1000 * 1000 -# sql = "set @@session.ob_query_timeout = {0}".format(timeout_ts) -# logging.info(sql) -# cur.execute(sql) -# sql = "alter system run job 'root_inspection'" -# logging.info(sql) -# cur.execute(sql) -# sql = "set @@session.ob_query_timeout = 10000000" -# logging.info(sql) -# cur.execute(sql) +# run_root_inspection(cur, timeout) #####========******####======== actions begin ========####******========#### # upgrade_syslog_level(conn, cur) # return @@ -1360,7 +1408,6 @@ # info_cnt = result[0][0] # if info_cnt > 0: # actions.set_parameter(cur, "syslog_level", "WDIAG") -# # except Exception, e: # logging.warn("upgrade syslog level failed!") # raise e @@ -1375,6 +1422,18 @@ #def get_tenant_ids(cur): # return [_[0] for _ in query(cur, 'select tenant_id from oceanbase.__all_tenant')] # +#def run_root_inspection(cur, timeout): +# +# query_timeout = actions.set_default_timeout_by_tenant(cur, timeout, 10, 600) +# +# actions.set_session_timeout(cur, query_timeout) +# +# sql = "alter system run job 'root_inspection'" +# logging.info(sql) +# cur.execute(sql) +# +# actions.set_session_timeout(cur, 10) +# #def upgrade_across_version(cur): # current_data_version = actions.get_current_data_version() # int_current_data_version = actions.get_version(current_data_version) @@ -1480,7 +1539,9 @@ # #def check_upgrade_job_result(cur, job_name, timeout, max_used_job_id): # try: -# times = (timeout if timeout > 0 else 3600) / 10 +# wait_timeout = actions.set_default_timeout_by_tenant(cur, timeout, 100, 3600) +# +# times = wait_timeout / 10 # while (times >= 0): # sql = """select job_status, rs_svr_ip, rs_svr_port, gmt_create from oceanbase.__all_rootservice_job # where job_type = '{0}' and job_id > {1} order by job_id desc limit 1 @@ -2217,7 +2278,8 @@ # fail_list.append("""still has tenant snapshot task, upgrade is not allowed temporarily""") # else: # logging.info('check tenant snapshot task success') -## 19. 检查是否有租户在升到4.3.0版本之前已将binlog_row_image设为MINIMAL +# +## 17. 检查是否有租户在升到4.3.0版本之前已将binlog_row_image设为MINIMAL #def check_variable_binlog_row_image(query_cur): ## 4.3.0.0之前的版本,MINIMAL模式生成的日志CDC无法正常消费(DELETE日志). ## 4.3.0版本开始,MINIMAL模式做了改进,支持CDC消费,需要在升级到4.3.0.0之后再打开. @@ -2405,7 +2467,7 @@ #' that all modules should be run. They are splitted by ",".\n' +\ #' For example: -m all, or --module=ddl,normal_dml,special_action\n' +\ #'-l, --log-file=name Log file path. If log file path is not given it\'s ' + os.path.splitext(sys.argv[0])[0] + '.log\n' +\ -#'-t, --timeout=name check timeout, default: 600(s).\n' + \ +#'-t, --timeout=name check timeout.\n' + \ #'-z, --zone=name If zone is not specified, check all servers status in cluster. \n' +\ #' Otherwise, only check servers status in specified zone. \n' + \ #'\n\n' +\ @@ -2467,8 +2529,7 @@ #Option('m', 'module', True, False, 'all'),\ ## 日志文件路径，不同脚本的main函数中中会改成不同的默认值 #Option('l', 'log-file', True, False),\ -## 一些检查的超时时间，默认是600s -#Option('t', 'timeout', True, False, '600'),\ +#Option('t', 'timeout', True, False, 0),\ #Option('z', 'zone', True, False, ''),\ #]\ # @@ -2620,13 +2681,38 @@ # else: # logging.info("zone is empty, check all servers in cluster") # +#def fetch_tenant_ids(query_cur): +# try: +# tenant_id_list = [] +# (desc, results) = query_cur.exec_query("""select distinct tenant_id from oceanbase.__all_tenant order by tenant_id desc""") +# for r in results: +# tenant_id_list.append(r[0]) +# return tenant_id_list +# except Exception, e: +# logging.exception('fail to fetch distinct tenant ids') +# raise e +# +#def set_default_timeout_by_tenant(query_cur, timeout, timeout_per_tenant, min_timeout): +# if timeout > 0: +# logging.info("use timeout from opt, timeout(s):{0}".format(timeout)) +# else: +# tenant_id_list = fetch_tenant_ids(query_cur) +# cal_timeout = len(tenant_id_list) * timeout_per_tenant +# timeout = (cal_timeout if cal_timeout > min_timeout else min_timeout) +# logging.info("use default timeout caculated by tenants, " +# "timeout(s):{0}, tenant_count:{1}, " +# "timeout_per_tenant(s):{2}, min_timeout(s):{3}" +# .format(timeout, len(tenant_id_list), timeout_per_tenant, min_timeout)) +# +# return timeout +# ##### START #### ## 0. 检查server版本是否严格一致 #def check_server_version_by_zone(query_cur, zone): # if zone == '': # logging.info("skip check server version by cluster") # else: -# sql = """select distinct(substring_index(build_version, '_', 1)) from __all_server where zone = '{0}'""".format(zone); +# sql = """select distinct(substring_index(build_version, '_', 1)) from oceanbase.__all_server where zone = '{0}'""".format(zone); # (desc, results) = query_cur.exec_query(sql); # if len(results) != 1: # raise MyError("servers build_version not match") @@ -2636,8 +2722,9 @@ ## 1. 检查paxos副本是否同步, paxos副本是否缺失 #def check_paxos_replica(query_cur, timeout): # # 1.1 检查paxos副本是否同步 -# sql = """select count(*) from GV$OB_LOG_STAT where in_sync = 'NO'""" -# check_until_timeout(query_cur, sql, 0, timeout) +# sql = """select count(*) from oceanbase.GV$OB_LOG_STAT where in_sync = 'NO'""" +# wait_timeout = set_default_timeout_by_tenant(query_cur, timeout, 10, 600) +# check_until_timeout(query_cur, sql, 0, wait_timeout) # # # 1.2 检查paxos副本是否有缺失 TODO # logging.info('check paxos replica success') @@ -2647,26 +2734,29 @@ # sql = """select count(*) from oceanbase.__all_server where (start_service_time <= 0 or status='inactive')""" # if zone != '': # sql += """ and zone = '{0}'""".format(zone) -# check_until_timeout(query_cur, sql, 0, timeout) +# wait_timeout = set_default_timeout_by_tenant(query_cur, timeout, 10, 600) +# check_until_timeout(query_cur, sql, 0, wait_timeout) # ## 3. 检查schema是否刷新成功 #def check_schema_status(query_cur, timeout): # sql = """select if (a.cnt = b.cnt, 1, 0) as passed from (select count(*) as cnt from oceanbase.__all_virtual_server_schema_info where refreshed_schema_version > 1 and refreshed_schema_version % 8 = 0) as a join (select count(*) as cnt from oceanbase.__all_server join oceanbase.__all_tenant) as b""" -# check_until_timeout(query_cur, sql, 1, timeout) +# wait_timeout = set_default_timeout_by_tenant(query_cur, timeout, 30, 600) +# check_until_timeout(query_cur, sql, 1, wait_timeout) # ## 4. check major finish #def check_major_merge(query_cur, timeout): # need_check = 0 -# (desc, results) = query_cur.exec_query("""select distinct value from GV$OB_PARAMETERs where name = 'enable_major_freeze';""") +# (desc, results) = query_cur.exec_query("""select distinct value from oceanbase.GV$OB_PARAMETERS where name = 'enable_major_freeze';""") # if len(results) != 1: # need_check = 1 # elif results[0][0] != 'True': # need_check = 1 # if need_check == 1: -# sql = """select count(1) from CDB_OB_MAJOR_COMPACTION where (GLOBAL_BROADCAST_SCN > LAST_SCN or STATUS != 'IDLE')""" -# check_until_timeout(query_cur, sql, 0, timeout) -# sql2 = """select /*+ query_timeout(1000000000) */ count(1) from __all_virtual_tablet_compaction_info where max_received_scn > finished_scn and max_received_scn > 0""" -# check_until_timeout(query_cur, sql2, 0, timeout) +# wait_timeout = set_default_timeout_by_tenant(query_cur, timeout, 30, 600) +# sql = """select count(1) from oceanbase.CDB_OB_MAJOR_COMPACTION where (GLOBAL_BROADCAST_SCN > LAST_SCN or STATUS != 'IDLE')""" +# check_until_timeout(query_cur, sql, 0, wait_timeout) +# sql2 = """select /*+ query_timeout(1000000000) */ count(1) from oceanbase.__all_virtual_tablet_compaction_info where max_received_scn > finished_scn and max_received_scn > 0""" +# check_until_timeout(query_cur, sql2, 0, wait_timeout) # #def check_until_timeout(query_cur, sql, value, timeout): # times = timeout / 10 @@ -2698,7 +2788,6 @@ # raise_on_warnings = True) # conn.autocommit = True # cur = conn.cursor(buffered=True) -# timeout = timeout if timeout > 0 else 600 # try: # query_cur = QueryCursor(cur) # check_zone_valid(query_cur, zone) @@ -2795,8 +2884,14 @@ # # check compatible sync # parameter_count = int(server_count) * int(tenant_count) # current_data_version = actions.get_current_data_version() +# +# query_timeout = actions.set_default_timeout_by_tenant(cur, timeout, 2, 60) +# actions.set_session_timeout(cur, query_timeout) +# # sql = """select count(*) as cnt from oceanbase.__all_virtual_tenant_parameter_info where name = 'compatible' and value = '{0}' and tenant_id in ({1})""".format(current_data_version, tenant_ids_str) -# times = (timeout if timeout > 0 else 60) / 5 +# +# wait_timeout = actions.set_default_timeout_by_tenant(cur, timeout, 10, 60) +# times = wait_timeout / 5 # while times >= 0: # logging.info(sql) # cur.execute(sql) @@ -2816,6 +2911,8 @@ # raise e # time.sleep(5) # +# actions.set_session_timeout(cur, 10) +# # # check target_data_version/current_data_version from __all_core_table # int_current_data_version = actions.get_version(current_data_version) # sql = "select count(*) from __all_virtual_core_table where column_name in ('target_data_version', 'current_data_version') and column_value = {0} and tenant_id in ({1})".format(int_current_data_version, tenant_ids_str) @@ -2830,16 +2927,20 @@ # logging.info("all tenant's target_data_version/current_data_version are match with {0}".format(current_data_version)) # ## 3 检查内部表自检是否成功 -#def check_root_inspection(query_cur, timeout): +#def check_root_inspection(cur, query_cur, timeout): # sql = "select count(*) from oceanbase.__all_virtual_upgrade_inspection where info != 'succeed'" -# times = timeout if timeout > 0 else 180 -# while times > 0 : +# +# wait_timeout = actions.set_default_timeout_by_tenant(cur, timeout, 10, 600) +# +# times = wait_timeout / 10 +# while times >= 0 : # (desc, results) = query_cur.exec_query(sql) # if results[0][0] == 0: # break # time.sleep(10) # times -= 1 -# if times == 0: +# +# if times == -1: # logging.warn('check root inspection failed!') # raise e # logging.info('check root inspection success') @@ -2867,7 +2968,7 @@ # try: # check_cluster_version(cur, timeout) # check_data_version(cur, query_cur, timeout) -# check_root_inspection(query_cur, timeout) +# check_root_inspection(cur, query_cur, timeout) # enable_ddl(cur, timeout) # enable_rebalance(cur, timeout) # enable_rereplication(cur, timeout) diff --git a/unittest/logservice/test_log_config_mgr.cpp b/unittest/logservice/test_log_config_mgr.cpp index f5a47693fc..fbce151fda 100755 --- a/unittest/logservice/test_log_config_mgr.cpp +++ b/unittest/logservice/test_log_config_mgr.cpp @@ -131,8 +131,6 @@ TEST_F(TestLogConfigMgr, test_set_initial_member_list) EXPECT_EQ(OB_NOT_SUPPORTED, cm.set_initial_member_list(init_member_list, ObMember(addr1, 0), 2, learner_list, 1, config_version)); // arb_member overlaps with member_list EXPECT_EQ(OB_INVALID_ARGUMENT, cm.set_initial_member_list(init_member_list, ObMember(addr2, 0), 2, learner_list, 1, config_version)); - // invalid replica_num - EXPECT_EQ(OB_INVALID_ARGUMENT, cm.set_initial_member_list(init_member_list, ObMember(addr3, 0), 3, learner_list, 1, config_version)); // arb_member overlaps with learners learner_list.add_learner(ObMember(addr4, 0)); EXPECT_EQ(OB_INVALID_ARGUMENT, cm.set_initial_member_list(init_member_list, ObMember(addr4, 0), 2, learner_list, 1, config_version)); diff --git a/unittest/share/index_usage/test_index_usage.cpp b/unittest/share/index_usage/test_index_usage.cpp index 6a58cbc7cd..73b3951f72 100644 --- a/unittest/share/index_usage/test_index_usage.cpp +++ b/unittest/share/index_usage/test_index_usage.cpp @@ -112,8 +112,7 @@ TEST_F(TestIndexUsageInfo, test_sample_filter) int64_t count = 0; mgr_.is_sample_mode_=true; for (int64_t i = 0; i < 20; i++) { - int64_t random_num = common::ObClockGenerator::getClock(); - is_filter = mgr_.sample_filterd(random_num); + is_filter = mgr_.sample_filterd(i); if (!is_filter) { count++; } @@ -122,8 +121,7 @@ TEST_F(TestIndexUsageInfo, test_sample_filter) mgr_.is_sample_mode_=false; count = 0; for (int64_t i = 0; i < 20; i++) { - int64_t random_num = common::ObClockGenerator::getClock(); - is_filter = mgr_.sample_filterd(random_num); + is_filter = mgr_.sample_filterd(i); if (!is_filter) { count++; } diff --git a/unittest/share/test_json_path.cpp b/unittest/share/test_json_path.cpp index 371a59a43a..031ca0370d 100644 --- a/unittest/share/test_json_path.cpp +++ b/unittest/share/test_json_path.cpp @@ -54,9 +54,9 @@ TEST_F(TestJsonPath, test_is_mysql_terminator_mysql) for(int i = 0; i < sizeof(ch); ++i) { if (i <= 3) { - ASSERT_EQ(true, ObJsonPathUtil::is_mysql_terminator(ch[i])); + ASSERT_EQ(true, ObJsonPathUtil::is_key_name_terminator(ch[i])); } else { - ASSERT_EQ(false, ObJsonPathUtil::is_mysql_terminator(ch[i])); + ASSERT_EQ(false, ObJsonPathUtil::is_key_name_terminator(ch[i])); } } } diff --git a/unittest/storage/blocksstable/ob_multi_version_sstable_test.h b/unittest/storage/blocksstable/ob_multi_version_sstable_test.h index aa4af5df83..2065373232 100644 --- a/unittest/storage/blocksstable/ob_multi_version_sstable_test.h +++ b/unittest/storage/blocksstable/ob_multi_version_sstable_test.h @@ -501,8 +501,8 @@ void ObMultiVersionSSTableTest::prepare_data_end( ObTabletCreateSSTableParam param; table_key_.table_type_ = table_type; - param.data_block_ids_ = res.data_block_ids_; - param.other_block_ids_ = res.other_block_ids_; + ASSERT_EQ(OB_SUCCESS, param.data_block_ids_.assign(res.data_block_ids_)); + ASSERT_EQ(OB_SUCCESS, param.other_block_ids_.assign(res.other_block_ids_)); param.table_key_ = table_key_; param.schema_version_ = SCHEMA_VERSION; param.create_snapshot_version_ = 0; diff --git a/unittest/storage/tx/it/test_tx.cpp b/unittest/storage/tx/it/test_tx.cpp index 87d09c37fb..f00c901a30 100644 --- a/unittest/storage/tx/it/test_tx.cpp +++ b/unittest/storage/tx/it/test_tx.cpp @@ -2405,10 +2405,14 @@ TEST_F(ObTestTx, interrupt_get_read_snapshot) PREPARE_TX(n1, tx); ObTxReadSnapshot snapshot; n1->get_ts_mgr_().inject_get_gts_error(OB_EAGAIN); - ASYNC_DO(acq_snapshot, n1->get_read_snapshot(tx, ObTxIsolationLevel::RC, n1->ts_after_ms(20 * 1000), snapshot)); - ASSERT_EQ(OB_SUCCESS, n1->interrupt(tx, OB_TRANS_KILLED)); - ASYNC_WAIT(acq_snapshot, 2000 * 1000, wait_ret); - ASSERT_EQ(OB_ERR_INTERRUPTED, wait_ret); + int ret = OB_SUCCESS; + do { + ASYNC_DO(acq_snapshot, n1->get_read_snapshot(tx, ObTxIsolationLevel::RC, n1->ts_after_ms(20 * 1000), snapshot)); + ASSERT_EQ(OB_SUCCESS, n1->interrupt(tx, OB_TRANS_KILLED)); + ASYNC_WAIT(acq_snapshot, 2000 * 1000, wait_ret); + ret = wait_ret; + } while (OB_GTS_NOT_READY == ret); + ASSERT_EQ(OB_ERR_INTERRUPTED, ret); ROLLBACK_TX(n1, tx); } diff --git a/unittest/storage/tx/mock_utils/basic_fake_define.h b/unittest/storage/tx/mock_utils/basic_fake_define.h index 51097a372f..25b1b2ebf7 100644 --- a/unittest/storage/tx/mock_utils/basic_fake_define.h +++ b/unittest/storage/tx/mock_utils/basic_fake_define.h @@ -70,7 +70,6 @@ public: ObTxData *tx_data = new (ptr) ObTxData(); tx_data->ref_cnt_ = 100; tx_data->tx_data_allocator_ = FAKE_ALLOCATOR; - tx_data->flag_ = 269381; tx_data_guard.init(tx_data); return OB_ISNULL(tx_data) ? OB_ALLOCATE_MEMORY_FAILED : OB_SUCCESS; } @@ -82,7 +81,6 @@ public: ObTxData *from = (ObTxData*)from_guard.tx_data(); to->ref_cnt_ = 100; to->tx_data_allocator_ = FAKE_ALLOCATOR; - to->flag_ = 269381; to_guard.init(to); OX (*to = *from); OZ (deep_copy_undo_status_list_(from->undo_status_list_, to->undo_status_list_)); @@ -271,6 +269,30 @@ public: return ret; } + int get_gts_sync(const uint64_t tenant_id, + const MonotonicTs stc, + const int64_t timeout_us, + share::SCN &scn, + MonotonicTs &receive_gts_ts) + { + int ret = OB_SUCCESS; + const int64_t expire_ts = ObClockGenerator::getClock() + timeout_us; + + do { + int64_t n = ObClockGenerator::getClock(); + if (n >= expire_ts) { + ret = OB_TIMEOUT; + } else if (OB_FAIL(get_gts(tenant_id, stc, NULL, scn, receive_gts_ts))) { + if (OB_EAGAIN == ret) { + ob_usleep(500); + } + } + } while (OB_EAGAIN == ret); + + return ret; + return get_gts(tenant_id, stc, NULL, scn, receive_gts_ts); + } + int get_gts(const uint64_t tenant_id, ObTsCbTask *task, share::SCN &scn) { if (get_gts_error_) { return get_gts_error_; } return OB_SUCCESS;