diff --git a/.gitignore b/.gitignore index 72a5b74c5..59275ebb1 100644 --- a/.gitignore +++ b/.gitignore @@ -172,6 +172,12 @@ src/pl/parser/_gen_pl_parser.output src/pl/parser/pl_parser_mysql_mode.output src/pl/parser/pl_parser_oracle_mode.output +############# close_modules ############# +close_modules/oracle_pl/pl/parser/*.output +close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode_lex.c +close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode_tab.c +close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode_tab.h + ############# tools ############# tools/ObRestore/mvn_repository tools/agentserver/agentserver diff --git a/CMakeLists.txt b/CMakeLists.txt index 661f2739b..3cebb2b67 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,12 +3,20 @@ cmake_minimum_required(VERSION 3.20) include(cmake/Utils.cmake) include(cmake/Env.cmake) -project("OceanBase_CE" - VERSION 4.2.0.0 - DESCRIPTION "OceanBase distributed database system" - HOMEPAGE_URL "https://open.oceanbase.com/" - LANGUAGES CXX C ASM) -message(STATUS "open source build enabled") +if(OB_BUILD_OPENSOURCE) + project("OceanBase_CE" + VERSION 4.2.0.0 + DESCRIPTION "OceanBase distributed database system" + HOMEPAGE_URL "https://open.oceanbase.com/" + LANGUAGES CXX C ASM) + message(STATUS "open source build enabled") +else() + project(OceanBase + VERSION 4.2.0.0 + DESCRIPTION "OceanBase distributed database system" + HOMEPAGE_URL "https://www.oceanbase.com/" + LANGUAGES CXX C ASM) +endif() if(ENABLE_COMPILE_DLL_MODE) @@ -94,6 +102,11 @@ message(STATUS "This is SOURCE dir " ${PROJECT_SOURCE_DIR}) set(CMAKE_POSITION_INDEPENDENT_CODE ON) +if(EXISTS ${CMAKE_SOURCE_DIR}/close_modules) + message(STATUS "will add close_modules sub directory") + add_subdirectory(close_modules) +endif() + add_subdirectory(deps/easy) add_subdirectory(deps/oblib) add_subdirectory(src/objit) @@ -141,10 +154,23 @@ cmake_dependent_option( include(CTest) if (OB_BUILD_UNITTEST) add_subdirectory(unittest) + if(OB_BUILD_CLOSE_MODULES) + add_subdirectory(mittest) + endif() elseif(OB_INCLUDE_UNITTEST) add_subdirectory(unittest EXCLUDE_FROM_ALL) + if(OB_BUILD_CLOSE_MODULES) + add_subdirectory(mittest EXCLUDE_FROM_ALL) + endif() endif() +if(OB_BUILD_CLOSE_MODULES) + if (OB_BUILD_TEST) + add_subdirectory(test) + elseif(OB_INCLUDE_TEST) + add_subdirectory(test EXCLUDE_FROM_ALL) + endif() +endif() if (OB_BUILD_TOOLS) add_subdirectory(tools) diff --git a/cmake/Env.cmake b/cmake/Env.cmake index 39d069638..4a9367df2 100644 --- a/cmake/Env.cmake +++ b/cmake/Env.cmake @@ -37,6 +37,9 @@ ob_define(OB_MAX_UNITY_BATCH_SIZE 30) # the global switch of unity build, defualt is 'ON' ob_define(OB_ENABLE_UNITY ON) +ob_define(OB_BUILD_OPENSOURCE ON) + + if(WITH_COVERAGE) # -ftest-coverage to generate .gcno file # -fprofile-arcs to generate .gcda file @@ -61,6 +64,88 @@ if(ENABLE_THIN_LTO) set(THIN_LTO_CONCURRENCY_LINK "-Wl,--thinlto-jobs=32,--lto-whole-program-visibility") endif() +set(ob_close_modules_static_name "") +set(ob_close_deps_static_name "") + +if (OB_BUILD_OPENSOURCE) + # 开源模式 + set(OB_BUILD_CLOSE_MODULES OFF) +else() + # 闭源模式 + set(OB_BUILD_CLOSE_MODULES ON) +endif() + +if(OB_BUILD_CLOSE_MODULES) + # SECURITY, 包含3个功能点 + ob_define(OB_BUILD_TDE_SECURITY ON) + ob_define(OB_BUILD_AUDIT_SECURITY ON) + ob_define(OB_BUILD_LABEL_SECURITY ON) + # 字符集 + ob_define(OB_BUILD_FULL_CHARSET ON) + # SPM功能 + ob_define(OB_BUILD_SPM ON) + + # oralce + ob_define(OB_BUILD_ORACLE_PARSER ON) + ob_define(OB_BUILD_ORACLE_PL ON) + ob_define(OB_BUILD_ORACLE_XML ON) + # dblink + ob_define(OB_BUILD_DBLINK ON) + # 仲裁功能 + ob_define(OB_BUILD_ARBITRATION ON) + + # 默认使用BABASSL + ob_define(OB_USE_BABASSL ON) + add_definitions(-DOB_USE_BABASSL) + # 默认使用OB_USE_DRCMSG + ob_define(OB_USE_DRCMSG ON) + add_definitions(-DOB_USE_DRCMSG) +endif() + +# 下面开始逻辑控制 +if(OB_BUILD_CLOSE_MODULES) + add_definitions(-DOB_BUILD_CLOSE_MODULES) +endif() + +if(OB_BUILD_TDE_SECURITY) + add_definitions(-DOB_BUILD_TDE_SECURITY) +endif() + +if(OB_BUILD_AUDIT_SECURITY) + add_definitions(-DOB_BUILD_AUDIT_SECURITY) +endif() + +if(OB_BUILD_LABEL_SECURITY) + add_definitions(-DOB_BUILD_LABEL_SECURITY) +endif() + +if(OB_BUILD_FULL_CHARSET) + add_definitions(-DOB_BUILD_FULL_CHARSET) +endif() + +if(OB_BUILD_SPM) + add_definitions(-DOB_BUILD_SPM) +endif() + +if(OB_BUILD_ORACLE_PARSER) + add_definitions(-DOB_BUILD_ORACLE_PARSER) +endif() + +if(OB_BUILD_ORACLE_PL) + add_definitions(-DOB_BUILD_ORACLE_PL) +endif() + +if(OB_BUILD_ORACLE_XML) + add_definitions(-DOB_BUILD_ORACLE_XML) +endif() + +if(OB_BUILD_ARBITRATION) + add_definitions(-DOB_BUILD_ARBITRATION) +endif() + +if(OB_BUILD_DBLINK) + add_definitions(-DOB_BUILD_DBLINK) +endif() # should not use initial-exec for tls-model if building OBCDC. if(NOT OB_BUILD_CDC) @@ -125,7 +210,36 @@ if (OB_USE_CLANG) set(CMAKE_SHARED_LINKER_FLAGS "${LD_OPT} -Wl,-z,noexecstack ${THIN_LTO_CONCURRENCY_LINK} ${REORDER_LINK_OPT}") set(CMAKE_EXE_LINKER_FLAGS "${LD_OPT} -Wl,-z,noexecstack -pie ${THIN_LTO_CONCURRENCY_LINK} ${REORDER_LINK_OPT} ${CMAKE_COVERAGE_EXE_LINKER_OPTIONS}") else() # not clang, use gcc +if(OB_BUILD_OPENSOURCE) message("gcc9 not support currently, please set OB_USE_CLANG ON and we will finish it as soon as possible") +else() + + if (OB_CC) + message(STATUS "Using OB_CC compiler: ${OB_CC}") + else() + find_program(OB_CC gcc + PATHS ${DEVTOOLS_DIR}/bin + NO_DEFAULT_PATH) + endif() + + if (OB_CXX) + message(STATUS "Using OB_CXX compiler: ${OB_CXX}") + else() + find_program(OB_CXX g++ + PATHS ${DEVTOOLS_DIR}/bin + NO_DEFAULT_PATH) + endif() + + if (OB_USE_LLD) + set(LD_OPT "-B${CMAKE_SOURCE_DIR}/rpm/.compile") + set(REORDER_COMP_OPT "-ffunction-sections") + set(REORDER_LINK_OPT "-Wl,--no-warn-symbol-ordering,--symbol-ordering-file,${HOTFUNC_PATH}") + endif() + set(CMAKE_CXX_FLAGS "${LD_OPT} -fdiagnostics-color ${REORDER_COMP_OPT}") + set(CMAKE_C_FLAGS "${LD_OPT} -fdiagnostics-color ${REORDER_COMP_OPT}") + set(CMAKE_SHARED_LINKER_FLAGS "-z noexecstack ${REORDER_LINK_OPT}") + set(CMAKE_EXE_LINKER_FLAGS "-z noexecstack ${REORDER_LINK_OPT}") +endif() endif() if (OB_BUILD_CCLS) @@ -143,6 +257,15 @@ else() message(FATAL_ERROR "can't find suitable compiler") endif() +find_program(OB_COMPILE_EXECUTABLE ob-compile) +if (NOT OB_COMPILE_EXECUTABLE) + message(STATUS "ob-compile not found, compile locally.") +else() + set(CMAKE_C_COMPILER_LAUNCHER ${OB_COMPILE_EXECUTABLE}) + set(CMAKE_CXX_COMPILER_LAUNCHER ${OB_COMPILE_EXECUTABLE}) + set(CMAKE_C_LINKER_LAUNCHER ${OB_COMPILE_EXECUTABLE}) + set(CMAKE_CXX_LINKER_LAUNCHER ${OB_COMPILE_EXECUTABLE}) +endif() option(OB_ENABLE_AVX2 "enable AVX2 and related instruction set support for x86_64" OFF) @@ -152,8 +275,6 @@ if( ${ARCHITECTURE} STREQUAL "x86_64" ) set(MTUNE_CFLAGS -mtune=core2) set(ARCH_LDFLAGS "") set(OCI_DEVEL_INC "${DEP_3RD_DIR}/usr/include/oracle/11.2/client64") - add_compile_options(-DRDMA_ENABLED) - set(rdma_lib_deps "reasy" ) else() set(MARCH_CFLAGS "-march=armv8-a+crc" ) set(MTUNE_CFLAGS "-mtune=generic" ) diff --git a/cmake/RPM.cmake b/cmake/RPM.cmake index 948b16d77..57194eddd 100644 --- a/cmake/RPM.cmake +++ b/cmake/RPM.cmake @@ -17,6 +17,7 @@ set(CPACK_PACKAGING_INSTALL_PREFIX /home/admin/oceanbase) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "OceanBase is a distributed relational database") set(CPACK_PACKAGE_VENDOR "OceanBase Inc.") set(CPACK_RPM_PACKAGE_RELEASE ${OB_RELEASEID}) +if (OB_BUILD_OPENSOURCE) set(CPACK_PACKAGE_NAME "oceanbase-ce") set(CPACK_PACKAGE_VERSION "${OceanBase_CE_VERSION}") set(CPACK_PACKAGE_VERSION_MAJOR "${OceanBase_CE_VERSION_MAJOR}") @@ -30,6 +31,14 @@ set(CPACK_RPM_UTILS_PACKAGE_PREFIX /usr) list(APPEND CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "/home") list(APPEND CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "/home/admin") list(APPEND CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "/home/admin/oceanbase") +else() +set(CPACK_PACKAGE_NAME "oceanbase") +set(CPACK_PACKAGE_VERSION "${OceanBase_VERSION}") +set(CPACK_PACKAGE_VERSION_MAJOR "${OceanBase_VERSION_MAJOR}") +set(CPACK_PACKAGE_VERSION_MINOR "${OceanBase_VERSION_MINOR}") +set(CPACK_PACKAGE_VERSION_PATCH "${OceanBase_VERSION_PATCH}") +set(CPACK_RPM_PACKAGE_URL "${OceanBase_HOMEPAGE_URL}") +endif() set(CPACK_RPM_PACKAGE_GROUP "Applications/Databases") set(CPACK_RPM_PACKAGE_DESCRIPTION "OceanBase is a distributed relational database") set(CPACK_RPM_PACKAGE_LICENSE "Mulan PubL v2.") @@ -54,11 +63,24 @@ set(CPACK_RPM_SPEC_MORE_DEFINE set(BITCODE_TO_ELF_LIST "") ## server +if (OB_BUILD_OPENSOURCE) install(PROGRAMS tools/import_time_zone_info.py ${CMAKE_BINARY_DIR}/src/observer/observer DESTINATION bin COMPONENT server) +else() +install(PROGRAMS + script/dooba/dooba + tools/import_time_zone_info.py + tools/import_srs_data.py + ${CMAKE_BINARY_DIR}/tools/ob_admin/ob_admin + tools/ob_admin/io_bench/bench_io.sh + ${CMAKE_BINARY_DIR}/src/observer/observer + $<$:${DEVTOOLS_DIR}/bin/obstack> + DESTINATION bin + COMPONENT server) +endif() install(FILES src/sql/fill_help_tables-ob.sql @@ -97,12 +119,22 @@ install( COMPONENT cdc ) +if(OB_BUILD_OPENSOURCE) install( FILES ${PROJECT_SOURCE_DIR}/src/logservice/libobcdc/tests/libobcdc.conf DESTINATION ${CMAKE_INSTALL_SYSCONFDIR} COMPONENT cdc ) +else() +install( + FILES + ${PROJECT_SOURCE_DIR}/src/logservice/libobcdc/tests/libobcdc.conf + ${PROJECT_SOURCE_DIR}/src/logservice/libobcdc/tests/timezone_info.conf + DESTINATION ${CMAKE_INSTALL_SYSCONFDIR} + COMPONENT cdc + ) +endif() endif() ## oceanbase-sql-parser @@ -331,6 +363,7 @@ if (OB_BUILD_LIBOBTABLE) COMPONENT table) endif() +if(OB_BUILD_OPENSOURCE) ## oceanbase-libs install(PROGRAMS deps/3rd/usr/local/oceanbase/deps/devel/lib/libaio.so.1 @@ -350,6 +383,7 @@ if(OB_BUILD_OBADMIN) COMPONENT utils ) endif() +endif() # install cpack to make everything work include(CPack) diff --git a/cmake/Utils.cmake b/cmake/Utils.cmake index df5ac5eb4..c070b6e25 100644 --- a/cmake/Utils.cmake +++ b/cmake/Utils.cmake @@ -1,3 +1,6 @@ +if(NOT OB_BUILD_OPENSOURCE) +include(cmake/Err.cmake) +endif() macro(ob_define VAR DEFAULT) if (NOT DEFINED ${VAR}) @@ -11,27 +14,26 @@ function(ob_replace_in_file INFILE OUTFILE MATCH-STRING REPLACE-STRING) file(WRITE ${OUTFILE} ${NEW-CONTENT}) endfunction() -# ob_set_subtarget usage demo -# ob_set_subtarget(ob_sql common -# sql1.cpp -# sql2.cpp -# sql3.cpp -# ) -# ob_set_subtarget(ob_sql executor -# executor/ob_executor1.cpp -# executor/ob_executor2.cpp -# executor/ob_executor3.cpp -# ) function(ob_set_subtarget target group) - list(APPEND "${target}_cache_objects_" ${ARGN}) + + list(LENGTH ${target}_cache_objects_ CCLS_TARGET_CURRENT_LENGTH) + + # 需要参与编译的源文件列表 + set(ARGN_NEED_LIST "") + + FOREACH(item ${ARGN}) + list(APPEND ARGN_NEED_LIST ${item}) + ENDFOREACH(item) + + list(APPEND "${target}_cache_objects_" ${ARGN_NEED_LIST}) set("${target}_cache_objects_" ${${target}_cache_objects_} PARENT_SCOPE) - # if need check ob cmake rules if (OB_CMAKE_RULES_CHECK) - FOREACH(item ${ARGN}) + FOREACH(item ${ARGN_NEED_LIST}) + # [E1001] Header files are not allowed in CMakeLists.txt string(REGEX MATCHALL "^.*\.h$" MATCH_OUTPUT ${item}) if(MATCH_OUTPUT) - message(FATAL_ERROR "Header files are not allowed in CMakeLists.txt\n") + message(FATAL_ERROR "\n${E1001}\n不允许把头文件${item}写到CMakeLists.txt文件中\n") endif() ENDFOREACH(item) endif() @@ -41,30 +43,55 @@ function(ob_set_subtarget target group) return() endif() - # ALONE group will not join unity build if(group STREQUAL "ALONE") return() endif() - set(i 0) - set(group_id 0) + if (NOT OB_BUILD_CCLS) + set(i 0) + set(group_id 0) + else() + # ccls构建，将更改分组方法，是以target为单位，而不是以group为单元 + set(i ${CCLS_TARGET_CURRENT_LENGTH}) + math(EXPR group_id "(${i} / ${OB_MAX_UNITY_BATCH_SIZE})") + endif() + set(ob_sub_objects "") - FOREACH(item ${ARGN}) + FOREACH(item ${ARGN_NEED_LIST}) math(EXPR i "(${i} + 1) % ${OB_MAX_UNITY_BATCH_SIZE}") list(APPEND ob_sub_objects ${item}) if (${i} EQUAL 0) - set_source_files_properties(${ob_sub_objects} PROPERTIES UNITY_GROUP "${target}_${group}/${group_id}") + if (NOT OB_BUILD_CCLS) + set_source_files_properties(${ob_sub_objects} PROPERTIES UNITY_GROUP "${target}_${group}/${group_id}") + else() + set_source_files_properties(${ob_sub_objects} PROPERTIES UNITY_GROUP "${target}/${group_id}") + endif() math(EXPR group_id "${group_id} + 1") set(ob_sub_objects "") endif() ENDFOREACH(item) + if (${i} GREATER 0) - set_source_files_properties(${ob_sub_objects} PROPERTIES UNITY_GROUP "${target}_${group}/${group_id}") + if (NOT OB_BUILD_CCLS) + set_source_files_properties(${ob_sub_objects} PROPERTIES UNITY_GROUP "${target}_${group}/${group_id}") + else() + set_source_files_properties(${ob_sub_objects} PROPERTIES UNITY_GROUP "${target}/${group_id}") + endif() endif() endfunction() +function (check_need_build_unity_target target need_build) + list(LENGTH ${target}_cache_objects_ TARGET_LENGTH) + if (TARGET_LENGTH EQUAL 0) + set(${need_build} FALSE PARENT_SCOPE) + else() + set(${need_build} TRUE PARENT_SCOPE) + endif() +endfunction() + + set(unity_after [[ #ifdef USING_LOG_PREFIX #undef USING_LOG_PREFIX diff --git a/deps/easy/CMakeLists.txt b/deps/easy/CMakeLists.txt index 671b94537..f295afbaa 100644 --- a/deps/easy/CMakeLists.txt +++ b/deps/easy/CMakeLists.txt @@ -20,11 +20,19 @@ set_property(GLOBAL PROPERTY EASY_INCLUDE_DIRS ${INNER_INCLUDE_DIRS} ) -target_include_directories( - easy_base INTERFACE - ${INNER_INCLUDE_DIRS} - ${DEP_DIR}/include -) +if(OB_USE_BABASSL) + target_include_directories( + easy_base INTERFACE + ${INNER_INCLUDE_DIRS} + ${DEP_3RD_DIR}/usr/local/babassl-ob/include + ) +else() + target_include_directories( + easy_base INTERFACE + ${INNER_INCLUDE_DIRS} + ${DEP_DIR}/include + ) +endif() if (OB_USE_CLANG) diff --git a/deps/easy/src/io/easy_ssl.c b/deps/easy/src/io/easy_ssl.c index 70bce0b8e..c0083e0d8 100644 --- a/deps/easy/src/io/easy_ssl.c +++ b/deps/easy/src/io/easy_ssl.c @@ -990,6 +990,18 @@ easy_ssl_ctx_t *easy_ssl_ctx_load(easy_pool_t *pool, const char *ssl_ca, goto error_exit; } +#ifdef OB_USE_BABASSL + if (is_babassl) { + if (!ssl_enc_cert || 0 == strlen(ssl_enc_cert)) { + easy_info_log("sm scene, no ssl_enc_cert"); + goto error_exit; + } + if (!ssl_enc_key || 0 == strlen(ssl_enc_key)) { + easy_info_log("sm scene, no ssl_enc_key"); + goto error_exit; + } + } +#endif if ((ss = (easy_ssl_ctx_t *)easy_pool_calloc(pool, sizeof(easy_ssl_ctx_t))) == NULL) { easy_error_log("easy_pool_calloc easy_ssl_ctx_t failed, size=%u", sizeof(easy_ssl_ctx_t)); @@ -1019,6 +1031,33 @@ easy_ssl_ctx_t *easy_ssl_ctx_load(easy_pool_t *pool, const char *ssl_ca, } if (is_from_file) { +#ifdef OB_USE_BABASSL + if (is_babassl) { + if (!SSL_CTX_use_sign_PrivateKey_file(ss->ctx, ssl_key, + SSL_FILETYPE_PEM)) { + easy_info_log("SSL_CTX_use_sign_PrivateKey_file failed!"); + goto error_exit; + } + + if (!SSL_CTX_use_sign_certificate_file(ss->ctx, ssl_cert, + SSL_FILETYPE_PEM)) { + easy_info_log("SSL_CTX_use_sign_certificate_file failed!"); + goto error_exit; + } + + if (!SSL_CTX_use_enc_PrivateKey_file(ss->ctx, ssl_enc_key, + SSL_FILETYPE_PEM)) { + easy_info_log("SSL_CTX_use_enc_PrivateKey_file failed!"); + goto error_exit; + } + + if (!SSL_CTX_use_enc_certificate_file(ss->ctx, ssl_enc_cert, + SSL_FILETYPE_PEM)) { + easy_info_log("SSL_CTX_use_enc_certificate_file failed!"); + goto error_exit; + } + } +#endif if(!is_babassl) { if (easy_ssl_client_certificate_for_mysql(ss, ssl_ca, 1) != EASY_OK) { easy_error_log("easy_ssl_client_certificate_for_mysql failed, client_certificate=%s", ssl_ca); @@ -1032,6 +1071,42 @@ easy_ssl_ctx_t *easy_ssl_ctx_load(easy_pool_t *pool, const char *ssl_ca, } } } else { +#ifdef OB_USE_BABASSL + if (is_babassl) { + if (NULL == (pkey = easy_ssl_read_sm_pkey(ssl_key))) { + goto error_exit; + } + if (!SSL_CTX_use_sign_PrivateKey(ss->ctx, pkey)) { + easy_info_log("SSL_CTX_use_sign_PrivateKey failed!"); + goto error_exit; + } + + if (NULL == (x509 = easy_ssl_get_sm_cert(ssl_cert))) { + goto error_exit; + } + if (!SSL_CTX_use_sign_certificate(ss->ctx, x509)) { + easy_info_log("SSL_CTX_use_sign_certificate failed"); + goto error_exit; + } + + if (NULL == (pkey = easy_ssl_read_sm_pkey(ssl_enc_key))) { + goto error_exit; + } + if (!SSL_CTX_use_enc_PrivateKey(ss->ctx, pkey)) { + easy_info_log("SSL_CTX_use_enc_PrivateKey failed!"); + goto error_exit; + } + + if (NULL == (x509 = easy_ssl_get_sm_cert(ssl_enc_cert))) { + goto error_exit; + } + + if (!SSL_CTX_use_enc_certificate(ss->ctx, x509)) { + easy_info_log("SSL_CTX_use_enc_certificate failed"); + goto error_exit; + } + } +#endif if (!is_babassl) { if (easy_ssl_client_certificate_for_mysql_memory(ss, ssl_ca) != EASY_OK) { easy_error_log("easy_ssl_client_certificate_for_mysql_memory failed, client_certificate=%s", ssl_ca); @@ -1505,8 +1580,19 @@ static int easy_ssl_ctx_create(easy_ssl_ctx_t *ssl) static int easy_ssl_ctx_create_for_mysql(easy_ssl_ctx_t *ssl, int is_babassl) { +#ifdef OB_USE_BABASSL + if (is_babassl) { + ssl->ctx = SSL_CTX_new(NTLS_method()); + if (NULL != ssl->ctx) { + SSL_CTX_enable_ntls(ssl->ctx); + } + } else { + ssl->ctx = SSL_CTX_new(SSLv23_method()); + } +#else (void)is_babassl; ssl->ctx = SSL_CTX_new(SSLv23_method()); +#endif if (ssl->ctx == NULL) { easy_ssl_error(EASY_LOG_ERROR, "SSL_CTX_new() failed"); @@ -1733,6 +1819,7 @@ static int easy_ssl_dhparam(easy_ssl_ctx_t *ssl, char *file) return EASY_ERROR; } +#ifndef OB_USE_BABASSL dh->p = BN_bin2bn(dh1024_p, sizeof(dh1024_p), NULL); dh->g = BN_bin2bn(dh1024_g, sizeof(dh1024_g), NULL); @@ -1741,6 +1828,18 @@ static int easy_ssl_dhparam(easy_ssl_ctx_t *ssl, char *file) DH_free(dh); return EASY_ERROR; } +#else + if (1 != DH_set0_pqg(dh, BN_bin2bn(dh1024_p, sizeof(dh1024_p), NULL), NULL, NULL)) { + easy_ssl_error(EASY_LOG_ERROR, "BN_bin2bn() failed"); + DH_free(dh); + return EASY_ERROR; + } + if (1 != DH_set0_pqg(dh, NULL, NULL, BN_bin2bn(dh1024_g, sizeof(dh1024_g), NULL))) { + easy_ssl_error(EASY_LOG_ERROR, "BN_bin2bn() failed"); + DH_free(dh); + return EASY_ERROR; + } +#endif SSL_CTX_set_tmp_dh(ssl->ctx, dh); DH_free(dh); diff --git a/deps/oblib/src/CMakeLists.txt b/deps/oblib/src/CMakeLists.txt index 55d20abad..acec02ba5 100644 --- a/deps/oblib/src/CMakeLists.txt +++ b/deps/oblib/src/CMakeLists.txt @@ -3,11 +3,9 @@ add_library(oblib_base_base_base INTERFACE) get_property(EASY_INCLUDE_DIRS GLOBAL PROPERTY "EASY_INCLUDE_DIRS" ) - target_include_directories( oblib_base_base_base INTERFACE ${CMAKE_SOURCE_DIR} - ${CMAKE_SOURCE_DIR}/deps/oblib/src/lib/oracleclient ${CMAKE_SOURCE_DIR}/deps/easy/src ${CMAKE_SOURCE_DIR}/deps/oblib/src ${CMAKE_SOURCE_DIR}/deps/oblib/src/common @@ -18,16 +16,88 @@ target_include_directories( ${CMAKE_SOURCE_DIR}/src/objit/src ${DEP_DIR}/include ${DEP_DIR}/include/libxml2 - ${DEP_DIR}/include/mariadb - ${DEP_3RD_DIR}/usr/local/include ${DEVTOOLS_DIR} ${DEP_3RD_DIR}/usr/local/include ${DEP_3RD_DIR}/usr/include/ - ${DEP_3RD_DIR}/usr/local/include ${DEP_DIR}/include/apr-1/ ${DEP_DIR}/include/icu/common ${USSL_INCLUDE_DIRS} +) + +if(OB_BUILD_OPENSOURCE) + target_include_directories( + oblib_base_base_base INTERFACE + ${DEP_DIR}/include/mariadb + ) +endif() + +if(OB_BUILD_SPM) + target_include_directories( + oblib_base_base_base INTERFACE + ${CMAKE_SOURCE_DIR}/close_modules/spm ) +endif() + + +if(OB_BUILD_DBLINK) + target_include_directories( + oblib_base_base_base INTERFACE + ${DEP_3RD_DIR}/u01/obclient/include + ${CMAKE_SOURCE_DIR}/close_modules/dblink/deps/oblib/src + ${OCI_DEVEL_INC} + ) +endif() + +if(OB_BUILD_ARBITRATION) + target_include_directories( + oblib_base_base_base INTERFACE + ${CMAKE_SOURCE_DIR}/close_modules/arbitration + ) +endif() + +if(OB_BUILD_ORACLE_PL) + target_include_directories( + oblib_base_base_base INTERFACE + ${CMAKE_SOURCE_DIR}/close_modules/oracle_pl + ) +endif() + +if(OB_BUILD_TDE_SECURITY) + target_include_directories( + oblib_base_base_base INTERFACE + ${CMAKE_SOURCE_DIR}/close_modules/tde_security + ) +endif() + +if(OB_BUILD_ORACLE_XML) + target_include_directories( + oblib_base_base_base INTERFACE + ${CMAKE_SOURCE_DIR}/close_modules/xml + ${CMAKE_SOURCE_DIR}/close_modules/xml/deps/oblib/src/ + ) +endif() + +if(OB_BUILD_AUDIT_SECURITY) + target_include_directories( + oblib_base_base_base INTERFACE + ${CMAKE_SOURCE_DIR}/close_modules/audit_security + ) +endif() + +if(OB_BUILD_ORACLE_XML) + target_include_directories( + oblib_base_base_base INTERFACE + ${CMAKE_SOURCE_DIR}/close_modules/charset + ${CMAKE_SOURCE_DIR}/close_modules/charset/deps/oblib/src/ + ) +endif() + +if(OB_USE_BABASSL) + target_include_directories( + oblib_base_base_base INTERFACE + ${DEP_3RD_DIR}/usr/local/babassl-ob/include + ) +endif() if (OB_USE_CLANG) # The following clang warnings should be fixed later: -Wno-unused-variable -Wno-invalid-offsetof @@ -114,6 +184,7 @@ endif() target_compile_features(oblib_base_base INTERFACE cxx_std_11) +if(OB_BUILD_OPENSOURCE) set(LGPL_DEPS "-L${DEP_DIR}/lib/mariadb -lmariadb") if (OB_STATIC_LINK_LGPL_DEPS) set(LGPL_DEPS "-L${DEP_DIR}/lib/mariadb -l:libmariadbclient.a") @@ -140,6 +211,30 @@ target_link_libraries(oblib_base_base_base -laio -lpthread -lcurl -ldl -lrt ${ARCH_LDFLAGS} ) +else() +set(ignoreMe OB_STATIC_LINK_LGPL_DEPS) +target_link_libraries(oblib_base_base_base + INTERFACE + oss + easy + ${DEP_DIR}/lib/libisal.a + $<$:${DEP_DIR}/lib/libunwind.a> + ${DEP_3RD_DIR}/usr/local/babassl-ob/lib/libssl.a + ${DEP_3RD_DIR}/usr/local/babassl-ob/lib/libcrypto.a + ${DEP_3RD_DIR}/u01/obclient/lib/libobclnt.a + ${DEP_DIR}/lib/libs2.a + ${DEP_DIR}/lib/libz.a + ${DEP_DIR}/lib/libicui18n.a + ${DEP_DIR}/lib/libicustubdata.a + ${DEP_DIR}/lib/libicuuc.a + -L${DEP_DIR}/var/usr/lib64 + -L${DEP_DIR}/var/usr/lib + -L${DEP_3RD_DIR}/usr/lib + -L${DEP_3RD_DIR}/usr/lib64 + -laio -lpthread -lcurl -ldl -lrt + ${ARCH_LDFLAGS} + ) +endif() target_link_libraries(oblib_base_base INTERFACE oblib_base_base_base) add_library(oblib_base INTERFACE) diff --git a/deps/oblib/src/lib/CMakeLists.txt b/deps/oblib/src/lib/CMakeLists.txt index 60a40e9cc..430dbb70e 100644 --- a/deps/oblib/src/lib/CMakeLists.txt +++ b/deps/oblib/src/lib/CMakeLists.txt @@ -194,7 +194,6 @@ ob_set_subtarget(oblib_lib oblog oblog/ob_syslog_rate_limiter.cpp ) - ob_set_subtarget(oblib_lib signal signal/ob_signal_handlers.cpp signal/ob_signal_processor.cpp @@ -287,6 +286,17 @@ add_library(malloc_hook STATIC alloc/malloc_hook.h) target_link_libraries(malloc_hook oblib_base) +if(OB_BUILD_ORACLE_XML) +target_link_libraries(oblib_lib + PUBLIC ob_malloc compress restore + ${DEP_DIR}/lib/libxml2.a + ${DEP_DIR}/lib/liblzma.a + ${DEP_3RD_DIR}/usr/local/lib/libxslt.a + ${DEP_3RD_DIR}/usr/local/lib/libexslt.a + ${ob_close_deps_static_name} +) +else() target_link_libraries(oblib_lib PUBLIC ob_malloc ) +endif() diff --git a/deps/oblib/src/lib/allocator/ob_mod_define.h b/deps/oblib/src/lib/allocator/ob_mod_define.h index 388870e96..cf46e8ffd 100644 --- a/deps/oblib/src/lib/allocator/ob_mod_define.h +++ b/deps/oblib/src/lib/allocator/ob_mod_define.h @@ -53,7 +53,6 @@ LABEL_ITEM_DEF(OB_FIFO_ALLOC, FifoAlloc) //commonmodules LABEL_ITEM_DEF(OB_OBJ_FREELISTS, ObjFreelists) LABEL_ITEM_DEF(OB_COMMON_NETWORK, CommonNetwork) -LABEL_ITEM_DEF(OB_RDMA_MYSQL, MysqlRdmaNet) LABEL_ITEM_DEF(OB_THREAD_BUFFER, ThreadBuffer) LABEL_ITEM_DEF(OB_KVSTORE_CACHE, KvstoreCache) LABEL_ITEM_DEF(OB_KVSTORE_CACHE_ITERATOR, KvstorCacheIter) @@ -677,7 +676,7 @@ struct InnerModIds #undef LABEL_ITEM_DEF }; enum { LABEL_COUNT_LIMIT = InnerModIds::OB_MOD_END }; - STATIC_ASSERT(LABEL_COUNT_LIMIT == 453, "forbidden to add new label!!!"); + STATIC_ASSERT(LABEL_COUNT_LIMIT == 452, "forbidden to add new label!!!"); }; #define ObNewModIds ObModIds diff --git a/deps/oblib/src/lib/charset/ob_charset.cpp b/deps/oblib/src/lib/charset/ob_charset.cpp index a7095f907..42a88dcff 100644 --- a/deps/oblib/src/lib/charset/ob_charset.cpp +++ b/deps/oblib/src/lib/charset/ob_charset.cpp @@ -293,8 +293,14 @@ const ObCollationWrapper ObCharset::collation_wrap_arr_[ObCharset::VALID_COLLATI {CS_TYPE_GBK_BIN, CHARSET_GBK, CS_TYPE_GBK_BIN, false, true, 1}, {CS_TYPE_UTF16_GENERAL_CI, CHARSET_UTF16, CS_TYPE_UTF16_GENERAL_CI, true, true, 1}, {CS_TYPE_UTF16_BIN, CHARSET_UTF16, CS_TYPE_UTF16_BIN, false, true, 1}, +#ifndef OB_BUILD_FULL_CHARSET {CS_TYPE_INVALID, CHARSET_INVALID, CS_TYPE_INVALID, false, false, 1}, {CS_TYPE_INVALID, CHARSET_INVALID, CS_TYPE_INVALID, false, false, 1}, +#else + //{CS_TYPE_UTF8MB4_ZH_0900_AS_CS, CHARSET_UTF8MB4, CS_TYPE_UTF8MB4_ZH_0900_AS_CS, false, true, 0}, + {CS_TYPE_UTF8MB4_UNICODE_CI, CHARSET_UTF8MB4, CS_TYPE_UTF8MB4_UNICODE_CI, false, true, 1}, + {CS_TYPE_UTF16_UNICODE_CI, CHARSET_UTF16, CS_TYPE_UTF16_UNICODE_CI, false, true, 1}, +#endif {CS_TYPE_GB18030_CHINESE_CI, CHARSET_GB18030, CS_TYPE_GB18030_CHINESE_CI, true, true, 1}, {CS_TYPE_GB18030_BIN, CHARSET_GB18030, CS_TYPE_GB18030_BIN, false, true, 1}, {CS_TYPE_LATIN1_SWEDISH_CI, CHARSET_LATIN1, CS_TYPE_LATIN1_SWEDISH_CI,true, true, 1}, @@ -330,7 +336,11 @@ ObCharsetInfo *ObCharset::charset_arr[CS_TYPE_MAX] = { &ob_charset_gbk_bin, // 87 NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, // 88 NULL, NULL, NULL, NULL, NULL, // 96 +#ifdef OB_BUILD_FULL_CHARSET + &ob_charset_utf16_unicode_ci, // 101 +#else NULL, +#endif NULL, NULL, // 102 NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, // 104 NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, // 112 @@ -350,7 +360,11 @@ ObCharsetInfo *ObCharset::charset_arr[CS_TYPE_MAX] = { &ob_charset_gb18030_2022_pinyin_cs, &ob_charset_gb18030_2022_radical_ci,// 218 &ob_charset_gb18030_2022_radical_cs, &ob_charset_gb18030_2022_stroke_ci, // 220 &ob_charset_gb18030_2022_stroke_cs, NULL, // 222 +#ifdef OB_BUILD_FULL_CHARSET + &ob_charset_utf8mb4_unicode_ci, // 224 +#else NULL, +#endif NULL, NULL, NULL, NULL, NULL, NULL, NULL, // 225 NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, // 232 NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, // 240 @@ -494,6 +508,31 @@ uint64_t ObCharset::strntoullrnd(const char *str, return result; } +#ifdef OB_BUILD_FULL_CHARSET +/* + Convert integer to its string representation in given scale of notation. + + SYNOPSIS + int2str() + val - value to convert + dst - points to buffer where string representation should be stored + radix - radix of scale of notation + upcase - set to 1 if we should use upper-case digits + + DESCRIPTION + Converts the (long) integer value to its character form and moves it to + the destination buffer followed by a terminating NUL. + If radix is -2..-36, val is taken to be SIGNED, if radix is 2..36, val is + taken to be UNSIGNED. That is, val is signed if and only if radix is. + All other radixes treated as bad and nothing will be changed in this case. + + For conversion to decimal representation (radix is -10 or 10) one can use + optimized int10_to_str() function. + + RETURN VALUE + Pointer to ending NUL character or NullS if radix is bad. +*/ +#endif //============================================================= char* ObCharset::lltostr(int64_t val, char *dst, int radix, int upcase) @@ -1063,6 +1102,10 @@ int ObCharset::well_formed_len(ObCollationType collation_type, const char *str, return ret; } +#ifdef OB_BUILD_FULL_CHARSET +// Be careful with this function. The return value may be out of range. +// Refer to +#endif size_t ObCharset::charpos(const ObCollationType collation_type, const char *str, const int64_t str_len, @@ -1546,6 +1589,14 @@ ObCollationType ObCharset::collation_type(const ObString &cs_name) collation_type = CS_TYPE_UTF16_GENERAL_CI; } else if (0 == cs_name.case_compare(ob_charset_utf16_bin.name)) { collation_type = CS_TYPE_UTF16_BIN; +#ifdef OB_BUILD_FULL_CHARSET + } else if (0 == cs_name.case_compare("utf8_unicode_ci")) { + collation_type = CS_TYPE_UTF8MB4_UNICODE_CI; + } else if (0 == cs_name.case_compare(ob_charset_utf16_unicode_ci.name)) { + collation_type = CS_TYPE_UTF16_UNICODE_CI; + } else if (0 == cs_name.case_compare(ob_charset_utf8mb4_unicode_ci.name)) { + collation_type = CS_TYPE_UTF8MB4_UNICODE_CI; +#endif } else if (0 == cs_name.case_compare(ob_charset_gb18030_bin.name)) { collation_type = CS_TYPE_GB18030_BIN; } else if (0 == cs_name.case_compare(ob_charset_gb18030_chinese_ci.name)) { @@ -1588,6 +1639,9 @@ bool ObCharset::is_valid_collation(ObCharsetType charset_type, ObCollationType c if (CHARSET_UTF8MB4 == charset_type) { if (CS_TYPE_UTF8MB4_BIN == collation_type || CS_TYPE_UTF8MB4_GENERAL_CI == collation_type +#ifdef OB_BUILD_FULL_CHARSET + || CS_TYPE_UTF8MB4_UNICODE_CI == collation_type +#endif ) { ret = true; } @@ -1601,6 +1655,9 @@ bool ObCharset::is_valid_collation(ObCharsetType charset_type, ObCollationType c } else if (CHARSET_UTF16 == charset_type) { if (CS_TYPE_UTF16_GENERAL_CI == collation_type || CS_TYPE_UTF16_BIN == collation_type +#ifdef OB_BUILD_FULL_CHARSET + || CS_TYPE_UTF16_UNICODE_CI == collation_type +#endif ) { ret = true; } @@ -1693,6 +1750,11 @@ bool ObCharset::is_valid_collation(int64_t collation_type_int) || CS_TYPE_LATIN1_SWEDISH_CI == collation_type || CS_TYPE_LATIN1_BIN == collation_type || is_gb18030_2022(collation_type) +#ifdef OB_BUILD_FULL_CHARSET + || CS_TYPE_UTF8MB4_UNICODE_CI == collation_type + || CS_TYPE_UTF16_UNICODE_CI == collation_type + || (CS_TYPE_EXTENDED_MARK < collation_type && collation_type < CS_TYPE_MAX) +#endif ; } @@ -1913,6 +1975,57 @@ int ObCharset::result_collation( return ret; } +#ifdef OB_BUILD_FULL_CHARSET +/** note from mysql: + Aggregate two collations together taking + into account their coercibility (aka derivation):. + + 0 == DERIVATION_EXPLICIT - an explicitly written COLLATE clause @n + 1 == DERIVATION_NONE - a mix of two different collations @n + 2 == DERIVATION_IMPLICIT - a column @n + 3 == DERIVATION_COERCIBLE - a string constant. + + The most important rules are: + -# If collations are the same: + chose this collation, and the strongest derivation. + -# If collations are different: + - Character sets may differ, but only if conversion without + data loss is possible. The caller provides flags whether + character set conversion attempts should be done. If no + flags are substituted, then the character sets must be the same. + Currently processed flags are: + MY_COLL_ALLOW_SUPERSET_CONV - allow conversion to a superset + MY_COLL_ALLOW_COERCIBLE_CONV - allow conversion of a coercible value + - two EXPLICIT collations produce an error, e.g. this is wrong: + CONCAT(expr1 collate latin1_swedish_ci, expr2 collate latin1_german_ci) + - the side with smaller derivation value wins, + i.e. a column is stronger than a string constant, + an explicit COLLATE clause is stronger than a column. + - if derivations are the same, we have DERIVATION_NONE, + we'll wait for an explicit COLLATE clause which possibly can + come from another argument later: for example, this is valid, + but we don't know yet when collecting the first two arguments: + @code + CONCAT(latin1_swedish_ci_column, + latin1_german1_ci_column, + expr COLLATE latin1_german2_ci) + @endcode +*/ + +/** this function is to determine use which charset when compare + * We consider only three charsets(binary, gbk and utf8mb4), so the rule is simpler. Especially, + * res_level can not be CS_LEVEL_NONE. + * + * MySQL uses coercibility values with the following rules to resolve ambiguities: + * 1. Use the collation with the lowest coercibility value. + * 2. If both sides have the same coercibility, then: + * 2.a If both sides are Unicode, or both sides are not Unicode, it is an error. + * 2.b If one of the sides has a Unicode character set, and another side has a non-Unicode character set, the side with Unicode character set wins, + * and automatic character set conversion is applied to the non-Unicode side. + * 2.c For an operation with operands from the same character set but that mix a _bin collation and a _ci or _cs collation, the _bin collation is used. + * This is similar to how operations that mix nonbinary and binary strings evaluate the operands as binary strings, except that it is for collations rather than data types. +*/ +#endif int ObCharset::aggregate_collation( const ObCollationLevel collation_level1, const ObCollationType collation_type1, @@ -3080,6 +3193,7 @@ int ObCharset::get_nls_charset_id_by_charset_type(ObCharsetType charset_type) return static_cast(ret_id); } +#ifndef OB_BUILD_FULL_CHARSET int ObCharset::init_charset() { @@ -3090,6 +3204,198 @@ int ObCharset::init_charset() return ret; } +#else + +static void ob_charset_error_reporter(enum loglevel level, uint ecode, ...) { + //UNUSED(level); + UNUSED(ecode); + switch (level) { + case ERROR_LEVEL: + LIB_LOG_RET(ERROR, OB_ERROR, "fail to init charset", K(ecode)); + break; + case WARNING_LEVEL: + LIB_LOG_RET(WARN, OB_ERROR, "fail to init charset", K(ecode)); + break; + case INFORMATION_LEVEL: + LIB_LOG(INFO, "fail to init charset", K(ecode)); + break; + default: + break; + } +} + +#define CHARSET_INIT_MEM_ATTR "CharsetInit" + +static void *charset_malloc(size_t size) { + return ob_malloc(size, CHARSET_INIT_MEM_ATTR); +} + +static void *charset_realloc(void *ptr, size_t size) { + ObMemAttr attr; + attr.label_ = CHARSET_INIT_MEM_ATTR; + return ob_realloc(ptr, size, attr); +} + +static void charset_free(void *ptr) { + return ob_free(ptr); +} + +/** + Initialize character set loader to use mysys memory management functions. + @param loader Loader to initialize +*/ +void ob_charset_loader_init_mysys(ObCharsetLoader *loader) +{ + loader->errcode = 0; + loader->errarg[0] = '\0'; + loader->once_alloc = charset_malloc; + loader->mem_malloc = charset_malloc; + loader->mem_realloc = charset_realloc; + loader->mem_free = charset_free; + loader->reporter = ob_charset_error_reporter; + loader->add_collation = NULL; +} + +int ObCharset::copy_zh_cs(ObCharsetInfo *from_cs, ObCollationType to_coll_type, ObCharsetInfo *&to_cs) +{ + int ret = OB_SUCCESS; + to_cs = NULL; + if (OB_ISNULL(to_cs = static_cast(charset_malloc(sizeof(ObCharsetInfo))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc charset", K(ret)); + } else { + ObCollationType bin_coll = get_default_collation_oracle(charset_type_by_coll(to_coll_type)); + if (!is_valid_collation(to_coll_type) || !is_valid_collation(bin_coll)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected bin coll", K(ret), K(to_coll_type), K(bin_coll)); + } else { + *to_cs = *charset_arr[bin_coll]; + to_cs->uca = from_cs->uca; + to_cs->tailoring = from_cs->tailoring; + to_cs->coll_param = from_cs->coll_param; + to_cs->levels_for_compare = 3; + to_cs->coll = from_cs->coll; + to_cs->pad_attribute = NO_PAD; + //TODO + //for now, the collations are used for nlssort and not exposed to user + //the cs attributes are not all correct, such as names and number + } + } + return ret; +} + +int ObCharset::init_charset() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(init_gb18030_2022())) { + LOG_WARN("failed to init gb18030 2022", K(ret)); + } + + auto add_coll = [&ret](ObCollationType coll_type, ObCharsetInfo *cs)->void { + if (OB_SUCC(ret)) { + if (OB_ISNULL(cs) || !is_valid_collation(coll_type)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), K(cs), K(coll_type)); + } else { + charset_arr[coll_type] = cs; + cs->state |= OB_CS_COMPILED; + } + } + }; + + + ObCharsetLoader loader; + ob_charset_loader_init_mysys(&loader); + + if (OB_SUCC(ret)) { + auto *utf8_pinyin = &ob_charset_utf8mb4_zh_0900_as_cs; + ObCollationHandler *pinyin_coll = ob_charset_utf8mb4_zh_0900_as_cs.coll; + + if (pinyin_coll->init(utf8_pinyin, &loader)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to init charset", K(ret)); + } else { + ObCollationType pinyin_colls[] = { + CS_TYPE_GBK_ZH_0900_AS_CS, CS_TYPE_UTF8MB4_ZH_0900_AS_CS, + CS_TYPE_GB18030_ZH_0900_AS_CS, CS_TYPE_UTF16_ZH_0900_AS_CS, + CS_TYPE_GB18030_2022_ZH_0900_AS_CS + }; + add_coll(CS_TYPE_UTF8MB4_ZH_0900_AS_CS, utf8_pinyin); + + for (int i = 0; OB_SUCC(ret) && i < array_elements(pinyin_colls); i++) { + if (NULL == charset_arr[pinyin_colls[i]]) { + ObCharsetInfo *new_cs = NULL; + if (OB_FAIL(copy_zh_cs(utf8_pinyin, pinyin_colls[i], new_cs))) { + LOG_WARN("fail to copy zh cs", K(ret)); + } else { + add_coll(pinyin_colls[i], new_cs); + } + } + } + } + } + + if (OB_SUCC(ret)) { + auto *utf8_radical = &ob_charset_utf8mb4_zh2_0900_as_cs; + ObCollationHandler *radical_coll = ob_charset_utf8mb4_zh2_0900_as_cs.coll; + if (radical_coll->init(utf8_radical, &loader)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to init charset", K(ret)); + } else { + ObCollationType radical_colls[] = { + CS_TYPE_GBK_ZH2_0900_AS_CS, CS_TYPE_UTF8MB4_ZH2_0900_AS_CS, + CS_TYPE_GB18030_ZH2_0900_AS_CS, CS_TYPE_UTF16_ZH2_0900_AS_CS, + CS_TYPE_GB18030_2022_ZH2_0900_AS_CS + }; + add_coll(CS_TYPE_UTF8MB4_ZH2_0900_AS_CS, utf8_radical); + + for (int i = 0; OB_SUCC(ret) && i < array_elements(radical_colls); i++) { + if (NULL == charset_arr[radical_colls[i]]) { + ObCharsetInfo *new_cs = NULL; + if (OB_FAIL(copy_zh_cs(utf8_radical, radical_colls[i], new_cs))) { + LOG_WARN("fail to copy zh cs", K(ret)); + } else { + add_coll(radical_colls[i], new_cs); + } + } + } + } + } + + if (OB_SUCC(ret)) { + auto *utf8_stroke = &ob_charset_utf8mb4_zh3_0900_as_cs; + ObCollationHandler *stroke_coll = ob_charset_utf8mb4_zh3_0900_as_cs.coll; + if (stroke_coll->init(utf8_stroke, &loader)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to init charset", K(ret)); + } else { + ObCollationType stroke_colls[] = { + CS_TYPE_GBK_ZH3_0900_AS_CS, CS_TYPE_UTF8MB4_ZH3_0900_AS_CS, + CS_TYPE_GB18030_ZH3_0900_AS_CS, CS_TYPE_UTF16_ZH3_0900_AS_CS, + CS_TYPE_GB18030_2022_ZH3_0900_AS_CS + }; + add_coll(CS_TYPE_UTF8MB4_ZH3_0900_AS_CS, utf8_stroke); + + for (int i = 0; OB_SUCC(ret) && i < array_elements(stroke_colls); i++) { + if (NULL == charset_arr[stroke_colls[i]]) { + ObCharsetInfo *new_cs = NULL; + if (OB_FAIL(copy_zh_cs(utf8_stroke, stroke_colls[i], new_cs))) { + LOG_WARN("fail to copy zh cs", K(ret)); + } else { + add_coll(stroke_colls[i], new_cs); + } + } + } + } + } + + //init utf8_0900_binary + add_coll(CS_TYPE_UTF8MB4_0900_BIN, &ob_charset_utf8mb4_0900_bin); + + return ret; +} + +#endif ObString ObCharsetUtils::const_str_for_ascii_[CHARSET_MAX][INT8_MAX + 1]; diff --git a/deps/oblib/src/lib/charset/ob_ctype.h b/deps/oblib/src/lib/charset/ob_ctype.h index 9fb051a41..b15f73aad 100644 --- a/deps/oblib/src/lib/charset/ob_ctype.h +++ b/deps/oblib/src/lib/charset/ob_ctype.h @@ -190,6 +190,79 @@ typedef struct ObUnicaseInfo const ObUnicaseInfoChar **page; } ObUnicaseInfo; +#ifdef OB_BUILD_FULL_CHARSET +// OB_CHARSET_HANDLER +// ================== + +// OB_CHARSET_HANDLER is a collection of character-set +// related routines. Defined in m_ctype.h. Have the +// following set of functions: + +// Multi-byte routines +// ------------------ +// ismbchar() - detects whether the given string is a multi-byte sequence +// mbcharlen() - returns length of multi-byte sequence starting with +// the given character +// numchars() - returns number of characters in the given string, e.g. +// in SQL function CHAR_LENGTH(). +// charpos() - calculates the offset of the given position in the string. +// Used in SQL functions LEFT(), RIGHT(), SUBSTRING(), +// INSERT() + +// well_formed_len() +// - returns length of a given multi-byte string in bytes +// Used in INSERTs to shorten the given string so it +// a) is "well formed" according to the given character set +// b) can fit into the given data type + +// lengthsp() - returns the length of the given string without trailing spaces. + + +// Unicode conversion routines +// --------------------------- +// mb_wc - converts the left multi-byte sequence into its Unicode code. +// mc_mb - converts the given Unicode code into multi-byte sequence. + + +// Case and sort conversion +// ------------------------ +// caseup_str - converts the given 0-terminated string to uppercase +// casedn_str - converts the given 0-terminated string to lowercase +// caseup - converts the given string to lowercase using length +// casedn - converts the given string to lowercase using length + +// Number-to-string conversion routines +// ------------------------------------ +// snprintf() +// long10_to_str() +// longlong10_to_str() + +// The names are pretty self-describing. + +// String padding routines +// ----------------------- +// fill() - writes the given Unicode value into the given string +// with the given length. Used to pad the string, usually +// with space character, according to the given charset. + +// String-to-number conversion routines +// ------------------------------------ +// strntol() +// strntoul() +// strntoll() +// strntoull() +// strntod() + +// These functions are almost the same as their STDLIB counterparts, +// but also: +// - accept length instead of 0-terminator +// - are character set dependent + +// Simple scanner routines +// ----------------------- +// scan() - to skip leading spaces in the given string. +// Used when a string value is inserted into a numeric field. +#endif typedef struct ObCharsetHandler { //my_bool (*init)(struct ObCharsetInfo *, MY_CHARSET_LOADER *loader); @@ -257,8 +330,22 @@ typedef struct ObCharsetHandler size_t (*scan)(const struct ObCharsetInfo *, const char *b, const char *e, int sq); } ObCharsetHandler; - - +#ifdef OB_BUILD_FULL_CHARSET +// OB_COLLATION_HANDLER +// ==================== +// strnncoll() - compares two strings according to the given collation +// strnncollsp() - like the above but ignores trailing spaces for PAD SPACE +// collations. For NO PAD collations, identical to strnncoll. +// strnxfrm() - makes a sort key suitable for memcmp() corresponding +// to the given string +// like_range() - creates a LIKE range, for optimizer +// wildcmp() - wildcard comparison, for LIKE +// strcasecmp() - 0-terminated string comparison +// instr() - finds the first substring appearance in the string +// hash_sort() - calculates hash value taking into account +// the collation rules, e.g. case-insensitivity, +// accent sensitivity, etc. +#endif static const int HASH_BUFFER_LENGTH = 128; typedef uint64_t (*hash_algo)(const void* input, uint64_t length, uint64_t seed); @@ -353,8 +440,15 @@ struct ObCharsetInfo ObCharsetHandler *cset; ObCollationHandler *coll; +#ifdef OB_BUILD_FULL_CHARSET + /** + If this collation is PAD_SPACE, it collates as if all inputs were + padded with a given number of spaces at the end (see the "num_codepoints" + flag to strnxfrm). NO_PAD simply compares unextended strings. - + Note that this is fundamentally about the behavior of coll->strnxfrm. + */ +#endif enum ObCharsetPadAttr pad_attribute; }; @@ -438,6 +532,16 @@ extern ObCharsetInfo ob_charset_gb18030_2022_radical_cs; extern ObCharsetInfo ob_charset_gb18030_2022_stroke_ci; extern ObCharsetInfo ob_charset_gb18030_2022_stroke_cs; extern ObCharsetInfo ob_charset_gb18030_2022_bin; +#ifdef OB_BUILD_FULL_CHARSET +extern ObCharsetInfo ob_charset_utf8mb4_unicode_ci; +extern ObCharsetInfo ob_charset_utf16_unicode_ci; +extern ObCharsetInfo ob_charset_utf8mb4_zh_0900_as_cs; +extern ObCharsetInfo ob_charset_utf8mb4_zh2_0900_as_cs; +extern ObCharsetInfo ob_charset_utf8mb4_zh3_0900_as_cs; +extern ObCharsetInfo ob_charset_utf8mb4_0900_bin; +extern ObCharsetInfo ob_charset_latin1; +extern ObCharsetInfo ob_charset_latin1_bin; +#endif extern ObCollationHandler ob_collation_mb_bin_handler; extern ObCharsetHandler ob_charset_utf8mb4_handler; diff --git a/deps/oblib/src/lib/charset/ob_ctype_bin_os.cc b/deps/oblib/src/lib/charset/ob_ctype_bin_os.cc index e381d1cfd..6f92da9e6 100644 --- a/deps/oblib/src/lib/charset/ob_ctype_bin_os.cc +++ b/deps/oblib/src/lib/charset/ob_ctype_bin_os.cc @@ -18,6 +18,7 @@ * - initial release * */ +#ifndef OB_BUILD_FULL_CHARSET #include "lib/charset/ob_ctype.h" @@ -522,3 +523,5 @@ ObCharsetInfo ob_charset_bin = #undef likeconv #undef INC_PTR + +#endif \ No newline at end of file diff --git a/deps/oblib/src/lib/charset/ob_ctype_gb18030_os.cc b/deps/oblib/src/lib/charset/ob_ctype_gb18030_os.cc index cd301b716..e46d837d0 100644 --- a/deps/oblib/src/lib/charset/ob_ctype_gb18030_os.cc +++ b/deps/oblib/src/lib/charset/ob_ctype_gb18030_os.cc @@ -15,6 +15,7 @@ * * Authors: */ +#ifndef OB_BUILD_FULL_CHARSET #include "lib/charset/ob_ctype.h" #include "lib/charset/ob_charset.h" @@ -20760,4 +20761,6 @@ ObCharsetInfo ob_charset_gb18030_2022_stroke_cs = &ob_charset_gb18030_2022_handler, &ob_collation_2022_stroke_cs_handler, PAD_SPACE -}; \ No newline at end of file +}; + +#endif \ No newline at end of file diff --git a/deps/oblib/src/lib/charset/ob_ctype_gbk_os.cc b/deps/oblib/src/lib/charset/ob_ctype_gbk_os.cc index 27692edbe..4f70bd2c5 100644 --- a/deps/oblib/src/lib/charset/ob_ctype_gbk_os.cc +++ b/deps/oblib/src/lib/charset/ob_ctype_gbk_os.cc @@ -15,6 +15,7 @@ * * Authors: */ +#ifndef OB_BUILD_FULL_CHARSET #include "lib/charset/ob_ctype.h" @@ -10940,3 +10941,4 @@ ObCharsetInfo ob_charset_gbk_bin= PAD_SPACE }; +#endif \ No newline at end of file diff --git a/deps/oblib/src/lib/charset/ob_ctype_latin1_os.cc b/deps/oblib/src/lib/charset/ob_ctype_latin1_os.cc index 660289ad9..e7d876c3c 100644 --- a/deps/oblib/src/lib/charset/ob_ctype_latin1_os.cc +++ b/deps/oblib/src/lib/charset/ob_ctype_latin1_os.cc @@ -18,6 +18,8 @@ * - initial release * */ +#ifndef OB_BUILD_FULL_CHARSET + #include "lib/charset/ob_mysql_global.h" #include "lib/charset/ob_ctype.h" #include "lib/utility/ob_macro_utils.h" @@ -406,3 +408,6 @@ ObCharsetInfo ob_charset_latin1_bin = { &ob_charset_latin1_handler, &ob_collation_8bit_bin_handler, PAD_SPACE}; + + +#endif \ No newline at end of file diff --git a/deps/oblib/src/lib/charset/ob_ctype_mb_os.cc b/deps/oblib/src/lib/charset/ob_ctype_mb_os.cc index dbb55e8b3..7f9de5ecc 100644 --- a/deps/oblib/src/lib/charset/ob_ctype_mb_os.cc +++ b/deps/oblib/src/lib/charset/ob_ctype_mb_os.cc @@ -15,6 +15,7 @@ * * Authors: */ +#ifndef OB_BUILD_FULL_CHARSET #include "lib/charset/ob_ctype.h" @@ -769,3 +770,5 @@ ObCollationHandler ob_collation_mb_bin_handler = { #undef INC_PTR #undef likeconv + +#endif \ No newline at end of file diff --git a/deps/oblib/src/lib/charset/ob_ctype_os.cc b/deps/oblib/src/lib/charset/ob_ctype_os.cc index c4f844794..c9f70a857 100644 --- a/deps/oblib/src/lib/charset/ob_ctype_os.cc +++ b/deps/oblib/src/lib/charset/ob_ctype_os.cc @@ -15,6 +15,7 @@ * * Authors: */ +#ifndef OB_BUILD_FULL_CHARSET #include "lib/charset/ob_ctype.h" @@ -120,3 +121,5 @@ ob_convert(char *to, uint32 to_length, const ObCharsetInfo *to_cs, return 0; } + +#endif \ No newline at end of file diff --git a/deps/oblib/src/lib/charset/ob_ctype_simple_os.cc b/deps/oblib/src/lib/charset/ob_ctype_simple_os.cc index 62f5eef29..8db9ccd33 100644 --- a/deps/oblib/src/lib/charset/ob_ctype_simple_os.cc +++ b/deps/oblib/src/lib/charset/ob_ctype_simple_os.cc @@ -18,6 +18,7 @@ * - initial release * */ +#ifndef OB_BUILD_FULL_CHARSET #include "lib/charset/ob_ctype.h" #include "lib/charset/ob_dtoa.h" @@ -1097,4 +1098,6 @@ ObCollationHandler ob_collation_8bit_simple_ci_handler = { ob_propagate_simple}; #undef likeconv -#undef INC_PTR \ No newline at end of file +#undef INC_PTR + +#endif \ No newline at end of file diff --git a/deps/oblib/src/lib/charset/ob_ctype_utf16_os.cc b/deps/oblib/src/lib/charset/ob_ctype_utf16_os.cc index 61bda532c..18dcb4f10 100644 --- a/deps/oblib/src/lib/charset/ob_ctype_utf16_os.cc +++ b/deps/oblib/src/lib/charset/ob_ctype_utf16_os.cc @@ -15,6 +15,7 @@ * * Authors: */ +#ifndef OB_BUILD_FULL_CHARSET #include "lib/charset/ob_ctype.h" #include "lib/charset/ob_dtoa.h" @@ -1253,3 +1254,5 @@ ObCharsetInfo ob_charset_utf16_general_ci= &ob_collation_utf16_general_ci_handler, PAD_SPACE }; + +#endif \ No newline at end of file diff --git a/deps/oblib/src/lib/charset/ob_ctype_utf8_os.cc b/deps/oblib/src/lib/charset/ob_ctype_utf8_os.cc index be26158ab..c5ae98a43 100644 --- a/deps/oblib/src/lib/charset/ob_ctype_utf8_os.cc +++ b/deps/oblib/src/lib/charset/ob_ctype_utf8_os.cc @@ -18,6 +18,7 @@ * - initial release * */ +#ifndef OB_BUILD_FULL_CHARSET #include "lib/charset/ob_ctype.h" #include "lib/charset/ob_dtoa.h" @@ -2579,3 +2580,5 @@ ObCharsetInfo ob_charset_utf8mb4_bin= PAD_SPACE }; + +#endif \ No newline at end of file diff --git a/deps/oblib/src/lib/charset/ob_dtoa_os.cc b/deps/oblib/src/lib/charset/ob_dtoa_os.cc index 559a75713..69351d254 100644 --- a/deps/oblib/src/lib/charset/ob_dtoa_os.cc +++ b/deps/oblib/src/lib/charset/ob_dtoa_os.cc @@ -19,6 +19,7 @@ * */ +#ifndef OB_BUILD_FULL_CHARSET #include "lib/charset/ob_dtoa.h" #include "lib/charset/ob_mysql_global.h" @@ -2486,3 +2487,5 @@ ret1: } #undef P + +#endif \ No newline at end of file diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_connection.cpp b/deps/oblib/src/lib/mysqlclient/ob_mysql_connection.cpp index 5303a3a03..bcddea483 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_connection.cpp +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_connection.cpp @@ -134,6 +134,9 @@ int ObMySQLConnection::connect(const char *user, const char *pass, const char *d LOG_INFO("connecting to mysql server", "ip", host, "port", addr.get_port()); mysql_init(&mysql_); timeout_ = timeout; +#ifdef OB_BUILD_TDE_SECURITY + int64_t ssl_enforce = 1; +#endif mysql_options(&mysql_, MYSQL_OPT_CONNECT_TIMEOUT, &timeout_); if (read_write_no_timeout) { int64_t zero_second = 0; @@ -157,6 +160,9 @@ int ObMySQLConnection::connect(const char *user, const char *pass, const char *d default: mysql_options4(&mysql_, MYSQL_OPT_CONNECT_ATTR_ADD, OB_SQL_REQUEST_LEVEL, OB_SQL_REQUEST_LEVEL0); } +#ifdef OB_BUILD_TDE_SECURITY + mysql_options(&mysql_, MYSQL_OPT_SSL_ENFORCE, &ssl_enforce); +#endif int32_t port = addr.get_port(); MYSQL *mysql = mysql_real_connect(&mysql_, host, user, pass, db, port, NULL, 0); if (OB_ISNULL(mysql)) { @@ -204,6 +210,12 @@ int ObMySQLConnection::connect(const char *user, const char *pass, const char *d close(); LOG_INFO("connecting to mysql server", "ip", host, "port", root_->get_server().get_port()); mysql_init(&mysql_); +#ifdef OB_BUILD_TDE_SECURITY + int64_t ssl_enforce = 1; + if (! use_ssl) { + ssl_enforce = 0; + } +#endif mysql_options(&mysql_, MYSQL_OPT_CONNECT_TIMEOUT, &timeout_); if (read_write_no_timeout) { int64_t zero_second = 0; @@ -227,6 +239,9 @@ int ObMySQLConnection::connect(const char *user, const char *pass, const char *d default: mysql_options4(&mysql_, MYSQL_OPT_CONNECT_ATTR_ADD, OB_SQL_REQUEST_LEVEL, OB_SQL_REQUEST_LEVEL0); } +#ifdef OB_BUILD_TDE_SECURITY + mysql_options(&mysql_, MYSQL_OPT_SSL_ENFORCE, &ssl_enforce); +#endif int32_t port = root_->get_server().get_port(); MYSQL *mysql = mysql_real_connect(&mysql_, host, user, pass, db, port, NULL, 0); if (OB_ISNULL(mysql)) { diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_connection_pool.h b/deps/oblib/src/lib/mysqlclient/ob_mysql_connection_pool.h index d8aebb80a..5289ab161 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_connection_pool.h +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_connection_pool.h @@ -22,6 +22,9 @@ #include "lib/mysqlclient/ob_mysql_connection.h" #include "lib/net/ob_addr.h" #include "lib/mysqlclient/ob_isql_connection_pool.h" +#ifdef OB_BUILD_DBLINK +#include "lib/oracleclient/ob_oracle_oci_connection.h" +#endif namespace oceanbase { @@ -253,12 +256,18 @@ public: inline void set_link_type(DblinkDriverProto link_type) { link_type_= link_type; } DblinkDriverProto &get_link_type() { return link_type_; } ObMySQLConnectionPool &get_mysql_pool() { return mysql_pool_; } +#ifdef OB_BUILD_DBLINK + ObOciConnectionPool &get_oci_pool() { return oci_pool_; } +#endif inline void stop() {} private: // ObISQLConnectionPool &get_pool_from_type(DblinkDriverProto &link_type); // ObDbLinkMySQLConnectionPool mysql_pool_; ObMySQLConnectionPool mysql_pool_; +#ifdef OB_BUILD_DBLINK + ObOciConnectionPool oci_pool_; +#endif DblinkDriverProto link_type_; }; diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp index d674c35dc..1bdc18c81 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp @@ -18,6 +18,9 @@ #include "lib/mysqlclient/ob_isql_connection_pool.h" #include "lib/mysqlclient/ob_mysql_proxy.h" #include "common/sql_mode/ob_sql_mode_utils.h" +#ifdef OB_BUILD_DBLINK +#include "lib/oracleclient/ob_oci_environment.h" +#endif using namespace oceanbase::common; using namespace oceanbase::common::sqlclient; @@ -411,6 +414,11 @@ int ObDbLinkProxy::switch_dblink_conn_pool(DblinkDriverProto type, ObISQLConnect case DBLINK_DRV_OB: dblink_conn_pool = static_cast(&(link_pool_->get_mysql_pool())); break; +#ifdef OB_BUILD_DBLINK + case DBLINK_DRV_OCI : + dblink_conn_pool = static_cast(&(link_pool_->get_oci_pool())); + break; +#endif default: ret = OB_ERR_UNEXPECTED; LOG_WARN("unknown dblink type", K(ret), K(type)); @@ -534,6 +542,32 @@ int ObDbLinkProxy::execute_init_sql(const sqlclient::dblink_param_ctx ¶m_ctx } } } +#ifdef OB_BUILD_DBLINK + else if (DBLINK_DRV_OCI == param_ctx.link_type_) { + static sql_ptr_type sql_ptr_ora[] = { + "alter session set nls_date_format='YYYY-MM-DD HH24:MI:SS'", + "alter session set nls_timestamp_format = 'YYYY-MM-DD HH24:MI:SS.FF'", + "alter session set nls_timestamp_tz_format = 'YYYY-MM-DD HH24:MI:SS.FF TZR TZD'" + }; + // oracle init + OciStatement stmt; + ObOciConnection *conn = static_cast(dblink_conn); + if (OB_ISNULL(conn)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("null oci connection", K(ret)); + } + int64_t affected_rows = 0; //no use + for (int i = 0; OB_SUCC(ret) && i < sizeof(sql_ptr_ora) / sizeof(sql_ptr_type); ++i) { + if (OB_FAIL(stmt.init_stmt(conn->get_oci_connection()))) { + LOG_WARN("init oci statement failed", K(ret), K(param_ctx)); + } else if (OB_FAIL(stmt.set_sql_text(ObString(sql_ptr_ora[i])))) { + LOG_WARN("failed to set sql text", K(ret), K(ObString(sql_ptr_ora[i]))); + } else if (OB_FAIL(stmt.execute_update(affected_rows))) { + LOG_WARN("execute sql failed", K(ret), K(param_ctx)); + } + } + } +#endif return ret; } @@ -603,6 +637,15 @@ int ObDbLinkProxy::clean_dblink_connection(uint64_t tenant_id) if (OB_FAIL(link_pool_->get_mysql_pool().clean_dblink_connection(tenant_id))) { LOG_WARN("clean mysql pool failed", K(ret)); } +#ifdef OB_BUILD_DBLINK + int tmp_ret = ret; + if (OB_FAIL(link_pool_->get_oci_pool().clean_dblink_connection(tenant_id))) { + LOG_WARN("clean oci pool failed", K(ret)); + } + if (OB_SUCC(ret) && OB_UNLIKELY(OB_SUCCESS != tmp_ret)) { + ret = tmp_ret; + } +#endif } return ret; -} \ No newline at end of file +} diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h index 0b1fe698a..a4ba3210a 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h @@ -17,6 +17,9 @@ #include "lib/mysqlclient/ob_mysql_result.h" #include "lib/mysqlclient/ob_mysql_statement.h" #include "lib/mysqlclient/ob_mysql_connection_pool.h" +#ifdef OB_BUILD_DBLINK +#include "lib/oracleclient/ob_oracle_oci_connection.h" +#endif namespace oceanbase { diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_result_impl.cpp b/deps/oblib/src/lib/mysqlclient/ob_mysql_result_impl.cpp index 0b6ddeb12..b95051202 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_result_impl.cpp +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_result_impl.cpp @@ -815,6 +815,35 @@ int ObMySQLResultImpl::get_col_meta(const int64_t col_idx, bool old_max_length, ObDataType &data_type) const { int ret = OB_SUCCESS; +#ifdef OB_BUILD_DBLINK + ObObjType ob_type; + if (OB_ISNULL(fields_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("check fields_ failed", K(ret)); + } else if (col_idx < 0 || col_idx >= result_column_count_) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid column idx", K(col_idx), K_(result_column_count)); + } else if (OB_FAIL(get_ob_type(ob_type, static_cast(fields_[col_idx].type), + fields_[col_idx].flags & UNSIGNED_FLAG))) { + LOG_WARN("failed to get ob type", K(ret), "mysql_type", fields_[col_idx].type); + } else { + int16_t precision = fields_[col_idx].precision; + int16_t scale = fields_[col_idx].decimals; + int32_t length = fields_[col_idx].length; + name.assign_ptr(fields_[col_idx].name, STRLEN(fields_[col_idx].name)); + data_type.meta_.set_type(ob_type); + data_type.meta_.set_collation_type(static_cast(fields_[col_idx].charsetnr)); + //data_type.meta_.set_autoincrement(fields_[col_idx].flags & AUTO_INCREMENT_FLAG); + data_type.set_zero_fill(fields_[col_idx].flags & ZEROFILL_FLAG); + format_precision_scale_length(precision, scale, length, + ob_type, data_type.meta_.get_collation_type(), + DBLINK_DRV_OB, old_max_length); + data_type.set_precision(precision); + data_type.set_scale(scale); + data_type.set_length(length); + LOG_DEBUG("get col type from obclient", K(ob_type), K(data_type), K(ret)); + } +#endif return ret; } diff --git a/deps/oblib/src/lib/mysqlclient/ob_tenant_oci_envs.h b/deps/oblib/src/lib/mysqlclient/ob_tenant_oci_envs.h index 5e9bceb58..53c63fd7d 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_tenant_oci_envs.h +++ b/deps/oblib/src/lib/mysqlclient/ob_tenant_oci_envs.h @@ -6,12 +6,14 @@ namespace common { namespace sqlclient { +#ifndef OB_BUILD_DBLINK class ObTenantOciEnvs { public: static int mtl_init(ObTenantOciEnvs *&tenant_oci_envs) { return OB_SUCCESS; } static void mtl_destroy(ObTenantOciEnvs *&tenant_oci_envs) { } }; +#endif } //sqlclient } // namespace common diff --git a/deps/oblib/src/lib/ob_name_def.h b/deps/oblib/src/lib/ob_name_def.h index 6a4a984b1..505f6aecf 100644 --- a/deps/oblib/src/lib/ob_name_def.h +++ b/deps/oblib/src/lib/ob_name_def.h @@ -844,6 +844,11 @@ #define N_DES_ENCRYPT "des_encrypt" #define N_ENCRYPT "encrypt" +//*********** for spm +#define N_SPM_LOAD_PLANS_FROM_PLAN_CACHE "spm_load_plans_from_plan_cache" +#define N_SPM_ALTER_BASELINE "spm_alter_baseline" +#define N_SPM_DROP_BASELINE "spm_drop_baseline" +//*********** for spm end #define N_UID "uid" #define N_PL_INTEGER_CHECKER "pl_integer_checker" diff --git a/deps/oblib/src/lib/oblog/ob_log.cpp b/deps/oblib/src/lib/oblog/ob_log.cpp index 095104caa..6a9b415f2 100644 --- a/deps/oblib/src/lib/oblog/ob_log.cpp +++ b/deps/oblib/src/lib/oblog/ob_log.cpp @@ -1692,6 +1692,43 @@ int64_t ObLogger::get_wait_us(const int32_t level) return ret_timeout_us; } +#ifdef OB_BUILD_AUDIT_SECURITY +int ObLogger::async_audit_dump(const common::ObBasebLogPrint &info) +{ + int ret = OB_SUCCESS; + if (OB_LIKELY(is_async_log_used()) + && OB_LIKELY(info.get_data_length() > 0)) { + const int32_t level = OB_LOG_LEVEL_INFO; + ObPLogItem *log_item = NULL; + set_disable_logging(true); + //1. fill log buffer + if (OB_FAIL(alloc_log_item(level, MAX_LOG_SIZE, log_item))) { + LOG_STDERR("alloc_log_item error, ret=%d\n", ret); + } else if (OB_ISNULL(log_item)) { + ret = OB_ERR_UNEXPECTED; + } else { + int64_t pos = log_item->get_data_len(); + log_item->set_timestamp(info.get_timestamp()); + log_item->set_fd_type(FD_AUDIT_FILE); + if (OB_FAIL(info.print_data(log_item->get_buf(), log_item->get_buf_size(), pos))) { + LOG_STDERR("print_data error ret = %d\n", ret); + } else if (FALSE_IT(check_log_end(*log_item, pos))) { + } else if (OB_FAIL(append_log(*log_item))) { + LOG_STDERR("append_log error ret = %d\n", ret); + } + } + + //3. stat + if (OB_FAIL(ret)) { + inc_dropped_log_count(level); + free_log_item(log_item); + log_item = NULL; + } + set_disable_logging(false); + } + return ret; +} +#endif int ObLogger::alloc_log_item(const int32_t level, const int64_t size, ObPLogItem *&log_item) { diff --git a/deps/oblib/src/lib/oblog/ob_log.h b/deps/oblib/src/lib/oblog/ob_log.h index c67faf8fc..bdf91e467 100644 --- a/deps/oblib/src/lib/oblog/ob_log.h +++ b/deps/oblib/src/lib/oblog/ob_log.h @@ -510,6 +510,9 @@ public: } +#ifdef OB_BUILD_AUDIT_SECURITY + int async_audit_dump(const common::ObBasebLogPrint &info); +#endif //@brief Check whether the level to print. bool __attribute__((weak, noinline, cold)) need_to_print(const int32_t level) { return (level <= get_log_level()); } diff --git a/deps/oblib/src/lib/oblog/ob_log_module.h b/deps/oblib/src/lib/oblog/ob_log_module.h index bab8b8074..c20473def 100644 --- a/deps/oblib/src/lib/oblog/ob_log_module.h +++ b/deps/oblib/src/lib/oblog/ob_log_module.h @@ -67,7 +67,6 @@ DEFINE_LOG_SUB_MOD(EASY) // libeasy DEFINE_LOG_SUB_MOD(DETECT) // dead lock DEFINE_LOG_SUB_MOD(PALF) // palf DEFINE_LOG_SUB_MOD(STANDBY) // primary and standby cluster -DEFINE_LOG_SUB_MOD(REASY) // libreasy DEFINE_LOG_SUB_MOD(COORDINATOR) // leader coordinator DEFINE_LOG_SUB_MOD(FLT) // trace DEFINE_LOG_SUB_MOD(OBTRACE) // trace @@ -186,6 +185,7 @@ DEFINE_LOG_SUB_MOD(CG) // code_generator DEFINE_LOG_SUB_MOD(MONITOR) // monitor DEFINE_LOG_SUB_MOD(DTL) // data transfer layer DEFINE_LOG_SUB_MOD(DAS) // data access service +DEFINE_LOG_SUB_MOD(SPM) // sql plan baseline DEFINE_LOG_SUB_MOD(QRR) // query rewrite rule LOG_MOD_END(SQL) @@ -784,11 +784,14 @@ LOG_MOD_END(PL) info_string, ##args) #define _SQL_DAS_LOG(level, _fmt_, args...) _OB_SUB_MOD_LOG(SQL, DAS, level, \ _fmt_, ##args) +#define SQL_SPM_LOG(level, info_string, args...) OB_SUB_MOD_LOG(SQL, SPM, level, \ + info_string, ##args) +#define _SQL_SPM_LOG(level, _fmt_, args...) _OB_SUB_MOD_LOG(SQL, SPM, level, \ + _fmt_, ##args) #define SQL_QRR_LOG(level, info_string, args...) OB_SUB_MOD_LOG(SQL, QRR, level, \ info_string, ##args) #define _SQL_QRR_LOG(level, _fmt_, args...) _OB_SUB_MOD_LOG(SQL, QRR, level, \ _fmt_, ##args) - #define DETECT_LOG_LOG(level, info_string, args...) OB_SUB_MOD_LOG(DETECT, LOG,level, \ info_string, ##args) #define _DETECT_LOG_LOG(level, _fmt_, args...) _OB_SUB_MOD_LOG(DETECT, LOG,level, \ diff --git a/deps/oblib/src/lib/ssl/ob_ssl_config.cpp b/deps/oblib/src/lib/ssl/ob_ssl_config.cpp index b008a41f6..621bdbf2c 100644 --- a/deps/oblib/src/lib/ssl/ob_ssl_config.cpp +++ b/deps/oblib/src/lib/ssl/ob_ssl_config.cpp @@ -208,6 +208,44 @@ static int ob_ssl_set_verify_mode_and_load_CA(SSL_CTX* ctx, const ObSSLConfig& s return ret; } +#ifdef OB_USE_BABASSL +static int ob_ssl_load_cert_and_pkey_for_sm_memory(SSL_CTX* ctx, const ObSSLConfig& ssl_config) +{ + int ret = OB_SUCCESS; + EVP_PKEY *pkey = NULL; + X509 *x509 = NULL; + if (NULL == (pkey = ob_ssl_get_sm_pkey_memory(ssl_config.sign_private_key_))) { + ret = OB_ERR_UNEXPECTED; + COMMON_LOG(WARN, "ob_ssl_get_sm_pkey_memory failed", K(ssl_config.sign_private_key_), K(ret)); + } else if (!SSL_CTX_use_sign_PrivateKey(ctx, pkey)) { + ret = OB_ERR_UNEXPECTED; + EVP_PKEY_free(pkey); + COMMON_LOG(WARN, "SSL_CTX_use_sign_PrivateKey failed", K(ssl_config.sign_private_key_), K(ret), K(ERR_error_string(ERR_get_error(), NULL))); + } else if (NULL == (x509 = ob_ssl_get_sm_cert_memory(ssl_config.sign_cert_))){ + ret = OB_ERR_UNEXPECTED; + COMMON_LOG(WARN, "ob_ssl_get_sm_cert_memory failed", K(ssl_config.sign_cert_), K(ret)); + } else if (!SSL_CTX_use_sign_certificate(ctx, x509)) { + ret = OB_ERR_UNEXPECTED; + X509_free(x509); + COMMON_LOG(WARN, "SSL_CTX_use_sign_certificate failed", K(ssl_config.sign_cert_), K(ret), K(ERR_error_string(ERR_get_error(), NULL))); + } else if (NULL == (pkey = ob_ssl_get_sm_pkey_memory(ssl_config.enc_private_key_))) { + ret = OB_ERR_UNEXPECTED; + COMMON_LOG(WARN, "ob_ssl_get_sm_pkey_memory failed", K(ssl_config.enc_private_key_), K(ret)); + } else if (!SSL_CTX_use_enc_PrivateKey(ctx, pkey)) { + ret = OB_ERR_UNEXPECTED; + EVP_PKEY_free(pkey); + COMMON_LOG(WARN, "SSL_CTX_use_enc_PrivateKey failed", K(ssl_config.enc_private_key_), K(ret), K(ERR_error_string(ERR_get_error(), NULL))); + } else if (NULL == (x509 = ob_ssl_get_sm_cert_memory(ssl_config.enc_cert_))) { + ret = OB_ERR_UNEXPECTED; + COMMON_LOG(WARN, "ob_ssl_get_sm_cert_memory failed", K(ssl_config.enc_cert_), K(ret)); + } else if (!SSL_CTX_use_enc_certificate(ctx, x509)) { + ret = OB_ERR_UNEXPECTED; + X509_free(x509); + COMMON_LOG(WARN, "SSL_CTX_use_enc_certificate failed", K(ssl_config.enc_cert_), K(ret), K(ERR_error_string(ERR_get_error(), NULL))); + } + return ret; +} +#endif static int ob_ssl_load_cert_and_pkey_for_intl_memory(SSL_CTX* ctx, const ObSSLConfig& ssl_config) { @@ -280,6 +318,21 @@ static int ob_ssl_load_cert_and_pkey(SSL_CTX* ctx, const ObSSLConfig& ssl_config int ret = OB_SUCCESS; if (ssl_config.is_from_file_) { if (ssl_config.is_sm_) { +#ifdef OB_USE_BABASSL + if (!SSL_CTX_use_sign_PrivateKey_file(ctx, ssl_config.sign_private_key_, SSL_FILETYPE_PEM)) { + ret = OB_ERR_UNEXPECTED; + COMMON_LOG(WARN, "SSL_CTX_use_sign_PrivateKey_file failed", K(ssl_config.sign_private_key_), K(ret)); + } else if (!SSL_CTX_use_sign_certificate_file(ctx, ssl_config.sign_cert_, SSL_FILETYPE_PEM)) { + ret = OB_ERR_UNEXPECTED; + COMMON_LOG(WARN, "SSL_CTX_use_sign_certificate_file failed", K(ssl_config.sign_cert_), K(ret)); + } else if (!SSL_CTX_use_enc_PrivateKey_file(ctx, ssl_config.enc_private_key_, SSL_FILETYPE_PEM)) { + ret = OB_ERR_UNEXPECTED; + COMMON_LOG(WARN, "SSL_CTX_use_enc_PrivateKey_file failed", K(ssl_config.enc_private_key_), K(ret)); + } else if (!SSL_CTX_use_enc_certificate_file(ctx, ssl_config.enc_cert_, SSL_FILETYPE_PEM)) { + ret = OB_ERR_UNEXPECTED; + COMMON_LOG(WARN, "SSL_CTX_use_enc_certificate_file failed", K(ssl_config.enc_cert_), K(ret)); + } +#endif } else { if (SSL_CTX_use_certificate_chain_file(ctx, ssl_config.sign_cert_) <= 0) { ret = OB_ERR_UNEXPECTED; @@ -294,6 +347,11 @@ static int ob_ssl_load_cert_and_pkey(SSL_CTX* ctx, const ObSSLConfig& ssl_config } } else { if (ssl_config.is_sm_) { +#ifdef OB_USE_BABASSL + if (OB_FAIL(ob_ssl_load_cert_and_pkey_for_sm_memory(ctx, ssl_config))) { + COMMON_LOG(WARN, "ob_ssl_load_cert_and_pkey_for_sm_memory failed", K(ret)); + } +#endif } else { if (OB_FAIL(ob_ssl_load_cert_and_pkey_for_intl_memory(ctx, ssl_config))) { COMMON_LOG(WARN, "ob_ssl_load_cert_and_pkey_for_intl_memory failed", K(ret)); @@ -312,6 +370,12 @@ static SSL_CTX* ob_ssl_create_ssl_ctx(const ObSSLConfig& ssl_config) SSL_CTX *ctx = NULL; if (ssl_config.is_sm_) { +#ifdef OB_USE_BABASSL + ctx = SSL_CTX_new(NTLS_method()); + if (NULL != ctx) { + SSL_CTX_enable_ntls(ctx); + } +#endif } else { ctx = SSL_CTX_new(SSLv23_method()); } diff --git a/deps/oblib/src/rpc/ob_request.h b/deps/oblib/src/rpc/ob_request.h index 540a56565..659d852b7 100644 --- a/deps/oblib/src/rpc/ob_request.h +++ b/deps/oblib/src/rpc/ob_request.h @@ -39,7 +39,7 @@ class ObRequest: public common::ObLink public: friend class ObSqlRequestOperator; enum Type { OB_RPC, OB_MYSQL, OB_TASK, OB_TS_TASK, OB_SQL_TASK, OB_SQL_SOCK_TASK }; - enum TransportProto { TRANSPORT_PROTO_EASY = 0, TRANSPORT_PROTO_POC = 1, TRANSPORT_PROTO_RDMA = 2 }; + enum TransportProto { TRANSPORT_PROTO_EASY = 0, TRANSPORT_PROTO_POC = 1 }; enum Stat { OB_EASY_REQUEST_EZ_RECV = 0, OB_EASY_REQUEST_RPC_DELIVER = 1, diff --git a/deps/oblib/src/rpc/obmysql/ob_mysql_util.cpp b/deps/oblib/src/rpc/obmysql/ob_mysql_util.cpp index 7c0bb715d..ce898ed4c 100644 --- a/deps/oblib/src/rpc/obmysql/ob_mysql_util.cpp +++ b/deps/oblib/src/rpc/obmysql/ob_mysql_util.cpp @@ -25,6 +25,10 @@ #include "lib/json_type/ob_json_bin.h" #include "lib/json_type/ob_json_base.h" #include "lib/geo/ob_geo_bin.h" +#ifdef OB_BUILD_ORACLE_XML +#include "lib/xml/ob_xml_util.h" +#include "lib/xml/ob_xml_bin.h" +#endif using namespace oceanbase::common; namespace oceanbase @@ -1075,7 +1079,46 @@ int ObMySQLUtil::urowid_cell_str(char *buf, const int64_t len, const ObURowIDDat int ObMySQLUtil::sql_utd_cell_str(uint64_t tenant_id, char *buf, const int64_t len, const ObString &val, int64_t &pos) { INIT_SUCC(ret); +#ifdef OB_BUILD_ORACLE_XML + lib::ObMemAttr mem_attr(tenant_id, "XMLCodeGen"); + lib::ObMallocHookAttrGuard malloc_guard(mem_attr); + ObArenaAllocator allocator(mem_attr); + ObMulModeNodeType node_type = M_MAX_TYPE; + ObStringBuffer jbuf(&allocator); + ParamPrint param_list; + param_list.indent = 2; + ObIMulModeBase *node = NULL; + ObXmlNode *xml_node = NULL; + ObMulModeMemCtx* xml_mem_ctx = nullptr; + if (OB_ISNULL(buf)) { + ret = OB_INVALID_ARGUMENT; + OB_LOG(WARN, "invalid input args", K(ret), KP(buf)); + } else if (val.length() == 0) { + if (OB_FAIL(ObMySQLUtil::store_null(buf, len, pos))) { + OB_LOG(WARN, "fail to set null string", K(pos), K(len)); + } + } else { + int64_t new_length = val.length(); + if (OB_LIKELY(new_length < len - pos)) { + int64_t pos_bk = pos; + if (OB_FAIL(ObMySQLUtil::store_length(buf, len, new_length, pos))) { + LOG_WARN("xml_cell_str store length failed", K(ret), K(len), K(new_length), K(pos)); + } else { + if (OB_LIKELY(new_length <= len - pos)) { + MEMCPY(buf + pos, val.ptr(), val.length()); + pos += new_length; + } else { + pos = pos_bk; + ret = OB_SIZE_OVERFLOW; + } + } + } else { + ret = OB_SIZE_OVERFLOW; + } + } +#else ret = OB_NOT_SUPPORTED; +#endif return ret; } diff --git a/deps/oblib/src/rpc/obmysql/ob_rdma_server.h b/deps/oblib/src/rpc/obmysql/ob_rdma_server.h deleted file mode 100644 index a171b2e9c..000000000 --- a/deps/oblib/src/rpc/obmysql/ob_rdma_server.h +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#ifndef OCEANBASE_OBMYSQL_OB_RDMA_SERVER_H_ -#define OCEANBASE_OBMYSQL_OB_RDMA_SERVER_H_ -#include -#include -#include "ob_mysql_rdma_handler.h" -#include "reasy_header_obp.h" -#include "lib/lock/ob_drw_lock.h" - - -using namespace oceanbase; - -namespace oceanbase -{ -namespace obmysql -{ - -#define OB_MYSQL_SESSION_GROUP_COUNT 128 -#define OB_MYSQL_MCONN_HASHMAP_COUNT 8 - -#define OB_MYSQL_HASH_CODE(x, y) \ - ({ \ - int res; \ - res = ((x >> 5) + (x >> 10) + y) & (OB_MYSQL_SESSION_GROUP_COUNT - 1); \ - res; \ - }) - -extern bool checkLogLevel(enum ObRdmaDebugLogLevelReq req); - -typedef common::ObLinkHashMap, common::RefHandle, 128> ObRdmaConnMap; - - -struct hash_A{ - size_t operator()(const class ObMysqlSessionID & A)const{ - return A.sessid_; - } -}; - -struct equal_A{ - bool operator()(const class ObMysqlSessionID & a1, const class ObMysqlSessionID & a2)const{ - return a1.sessid_ == a2.sessid_; - } -}; - -class ObRdmaServer -{ -public: - ObRdmaServer(); - virtual ~ObRdmaServer(); - int init(ObMySQLRdmaHandler& handler, const common::ObAddr &addr, - uint32_t port, int io_thread_count); - int start(); - void wait(); - int stop(); - void destroy(); - - int create_session(struct reasy_rdma_buffer *req); - int destroy_session(struct reasy_rdma_buffer *req); - int process(struct reasy_rdma_buffer *req); - int update_credit(struct reasy_rdma_buffer *req); - int send_keepalive(struct reasy_rdma_buffer *req); - int reg_mconn(ObMysqlRdmaConnection& mconn); - int unreg_mconn(ObMysqlRdmaConnection& mconn); - int get_mconn(struct ObRdmaMysqlHeader *rheader, struct reasy_connection_desc *rdesc, ObMysqlRdmaConnection *&mconn); - void put_mconn(ObMysqlRdmaConnection *mconn); - static int send_response(ObMysqlRdmaConnection& mconn, char *buffer, size_t length); - static int encode_reasy_header(ObMysqlRdmaConnection& mconn, reasy_rdma_buffer *rsp, - uint8 type, uint32_t len); - static int send_response(ObMysqlRdmaConnection& mconn, reasy_rdma_buffer *rsp, - uint8 type, uint32_t len); - - inline int get_map_pos(int64_t ver_id) - { - return io_thread_count_ == 0 ? 0 : ver_id % io_thread_count_; - } - -private: - ObMySQLRdmaHandler *handler_; - common::ObAddr listen_addr_; - uint32_t listen_port_; - struct reasy_io_t *rio_; - bool is_inited_; - bool started_; - int io_thread_count_; - __gnu_cxx::hash_map mconn_hmap_[OB_MYSQL_MCONN_HASHMAP_COUNT]; - common::DRWLock sess_grp_locks_[OB_MYSQL_SESSION_GROUP_COUNT]; -}; -extern ObRdmaServer* global_rdma_server; - -}; // end namespace obmysql -}; // end namespace oceanbase - -#endif /* #define OCEANBASE_OBMYSQL_OB_RDMA_SERVER_H_ */ diff --git a/deps/oblib/src/rpc/obmysql/ob_reasy_sql_request_operator.h b/deps/oblib/src/rpc/obmysql/ob_reasy_sql_request_operator.h deleted file mode 100644 index 22410a8e5..000000000 --- a/deps/oblib/src/rpc/obmysql/ob_reasy_sql_request_operator.h +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#ifndef OCEANBASE_RPC_OB_REASY_SQL_REQUEST_OPERATOR_H_ -#define OCEANBASE_RPC_OB_REASY_SQL_REQUEST_OPERATOR_H_ -#include "rpc/ob_sql_request_operator.h" - -namespace oceanbase -{ -namespace obmysql -{ - -class ObReasySqlRequestOperator: public rpc::ObISqlRequestOperator -{ -public: - ObReasySqlRequestOperator() {} - virtual ~ObReasySqlRequestOperator() {} - virtual void *get_sql_session(rpc::ObRequest* req) override; - virtual SSL *get_sql_ssl_st(rpc::ObRequest* req) override; - virtual char* alloc_sql_response_buffer(rpc::ObRequest* req, int64_t size) override; - virtual char *sql_reusable_alloc(rpc::ObRequest* req, const int64_t size) override; - virtual common::ObAddr get_peer(const rpc::ObRequest* req) override; - virtual void disconnect_sql_conn(rpc::ObRequest* req) override; - virtual void finish_sql_request(rpc::ObRequest* req) override; - virtual int write_response(rpc::ObRequest* req, const char* buf, int64_t sz) override; - virtual int async_write_response(rpc::ObRequest* req, const char* buf, int64_t sz) override; - virtual void get_sock_desc(rpc::ObRequest* req, rpc::ObSqlSockDesc& desc) override; - virtual void disconnect_by_sql_sock_desc(rpc::ObSqlSockDesc& desc) override; - virtual void destroy(rpc::ObRequest* req) override; - virtual void set_sql_session_to_sock_desc(rpc::ObRequest* req, void* sess) override; -}; - -}; // end namespace rpc -}; // end namespace oceanbase - -#endif /* OCEANBASE_RPC_OB_EASY_SQL_REQUEST_OPERATOR_H_ */ - diff --git a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h index 09b76da9e..ac8f913e5 100644 --- a/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h +++ b/deps/oblib/src/rpc/obrpc/ob_rpc_packet_list.h @@ -269,6 +269,12 @@ PCODE_DEF(OB_RUN_UPGRADE_JOB, 0x2A1) PCODE_DEF(OB_ADMIN_WASH_MEMORY_FRAGMENTATION, 0x2A2) PCODE_DEF(OB_CHECKPOINT_SLOG, 0x2A3) PCODE_DEF(OB_UPGRADE_TABLE_SCHEMA, 0x2A4) +#ifdef OB_BUILD_ARBITRATION +PCODE_DEF(OB_ADMIN_ADD_ARBITRATION_SERVICE, 0x2A5) +PCODE_DEF(OB_ADMIN_REMOVE_ARBITRATION_SERVICE, 0x2A6) +PCODE_DEF(OB_ADMIN_REPLACE_ARBITRATION_SERVICE, 0x2A7) +PCODE_DEF(OB_REMOVE_CLUSTER_INFO_FROM_ARB_SERVER, 0x2A8) +#endif PCODE_DEF(OB_CREATE_OUTLINE, 0x350) PCODE_DEF(OB_DROP_OUTLINE, 0x351) @@ -556,6 +562,10 @@ PCODE_DEF(OB_RS_CANCEL_EVOLVE_TASK, 0x57B) PCODE_DEF(OB_START_TRANSFER_TASK, 0x57C) PCODE_DEF(OB_FINISH_TRANSFER_TASK, 0x57D) +#ifdef OB_BUILD_ARBITRATION +PCODE_DEF(OB_ADD_ARB, 0x57E) +PCODE_DEF(OB_REMOVE_ARB, 0x57F) +#endif //// DTL // @@ -777,11 +787,18 @@ PCODE_DEF(OB_CHANGE_LS_ACCESS_MODE, 0x928) PCODE_DEF(OB_IN_TRANS_LOCK_TABLE, 0x929) PCODE_DEF(OB_IN_TRANS_LOCK_TABLET, 0x92A) PCODE_DEF(OB_HIGH_PRIORITY_TABLE_LOCK_TASK, 0x92B) +#ifdef OB_BUILD_ARBITRATION +PCODE_DEF(OB_CREATE_ARB, 0x92C) +PCODE_DEF(OB_DELETE_ARB, 0x92D) +#endif PCODE_DEF(OB_REMOVE_OBJ_LOCK, 0x92E) PCODE_DEF(OB_ARB_GC_NOTIFY, 0x92F) PCODE_DEF(OB_UPDATE_OBJ_LOCK, 0x930) PCODE_DEF(OB_BATCH_TABLE_LOCK_TASK, 0x931) PCODE_DEF(OB_HIGH_PRIORITY_BATCH_TABLE_LOCK_TASK, 0x932) +#ifdef OB_BUILD_ARBITRATION +PCODE_DEF(OB_ARB_CLUSTER_OP, 0x933) +#endif // ddl PCODE_DEF(OB_REMOTE_WRITE_DDL_REDO_LOG, 0x950) @@ -943,6 +960,10 @@ PCODE_DEF(OB_LOG_GET_PALF_STAT, 0x151D) PCODE_DEF(OB_LOG_NOTIFY_FETCH_LOG, 0x151E) PCODE_DEF(OB_LOG_GET_STAT, 0x151F) PCODE_DEF(OB_LOG_FORCE_SET_LS_AS_SINGLE_REPLICA, 0x1520) +#ifdef OB_BUILD_ARBITRATION +PCODE_DEF(OB_LOG_FORCE_CLEAR_ARB_CLUSTER_INFO, 0x1521) +PCODE_DEF(OB_LOG_GET_ARB_MEMBER_INFO, 0x1522) +#endif PCODE_DEF(OB_LOG_BATCH_FETCH_RESP, 0X1523) // 1531-1550 for obesi diff --git a/deps/ussl-hook/ssl/ssl_config.c b/deps/ussl-hook/ssl/ssl_config.c index a1631e88e..67841a926 100644 --- a/deps/ussl-hook/ssl/ssl_config.c +++ b/deps/ussl-hook/ssl/ssl_config.c @@ -132,7 +132,39 @@ static void __attribute__((constructor(103))) setup() pthread_rwlock_init(&g_ssl_ctx_rwlock, NULL); } +#ifdef OB_USE_BABASSL +static X509 *ob_ssl_get_sm_cert_memory(const char *cert) +{ + BIO *bio = NULL; + X509 *x509 = NULL; + if (NULL == (bio = BIO_new_mem_buf(cert, -1))) { + ussl_log_error("BIO_new_mem_buf failed, errno:%d", errno); + } else if (NULL == (x509 = PEM_read_bio_X509(bio, NULL, NULL, NULL))) { + ussl_log_error("PEM_read_bio_X509 failed, errno:%d", errno); + } + if (NULL != bio) { + BIO_free(bio); + } + return x509; +} +#endif +#ifdef OB_USE_BABASSL +static EVP_PKEY *ob_ssl_get_sm_pkey_memory(const char *key) +{ + BIO *bio = NULL; + EVP_PKEY *pkey = NULL; + if (NULL == (bio = BIO_new_mem_buf(key, strlen(key)))) { + ussl_log_error("BIO_new_mem_buf failed, errno:%d", errno); + } else if (NULL == (pkey = PEM_read_bio_PrivateKey(bio, NULL, NULL, NULL))) { + ussl_log_error("PEM_read_bio_PrivateKey failed, errno:%d", errno); + } + if (NULL != bio) { + BIO_free(bio); + } + return pkey; +} +#endif static int ob_ssl_config_check(const ssl_config_item_t *ssl_config) { @@ -211,6 +243,59 @@ static int ob_ssl_set_verify_mode_and_load_CA(SSL_CTX *ctx, const ssl_config_ite return ret; } +#ifdef OB_USE_BABASSL +static int ob_ssl_load_cert_and_pkey_for_sm_memory(SSL_CTX *ctx, + const ssl_config_item_t *ssl_config) +{ + int ret = 0; + EVP_PKEY *sign_pkey = NULL; + EVP_PKEY *enc_pkey = NULL; + X509 *sign_x509 = NULL; + X509 *enc_x509 = NULL; + if (NULL == (sign_pkey = ob_ssl_get_sm_pkey_memory(ssl_config->sign_private_key))) { + ret = EINVAL; + ussl_log_warn("ob_ssl_get_sm_pkey_memory failed, ret:%d", ret); + } else if (!SSL_CTX_use_sign_PrivateKey(ctx, sign_pkey)) { + ret = EINVAL; + ussl_log_warn("SSL_CTX_use_sign_PrivateKey failed, ret:%d, err:%s", ret, + ERR_error_string(ERR_get_error(), NULL)); + } else if (NULL == (sign_x509 = ob_ssl_get_sm_cert_memory(ssl_config->sign_cert))) { + ret = EINVAL; + ussl_log_warn("ob_ssl_get_sm_cert_memory failed, ret:%d", ret); + } else if (!SSL_CTX_use_sign_certificate(ctx, sign_x509)) { + ret = EINVAL; + ussl_log_warn("SSL_CTX_use_sign_certificate failed, ret:%d, err:%s", ret, + ERR_error_string(ERR_get_error(), NULL)); + } else if (NULL == (enc_pkey = ob_ssl_get_sm_pkey_memory(ssl_config->enc_private_key))) { + ret = EINVAL; + ussl_log_warn("ob_ssl_get_sm_pkey_memory failed, ret:%d", ret); + } else if (!SSL_CTX_use_enc_PrivateKey(ctx, enc_pkey)) { + ret = EINVAL; + ussl_log_warn("SSL_CTX_use_enc_PrivateKey failed, ret:%d, err:%s", ret, + ERR_error_string(ERR_get_error(), NULL)); + } else if (NULL == (enc_x509 = ob_ssl_get_sm_cert_memory(ssl_config->enc_cert))) { + ret = EINVAL; + ussl_log_warn("ob_ssl_get_sm_cert_memory failed, ret:%d", ret); + } else if (!SSL_CTX_use_enc_certificate(ctx, enc_x509)) { + ret = EINVAL; + ussl_log_warn("SSL_CTX_use_enc_certificate failed,ret:%d, err:%s", ret, + ERR_error_string(ERR_get_error(), NULL)); + } + if (NULL != sign_pkey) { + EVP_PKEY_free(sign_pkey); + } + if (NULL != enc_pkey) { + EVP_PKEY_free(enc_pkey); + } + if (NULL != sign_x509) { + X509_free(sign_x509); + } + if (NULL != enc_x509) { + X509_free(enc_x509); + } + return ret; +} +#endif static int ob_ssl_load_cert_and_pkey_for_intl_memory(SSL_CTX *ctx, const ssl_config_item_t *ssl_config) @@ -284,6 +369,22 @@ static int ob_ssl_load_cert_and_pkey(SSL_CTX *ctx, const ssl_config_item_t *ssl_ int ret = 0; if (ssl_config->is_from_file) { if (ssl_config->is_sm) { +#ifdef OB_USE_BABASSL + if (!SSL_CTX_use_sign_PrivateKey_file(ctx, ssl_config->sign_private_key, SSL_FILETYPE_PEM)) { + ret = EINVAL; + ussl_log_warn("SSL_CTX_use_sign_PrivateKey_file failed,ret:%d", ret); + } else if (!SSL_CTX_use_sign_certificate_file(ctx, ssl_config->sign_cert, SSL_FILETYPE_PEM)) { + ret = EINVAL; + ussl_log_warn("SSL_CTX_use_sign_certificate_file failed, ret:%d", ret); + } else if (!SSL_CTX_use_enc_PrivateKey_file(ctx, ssl_config->enc_private_key, + SSL_FILETYPE_PEM)) { + ret = EINVAL; + ussl_log_warn("SSL_CTX_use_enc_PrivateKey_file failed, ret:%d", ret); + } else if (!SSL_CTX_use_enc_certificate_file(ctx, ssl_config->enc_cert, SSL_FILETYPE_PEM)) { + ret = EINVAL; + ussl_log_warn("SSL_CTX_use_enc_certificate_file failed, ret:%d", ret); + } +#endif } else { if (SSL_CTX_use_certificate_chain_file(ctx, ssl_config->sign_cert) <= 0) { ret = EINVAL; @@ -299,6 +400,11 @@ static int ob_ssl_load_cert_and_pkey(SSL_CTX *ctx, const ssl_config_item_t *ssl_ } } else { if (ssl_config->is_sm) { +#ifdef OB_USE_BABASSL + if (0 != (ret = ob_ssl_load_cert_and_pkey_for_sm_memory(ctx, ssl_config))) { + ussl_log_warn("ob_ssl_load_cert_and_pkey_for_sm_memory failed, ret:%d", ret); + } +#endif } else { if (0 != (ret = ob_ssl_load_cert_and_pkey_for_intl_memory(ctx, ssl_config))) { ussl_log_warn("ob_ssl_load_cert_and_pkey_for_intl_memory failed, ret:%d", ret); @@ -327,6 +433,12 @@ static SSL_CTX *ob_ssl_create_ssl_ctx(const ssl_config_item_t *ssl_config, int t } if (ssl_config->is_sm) { +#ifdef OB_USE_BABASSL + ctx = SSL_CTX_new(NTLS_method()); + if (NULL != ctx) { + SSL_CTX_enable_ntls(ctx); + } +#endif } else { ctx = SSL_CTX_new(SSLv23_method()); } diff --git a/mittest/mtlenv/storage/CMakeLists.txt b/mittest/mtlenv/storage/CMakeLists.txt index 1d7524fc4..f0f2c75e5 100644 --- a/mittest/mtlenv/storage/CMakeLists.txt +++ b/mittest/mtlenv/storage/CMakeLists.txt @@ -8,6 +8,9 @@ storage_dml_unittest(test_tablet_status_cache test_tablet_status_cache.cpp) storage_dml_unittest(test_tablet_member_load_and_free test_tablet_member_load_and_free.cpp) storage_dml_unittest(test_ls_migration_param test_ls_migration_param.cpp) storage_dml_unittest(test_ls_tablet_service test_ls_tablet_service.cpp) +if(OB_BUILD_CLOSE_MODULES) + storage_unittest(test_memtable test_memtable_v2.cpp) +endif() #storage_dml_unittest(test_lob_manager test_lob_manager.cpp) storage_unittest(test_trans test_trans.cpp) #storage_unittest(test_ls_restore_task_mgr) diff --git a/mittest/mtlenv/test_tx_data_table.cpp b/mittest/mtlenv/test_tx_data_table.cpp index ae0f12420..127506a06 100644 --- a/mittest/mtlenv/test_tx_data_table.cpp +++ b/mittest/mtlenv/test_tx_data_table.cpp @@ -37,7 +37,7 @@ static int64_t const_data_num; int64_t tx_data_num CACHE_ALIGNED = 0; int64_t inserted_cnt = 0; -share::SCN insert_start_scn = share::SCN::min_scn(); +oceanbase::share::SCN insert_start_scn = oceanbase::share::SCN::min_scn(); const int64_t ONE_SEC_NS = 1000LL * 1000LL * 1000LL; const int64_t MOD_NS = 1000LL * ONE_SEC_NS; diff --git a/mittest/simple_server/errsim/storage_ha/errsim_test_transfer_handler.cpp b/mittest/simple_server/errsim/storage_ha/errsim_test_transfer_handler.cpp index 4bfbb59e7..5ef893cbc 100644 --- a/mittest/simple_server/errsim/storage_ha/errsim_test_transfer_handler.cpp +++ b/mittest/simple_server/errsim/storage_ha/errsim_test_transfer_handler.cpp @@ -27,10 +27,9 @@ #include "storage/tablet/ob_tablet.h" #include "storage/tx_storage/ob_ls_service.h" -using namespace unittest; - namespace oceanbase { +using namespace unittest; namespace storage { using namespace share::schema; @@ -1121,7 +1120,7 @@ TEST_F(TestTransferHandler, doing_commin_trans_failed) } // namespace oceanbase int main(int argc, char **argv) { - unittest::init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/mittest/simple_server/test_add_remove_replace_arbitration_service.cpp b/mittest/simple_server/test_add_remove_replace_arbitration_service.cpp index 84eeef11a..a603fddaa 100644 --- a/mittest/simple_server/test_add_remove_replace_arbitration_service.cpp +++ b/mittest/simple_server/test_add_remove_replace_arbitration_service.cpp @@ -213,7 +213,7 @@ TEST_F(TestAddRemoveReplaceArbitrationService, test_add_remove_replace) int main(int argc, char **argv) { - init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/mittest/simple_server/test_all_virtual_proxy_partition_info_default_value.cpp b/mittest/simple_server/test_all_virtual_proxy_partition_info_default_value.cpp index 4e26f1cd6..b3e706713 100644 --- a/mittest/simple_server/test_all_virtual_proxy_partition_info_default_value.cpp +++ b/mittest/simple_server/test_all_virtual_proxy_partition_info_default_value.cpp @@ -300,7 +300,7 @@ TEST_F(TestProxyDefaultValue, test_default_value_is_null) int main(int argc, char **argv) { - unittest::init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/mittest/simple_server/test_arbitration_service_replica_task_table_operator.cpp b/mittest/simple_server/test_arbitration_service_replica_task_table_operator.cpp index 9d781aa32..2fe5ad8f8 100644 --- a/mittest/simple_server/test_arbitration_service_replica_task_table_operator.cpp +++ b/mittest/simple_server/test_arbitration_service_replica_task_table_operator.cpp @@ -350,7 +350,7 @@ TEST_F(TestArbitrationServiceReplicaTaskTableOperator, test_operator) int main(int argc, char **argv) { - init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/mittest/simple_server/test_arbitration_service_rpc.cpp b/mittest/simple_server/test_arbitration_service_rpc.cpp index ffae68d76..72519edfe 100644 --- a/mittest/simple_server/test_arbitration_service_rpc.cpp +++ b/mittest/simple_server/test_arbitration_service_rpc.cpp @@ -93,7 +93,7 @@ TEST_F(TestArbitrationServiceRpc, test_argument) int main(int argc, char **argv) { - init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/mittest/simple_server/test_arbitration_service_table_operator.cpp b/mittest/simple_server/test_arbitration_service_table_operator.cpp index d54b1e6d8..3b1ec6c6d 100644 --- a/mittest/simple_server/test_arbitration_service_table_operator.cpp +++ b/mittest/simple_server/test_arbitration_service_table_operator.cpp @@ -204,7 +204,7 @@ TEST_F(TestArbitrationServiceTableOperator, test_single) int main(int argc, char **argv) { - init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/mittest/simple_server/test_change_arb_service_status.cpp b/mittest/simple_server/test_change_arb_service_status.cpp index d21133ccc..804c1460a 100644 --- a/mittest/simple_server/test_change_arb_service_status.cpp +++ b/mittest/simple_server/test_change_arb_service_status.cpp @@ -134,7 +134,7 @@ TEST_F(TestChangeArbServiceStatus, test_change_arb_service_status) int main(int argc, char **argv) { - init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/mittest/simple_server/test_create_tenant_with_arbitration_service.cpp b/mittest/simple_server/test_create_tenant_with_arbitration_service.cpp index 4fdb76885..98215768b 100644 --- a/mittest/simple_server/test_create_tenant_with_arbitration_service.cpp +++ b/mittest/simple_server/test_create_tenant_with_arbitration_service.cpp @@ -155,7 +155,7 @@ TEST_F(TestAddRemoveReplaceArbitrationService, test_add_remove_replace) int main(int argc, char **argv) { - init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/mittest/simple_server/test_get_stopped_zone_list.cpp b/mittest/simple_server/test_get_stopped_zone_list.cpp index 57e7d39f5..2c2bb8482 100644 --- a/mittest/simple_server/test_get_stopped_zone_list.cpp +++ b/mittest/simple_server/test_get_stopped_zone_list.cpp @@ -95,7 +95,7 @@ TEST_F(TestGetStoppedZoneList, GetStoppedZoneList) } // oceanbase int main(int argc, char **argv) { - init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/mittest/simple_server/test_location_service.cpp b/mittest/simple_server/test_location_service.cpp index 5b05c5d2d..bfeefa449 100644 --- a/mittest/simple_server/test_location_service.cpp +++ b/mittest/simple_server/test_location_service.cpp @@ -20,10 +20,9 @@ #include "lib/ob_errno.h" #include "share/location_cache/ob_location_service.h" // ObLocationService -using namespace unittest; - namespace oceanbase { +using namespace unittest; namespace share { using namespace common; @@ -306,7 +305,7 @@ TEST_F(TestLocationService, test_check_ls_exist) } // namespace oceanbase int main(int argc, char **argv) { - unittest::init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/mittest/simple_server/test_lock_table_persistence.cpp b/mittest/simple_server/test_lock_table_persistence.cpp index 3b88535d1..228ab0fc1 100644 --- a/mittest/simple_server/test_lock_table_persistence.cpp +++ b/mittest/simple_server/test_lock_table_persistence.cpp @@ -30,8 +30,8 @@ static const char *TEST_FILE_NAME = "test_lock_table_persistence"; static const char *BORN_CASE_NAME = "ObLockTableBeforeRestartTest"; static const char *RESTART_CASE_NAME = "ObLockTableAfterRestartTest"; -static share::SCN lock_scn; -static share::SCN unlock_scn; +static oceanbase::share::SCN lock_scn; +static oceanbase::share::SCN unlock_scn; namespace oceanbase { diff --git a/mittest/simple_server/test_ls_replica.cpp b/mittest/simple_server/test_ls_replica.cpp index 4a4dcf0b9..af1f05375 100644 --- a/mittest/simple_server/test_ls_replica.cpp +++ b/mittest/simple_server/test_ls_replica.cpp @@ -232,7 +232,7 @@ TEST_F(TestLSReplica, test_text2learnerlist) int main(int argc, char **argv) { - init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/mittest/simple_server/test_ls_status_operator.cpp b/mittest/simple_server/test_ls_status_operator.cpp index 9b7c9bc69..904d8e287 100644 --- a/mittest/simple_server/test_ls_status_operator.cpp +++ b/mittest/simple_server/test_ls_status_operator.cpp @@ -300,7 +300,7 @@ TEST_F(TestLSStatusOperator, add_tenant) int main(int argc, char **argv) { - init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/mittest/simple_server/test_ob_tablet_to_ls_operator.cpp b/mittest/simple_server/test_ob_tablet_to_ls_operator.cpp index 37e772dd8..16fc588a4 100644 --- a/mittest/simple_server/test_ob_tablet_to_ls_operator.cpp +++ b/mittest/simple_server/test_ob_tablet_to_ls_operator.cpp @@ -442,7 +442,7 @@ TEST_F(TestTabletToLSOperatorBatchGet, test_batch_get) int main(int argc, char **argv) { - init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/mittest/simple_server/test_schema_service_sql_impl.cpp b/mittest/simple_server/test_schema_service_sql_impl.cpp index d1d5f94ab..1b6f57674 100644 --- a/mittest/simple_server/test_schema_service_sql_impl.cpp +++ b/mittest/simple_server/test_schema_service_sql_impl.cpp @@ -20,10 +20,9 @@ #include "lib/ob_errno.h" #include "share/schema/ob_schema_service_sql_impl.h" -using namespace unittest; - namespace oceanbase { +using namespace unittest; namespace share { using namespace share::schema; @@ -194,7 +193,7 @@ TEST_F(TestSchemaServiceSqlImpl, test_get_table_latest_schema_versions) } // namespace oceanbase int main(int argc, char **argv) { - unittest::init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/mittest/simple_server/test_standby_balance_ls_group.cpp b/mittest/simple_server/test_standby_balance_ls_group.cpp index 6fd7f7959..c3a62a8b8 100644 --- a/mittest/simple_server/test_standby_balance_ls_group.cpp +++ b/mittest/simple_server/test_standby_balance_ls_group.cpp @@ -192,7 +192,7 @@ TEST_F(TestStandbyBalance, BalanceLSGroup) int main(int argc, char **argv) { - init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/mittest/simple_server/test_tenant_balance_operator.cpp b/mittest/simple_server/test_tenant_balance_operator.cpp index 68aaacdd8..32c756a61 100644 --- a/mittest/simple_server/test_tenant_balance_operator.cpp +++ b/mittest/simple_server/test_tenant_balance_operator.cpp @@ -602,7 +602,7 @@ TEST_F(TestBalanceOperator, ls_balance_helper) int main(int argc, char **argv) { - init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/mittest/simple_server/test_tenant_transfer_service.cpp b/mittest/simple_server/test_tenant_transfer_service.cpp index d27a29dee..bcb25b8e3 100644 --- a/mittest/simple_server/test_tenant_transfer_service.cpp +++ b/mittest/simple_server/test_tenant_transfer_service.cpp @@ -22,10 +22,9 @@ #include "share/transfer/ob_transfer_task_operator.h" // ObTransferTaskOperator #include "lib/allocator/page_arena.h" -using namespace unittest; - namespace oceanbase { +using namespace unittest; namespace rootserver { using namespace share::schema; @@ -389,7 +388,7 @@ TEST_F(TestTenantTransferService, test_offline_ddl_hidden_table) } // namespace oceanbase int main(int argc, char **argv) { - unittest::init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/mittest/simple_server/test_transfer_task_operator.cpp b/mittest/simple_server/test_transfer_task_operator.cpp index 10e43e4ee..9302b447f 100644 --- a/mittest/simple_server/test_transfer_task_operator.cpp +++ b/mittest/simple_server/test_transfer_task_operator.cpp @@ -18,10 +18,9 @@ #include "lib/ob_errno.h" #include "share/transfer/ob_transfer_task_operator.h" -using namespace unittest; - namespace oceanbase { +using namespace unittest; namespace share { using namespace schema; @@ -45,7 +44,7 @@ public: share::SCN start_scn_; share::SCN finish_scn_; ObTransferStatus status_; - TraceId trace_id_; + ObCurTraceId::TraceId trace_id_; ObTransferTask task_; transaction::tablelock::ObTableLockOwnerID lock_owner_id_; }; @@ -382,7 +381,7 @@ TEST_F(TestTransferTaskOperator, test_operator) } // namespace oceanbase int main(int argc, char **argv) { - unittest::init_log_and_gtest(argc, argv); + oceanbase::unittest::init_log_and_gtest(argc, argv); OB_LOGGER.set_log_level("INFO"); ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6ffc09544..2c1bc967a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,6 +2,7 @@ add_library(ob_base INTERFACE) target_include_directories( ob_base INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}) + target_compile_features(ob_base INTERFACE cxx_std_11) target_link_libraries(ob_base INTERFACE oblib_base objit_base ${OB_RELRO_FLAG} -Wl,-T,${CMAKE_SOURCE_DIR}/rpm/ld.lds) diff --git a/src/logservice/CMakeLists.txt b/src/logservice/CMakeLists.txt index 899e0bf9e..0d0eedf73 100644 --- a/src/logservice/CMakeLists.txt +++ b/src/logservice/CMakeLists.txt @@ -49,6 +49,8 @@ ob_set_subtarget(ob_logservice common ob_server_log_block_mgr.cpp ob_log_flashback_service.cpp ob_net_keepalive_adapter.cpp + ob_log_external_storage_handler.cpp + ob_log_external_storage_io_task.cpp ob_log_monitor.cpp ob_log_external_storage_handler.cpp ob_log_external_storage_io_task.cpp @@ -186,7 +188,6 @@ ob_set_subtarget(ob_logservice restoreservice restoreservice/ob_remote_log_writer.cpp ) - ob_set_subtarget(ob_logservice data_dictionary data_dictionary/ob_data_dict_utils.cpp data_dictionary/ob_data_dict_struct.cpp diff --git a/src/logservice/leader_coordinator/election_priority_impl/election_priority_v1.cpp b/src/logservice/leader_coordinator/election_priority_impl/election_priority_v1.cpp index ef2539279..79d94f7b4 100644 --- a/src/logservice/leader_coordinator/election_priority_impl/election_priority_v1.cpp +++ b/src/logservice/leader_coordinator/election_priority_impl/election_priority_v1.cpp @@ -189,6 +189,9 @@ int PriorityV1::refresh_(const share::ObLSID &ls_id) LsElectionReferenceInfo election_reference_info; SCN scn = SCN::min_scn(); if (observer::ObServer::get_instance().is_arbitration_mode()) { +#ifdef OB_BUILD_ARBITRATION + ret = OB_NO_NEED_UPDATE; +#endif } else if (OB_ISNULL(coordinator) || OB_ISNULL(detector)) { ret = OB_ERR_UNEXPECTED; COORDINATOR_LOG_(ERROR, "unexpected nullptr"); diff --git a/src/logservice/libobcdc/src/CMakeLists.txt b/src/logservice/libobcdc/src/CMakeLists.txt index 572cfb281..277cba392 100644 --- a/src/logservice/libobcdc/src/CMakeLists.txt +++ b/src/logservice/libobcdc/src/CMakeLists.txt @@ -1,7 +1,13 @@ add_library(obcdc_msg STATIC IMPORTED) -set_target_properties(obcdc_msg PROPERTIES - IMPORTED_LOCATION ${DEP_DIR}/lib/liboblogmsg.a) -set_property(GLOBAL PROPERTY CDC_MSG_HEADER_DIR ${DEP_DIR}/include/oblogmsg) +if (OB_USE_DRCMSG) + set_target_properties(obcdc_msg PROPERTIES + IMPORTED_LOCATION ${DEP_DIR}/lib/libdrcmsg.a) + set_property(GLOBAL PROPERTY CDC_MSG_HEADER_DIR ${DEP_DIR}/include/drcmsg) +else() + set_target_properties(obcdc_msg PROPERTIES + IMPORTED_LOCATION ${DEP_DIR}/lib/liboblogmsg.a) + set_property(GLOBAL PROPERTY CDC_MSG_HEADER_DIR ${DEP_DIR}/include/oblogmsg) +endif() add_library(rocksdb STATIC IMPORTED) set_target_properties(rocksdb PROPERTIES @@ -161,6 +167,7 @@ else() -Wl,-e,so_main easy ) + if(OB_BUILD_OPENSOURCE) set(LGPL_DEPS "-L${DEP_DIR}/lib/mariadb -lmariadb") if (OB_STATIC_LINK_LGPL_DEPS) set(LGPL_DEPS "-L${DEP_DIR}/lib/mariadb -l:libmariadbclient.a") @@ -169,6 +176,7 @@ else() PRIVATE ${LGPL_DEPS} ) + endif() add_dependencies(obcdc obcdc_objects) endif() set_target_properties(obcdc PROPERTIES diff --git a/src/logservice/libobcdc/src/libobcdc.h b/src/logservice/libobcdc/src/libobcdc.h index 54b8ac315..c3d2c8e67 100644 --- a/src/logservice/libobcdc/src/libobcdc.h +++ b/src/logservice/libobcdc/src/libobcdc.h @@ -27,8 +27,13 @@ #include // FNM_CASEFOLD #include #include +#ifndef OB_USE_DRCMSG #include "oblogmsg/LogRecord.h" typedef oceanbase::logmessage::ILogRecord ICDCRecord; +#else +#include +typedef IBinlogRecord ICDCRecord; +#endif namespace oceanbase { diff --git a/src/logservice/libobcdc/src/ob_log_binlog_record.cpp b/src/logservice/libobcdc/src/ob_log_binlog_record.cpp index 4104f3d92..2d3ce5f7c 100644 --- a/src/logservice/libobcdc/src/ob_log_binlog_record.cpp +++ b/src/logservice/libobcdc/src/ob_log_binlog_record.cpp @@ -14,6 +14,9 @@ #define USING_LOG_PREFIX OBLOG +#ifdef OB_USE_DRCMSG +#include // ITableMeta +#endif #include "ob_log_binlog_record.h" #include "ob_log_utils.h" diff --git a/src/logservice/libobcdc/src/ob_log_binlog_record.h b/src/logservice/libobcdc/src/ob_log_binlog_record.h index 99fa2f6bb..8bd100450 100644 --- a/src/logservice/libobcdc/src/ob_log_binlog_record.h +++ b/src/logservice/libobcdc/src/ob_log_binlog_record.h @@ -15,7 +15,12 @@ #ifndef OCEANBASE_LIBOBCDC_BINLOG_RECORD_ #define OCEANBASE_LIBOBCDC_BINLOG_RECORD_ +#ifndef OB_USE_DRCMSG #include "ob_cdc_msg_convert.h" +#else +#include // IBinlogRecord +#include // createBinlogRecord +#endif #include "lib/queue/ob_link.h" // ObLink #include "share/ob_define.h" diff --git a/src/logservice/libobcdc/src/ob_log_config.h b/src/logservice/libobcdc/src/ob_log_config.h index 25b85d62d..3bde218ed 100644 --- a/src/logservice/libobcdc/src/ob_log_config.h +++ b/src/logservice/libobcdc/src/ob_log_config.h @@ -177,6 +177,7 @@ public: // default value '0:not_skip' T_DEF_BOOL(skip_ob_version_compat_check, OB_CLUSTER_PARAMETER, 0, "0:not_skip, 1:skip") +#ifndef OB_USE_DRCMSG // default DFT_BR(LogRecordImpl), add DFT_BR_PB // passed in via IObLog::init interface // string LogMsgFactory::DFT_ColMeta = "ColMetaImpl"; @@ -185,6 +186,17 @@ public: // string LogMsgFactory::DFT_METAS = "MetaDataCollectionsImpl"; // string LogMsgFactory::DFT_LR = "LogRecordImpl"; DEF_STR(drc_message_factory_binlog_record_type, OB_CLUSTER_PARAMETER, "LogRecordImpl", "LogMsgFactory::DFT_BR"); +#else + // default DFT_BR(BinlogRecordImpl), add DFT_BR_PB + // passed in via IObLog::init interface + // string DRCMessageFactory::DFT_ColMeta = "ColMetaImpl"; + // string DRCMessageFactory::DFT_TableMeta = "TableMetaImpl"; + // string DRCMessageFactory::DFT_DBMeta = "DBMetaImpl"; + // string DRCMessageFactory::DFT_METAS = "MetaDataCollectionsImpl"; + // string DRCMessageFactory::DFT_BR = "BinlogRecordImpl"; + // string DRCMessageFactory::DFT_BR_PB = "BinlogRecordProtobuf"; + DEF_STR(drc_message_factory_binlog_record_type, OB_CLUSTER_PARAMETER, "BinlogRecordImpl", "DRCMessageFactory::DFT_BR"); +#endif // whether to check ObTraceId T_DEF_BOOL(need_verify_ob_trace_id, OB_CLUSTER_PARAMETER, 0, "0:disabled, 1:enabled"); diff --git a/src/logservice/libobcdc/src/ob_log_ddl_processor.h b/src/logservice/libobcdc/src/ob_log_ddl_processor.h index e1eb2e0e5..0093a358b 100644 --- a/src/logservice/libobcdc/src/ob_log_ddl_processor.h +++ b/src/logservice/libobcdc/src/ob_log_ddl_processor.h @@ -15,7 +15,11 @@ #ifndef OCEANBASE_LIBOBCDC_OB_LOG_DDL_PROCESSOR_H__ #define OCEANBASE_LIBOBCDC_OB_LOG_DDL_PROCESSOR_H__ +#ifndef OB_USE_DRCMSG #include "ob_cdc_msg_convert.h" +#else +#include // IBinlogRecord +#endif #include "lib/utility/ob_macro_utils.h" // DISALLOW_COPY_AND_ASSIGN #include "ob_log_utils.h" // _SEC_ diff --git a/src/logservice/libobcdc/src/ob_log_formatter.h b/src/logservice/libobcdc/src/ob_log_formatter.h index 5826bf314..0afce1fd1 100644 --- a/src/logservice/libobcdc/src/ob_log_formatter.h +++ b/src/logservice/libobcdc/src/ob_log_formatter.h @@ -15,7 +15,11 @@ #ifndef OCEANBASE_LIBOBCDC_FORMATTER_H__ #define OCEANBASE_LIBOBCDC_FORMATTER_H__ +#ifndef OB_USE_DRCMSG #include "ob_cdc_msg_convert.h" +#else +#include // binlogBuf +#endif #include "lib/allocator/ob_allocator.h" // ObIAllocator #include "lib/thread/ob_multi_fixed_queue_thread.h" // ObMQThread diff --git a/src/logservice/libobcdc/src/ob_log_meta_manager.h b/src/logservice/libobcdc/src/ob_log_meta_manager.h index 532c09632..014db6ae3 100644 --- a/src/logservice/libobcdc/src/ob_log_meta_manager.h +++ b/src/logservice/libobcdc/src/ob_log_meta_manager.h @@ -15,7 +15,12 @@ #ifndef OCEANBASE_LIBOBCDC_META_MANAGER_H__ #define OCEANBASE_LIBOBCDC_META_MANAGER_H__ +#ifndef OB_USE_DRCMSG #include "ob_cdc_msg_convert.h" +#else +#include // ITableMeta, IDBMeta +#include // DRCMessageFactory +#endif #include "share/ob_errno.h" // OB_SUCCESS #include "lib/lock/ob_spin_rwlock.h" // SpinRWLock, SpinRLockGuard, SpinWLockGuard #include "lib/allocator/page_arena.h" // DefaultPageAllocator diff --git a/src/logservice/libobcdc/src/ob_log_mysql_connector.cpp b/src/logservice/libobcdc/src/ob_log_mysql_connector.cpp index 6d17af075..2679f9611 100644 --- a/src/logservice/libobcdc/src/ob_log_mysql_connector.cpp +++ b/src/logservice/libobcdc/src/ob_log_mysql_connector.cpp @@ -204,6 +204,9 @@ int ObLogMySQLConnector::exec(MySQLQueryBase& query) bool ObLogMySQLConnector::is_oracle_mode() const { bool b_ret = false; +#ifdef OB_USE_DRCMSG + b_ret = OB_NOT_NULL(mysql_) && mysql_->oracle_mode; +#endif return b_ret; } @@ -262,6 +265,17 @@ int ObLogMySQLConnector::init_conn_(const MySQLConnConfig &cfg, K(mysql_error(mysql_)), K(write_timeout)); ret = OB_ERR_UNEXPECTED; } else { +#ifdef OB_BUILD_TDE_SECURITY + if (! enable_ssl_client_authentication) { + int64_t ssl_enforce = 0; + + if (0 != (mysql_options(mysql_, MYSQL_OPT_SSL_ENFORCE, &ssl_enforce))) { + LOG_ERROR("failed to set ssl mode for mysql conn", + K(mysql_error(mysql_)), K(ssl_enforce)); + ret = OB_ERR_UNEXPECTED; + } + } +#endif // CLIENT_MULTI_STATEMENTS: enable multiple-statement execution and multiple-result if (mysql_ != mysql_real_connect(mysql_, diff --git a/src/logservice/libobcdc/src/ob_log_rpc.cpp b/src/logservice/libobcdc/src/ob_log_rpc.cpp index 77769d6c9..f9da73917 100644 --- a/src/logservice/libobcdc/src/ob_log_rpc.cpp +++ b/src/logservice/libobcdc/src/ob_log_rpc.cpp @@ -20,6 +20,9 @@ #include "ob_log_config.h" // ObLogConfig #include "observer/ob_srv_network_frame.h" +#ifdef OB_BUILD_TDE_SECURITY +#include "share/ob_encrypt_kms.h" // ObSSLClient +#endif extern "C" { #include "ussl-hook.h" @@ -281,8 +284,25 @@ int ObLogRpc::reload_ssl_config() private_key = OB_CLIENT_SSL_KEY_FILE; } } else { +#ifndef OB_BUILD_TDE_SECURITY ret = OB_NOT_SUPPORTED; LOG_WARN("only support local file mode", K(ret)); +#else + share::ObSSLClient client; + + if (OB_FAIL(client.init(ssl_config.ptr(), ssl_config.length()))) { + OB_LOG(WARN, "kms client init", K(ret), K(ssl_config)); + } else if (OB_FAIL(client.check_param_valid())) { + OB_LOG(WARN, "kms client param is not valid", K(ret)); + } else { + use_bkmi = client.is_bkmi_mode(); + use_sm = client.is_sm_scene(); + ca_cert = client.get_root_ca().ptr(); + public_cert = client.public_cert_.content_.ptr(); + private_key = client.private_key_.content_.ptr(); + ssl_key_expired_time = client.public_cert_.key_expired_time_; + } +#endif } if (OB_SUCC(ret)) { diff --git a/src/logservice/libobcdc/src/ob_log_utils.h b/src/logservice/libobcdc/src/ob_log_utils.h index 5a2c9f663..d914771f6 100644 --- a/src/logservice/libobcdc/src/ob_log_utils.h +++ b/src/logservice/libobcdc/src/ob_log_utils.h @@ -15,7 +15,14 @@ #ifndef OCEANBASE_LIBOBCDC_UTILS_H__ #define OCEANBASE_LIBOBCDC_UTILS_H__ +#ifndef OB_USE_DRCMSG #include "ob_cdc_msg_convert.h" +#else +#include // RecordType +#include // IStrArray +#include // ITableMeta +#include +#endif #include #include "lib/allocator/ob_allocator.h" // ObIAllocator diff --git a/src/logservice/libobcdc/src/ob_obj2str_helper.cpp b/src/logservice/libobcdc/src/ob_obj2str_helper.cpp index c63c447d5..b077cf359 100644 --- a/src/logservice/libobcdc/src/ob_obj2str_helper.cpp +++ b/src/logservice/libobcdc/src/ob_obj2str_helper.cpp @@ -21,6 +21,9 @@ #include "sql/engine/expr/ob_datum_cast.h" // padding_char_for_cast #include "lib/alloc/ob_malloc_allocator.h" #include "lib/geo/ob_geo_utils.h" +#ifdef OB_BUILD_ORACLE_XML +#include "lib/xml/ob_xml_util.h" +#endif #include "sql/engine/expr/ob_expr_uuid.h" #include "sql/engine/expr/ob_expr_operator.h" #include "sql/engine/expr/ob_expr_res_type_map.h" @@ -452,7 +455,12 @@ int ObObj2strHelper::convert_xmltype_to_text_( common::ObString &str, common::ObIAllocator &allocator) const { +#ifdef OB_BUILD_ORACLE_XML + const ObString &data = obj.get_string(); + return ObXmlUtil::xml_bin_to_text(allocator, data, str); +#else return OB_NOT_SUPPORTED; +#endif } bool ObObj2strHelper::need_padding_(const lib::Worker::CompatMode &compat_mode, diff --git a/src/logservice/libobcdc/tests/ob_binlog_record_printer.h b/src/logservice/libobcdc/tests/ob_binlog_record_printer.h index 5f0e23125..488b7ada1 100644 --- a/src/logservice/libobcdc/tests/ob_binlog_record_printer.h +++ b/src/logservice/libobcdc/tests/ob_binlog_record_printer.h @@ -15,7 +15,14 @@ #ifndef OCEANBASE_LIBOBCDC_TESTS_BINLOG_RECORD_PRINTER_H__ #define OCEANBASE_LIBOBCDC_TESTS_BINLOG_RECORD_PRINTER_H__ +#ifndef OB_USE_DRCMSG #include "ob_cdc_msg_convert.h" +#else +#include // IBinlogRecord +#include // ITableMeta +#include // IStrArray +#include // binlogBuf +#endif #include "share/ob_define.h" // DISALLOW_COPY_AND_ASSIGN #include "ob_log_config.h" // TCONF diff --git a/src/logservice/logfetcher/ob_log_rpc.cpp b/src/logservice/logfetcher/ob_log_rpc.cpp index c921a6039..67374ff4b 100644 --- a/src/logservice/logfetcher/ob_log_rpc.cpp +++ b/src/logservice/logfetcher/ob_log_rpc.cpp @@ -20,6 +20,9 @@ #include "ob_log_config.h" // ObLogFetcherConfig #include "observer/ob_srv_network_frame.h" +#ifdef OB_BUILD_TDE_SECURITY +#include "share/ob_encrypt_kms.h" // ObSSLClient +#endif #include "logservice/data_dictionary/ob_data_dict_utils.h" /// The rpc proxy executes the RPC function with two error codes: @@ -255,8 +258,25 @@ int ObLogRpc::reload_ssl_config() private_key = OB_CLIENT_SSL_KEY_FILE; } } else { +#ifndef OB_BUILD_TDE_SECURITY ret = OB_NOT_SUPPORTED; LOG_WARN("only support local file mode", K(ret)); +#else + share::ObSSLClient client; + + if (OB_FAIL(client.init(ssl_config.ptr(), ssl_config.length()))) { + OB_LOG(WARN, "kms client init", K(ret), K(ssl_config)); + } else if (OB_FAIL(client.check_param_valid())) { + OB_LOG(WARN, "kms client param is not valid", K(ret)); + } else { + use_bkmi = client.is_bkmi_mode(); + use_sm = client.is_sm_scene(); + ca_cert = client.get_root_ca().ptr(); + public_cert = client.public_cert_.content_.ptr(); + private_key = client.private_key_.content_.ptr(); + ssl_key_expired_time = client.public_cert_.key_expired_time_; + } +#endif } if (OB_SUCC(ret)) { diff --git a/src/logservice/logrpc/ob_log_request_handler.cpp b/src/logservice/logrpc/ob_log_request_handler.cpp index 554498f96..8ab1031b5 100755 --- a/src/logservice/logrpc/ob_log_request_handler.cpp +++ b/src/logservice/logrpc/ob_log_request_handler.cpp @@ -98,6 +98,23 @@ int LogRequestHandler::get_rpc_proxy_(obrpc::ObLogServiceRpcProxy *&rpc_proxy) c return ret; } +#ifdef OB_BUILD_ARBITRATION +int LogRequestHandler::get_arb_service_(ObArbitrationService *&arb_service) const +{ + int ret = OB_SUCCESS; + logservice::ObLogService *log_service = NULL; + if (OB_ISNULL(log_service = MTL(logservice::ObLogService*))) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(WARN, "get_log_service failed", K(ret)); + } else if (OB_ISNULL(arb_service = log_service->get_arbitration_service())) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(WARN, "log_service.get_arbitration_service failed", K(ret)); + } else { + CLOG_LOG(TRACE, "get_arb_service_", KP(arb_service), KP(log_service), K(MTL_ID())); + } + return ret; +} +#endif int LogRequestHandler::get_flashback_service_(ObLogFlashbackService *&flashback_srv) const { @@ -236,6 +253,14 @@ int ConfigChangeCmdHandler::handle_config_change_cmd(const LogConfigChangeCmd &r case REMOVE_MEMBER_CMD: ret = palf_handle_->remove_member(req.removed_member_, req.new_replica_num_, req.timeout_us_); break; +#ifdef OB_BUILD_ARBITRATION + case ADD_ARB_MEMBER_CMD: + ret = palf_handle_->add_arb_member(req.added_member_, req.timeout_us_); + break; + case REMOVE_ARB_MEMBER_CMD: + ret = palf_handle_->remove_arb_member(req.removed_member_, req.timeout_us_); + break; +#endif case REPLACE_MEMBER_CMD: ret = palf_handle_->replace_member(req.added_member_, req.removed_member_, req.config_version_, req.timeout_us_); break; @@ -277,6 +302,26 @@ int ConfigChangeCmdHandler::handle_config_change_cmd(const LogConfigChangeCmd &r return ret; } +#ifdef OB_BUILD_ARBITRATION +template <> +int LogRequestHandler::handle_request(const LogServerProbeMsg &req) +{ + int ret = common::OB_SUCCESS; + ObArbitrationService *arb_service; + const common::ObAddr &server = req.src_; + if (false == req.is_valid()) { + ret = OB_INVALID_ARGUMENT; + CLOG_LOG(ERROR, "Invalid argument!!!", K(ret), K(req)); + } else if (OB_FAIL(get_arb_service_(arb_service))) { + CLOG_LOG(ERROR, "get_arb_service_ failed", K(ret), K(req)); + } else if (OB_FAIL(arb_service->handle_server_probe_msg(server, req))) { + CLOG_LOG(WARN, "handle_server_probe_msg failed", K(ret), K(req)); + } else { + CLOG_LOG(TRACE, "handle_server_probe_msg success", K(ret), K(server), K(req)); + } + return ret; +} +#endif template <> int LogRequestHandler::handle_request(const LogChangeAccessModeCmd &req) diff --git a/src/logservice/logrpc/ob_log_request_handler.h b/src/logservice/logrpc/ob_log_request_handler.h index 3c65f5d11..da116d6c3 100644 --- a/src/logservice/logrpc/ob_log_request_handler.h +++ b/src/logservice/logrpc/ob_log_request_handler.h @@ -33,6 +33,9 @@ class ObLogServiceRpcProxy; namespace logservice { +#ifdef OB_BUILD_ARBITRATION +class ObArbitrationService; +#endif class ObLogFlashbackService; class ObLogHandler; class ObLogReplayService; @@ -48,6 +51,9 @@ public: int handle_request(const ReqType &req); private: int get_palf_handle_guard_(const int64_t palf_id, palf::PalfHandleGuard &palf_handle_guard) const; +#ifdef OB_BUILD_ARBITRATION + int get_arb_service_(ObArbitrationService *&arb_service) const; +#endif int get_self_addr_(common::ObAddr &self) const; int get_rpc_proxy_(obrpc::ObLogServiceRpcProxy *&rpc_proxy) const; int get_flashback_service_(ObLogFlashbackService *&flashback_srv) const; diff --git a/src/logservice/logrpc/ob_log_rpc_processor.h b/src/logservice/logrpc/ob_log_rpc_processor.h index f5cb1c460..94537f60a 100644 --- a/src/logservice/logrpc/ob_log_rpc_processor.h +++ b/src/logservice/logrpc/ob_log_rpc_processor.h @@ -104,6 +104,12 @@ DEFINE_LOGSERVICE_SYNC_RPC_PROCESSOR(LogGetPalfStatReqP, LogGetPalfStatResp, obrpc::OB_LOG_GET_PALF_STAT); +#ifdef OB_BUILD_ARBITRATION +DEFINE_LOGSERVICE_RPC_PROCESSOR(LogServerProbeP, + obrpc::ObLogServiceRpcProxy, + LogServerProbeMsg, + obrpc::OB_LOG_ARB_PROBE_MSG); +#endif DEFINE_LOGSERVICE_RPC_PROCESSOR(LogChangeAccessModeP, obrpc::ObLogServiceRpcProxy, diff --git a/src/logservice/logrpc/ob_log_rpc_proxy.h b/src/logservice/logrpc/ob_log_rpc_proxy.h index 5539fb554..3de90997b 100644 --- a/src/logservice/logrpc/ob_log_rpc_proxy.h +++ b/src/logservice/logrpc/ob_log_rpc_proxy.h @@ -33,6 +33,10 @@ public: (logservice::LogChangeAccessModeCmd)); RPC_AP(PR3 send_log_flashback_msg, OB_LOG_FLASHBACK_CMD, (logservice::LogFlashbackMsg)); +#ifdef OB_BUILD_ARBITRATION + RPC_S(PR5 create_arb, OB_CREATE_ARB, (obrpc::ObCreateArbArg), obrpc::ObCreateArbResult); + RPC_S(PR5 delete_arb, OB_DELETE_ARB, (obrpc::ObDeleteArbArg), obrpc::ObDeleteArbResult); +#endif RPC_S(PR3 get_palf_stat, OB_LOG_GET_PALF_STAT, (logservice::LogGetPalfStatReq), logservice::LogGetPalfStatResp); }; diff --git a/src/logservice/logrpc/ob_log_rpc_req.cpp b/src/logservice/logrpc/ob_log_rpc_req.cpp index cf3210f7e..4c10248c2 100644 --- a/src/logservice/logrpc/ob_log_rpc_req.cpp +++ b/src/logservice/logrpc/ob_log_rpc_req.cpp @@ -151,6 +151,9 @@ bool LogConfigChangeCmd::is_valid() const bool LogConfigChangeCmd::is_remove_member_list() const { return REMOVE_MEMBER_CMD == cmd_type_ +#ifdef OB_BUILD_ARBITRATION + || REMOVE_ARB_MEMBER_CMD == cmd_type_ +#endif || REPLACE_MEMBER_CMD == cmd_type_ || SWITCH_TO_LEARNER_CMD == cmd_type_ || REPLACE_MEMBER_WITH_LEARNER_CMD == cmd_type_; @@ -159,6 +162,9 @@ bool LogConfigChangeCmd::is_remove_member_list() const bool LogConfigChangeCmd::is_add_member_list() const { return ADD_MEMBER_CMD == cmd_type_ +#ifdef OB_BUILD_ARBITRATION + || ADD_ARB_MEMBER_CMD == cmd_type_ +#endif || REPLACE_MEMBER_CMD == cmd_type_ || SWITCH_TO_ACCEPTOR_CMD == cmd_type_ || REPLACE_MEMBER_WITH_LEARNER_CMD == cmd_type_; diff --git a/src/logservice/logrpc/ob_log_rpc_req.h b/src/logservice/logrpc/ob_log_rpc_req.h index c516f6858..28dd90ae7 100644 --- a/src/logservice/logrpc/ob_log_rpc_req.h +++ b/src/logservice/logrpc/ob_log_rpc_req.h @@ -30,6 +30,10 @@ enum LogConfigChangeCmdType { CHANGE_REPLICA_NUM_CMD, ADD_MEMBER_CMD, REMOVE_MEMBER_CMD, +#ifdef OB_BUILD_ARBITRATION + ADD_ARB_MEMBER_CMD, + REMOVE_ARB_MEMBER_CMD, +#endif REPLACE_MEMBER_CMD, ADD_LEARNER_CMD, REMOVE_LEARNER_CMD, @@ -50,6 +54,10 @@ inline const char *log_config_change_cmd2str(const LogConfigChangeCmdType state) { CHECK_CMD_TYPE_STR(ADD_MEMBER_CMD); CHECK_CMD_TYPE_STR(REMOVE_MEMBER_CMD); +#ifdef OB_BUILD_ARBITRATION + CHECK_CMD_TYPE_STR(ADD_ARB_MEMBER_CMD); + CHECK_CMD_TYPE_STR(REMOVE_ARB_MEMBER_CMD); +#endif CHECK_CMD_TYPE_STR(REPLACE_MEMBER_CMD); CHECK_CMD_TYPE_STR(ADD_LEARNER_CMD); CHECK_CMD_TYPE_STR(REMOVE_LEARNER_CMD); diff --git a/src/logservice/ob_log_handler.cpp b/src/logservice/ob_log_handler.cpp index dbabe2e54..7af10c229 100755 --- a/src/logservice/ob_log_handler.cpp +++ b/src/logservice/ob_log_handler.cpp @@ -339,6 +339,16 @@ int ObLogHandler::set_initial_member_list(const common::ObMemberList &member_lis return palf_handle_.set_initial_member_list(member_list, paxos_replica_num, learner_list); } +#ifdef OB_BUILD_ARBITRATION +int ObLogHandler::set_initial_member_list(const common::ObMemberList &member_list, + const common::ObMember &arb_member, + const int64_t paxos_replica_num, + const common::GlobalLearnerList &learner_list) +{ + RLockGuard guard(lock_); + return palf_handle_.set_initial_member_list(member_list, arb_member, paxos_replica_num, learner_list); +} +#endif int ObLogHandler::set_election_priority(palf::election::ElectionPriority *priority) { @@ -949,6 +959,199 @@ int ObLogHandler::switch_acceptor_to_learner(const common::ObMember &member, return ret; } +#ifdef OB_BUILD_ARBITRATION +int ObLogHandler::create_arb_member_(const common::ObMember &arb_member, + const int64_t timeout_us) +{ + int ret = OB_SUCCESS; + const int64_t conn_timeout_us = MIN(timeout_us, MIN_CONN_TIMEOUT_US); + int64_t mode_version = -1; + AccessMode access_mode = AccessMode::INVALID_ACCESS_MODE; + if (OB_FAIL(get_access_mode(mode_version, access_mode))) { + CLOG_LOG(WARN, "get_access_mode failed", KR(ret), K_(id), K(access_mode)); + } else if (AccessMode::INVALID_ACCESS_MODE == access_mode) { + ret = OB_ERR_UNEXPECTED; + CLOG_LOG(WARN, "invalid access_mode", KR(ret), K_(id), K(access_mode)); + } else { + share::ObTenantRole::Role role = share::ObTenantRole::Role::PRIMARY_TENANT; + if (AccessMode::APPEND != access_mode) { + role = share::ObTenantRole::Role::STANDBY_TENANT; + } + share::ObTenantRole tenant_role(role); + share::ObLSID ls_id(id_); + obrpc::ObCreateArbArg req; + obrpc::ObCreateArbResult resp; + if (OB_FAIL(req.init(MTL_ID(), ls_id, tenant_role))) { + CLOG_LOG(WARN, "ObCreateArbArg init failed", KR(ret), K_(id), K(arb_member), K(timeout_us)); + } else if (OB_FAIL(rpc_proxy_->to(arb_member.get_server()).timeout(conn_timeout_us).trace_time(true). + max_process_handler_time(timeout_us).by(MTL_ID()).create_arb(req, resp))) { + CLOG_LOG(WARN, "create_arb failed", KR(ret), K_(id), K(arb_member), K(timeout_us)); + } else { + CLOG_LOG(INFO, "create_arb success", KR(ret), K_(id), K(arb_member), K(timeout_us)); + } + } + return ret; +} + +int ObLogHandler::delete_arb_member_(const common::ObMember &arb_member, + const int64_t timeout_us) +{ + int ret = OB_SUCCESS; + const int64_t conn_timeout_us = MIN(timeout_us, MIN_CONN_TIMEOUT_US); + obrpc::ObDeleteArbArg req; + obrpc::ObDeleteArbResult resp; + share::ObLSID ls_id(id_); + if (OB_FAIL(req.init(MTL_ID(), ls_id))) { + CLOG_LOG(WARN, "ObDeleteArbArg init failed", KR(ret), K_(id), K(arb_member), K(timeout_us)); + } else if (OB_FAIL(rpc_proxy_->to(arb_member.get_server()).timeout(conn_timeout_us).trace_time(true). + max_process_handler_time(timeout_us).by(MTL_ID()).delete_arb(req, resp))) { + CLOG_LOG(WARN, "delete arb member failed", KR(ret), K_(id), K(arb_member), K(timeout_us)); + } + return ret; +} + +// @desc: add_arbitration_member interface +// | 1.add_arbitration_member() +// V +// [any_member] ----> [2. Sync create_arb_member] +// | +// [3. Sync LogConfigChangeCmd] -------> [leader] +// | +// [any_member] <----[5. Sync LogConfigChangeCmdResp] <---- | 4. one_stage_config_change_(ADD_ARB_MEMBER) +int ObLogHandler::add_arbitration_member(const common::ObMember &added_member, + const int64_t timeout_us) +{ + int ret = OB_SUCCESS; + const int64_t begin_ts = ObTimeUtility::current_time(); + int64_t current_timeout_us = timeout_us; + const int64_t abs_timeout_us = common::ObTimeUtility::current_time() + timeout_us / 2; + WLockGuardWithTimeout deps_guard(deps_lock_, abs_timeout_us, ret); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else if (OB_FAIL(ret)) { + CLOG_LOG(WARN, "get_lock failed", KR(ret), K_(id), K(added_member), K(timeout_us)); + } else if (is_in_stop_state_) { + ret = OB_NOT_RUNNING; + } else if (!added_member.is_valid() || timeout_us <= 0) { + ret = OB_INVALID_ARGUMENT; + CLOG_LOG(WARN, "invalid argument", KR(ret), K_(id), K(added_member), K(timeout_us)); + } else if (OB_FAIL(create_arb_member_(added_member, current_timeout_us))) { + CLOG_LOG(WARN, "create_arb_member_ failed", KR(ret), K_(id), K(added_member), K(current_timeout_us)); + } else if (FALSE_IT(current_timeout_us -= (ObTimeUtility::current_time() - begin_ts))) { + } else if (current_timeout_us <= 0) { + ret = OB_TIMEOUT; + CLOG_LOG(WARN, "add_arbitration_member tiemout", KR(ret), K_(id), K(added_member), K(timeout_us)); + } else { + common::ObMember dummy_member; + LogConfigChangeCmd req(self_, id_, added_member, dummy_member, 0, ADD_ARB_MEMBER_CMD, current_timeout_us); + if (OB_FAIL(submit_config_change_cmd_(req))) { + CLOG_LOG(WARN, " submit_config_change_cmd failed", KR(ret), K_(id), K(req), K(timeout_us), K(current_timeout_us)); + } else { + CLOG_LOG(INFO, "add_arbitration_member success", KR(ret), K_(id), K(added_member)); + } + } + return ret; +} + +// @desc: remove_arbitration_member interface +// | 1. remove_arbitration_member() +// V +// [any_member] -----[2. Sync LogConfigChangeCmd]----> [leader] +// | +// [4. Sync LogConfigChangeCmdResp] ---- | 3. one_stage_config_change_(REMOVE_ARB_MEMBER) +// | +// [5. Sync delete_arb_member] ----> [any_member] +int ObLogHandler::remove_arbitration_member(const common::ObMember &removed_member, + const int64_t timeout_us) +{ + int ret = OB_SUCCESS; + int64_t current_timeout_us = timeout_us; + const int64_t begin_ts = ObTimeUtility::current_time(); + const int64_t abs_timeout_us = common::ObTimeUtility::current_time() + timeout_us / 2; + WLockGuardWithTimeout deps_guard(deps_lock_, abs_timeout_us, ret); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else if (OB_FAIL(ret)) { + CLOG_LOG(WARN, "get_lock failed", KR(ret), K_(id), K(removed_member), K(timeout_us)); + } else if (is_in_stop_state_) { + ret = OB_NOT_RUNNING; + } else if (!removed_member.is_valid() || timeout_us <= 0) { + ret = OB_INVALID_ARGUMENT; + CLOG_LOG(WARN, "invalid argument", KR(ret), K_(id), K(removed_member), K(timeout_us)); + } else { + common::ObMember dummy_member; + LogConfigChangeCmd req(self_, id_, dummy_member, removed_member, 0, REMOVE_ARB_MEMBER_CMD, current_timeout_us); + if (OB_FAIL(submit_config_change_cmd_(req))) { + CLOG_LOG(WARN, " submit_config_change_cmd failed", KR(ret), K_(id), K(req), K(current_timeout_us)); + } else if (FALSE_IT(current_timeout_us -= (ObTimeUtility::current_time() - begin_ts))) { + } else if (current_timeout_us <= 0) { + ret = OB_TIMEOUT; + CLOG_LOG(WARN, "add_arbitration_member tiemout", KR(ret), K_(id), K(removed_member), K(timeout_us)); + } else if (OB_FAIL(delete_arb_member_(removed_member, current_timeout_us))) { + CLOG_LOG(WARN, "delete_arb_member_ failed", KR(ret), K_(id), K(removed_member), K(current_timeout_us)); + } else { + CLOG_LOG(INFO, "remove_arbitration_member success", KR(ret), K_(id), K(removed_member)); + } + } + return ret; +} + +// @desc: degrade_acceptor_to_learner interface +// | 1.degrade_acceptor_to_learner() +// V +// [leader] +int ObLogHandler::degrade_acceptor_to_learner(const palf::LogMemberAckInfoList °rade_servers, + const int64_t timeout_us) +{ + int ret = OB_SUCCESS; + const int64_t abs_timeout_us = common::ObTimeUtility::current_time() + timeout_us / 2; + WLockGuardWithTimeout deps_guard(deps_lock_, abs_timeout_us, ret); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else if (OB_FAIL(ret)) { + CLOG_LOG(WARN, "get_lock failed", KR(ret), K_(id), K(degrade_servers), K(timeout_us)); + } else if (is_in_stop_state_) { + ret = OB_NOT_RUNNING; + } else if (0 == degrade_servers.count() || + timeout_us <= 0) { + ret = OB_INVALID_ARGUMENT; + CLOG_LOG(WARN, "invalid argument", KR(ret), K_(id), K(degrade_servers), K(timeout_us)); + } else if (OB_FAIL(palf_handle_.degrade_acceptor_to_learner(degrade_servers, timeout_us))) { + CLOG_LOG(WARN, "degrade_acceptor_to_learner failed", KR(ret), K_(id), K(degrade_servers), K(timeout_us)); + } else { + CLOG_LOG(INFO, "degrade_acceptor_to_learner success", KR(ret), K_(id), K(degrade_servers)); + } + return ret; +} + +// @desc: upgrade_learner_to_acceptor interface +// | 1.upgrade_learner_to_acceptor() +// V +// [leader] +int ObLogHandler::upgrade_learner_to_acceptor(const palf::LogMemberAckInfoList &upgrade_servers, + const int64_t timeout_us) +{ + int ret = OB_SUCCESS; + const int64_t abs_timeout_us = common::ObTimeUtility::current_time() + timeout_us / 2; + WLockGuardWithTimeout deps_guard(deps_lock_, abs_timeout_us, ret); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else if (OB_FAIL(ret)) { + CLOG_LOG(WARN, "get_lock failed", KR(ret), K_(id), K(upgrade_servers), K(timeout_us)); + } else if (is_in_stop_state_) { + ret = OB_NOT_RUNNING; + } else if (0 == upgrade_servers.count() || + timeout_us <= 0) { + ret = OB_INVALID_ARGUMENT; + CLOG_LOG(WARN, "invalid argument", KR(ret), K_(id), K(upgrade_servers), K(timeout_us)); + } else if (OB_FAIL(palf_handle_.upgrade_learner_to_acceptor(upgrade_servers, timeout_us))) { + CLOG_LOG(WARN, "upgrade_learner_to_acceptor failed", KR(ret), K_(id), K(upgrade_servers), K(timeout_us)); + } else { + CLOG_LOG(INFO, "upgrade_learner_to_acceptor success", KR(ret), K_(id), K(upgrade_servers)); + } + return ret; +} +#endif int ObLogHandler::try_lock_config_change(const int64_t lock_owner, const int64_t timeout_us) diff --git a/src/logservice/ob_log_handler.h b/src/logservice/ob_log_handler.h index df8965837..c2e912c9c 100755 --- a/src/logservice/ob_log_handler.h +++ b/src/logservice/ob_log_handler.h @@ -88,6 +88,12 @@ public: virtual int set_initial_member_list(const common::ObMemberList &member_list, const int64_t paxos_replica_num, const common::GlobalLearnerList &learner_list) = 0; +#ifdef OB_BUILD_ARBITRATION + virtual int set_initial_member_list(const common::ObMemberList &member_list, + const common::ObMember &arb_member, + const int64_t paxos_replica_num, + const common::GlobalLearnerList &learner_list) = 0; +#endif virtual int set_region(const common::ObRegion ®ion) = 0; virtual int set_election_priority(palf::election::ElectionPriority *priority) = 0; virtual int reset_election_priority() = 0; @@ -141,6 +147,16 @@ public: virtual int switch_acceptor_to_learner(const common::ObMember &member, const int64_t paxos_replica_num, const int64_t timeout_us) = 0; +#ifdef OB_BUILD_ARBITRATION + virtual int add_arbitration_member(const common::ObMember &added_member, + const int64_t timeout_us) = 0; + virtual int remove_arbitration_member(const common::ObMember &removed_member, + const int64_t timeout_us) = 0; + virtual int degrade_acceptor_to_learner(const palf::LogMemberAckInfoList °rade_servers, + const int64_t timeout_us) = 0; + virtual int upgrade_learner_to_acceptor(const palf::LogMemberAckInfoList &upgrade_servers, + const int64_t timeout_us) = 0; +#endif virtual int try_lock_config_change(const int64_t lock_owner, const int64_t timeout_us) = 0; virtual int unlock_config_change(const int64_t lock_owner, const int64_t timeout_us) = 0; virtual int get_config_change_lock_stat(int64_t &lock_owner, bool &is_locked) = 0; @@ -275,6 +291,20 @@ public: int set_initial_member_list(const common::ObMemberList &member_list, const int64_t paxos_replica_num, const common::GlobalLearnerList &learner_list) override final; +#ifdef OB_BUILD_ARBITRATION + // @brief set the initial member list of paxos group which contains arbitration replica + // @param[in] ObMemberList, the initial member list, do not include arbitration replica + // @param[in] ObMember, the arbitration replica + // @param[in] int64_t, the paxos relica num(including arbitration replica) + // @param[in] GlobalLearnerList, the initial learner list + // @retval + // return OB_SUCCESS if success + // else return other errno + int set_initial_member_list(const common::ObMemberList &member_list, + const common::ObMember &arb_member, + const int64_t paxos_replica_num, + const common::GlobalLearnerList &learner_list) override final; +#endif int set_region(const common::ObRegion ®ion) override final; int set_election_priority(palf::election::ElectionPriority *priority) override final; int reset_election_priority() override final; @@ -518,6 +548,54 @@ public: const int64_t new_replica_num, const int64_t timeout_us) override final; +#ifdef OB_BUILD_ARBITRATION + // @brief, add an arbitration member to paxos group + // @param[in] common::ObMember &member: arbitration member which will be added + // @param[in] const int64_t timeout_us: add member timeout, us + // @return + // - OB_SUCCESS: add arbitration member successfully + // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change + // - OB_TIMEOUT: add arbitration member timeout + // - other: bug + int add_arbitration_member(const common::ObMember &added_member, + const int64_t timeout_us) override final; + + // @brief, remove an arbitration member from paxos group + // @param[in] common::ObMember &member: arbitration member which will be removed + // @param[in] const int64_t timeout_us: remove member timeout, us + // @return + // - OB_SUCCESS: remove arbitration member successfully + // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change + // - OB_TIMEOUT: remove arbitration member timeout + // - other: bug + int remove_arbitration_member(const common::ObMember &arb_member, + const int64_t timeout_us) override final; + // @brief: degrade an acceptor(full replica) to learner(special read only replica) in this cluster + // @param[in] const common::ObMemberList &member_list: acceptors will be degraded to learner + // @param[in] const int64_t timeout_us + // @return + // - OB_SUCCESS + // - OB_INVALID_ARGUMENT: invalid argument + // - OB_TIMEOUT: timeout + // - OB_NOT_MASTER: not leader + // - OB_EAGAIN: need retry + // - OB_OP_NOT_ALLOW: can not do degrade because of pre check fails + // if last_ack_ts of any server in LogMemberAckInfoList has changed, can not degrade + int degrade_acceptor_to_learner(const palf::LogMemberAckInfoList °rade_servers, const int64_t timeout_us) override final; + + // @brief: upgrade a learner(special read only replica) to acceptor(full replica) in this cluster + // @param[in] const common::ObMemberList &learner_list: learners will be upgraded to acceptors + // @param[in] const int64_t timeout_us + // @return + // - OB_SUCCESS + // - OB_INVALID_ARGUMENT: invalid argument + // - OB_TIMEOUT: timeout + // - OB_NOT_MASTER: not leader + // - OB_EAGAIN: need retry + // - OB_OB_NOT_ALLOW: can not do upgrade because of pre check fails + // if lsn of any server in LogMemberAckInfoList is less than match_lsn in palf, can not upgrade + int upgrade_learner_to_acceptor(const palf::LogMemberAckInfoList &upgrade_servers, const int64_t timeout_us) override final; +#endif //---------config change lock related--------// //@brief: try lock config change which will forbidden changing on memberlist @@ -611,6 +689,10 @@ private: int submit_config_change_cmd_(const LogConfigChangeCmd &req); int submit_config_change_cmd_(const LogConfigChangeCmd &req, LogConfigChangeCmdResp &resp); +#ifdef OB_BUILD_ARBITRATION + int create_arb_member_(const common::ObMember &arb_member, const int64_t timeout_us); + int delete_arb_member_(const common::ObMember &arb_member, const int64_t timeout_us); +#endif DISALLOW_COPY_AND_ASSIGN(ObLogHandler); private: common::ObAddr self_; diff --git a/src/logservice/ob_log_monitor.cpp b/src/logservice/ob_log_monitor.cpp index a38596df4..fe02612f3 100644 --- a/src/logservice/ob_log_monitor.cpp +++ b/src/logservice/ob_log_monitor.cpp @@ -301,6 +301,33 @@ int ObLogMonitor::add_log_write_stat(const int64_t palf_id, const int64_t log_wr } // =========== PALF Performance Statistic =========== +#ifdef OB_BUILD_ARBITRATION +// =========== Arbitration Event Reporting =========== +#define ARBSRV_MONITOR_EVENT_FMT_PREFIX "ARB", type_to_string_(event), "TENANT_ID", mtl_id, "LS_ID", palf_id +int ObLogMonitor::record_degrade_event(const int64_t palf_id, const char *degraded_list, const char *reasons) +{ + int ret = OB_SUCCESS; + const int64_t mtl_id = MTL_ID(); + const EventType event = EventType::DEGRADE; + SERVER_EVENT_ADD_WITH_RETRY(ARBSRV_MONITOR_EVENT_FMT_PREFIX, + "DEGRADED LIST", degraded_list, + "REASONS", reasons); + return ret; +} + +int ObLogMonitor::record_upgrade_event(const int64_t palf_id, const char *upgraded_list, const char *reasons) +{ + int ret = OB_SUCCESS; + const int64_t mtl_id = MTL_ID(); + const EventType event = EventType::UPGRADE; + SERVER_EVENT_ADD_WITH_RETRY(ARBSRV_MONITOR_EVENT_FMT_PREFIX, + "UPGRADED LIST", upgraded_list, + "REASONS", reasons); + return ret; +} +#undef ARBSRV_MONITOR_EVENT_FMT_PREFIX +// =========== Arbitration Event Reporting =========== +#endif #undef LOG_MONITOR_EVENT_FMT_PREFIX } // end namespace logservice diff --git a/src/logservice/ob_log_monitor.h b/src/logservice/ob_log_monitor.h index d6d0626bc..747579d53 100644 --- a/src/logservice/ob_log_monitor.h +++ b/src/logservice/ob_log_monitor.h @@ -13,6 +13,9 @@ #ifndef OCEANBASE_LOGSERVICE_OB_LOG_MONITOR_H_ #define OCEANBASE_LOGSERVICE_OB_LOG_MONITOR_H_ +#ifdef OB_BUILD_ARBITRATION +#include "logservice/ob_arbitration_service.h" +#endif #include "palf/palf_callback.h" namespace oceanbase @@ -21,6 +24,9 @@ namespace logservice { class ObLogMonitor : public palf::PalfMonitorCb +#ifdef OB_BUILD_ARBITRATION +, public IObArbitrationMonitor +#endif { public: ObLogMonitor() { } @@ -79,6 +85,13 @@ public: // =========== PALF Performance Statistic =========== int add_log_write_stat(const int64_t palf_id, const int64_t log_write_size) override final; // =========== PALF Performance Statistic =========== +#ifdef OB_BUILD_ARBITRATION +public: + // =========== Arbitration Event Reporting =========== + int record_degrade_event(const int64_t palf_id, const char *degraded_list, const char *reasons) override final; + int record_upgrade_event(const int64_t palf_id, const char *upgraded_list, const char *reasons) override final; + // =========== Arbitration Event Reporting =========== +#endif private: enum EventType { diff --git a/src/logservice/ob_log_service.cpp b/src/logservice/ob_log_service.cpp index 0163c7a5b..e43798be8 100644 --- a/src/logservice/ob_log_service.cpp +++ b/src/logservice/ob_log_service.cpp @@ -59,6 +59,9 @@ ObLogService::ObLogService() : ls_adapter_(), rpc_proxy_(), reporter_(), +#ifdef OB_BUILD_ARBITRATION + arb_service_(), +#endif restore_service_(), flashback_service_(), monitor_() @@ -127,6 +130,10 @@ int ObLogService::start() CLOG_LOG(WARN, "failed to start cdc_service_", K(ret)); } else if (OB_FAIL(restore_service_.start())) { CLOG_LOG(WARN, "failed to start restore_service_", K(ret)); +#ifdef OB_BUILD_ARBITRATION + } else if (OB_FAIL(arb_service_.start())) { + CLOG_LOG(WARN, "failed to start arb_service_", K(ret)); +#endif } else { is_running_ = true; FLOG_INFO("ObLogService is started"); @@ -143,6 +150,9 @@ void ObLogService::stop() (void)role_change_service_.stop(); (void)cdc_service_.stop(); (void)restore_service_.stop(); +#ifdef OB_BUILD_ARBITRATION + (void)arb_service_.stop(); +#endif FLOG_INFO("ObLogService is stopped"); } @@ -153,6 +163,9 @@ void ObLogService::wait() role_change_service_.wait(); cdc_service_.wait(); restore_service_.wait(); +#ifdef OB_BUILD_ARBITRATION + arb_service_.wait(); +#endif } void ObLogService::destroy() @@ -167,6 +180,9 @@ void ObLogService::destroy() rpc_proxy_.destroy(); reporter_.destroy(); restore_service_.destroy(); +#ifdef OB_BUILD_ARBITRATION + arb_service_.destroy(); +#endif flashback_service_.destroy(); if (NULL != palf_env_) { PalfEnv::destroy_palf_env(palf_env_); @@ -250,6 +266,10 @@ int ObLogService::init(const PalfOptions &options, CLOG_LOG(WARN, "failed to init cdc_service_", K(ret)); } else if (OB_FAIL(restore_service_.init(transport, ls_service, this))) { CLOG_LOG(WARN, "failed to init restore_service_", K(ret)); +#ifdef OB_BUILD_ARBITRATION + } else if (OB_FAIL(arb_service_.init(self, palf_env_, &rpc_proxy_, net_keepalive_adapter, &monitor_))) { + CLOG_LOG(WARN, "failed to init arb_service_", K(ret), K(self), KP(palf_env_)); +#endif } else if (OB_FAIL(flashback_service_.init(self, &location_adapter_, &rpc_proxy_, sql_proxy))) { CLOG_LOG(WARN, "failed to init flashback_service_", K(ret)); } else { @@ -720,6 +740,22 @@ int ObLogService::diagnose_apply(const share::ObLSID &id, return ret; } +#ifdef OB_BUILD_ARBITRATION +int ObLogService::diagnose_arb_srv(const share::ObLSID &id, + LogArbSrvDiagnoseInfo &diagnose_info) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + CLOG_LOG(WARN, "log_service is not inited", K(ret)); + } else if (OB_FAIL(arb_service_.diagnose(id, diagnose_info))) { + CLOG_LOG(WARN, "arb_service_ diagnose failed", K(ret), K(id)); + } else { + // do nothing + } + return ret; +} +#endif int ObLogService::get_io_start_time(int64_t &last_working_time) { diff --git a/src/logservice/ob_log_service.h b/src/logservice/ob_log_service.h index c81e0800f..15706ab3a 100644 --- a/src/logservice/ob_log_service.h +++ b/src/logservice/ob_log_service.h @@ -25,7 +25,11 @@ #include "rcservice/ob_role_change_service.h" #include "restoreservice/ob_log_restore_service.h" // ObLogRestoreService #include "replayservice/ob_log_replay_service.h" +#ifndef OB_BUILD_ARBITRATION #include "ob_net_keepalive_adapter.h" +#else +#include "logservice/ob_arbitration_service.h" +#endif #include "ob_reporter_adapter.h" #include "ob_ls_adapter.h" #include "ob_location_adapter.h" @@ -200,6 +204,9 @@ public: int diagnose_role_change(RCDiagnoseInfo &diagnose_info); int diagnose_replay(const share::ObLSID &id, ReplayDiagnoseInfo &diagnose_info); int diagnose_apply(const share::ObLSID &id, ApplyDiagnoseInfo &diagnose_info); +#ifdef OB_BUILD_ARBITRATION + int diagnose_arb_srv(const share::ObLSID &id, LogArbSrvDiagnoseInfo &diagnose_info); +#endif int get_io_start_time(int64_t &last_working_time); int check_disk_space_enough(bool &is_disk_enough); @@ -209,6 +216,9 @@ public: cdc::ObCdcService *get_cdc_service() { return &cdc_service_; } ObLogRestoreService *get_log_restore_service() { return &restore_service_; } ObLogReplayService *get_log_replay_service() { return &replay_service_; } +#ifdef OB_BUILD_ARBITRATION + ObArbitrationService *get_arbitration_service() { return &arb_service_; } +#endif obrpc::ObLogServiceRpcProxy *get_rpc_proxy() { return &rpc_proxy_; } ObLogFlashbackService *get_flashback_service() { return &flashback_service_; } private: @@ -235,6 +245,9 @@ private: obrpc::ObLogServiceRpcProxy rpc_proxy_; ObLogReporterAdapter reporter_; cdc::ObCdcService cdc_service_; +#ifdef OB_BUILD_ARBITRATION + ObArbitrationService arb_service_; +#endif ObLogRestoreService restore_service_; ObLogFlashbackService flashback_service_; ObLogMonitor monitor_; diff --git a/src/logservice/palf/election/algorithm/election_impl.cpp b/src/logservice/palf/election/algorithm/election_impl.cpp index 1450164ab..b46585e28 100644 --- a/src/logservice/palf/election/algorithm/election_impl.cpp +++ b/src/logservice/palf/election/algorithm/election_impl.cpp @@ -485,6 +485,15 @@ uint64_t ElectionImpl::get_ls_biggest_min_cluster_version_ever_seen_() const int ret = OB_SUCCESS; uint64_t ls_biggest_min_cluster_version_ever_seen = 0; if (observer::ObServer::get_instance().is_arbitration_mode()) { +#ifdef OB_BUILD_ARBITRATION + if (CLUSTER_CURRENT_VERSION < ls_biggest_min_cluster_version_ever_seen_.version_) { + ret = OB_ERR_UNEXPECTED; + LOG_NONE(ERROR, "ls_biggest_min_cluster_version_ever_seen_ greater than arb binary version"); + } else if (ls_biggest_min_cluster_version_ever_seen_.version_ == 0) { + LOG_NONE(WARN, "ls_biggest_min_cluster_version_ever_seen_ not setted yet"); + } + ls_biggest_min_cluster_version_ever_seen = ls_biggest_min_cluster_version_ever_seen_.version_; +#endif } else { ls_biggest_min_cluster_version_ever_seen = std::max(GET_MIN_CLUSTER_VERSION(), ls_biggest_min_cluster_version_ever_seen_.version_); diff --git a/src/logservice/palf/log_engine.cpp b/src/logservice/palf/log_engine.cpp index d7b5e6186..fac606ebd 100644 --- a/src/logservice/palf/log_engine.cpp +++ b/src/logservice/palf/log_engine.cpp @@ -1008,6 +1008,21 @@ int LogEngine::submit_config_change_pre_check_req(const common::ObAddr &server, return ret; } +#ifdef OB_BUILD_ARBITRATION +int LogEngine::sync_get_arb_member_info(const common::ObAddr &server, + const int64_t timeout_us, + LogGetArbMemberInfoResp &resp) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else { + ret = log_net_service_.submit_get_arb_member_info_req( + server, timeout_us, resp); + } + return ret; +} +#endif int LogEngine::submit_fetch_log_req(const ObAddr &server, const FetchLogType fetch_type, diff --git a/src/logservice/palf/log_engine.h b/src/logservice/palf/log_engine.h index 68334dd22..606e4756f 100644 --- a/src/logservice/palf/log_engine.h +++ b/src/logservice/palf/log_engine.h @@ -330,6 +330,11 @@ public: const bool need_purge_throttling, const int64_t timeout_us, LogGetMCStResp &resp); +#ifdef OB_BUILD_ARBITRATION + virtual int sync_get_arb_member_info(const common::ObAddr &server, + const int64_t timeout_us, + LogGetArbMemberInfoResp &resp); +#endif // @brief: this function used to submit fetch log request to sepcified server // @param[in] server: the address of remote server(data source) diff --git a/src/logservice/palf/log_net_service.cpp b/src/logservice/palf/log_net_service.cpp index 5d045835a..befb30e67 100644 --- a/src/logservice/palf/log_net_service.cpp +++ b/src/logservice/palf/log_net_service.cpp @@ -274,6 +274,22 @@ int LogNetService::submit_config_change_pre_check_req( return ret; } +#ifdef OB_BUILD_ARBITRATION +int LogNetService::submit_get_arb_member_info_req( + const common::ObAddr &server, + const int64_t timeout_us, + LogGetArbMemberInfoResp &resp) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else { + LogGetArbMemberInfoReq req(palf_id_); + ret = post_sync_request_to_server_(server, timeout_us, req, resp); + } + return ret; +} +#endif int LogNetService::submit_register_parent_req( const common::ObAddr &server, diff --git a/src/logservice/palf/log_net_service.h b/src/logservice/palf/log_net_service.h index 27dfa783d..97ffed6a3 100644 --- a/src/logservice/palf/log_net_service.h +++ b/src/logservice/palf/log_net_service.h @@ -199,6 +199,12 @@ public: const int64_t timeout_us, LogGetMCStResp &resp); +#ifdef OB_BUILD_ARBITRATION + int submit_get_arb_member_info_req( + const common::ObAddr &server, + const int64_t timeout_us, + LogGetArbMemberInfoResp &resp); +#endif int submit_register_parent_req( const common::ObAddr &server, diff --git a/src/logservice/palf/log_req.cpp b/src/logservice/palf/log_req.cpp index e6bff4be5..a2ffe897f 100644 --- a/src/logservice/palf/log_req.cpp +++ b/src/logservice/palf/log_req.cpp @@ -772,6 +772,88 @@ void LogGetStatResp::reset() OB_SERIALIZE_MEMBER(LogGetStatResp, max_scn_, end_lsn_); // ================= LogGetStatResp end ================ +#ifdef OB_BUILD_ARBITRATION +LogGetArbMemberInfoReq::LogGetArbMemberInfoReq() + : palf_id_(-1) +{} + +LogGetArbMemberInfoReq::LogGetArbMemberInfoReq(const int64_t palf_id) + : palf_id_(palf_id) +{} + +LogGetArbMemberInfoReq::~LogGetArbMemberInfoReq() +{ + reset(); +} + +bool LogGetArbMemberInfoReq::is_valid() const +{ + return is_valid_palf_id(palf_id_); +} + +void LogGetArbMemberInfoReq::reset() +{ + palf_id_ = -1; +} + +OB_SERIALIZE_MEMBER(LogGetArbMemberInfoReq, palf_id_); + +LogGetArbMemberInfoResp::LogGetArbMemberInfoResp() + : arb_member_info_() +{} + +LogGetArbMemberInfoResp::~LogGetArbMemberInfoResp() +{ + reset(); +} + +bool LogGetArbMemberInfoResp::is_valid() const +{ + return arb_member_info_.is_valid(); +} + +void LogGetArbMemberInfoResp::reset() +{ + arb_member_info_.reset(); +} + +OB_SERIALIZE_MEMBER(LogGetArbMemberInfoResp, arb_member_info_); + +ArbMemberInfo::ArbMemberInfo() + : palf_id_(INVALID_PALF_ID), + arb_server_(), + log_proposal_id_(INVALID_PROPOSAL_ID), + config_version_(), + mode_version_(INVALID_PROPOSAL_ID), + access_mode_(AccessMode::INVALID_ACCESS_MODE), + paxos_member_list_(), + paxos_replica_num_(-1), + arbitration_member_(), + degraded_list_() { } + +bool ArbMemberInfo::is_valid() const +{ + return (arb_server_.is_valid() && palf_id_ != INVALID_PALF_ID); +} + +void ArbMemberInfo::reset() +{ + palf_id_ = INVALID_PALF_ID; + arb_server_.reset(); + log_proposal_id_ = INVALID_PROPOSAL_ID; + config_version_.reset(); + mode_version_ = INVALID_PROPOSAL_ID; + access_mode_ = AccessMode::INVALID_ACCESS_MODE; + paxos_member_list_.reset(); + paxos_replica_num_ = -1; + arbitration_member_.reset(); + degraded_list_.reset(); +} + +OB_SERIALIZE_MEMBER(ArbMemberInfo, palf_id_, arb_server_, log_proposal_id_, + config_version_, mode_version_, access_mode_, paxos_member_list_, + paxos_replica_num_, arbitration_member_, degraded_list_); +#endif LogBatchFetchResp::LogBatchFetchResp() : msg_proposal_id_(INVALID_PROPOSAL_ID), diff --git a/src/logservice/palf/log_req.h b/src/logservice/palf/log_req.h index ae44c2aab..6b0943afd 100644 --- a/src/logservice/palf/log_req.h +++ b/src/logservice/palf/log_req.h @@ -446,6 +446,67 @@ public: LSN end_lsn_; }; +#ifdef OB_BUILD_ARBITRATION +struct LogGetArbMemberInfoReq { + OB_UNIS_VERSION(1); +public: + LogGetArbMemberInfoReq(); + LogGetArbMemberInfoReq(const int64_t palf_id); + ~LogGetArbMemberInfoReq(); + bool is_valid() const; + void reset(); + TO_STRING_KV(K_(palf_id)); + int64_t palf_id_; +}; + +struct ArbMemberInfo { + OB_UNIS_VERSION(1); +public: + ArbMemberInfo(); + ~ArbMemberInfo() { reset(); } + bool is_valid() const; + void reset(); + ArbMemberInfo &operator=(const ArbMemberInfo &other) + { + this->palf_id_ = other.palf_id_; + this->arb_server_ = other.arb_server_; + this->log_proposal_id_ = other.log_proposal_id_; + this->config_version_ = other.config_version_; + this->mode_version_ = other.mode_version_; + this->access_mode_ = other.access_mode_; + this->paxos_member_list_ = other.paxos_member_list_; + this->paxos_replica_num_ = other.paxos_replica_num_; + this->arbitration_member_ = other.arbitration_member_; + this->degraded_list_ = other.degraded_list_; + return *this; + } + TO_STRING_KV(K_(palf_id), K_(arb_server), K_(log_proposal_id), K_(config_version), K_(mode_version), + K_(access_mode), K_(paxos_member_list), K_(paxos_replica_num), K_(arbitration_member), + K_(degraded_list)); +public: + int64_t palf_id_; + common::ObAddr arb_server_; + int64_t log_proposal_id_; + LogConfigVersion config_version_; + int64_t mode_version_; + AccessMode access_mode_; + ObMemberList paxos_member_list_; + int64_t paxos_replica_num_; + common::ObMember arbitration_member_; + common::GlobalLearnerList degraded_list_; +}; + +struct LogGetArbMemberInfoResp { + OB_UNIS_VERSION(1); +public: + LogGetArbMemberInfoResp(); + ~LogGetArbMemberInfoResp(); + bool is_valid() const; + void reset(); + TO_STRING_KV(K_(arb_member_info)); + ArbMemberInfo arb_member_info_; +}; +#endif } // end namespace palf } // end namespace oceanbase diff --git a/src/logservice/palf/log_request_handler.cpp b/src/logservice/palf/log_request_handler.cpp index e84b78396..ec25d2d04 100644 --- a/src/logservice/palf/log_request_handler.cpp +++ b/src/logservice/palf/log_request_handler.cpp @@ -520,6 +520,29 @@ int LogRequestHandler::handle_sync_request( return ret; } +#ifdef OB_BUILD_ARBITRATION +template <> +int LogRequestHandler::handle_sync_request( + const int64_t palf_id, + const ObAddr &server, + const LogGetArbMemberInfoReq &req, + LogGetArbMemberInfoResp &resp) +{ + int ret = common::OB_SUCCESS; + IPalfHandleImplGuard guard; + if (false == is_valid_palf_id(palf_id) || false == req.is_valid()) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(ERROR, "Invalid argument!!!", K(ret), K(palf_id), K(req), KPC(palf_env_impl_)); + } else if (OB_FAIL(palf_env_impl_->get_palf_handle_impl(palf_id, guard))) { + PALF_LOG(WARN, "PalfEnvImpl get_palf_handle_impl failed", K(ret), K(palf_id), K(server)); + } else if (OB_FAIL(guard.get_palf_handle_impl()->get_arb_member_info(resp.arb_member_info_))) { + PALF_LOG(WARN, "get_arb_member_info failed", K(ret), K(palf_id), K(server)); + } else { + PALF_LOG(INFO, "get_arb_member_info succ", K(ret), K(palf_id), K(server), K(req), K(resp)); + } + return ret; +} +#endif } // end namespace palf } // end namespace oceanbase diff --git a/src/logservice/palf/log_rpc_processor.h b/src/logservice/palf/log_rpc_processor.h index 0f14eb65c..5018d9fe7 100644 --- a/src/logservice/palf/log_rpc_processor.h +++ b/src/logservice/palf/log_rpc_processor.h @@ -149,6 +149,13 @@ DEFINE_SYNC_RPC_PROCESSOR(LogGetStatP, LogGetStatResp, obrpc::OB_LOG_GET_STAT); +#ifdef OB_BUILD_ARBITRATION +DEFINE_SYNC_RPC_PROCESSOR(ObRpcGetArbMemberInfoP, + obrpc::LogRpcProxyV2, + LogGetArbMemberInfoReq, + LogGetArbMemberInfoResp, + obrpc::OB_LOG_GET_ARB_MEMBER_INFO); +#endif } // end namespace palf } // end namespace oceanbase diff --git a/src/logservice/palf/log_rpc_proxy.cpp b/src/logservice/palf/log_rpc_proxy.cpp index 91758ee83..decf9f4bc 100644 --- a/src/logservice/palf/log_rpc_proxy.cpp +++ b/src/logservice/palf/log_rpc_proxy.cpp @@ -112,5 +112,10 @@ DEFINE_SYNC_RPC_PROXY_POST_FUNCTION(get_mc_st, DEFINE_SYNC_RPC_PROXY_POST_FUNCTION(get_log_stat, LogGetStatReq, LogGetStatResp); +#ifdef OB_BUILD_ARBITRATION +DEFINE_SYNC_RPC_PROXY_POST_FUNCTION(get_remote_arb_member_info, + LogGetArbMemberInfoReq, + LogGetArbMemberInfoResp); +#endif } // end namespace obrpc } // end namespace oceanbase diff --git a/src/logservice/palf/log_rpc_proxy.h b/src/logservice/palf/log_rpc_proxy.h index c61feb861..cbf29719a 100644 --- a/src/logservice/palf/log_rpc_proxy.h +++ b/src/logservice/palf/log_rpc_proxy.h @@ -105,6 +105,13 @@ public: LogGetStatReq, LogGetStatResp, OB_LOG_GET_STAT); +#ifdef OB_BUILD_ARBITRATION + DECLARE_SYNC_RPC_PROXY_POST_FUNCTION(PR5, + get_remote_arb_member_info, + LogGetArbMemberInfoReq, + LogGetArbMemberInfoResp, + OB_LOG_GET_ARB_MEMBER_INFO); +#endif }; } // end namespace obrpc } // end namespace oceanbase diff --git a/src/logservice/palf/palf_handle.cpp b/src/logservice/palf/palf_handle.cpp index cd40d1efb..c89615b9d 100755 --- a/src/logservice/palf/palf_handle.cpp +++ b/src/logservice/palf/palf_handle.cpp @@ -86,6 +86,28 @@ int PalfHandle::set_initial_member_list(const common::ObMemberList &member_list, return palf_handle_impl_->set_initial_member_list(member_list, paxos_replica_num, learner_list); } +#ifdef OB_BUILD_ARBITRATION +int PalfHandle::set_initial_member_list(const common::ObMemberList &member_list, + const common::ObMember &arb_member, + const int64_t paxos_replica_num, + const common::GlobalLearnerList &learner_list) +{ + CHECK_VALID; + return palf_handle_impl_->set_initial_member_list(member_list, arb_member, paxos_replica_num, learner_list); +} + +int PalfHandle::get_remote_arb_member_info(ArbMemberInfo &arb_member_info) const +{ + CHECK_VALID; + return palf_handle_impl_->get_remote_arb_member_info(arb_member_info); +} + +int PalfHandle::get_arbitration_member(common::ObMember &arb_member) const +{ + CHECK_VALID; + return palf_handle_impl_->get_arbitration_member(arb_member); +} +#endif int PalfHandle::set_region(const common::ObRegion ®ion) { @@ -408,6 +430,35 @@ int PalfHandle::replace_member_with_learner(const common::ObMember &added_member return palf_handle_impl_->replace_member_with_learner(added_member, removed_member, config_version, timeout_us); } +#ifdef OB_BUILD_ARBITRATION +int PalfHandle::add_arb_member(const common::ObMember &member, + const int64_t timeout_us) +{ + CHECK_VALID; + return palf_handle_impl_->add_arb_member(member, timeout_us); +} + +int PalfHandle::remove_arb_member(const common::ObMember &member, + const int64_t timeout_us) +{ + CHECK_VALID; + return palf_handle_impl_->remove_arb_member(member, timeout_us); +} + +int PalfHandle::degrade_acceptor_to_learner(const LogMemberAckInfoList °rade_servers, + const int64_t timeout_us) +{ + CHECK_VALID; + return palf_handle_impl_->degrade_acceptor_to_learner(degrade_servers, timeout_us); +} + +int PalfHandle::upgrade_learner_to_acceptor(const LogMemberAckInfoList &upgrade_servers, + const int64_t timeout_us) +{ + CHECK_VALID; + return palf_handle_impl_->upgrade_learner_to_acceptor(upgrade_servers, timeout_us); +} +#endif int PalfHandle::change_leader_to(const common::ObAddr &dst_addr) { diff --git a/src/logservice/palf/palf_handle.h b/src/logservice/palf/palf_handle.h index 7af18e1e5..bff9f7092 100755 --- a/src/logservice/palf/palf_handle.h +++ b/src/logservice/palf/palf_handle.h @@ -65,6 +65,21 @@ public: int set_initial_member_list(const common::ObMemberList &member_list, const int64_t paxos_replica_num, const common::GlobalLearnerList &learner_list); +#ifdef OB_BUILD_ARBITRATION + // @brief set the initial member list of paxos group which contains + // arbitration replica after creating palf successfully, + // it can only be called once + // @param[in] ObMemberList, the initial member list, do not include arbitration replica + // @param[in] ObMember, the arbitration replica + // @param[in] int64_t, the paxos relica num(including arbitration replica) + // @retval + // return OB_SUCCESS if success + // else return other errno + int set_initial_member_list(const common::ObMemberList &member_list, + const common::ObMember &arb_member, + const int64_t paxos_replica_num, + const common::GlobalLearnerList &learner_list); +#endif int set_region(const common::ObRegion ®ion); int set_paxos_member_region_map(const common::ObArrayHashMap ®ion_map); //================ 文件访问相关接口 ======================= @@ -336,6 +351,72 @@ public: const common::ObMember &removed_member, const LogConfigVersion &config_version, const int64_t timeout_us); +#ifdef OB_BUILD_ARBITRATION + + // @brief, add an arbitration member to paxos group + // @param[in] common::ObMember &member: arbitration member which will be added + // @param[in] const int64_t timeout_us: add member timeout, us + // @return + // - OB_SUCCESS: add arbitration member successfully + // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change + // - OB_TIMEOUT: add arbitration member timeout + // - OB_NOT_MASTER: not leader or rolechange during membership changing + // - other: bug + int add_arb_member(const common::ObMember &added_member, + const int64_t timeout_us); + + // @brief, remove an arbitration member from paxos group + // @param[in] common::ObMember &member: arbitration member which will be removed + // @param[in] const int64_t timeout_us: remove member timeout, us + // @return + // - OB_SUCCESS: remove arbitration member successfully + // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change + // - OB_TIMEOUT: remove arbitration member timeout + // - OB_NOT_MASTER: not leader or rolechange during membership changing + // - other: bug + int remove_arb_member(const common::ObMember &arb_member, + const int64_t timeout_us); + // @brief: degrade an acceptor(full replica) to learner(special read only replica) in this cluster + // @param[in] const common::ObMemberList &member_list: acceptors will be degraded to learner + // @param[in] const int64_t timeout_us + // @return + // - OB_SUCCESS + // - OB_INVALID_ARGUMENT: invalid argument + // - OB_TIMEOUT: timeout + // - OB_NOT_MASTER: not leader + // - OB_EAGAIN: need retry + // - OB_OP_NOT_ALLOW: can not do degrade because of pre check fails + // if last_ack_ts of any server in LogMemberAckInfoList has changed, can not degrade + int degrade_acceptor_to_learner(const LogMemberAckInfoList °rade_servers, const int64_t timeout_us); + + // @brief: upgrade a learner(special read only replica) to acceptor(full replica) in this cluster + // @param[in] const common::ObMemberList &learner_list: learners will be upgraded to acceptors + // @param[in] const int64_t timeout_us + // @return + // - OB_SUCCESS + // - OB_INVALID_ARGUMENT: invalid argument + // - OB_TIMEOUT: timeout + // - OB_NOT_MASTER: not leader + // - OB_EAGAIN: need retry + // - OB_OB_NOT_ALLOW: can not do upgrade because of pre check fails + // if lsn of any server in LogMemberAckInfoList is less than match_lsn in palf, can not upgrade + int upgrade_learner_to_acceptor(const LogMemberAckInfoList &upgrade_servers, const int64_t timeout_us); + // @brief: get arbitration member info from arbitration service by sync rpc. + // @param[in] ArbMemberInfo &arb_member_info: arbitration member info + // @return + // - OB_SUCCESS + // - OB_NOT_MASTER: self is not leader + // - OB_STATE_NOT_MATCH: palf does not have an arbitration member + // - OB_TIMEOUT: rpc timeout + // - OB_RPC_POST_ERROR: arbitration server is in easy's black_list + int get_remote_arb_member_info(ArbMemberInfo &arb_member_info) const; + // @brief: get arbitration member + // @param[in/out] common::ObMember &arb_member + // @return + // - OB_SUCCESS + // - OB_NOT_INIT + int get_arbitration_member(common::ObMember &arb_member) const; +#endif int revoke_leader(const int64_t proposal_id); int change_leader_to(const common::ObAddr &dst_addr); // @brief: change AccessMode of palf. diff --git a/src/logservice/palf/palf_handle_impl.cpp b/src/logservice/palf/palf_handle_impl.cpp index 02934becd..b3840f940 100755 --- a/src/logservice/palf/palf_handle_impl.cpp +++ b/src/logservice/palf/palf_handle_impl.cpp @@ -288,6 +288,39 @@ int PalfHandleImpl::set_initial_member_list( return ret; } +#ifdef OB_BUILD_ARBITRATION +int PalfHandleImpl::set_initial_member_list( + const common::ObMemberList &member_list, + const common::ObMember &arb_member, + const int64_t paxos_replica_num, + const common::GlobalLearnerList &learner_list) +{ + int ret = OB_SUCCESS; + LogConfigVersion config_version; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + PALF_LOG(WARN, "PalfHandleImpl has not inited!!!", K(ret)); + } else { + { + WLockGuard guard(lock_); + const int64_t proposal_id = state_mgr_.get_proposal_id(); + if (OB_FAIL(config_mgr_.set_initial_member_list(member_list, arb_member, paxos_replica_num, \ + learner_list, proposal_id, config_version))) { + PALF_LOG(WARN, "LogConfigMgr set_initial_member_list failed", K(ret), KPC(this)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(config_mgr_.wait_config_log_persistence(config_version))) { + PALF_LOG(WARN, "want_config_log_persistence failed", K(ret), KPC(this)); + } else { + PALF_EVENT("set_initial_member_list success", palf_id_, K(ret), K(member_list), K(arb_member), + K(learner_list), K(paxos_replica_num)); + report_set_initial_member_list_with_arb_(paxos_replica_num, member_list, arb_member); + } + } + return ret; +} +#endif int PalfHandleImpl::get_begin_lsn(LSN &lsn) const { @@ -923,6 +956,149 @@ int PalfHandleImpl::replace_member_with_learner(const common::ObMember &added_me return ret; } +#ifdef OB_BUILD_ARBITRATION +int PalfHandleImpl::add_arb_member( + const common::ObMember &member, + const int64_t timeout_us) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + PALF_LOG(WARN, "PalfHandleImpl not init", KR(ret), KPC(this)); + } else if (!member.is_valid() || timeout_us <= 0) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(WARN, "invalid argument", KR(ret), KPC(this), K(member), K(timeout_us)); + } else { + LogConfigChangeArgs args(member, 0, ADD_ARB_MEMBER); + if (OB_FAIL(one_stage_config_change_(args, timeout_us))) { + PALF_LOG(WARN, "add_arb_member failed", KR(ret), KPC(this), K(member)); + } else { + PALF_EVENT("add_arb_member success", palf_id_, KR(ret), KPC(this), K(member)); + report_add_arb_member_(member); + } + } + return ret; +} + +int PalfHandleImpl::remove_arb_member( + const common::ObMember &member, + const int64_t timeout_us) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + PALF_LOG(WARN, "PalfHandleImpl not init", KR(ret), KPC(this)); + } else if (!member.is_valid() || timeout_us <= 0) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(WARN, "invalid argument", KR(ret), KPC(this), K(member), K(timeout_us)); + } else { + LogConfigChangeArgs args(member, 0, REMOVE_ARB_MEMBER); + if (OB_FAIL(one_stage_config_change_(args, timeout_us))) { + PALF_LOG(WARN, "remove_arb_member failed", KR(ret), KPC(this), K(member)); + } else { + PALF_EVENT("remove_arb_member success", palf_id_, KR(ret), KPC(this), K(member)); + report_remove_arb_member_(member); + } + } + return ret; +} + +int PalfHandleImpl::degrade_acceptor_to_learner(const LogMemberAckInfoList °rade_servers, const int64_t timeout_us) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else if (0 == degrade_servers.count() || timeout_us <= 0) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(WARN, "invalid argument", KR(ret), KPC(this), K(degrade_servers), K(timeout_us)); + } else if (OB_FAIL(pre_check_before_degrade_upgrade_(degrade_servers, DEGRADE_ACCEPTOR_TO_LEARNER))) { + PALF_LOG(WARN, "pre_check_before_degrade_upgrade_ failed", KR(ret), KPC(this), K(degrade_servers)); + } else { + const int64_t begin_time_us = common::ObTimeUtility::current_time(); + for (int64_t i = 0; i < degrade_servers.count() && OB_SUCC(ret); i++) { + const ObMember &member = degrade_servers.at(i).member_; + LogConfigChangeArgs args(member, 0, DEGRADE_ACCEPTOR_TO_LEARNER); + const int64_t curr_time_us = common::ObTimeUtility::current_time(); + if (OB_FAIL(one_stage_config_change_(args, timeout_us + begin_time_us - curr_time_us))) { + PALF_LOG(WARN, "degrade_acceptor_to_learner failed", KR(ret), KPC(this), K(member), K(timeout_us)); + } else { + PALF_EVENT("degrade_acceptor_to_learner success", palf_id_, K(ret), K_(self), K(member), K(timeout_us)); + } + } + PALF_EVENT("degrade_acceptor_to_learner finish", palf_id_, K(ret), KPC(this), K(degrade_servers), K(timeout_us)); + } + return ret; +} + +int PalfHandleImpl::upgrade_learner_to_acceptor(const LogMemberAckInfoList &upgrade_servers, const int64_t timeout_us) +{ + int ret = OB_SUCCESS; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else if (0 == upgrade_servers.count() || timeout_us <= 0) { + ret = OB_INVALID_ARGUMENT; + PALF_LOG(WARN, "invalid argument", KR(ret), KPC(this), K(upgrade_servers), K(timeout_us)); + // } else if (OB_FAIL(pre_check_before_degrade_upgrade_(upgrade_servers, UPGRADE_LEARNER_TO_ACCEPTOR))) { + // PALF_LOG(WARN, "pre_check_before_degrade_upgrade_ failed", KR(ret), KPC(this), K(upgrade_servers)); + } else { + const int64_t begin_time_us = common::ObTimeUtility::current_time(); + for (int64_t i = 0; i < upgrade_servers.count() && OB_SUCC(ret); i++) { + const ObMember &member = upgrade_servers.at(i).member_; + LogConfigChangeArgs args(member, 0, UPGRADE_LEARNER_TO_ACCEPTOR); + const int64_t curr_time_us = common::ObTimeUtility::current_time(); + if (OB_FAIL(one_stage_config_change_(args, timeout_us + begin_time_us - curr_time_us))) { + PALF_LOG(WARN, "upgrade_learner_to_acceptor failed", KR(ret), KPC(this), K(member), K(timeout_us)); + } else { + PALF_LOG(INFO, "upgrade_learner_to_acceptor success", K(ret), KPC(this), K(member), K(timeout_us)); + } + } + PALF_EVENT("upgrade_learner_to_acceptor finish", palf_id_, K(ret), KPC(this), K(upgrade_servers), K(timeout_us)); + } + return ret; +} + +int PalfHandleImpl::get_remote_arb_member_info(ArbMemberInfo &arb_member_info) +{ + int ret = OB_SUCCESS; + ObMember arb_member; + (void) config_mgr_.get_arbitration_member(arb_member); + LogGetArbMemberInfoResp resp; + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else if (false == state_mgr_.is_leader_active()) { + // If self is not in state, return -4038. + ret = OB_NOT_MASTER; + } else if (!arb_member.is_valid()) { + // arb_member is invalid, return -4109. + ret = OB_STATE_NOT_MATCH; + } else if (OB_FAIL(log_engine_.sync_get_arb_member_info(arb_member.get_server(), \ + PALF_SYNC_RPC_TIMEOUT_US, resp))) { + PALF_LOG(WARN, "submit_get_arb_member_info_req failed", KR(ret), K_(palf_id), K_(self), + K(arb_member), K(resp)); + } else { + arb_member_info = resp.arb_member_info_; + } + return ret; +} + +int PalfHandleImpl::get_arb_member_info(palf::ArbMemberInfo &arb_member_info) const +{ + int ret = OB_SUCCESS; + return ret; +} + +int PalfHandleImpl::get_arbitration_member(common::ObMember &arb_member) const +{ + int ret = OB_SUCCESS; + RLockGuard guard(lock_); + if (IS_NOT_INIT) { + ret = OB_NOT_INIT; + } else if (OB_FAIL(config_mgr_.get_arbitration_member(arb_member))) { + PALF_LOG(WARN, "get_arbitration_member failed", K_(palf_id), K_(self), K(arb_member)); + } + return ret; +} +#endif int PalfHandleImpl::change_access_mode(const int64_t proposal_id, const int64_t mode_version, diff --git a/src/logservice/palf/palf_handle_impl.h b/src/logservice/palf/palf_handle_impl.h index fc2cbbc05..e52fa94e8 100755 --- a/src/logservice/palf/palf_handle_impl.h +++ b/src/logservice/palf/palf_handle_impl.h @@ -274,6 +274,22 @@ public: virtual int set_initial_member_list(const common::ObMemberList &member_list, const int64_t paxos_replica_num, const common::GlobalLearnerList &learner_list) = 0; +#ifdef OB_BUILD_ARBITRATION + // after creating palf which includes arbitration replica successfully, + // set initial memberlist(can only be called once) + // @param [in] ObMemberList, the initial member list, do not include arbitration replica + // @param [in] arb_member, arbitration replica + // @param [in] paxos_replica_num, number of paxos replicas + // @param [in] learner_list, learner_list + // @return :TODO + virtual int set_initial_member_list(const common::ObMemberList &member_list, + const common::ObMember &arb_member, + const int64_t paxos_replica_num, + const common::GlobalLearnerList &learner_list) = 0; + virtual int get_remote_arb_member_info(ArbMemberInfo &arb_member_info) = 0; + virtual int get_arb_member_info(ArbMemberInfo &arb_member_info) const = 0; + virtual int get_arbitration_member(common::ObMember &arb_member) const = 0; +#endif // set region for self // @param [common::ObRegion] region virtual int set_region(const common::ObRegion ®ion) = 0; @@ -491,6 +507,50 @@ public: const common::ObMember &removed_member, const LogConfigVersion &config_version, const int64_t timeout_us) = 0; +#ifdef OB_BUILD_ARBITRATION + // @brief, add an arbitration member to paxos group + // @param[in] common::ObMember &member: arbitration member which will be added + // @param[in] const int64_t timeout_us: add member timeout, us + // @return + // - OB_SUCCESS: add arbitration member successfully + // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change + // - OB_TIMEOUT: add arbitration member timeout + // - OB_NOT_MASTER: not leader or rolechange during membership changing + // - other: bug + virtual int add_arb_member(const common::ObMember &added_member, + const int64_t timeout_us) = 0; + // @brief, remove an arbitration member from paxos group + // @param[in] common::ObMember &member: arbitration member which will be removed + // @param[in] const int64_t timeout_us: remove member timeout, us + // @return + // - OB_SUCCESS: remove arbitration member successfully + // - OB_INVALID_ARGUMENT: invalid argumemt or not supported config change + // - OB_TIMEOUT: remove arbitration member timeout + // - OB_NOT_MASTER: not leader or rolechange during membership changing + // - other: bug + virtual int remove_arb_member(const common::ObMember &arb_member, + const int64_t timeout_us) = 0; + // @brief: degrade an acceptor(full replica) to learner(special read only replica) in this cluster + // @param[in] const common::ObMemberList &member_list: acceptors will be degraded to learner + // @param[in] const int64_t timeout_us + // @return + // - OB_SUCCESS + // - OB_INVALID_ARGUMENT: invalid argument + // - OB_TIMEOUT: timeout + // - OB_NOT_MASTER: not leader + virtual int degrade_acceptor_to_learner(const LogMemberAckInfoList °rade_servers, + const int64_t timeout_us) = 0; + // @brief: upgrade a learner(special read only replica) to acceptor(full replica) in this cluster + // @param[in] const common::ObMemberList &learner_list: learners will be upgraded to acceptors + // @param[in] const int64_t timeout_us + // @return + // - OB_SUCCESS + // - OB_INVALID_ARGUMENT: invalid argument + // - OB_TIMEOUT: timeout + // - OB_NOT_MASTER: not leader + virtual int upgrade_learner_to_acceptor(const LogMemberAckInfoList &upgrade_servers, + const int64_t timeout_us) = 0; +#endif // 设置日志文件的可回收位点，小于等于lsn的日志文件均可以安全回收 // @@ -802,6 +862,12 @@ public: int set_initial_member_list(const common::ObMemberList &member_list, const int64_t paxos_replica_num, const common::GlobalLearnerList &learner_list) override final; +#ifdef OB_BUILD_ARBITRATION + int set_initial_member_list(const common::ObMemberList &member_list, + const common::ObMember &arb_member, + const int64_t paxos_replica_num, + const common::GlobalLearnerList &learner_list) override final; +#endif int set_region(const common::ObRegion ®ion) override final; int set_paxos_member_region_map(const LogMemberRegionMap ®ion_map) override final; int submit_log(const PalfAppendOptions &opts, @@ -861,6 +927,19 @@ public: const common::ObMember &removed_member, const LogConfigVersion &config_version, const int64_t timeout_us) override final; +#ifdef OB_BUILD_ARBITRATION + int add_arb_member(const common::ObMember &added_member, + const int64_t timeout_us) override final; + int remove_arb_member(const common::ObMember &arb_member, + const int64_t timeout_us) override final; + int degrade_acceptor_to_learner(const LogMemberAckInfoList °rade_servers, + const int64_t timeout_us) override final; + int upgrade_learner_to_acceptor(const LogMemberAckInfoList &upgrade_servers, + const int64_t timeout_us) override final; + int get_remote_arb_member_info(ArbMemberInfo &arb_member_info) override final; + int get_arb_member_info(ArbMemberInfo &arb_member_info) const override final; + int get_arbitration_member(common::ObMember &arb_member) const override final; +#endif int set_base_lsn(const LSN &lsn) override final; int enable_sync() override final; int disable_sync() override final; diff --git a/src/observer/CMakeLists.txt b/src/observer/CMakeLists.txt index ccda880d1..78338eab5 100644 --- a/src/observer/CMakeLists.txt +++ b/src/observer/CMakeLists.txt @@ -396,7 +396,7 @@ add_library(oceanbase_static ${CMAKE_CURRENT_BINARY_DIR}/ob_version.cpp) target_link_libraries(oceanbase_static - PUBLIC ob_base "${ob_objects}" ob_sql_static ob_storage_static ob_share_static oblib objit) + PUBLIC ob_base "${ob_objects}" ob_sql_static ob_storage_static ob_share_static ${ob_close_modules_static_name} oblib objit) if (OB_GPERF_MODE) target_link_libraries(oceanbase_static @@ -422,6 +422,7 @@ else() ob_sql_static ob_storage_static ob_share_static + ${ob_close_modules_static_name} -Wl,--no-whole-archive -static-libgcc -static-libstdc++ objit) @@ -444,6 +445,7 @@ target_link_libraries(observer ob_sql_static ob_storage_static ob_share_static + ${ob_close_modules_static_name} -Wl,--end-group -static-libgcc -static-libstdc++ diff --git a/src/observer/dbms_job/ob_dbms_job_executor.cpp b/src/observer/dbms_job/ob_dbms_job_executor.cpp index 2c5819128..3e94b375e 100644 --- a/src/observer/dbms_job/ob_dbms_job_executor.cpp +++ b/src/observer/dbms_job/ob_dbms_job_executor.cpp @@ -24,6 +24,9 @@ #include "observer/ob_inner_sql_connection_pool.h" #include "sql/session/ob_sql_session_info.h" #include "sql/ob_sql.h" +#ifdef OB_BUILD_AUDIT_SECURITY +#include "pl/sys_package/ob_dbms_audit_mgmt.h" +#endif namespace oceanbase { @@ -137,9 +140,41 @@ int ObDBMSJobExecutor::run_dbms_job( uint64_t tenant_id, ObDBMSJobInfo &job_info, ObIAllocator &allocator) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSEDx(tenant_id, job_info, allocator); ret = OB_NOT_SUPPORTED; LOG_WARN("not support", K(ret)); +#else + ObSqlString what; + ObInnerSQLConnectionPool *pool = NULL; + ObInnerSQLConnection *conn = NULL; + SMART_VAR(ObSQLSessionInfo, session) { + int64_t affected_rows = 0; + CK (OB_LIKELY(inited_)); + CK (OB_NOT_NULL(sql_proxy_)); + CK (sql_proxy_->is_inited()); + if (OB_SUCC(ret)) { + if (job_info.is_mysql_audit_job()) { + pl::ObDbmsAuditMgmt::ObAuditParam audit_param; + OZ (pl::ObDbmsAuditMgmt::parse_mysql_job_param(job_info.get_what(), audit_param)); + OZ (pl::ObDbmsAuditMgmt::clean_audit_trail_impl(audit_param, allocator, sql_proxy_, tenant_id), audit_param); + } else { + ObOracleSqlProxy oracle_proxy(*(static_cast(sql_proxy_))); + CK (job_info.valid()); + CK (job_info.get_what().length() != 0); + OZ (what.append_fmt("BEGIN %.*s END;", + job_info.get_what().length(), job_info.get_what().ptr())); + CK (OB_NOT_NULL(pool = static_cast(oracle_proxy.get_pool()))); + OZ (init_env(job_info, session)); + OZ (pool->acquire_spi_conn(&session, conn)); + OZ (conn->execute_write(tenant_id, what.string().ptr(), affected_rows)); + if (OB_NOT_NULL(conn)) { + sql_proxy_->close(conn, ret); + } + } + } + } +#endif return ret; } diff --git a/src/observer/main.cpp b/src/observer/main.cpp index 239496977..8b80359a2 100644 --- a/src/observer/main.cpp +++ b/src/observer/main.cpp @@ -430,10 +430,6 @@ static void print_all_thread(const char* desc) MPRINT("============= [%s] finish to show unstopped thread =============", desc); } -extern "C" { -typedef void *(*reasy_pool_realloc_pt)(void *ptr, size_t size); -void reasy_pool_set_allocator(reasy_pool_realloc_pt alloc); -} int main(int argc, char *argv[]) { #ifdef ENABLE_SANITY diff --git a/src/observer/mysql/ob_query_driver.cpp b/src/observer/mysql/ob_query_driver.cpp index 808f10fcf..b6ebb115f 100644 --- a/src/observer/mysql/ob_query_driver.cpp +++ b/src/observer/mysql/ob_query_driver.cpp @@ -24,7 +24,11 @@ #include "share/ob_lob_access_utils.h" #include "lib/charset/ob_charset.h" #include "observer/mysql/obmp_stmt_prexecute.h" - +#ifdef OB_BUILD_ORACLE_XML +#include "lib/xml/ob_multi_mode_interface.h" +#include "lib/xml/ob_xml_util.h" +#include "sql/engine/expr/ob_expr_xml_func_helper.h" +#endif namespace oceanbase { @@ -205,6 +209,13 @@ int ObQueryDriver::response_query_result(ObResultSet &result, if (is_first_row) { is_first_row = false; can_retry = false; // 已经获取到第一行数据，不再重试了 +#ifdef OB_BUILD_SPM + if (OB_NOT_NULL(result.get_exec_context().get_physical_plan_ctx()) && + OB_NOT_NULL(sql_ctx) && sql_ctx->spm_ctx_.need_spm_timeout_) { + LOG_TRACE("reset to origin timeout because result is returning to user"); + result.get_exec_context().get_physical_plan_ctx()->set_spm_timeout_timestamp(0); + } +#endif if (OB_FAIL(response_query_header(result, has_more_result, false))) { LOG_WARN("fail to response query header", K(ret), K(row_num), K(can_retry)); } @@ -246,6 +257,10 @@ int ObQueryDriver::response_query_result(ObResultSet &result, } else if ((value.is_lob() || value.is_lob_locator() || value.is_json() || value.is_geometry()) && OB_FAIL(process_lob_locator_results(value, result))) { LOG_WARN("convert lob locator to longtext failed", K(ret)); +#ifdef OB_BUILD_ORACLE_XML + } else if (value.is_user_defined_sql_type() && OB_FAIL(ObXMLExprHelper::process_sql_udt_results(value, result))) { + LOG_WARN("convert udt to client format failed", K(ret), K(value.get_udt_subschema_id())); +#endif } } } diff --git a/src/observer/mysql/ob_query_retry_ctrl.cpp b/src/observer/mysql/ob_query_retry_ctrl.cpp index ebe25804a..b527d446a 100644 --- a/src/observer/mysql/ob_query_retry_ctrl.cpp +++ b/src/observer/mysql/ob_query_retry_ctrl.cpp @@ -792,10 +792,25 @@ void ObQueryRetryCtrl::switch_consumer_group_retry_proc(ObRetryParam &v) void ObQueryRetryCtrl::timeout_proc(ObRetryParam &v) { +#ifdef OB_BUILD_SPM + if (OB_UNLIKELY(v.err_ == OB_TIMEOUT && + ObSpmCacheCtx::STAT_FIRST_EXECUTE_PLAN == v.ctx_.spm_ctx_.spm_stat_ && + v.ctx_.spm_ctx_.need_spm_timeout_)) { + const_cast(v.ctx_).spm_ctx_.spm_stat_ = ObSpmCacheCtx::STAT_FALLBACK_EXECUTE_PLAN; + const_cast(v.ctx_).spm_ctx_.need_spm_timeout_ = false; + ObRetryObject retry_obj(v); + ObForceLocalRetryPolicy force_local_retry; + retry_obj.test(force_local_retry); + } else if (is_try_lock_row_err(v.session_.get_retry_info().get_last_query_retry_err())) { + v.client_ret_ = OB_ERR_EXCLUSIVE_LOCK_CONFLICT; + v.retry_type_ = RETRY_TYPE_NONE; + } +#else if (is_try_lock_row_err(v.session_.get_retry_info().get_last_query_retry_err())) { v.client_ret_ = OB_ERR_EXCLUSIVE_LOCK_CONFLICT; v.retry_type_ = RETRY_TYPE_NONE; } +#endif } /////// For inner SQL only /////////////// diff --git a/src/observer/mysql/ob_sync_cmd_driver.cpp b/src/observer/mysql/ob_sync_cmd_driver.cpp index b90969e01..2e64e3bc2 100644 --- a/src/observer/mysql/ob_sync_cmd_driver.cpp +++ b/src/observer/mysql/ob_sync_cmd_driver.cpp @@ -23,6 +23,9 @@ #include "share/ob_lob_access_utils.h" #include "observer/mysql/obmp_stmt_prexecute.h" #include "src/pl/ob_pl_user_type.h" +#ifdef OB_BUILD_ORACLE_XML +#include "sql/engine/expr/ob_expr_xml_func_helper.h" +#endif namespace oceanbase { @@ -330,6 +333,10 @@ int ObSyncCmdDriver::response_query_result(ObMySQLResultSet &result) } else if ((value.is_lob() || value.is_lob_locator() || value.is_json() || value.is_geometry()) && OB_FAIL(process_lob_locator_results(value, result))) { LOG_WARN("convert lob locator to longtext failed", K(ret)); +#ifdef OB_BUILD_ORACLE_XML + } else if (value.is_user_defined_sql_type() && OB_FAIL(ObXMLExprHelper::process_sql_udt_results(value, result))) { + LOG_WARN("convert udt to client format failed", K(ret), K(value.get_udt_subschema_id())); +#endif } } diff --git a/src/observer/mysql/obmp_base.cpp b/src/observer/mysql/obmp_base.cpp index d68df7365..cafc0d4f9 100644 --- a/src/observer/mysql/obmp_base.cpp +++ b/src/observer/mysql/obmp_base.cpp @@ -47,7 +47,9 @@ #include "share/ob_lob_access_utils.h" #include "sql/monitor/flt/ob_flt_utils.h" #include "sql/session/ob_sess_info_verify.h" +#ifdef OB_BUILD_ORACLE_XML #include "sql/engine/expr/ob_expr_xml_func_helper.h" +#endif namespace oceanbase { using namespace share; @@ -552,6 +554,13 @@ int ObMPBase::response_row(ObSQLSessionInfo &session, &allocator, &session))) { LOG_WARN("convert lob locator to longtext failed", K(ret)); +#ifdef OB_BUILD_ORACLE_XML + } else if (value.is_user_defined_sql_type() + && OB_FAIL(ObXMLExprHelper::process_sql_udt_results(value, + &allocator, + &session))) { + LOG_WARN("convert udt to client format failed", K(ret), K(value.get_udt_subschema_id())); +#endif } } } diff --git a/src/observer/mysql/obmp_connect.cpp b/src/observer/mysql/obmp_connect.cpp index 0df539fb1..6375f332b 100644 --- a/src/observer/mysql/obmp_connect.cpp +++ b/src/observer/mysql/obmp_connect.cpp @@ -338,6 +338,27 @@ int ObMPConnect::process() } if (NULL != session) { +#ifdef OB_BUILD_AUDIT_SECURITY + ObSqlString comment_text; + (void)comment_text.append_fmt("LOGIN: tenant_name=%.*s, user_name=%.*s, client_ip=%.*s, " + "sessid=%u, proxy_sessid=%lu, " + "capability=%X, proxy_capability=%lX, use_ssl=%s, protocol=%s", + tenant_name_.length(), tenant_name_.ptr(), + user_name_.length(), user_name_.ptr(), + host_name.length(), host_name.ptr(), + sessid, + proxy_sessid, + capability, + proxy_capability, + use_ssl ? "true" : "false", + get_cs_protocol_type_name(protoType)); + + (void)ObSecurityAuditUtils::handle_security_audit(*session, + stmt::T_LOGIN, + ObString::make_string("CONNECT"), + comment_text.string(), + proc_ret); +#endif // oracle temp table need to be refactored //if (OB_SUCCESS == proc_ret) { // proc_ret = session->drop_reused_oracle_temp_tables(); @@ -626,6 +647,10 @@ int ObMPConnect::load_privilege_info(ObSQLSessionInfo &session) LOG_WARN("failed to update database variables", K(ret)); } else if (OB_FAIL(session.update_max_packet_size())) { LOG_WARN("failed to update max packet size", K(ret)); +#ifdef OB_BUILD_AUDIT_SECURITY + } else if (OB_FAIL(check_audit_user(session_priv.tenant_id_, user_name_))) { + LOG_WARN("fail to check audit user privilege", K(ret)); +#endif } else if (OB_FAIL(get_client_attribute_capability(client_attr_cap_flags))) { LOG_WARN("failed to get client attribute capability", K(ret)); } else { @@ -850,6 +875,29 @@ int ObMPConnect::unlock_user_if_time_is_up_mysql(const uint64_t tenant_id, return ret; } +#ifdef OB_BUILD_AUDIT_SECURITY +int ObMPConnect::check_audit_user(const uint64_t tenant_id, ObString &user_name) +{ + int ret = OB_SUCCESS; + lib::Worker::CompatMode compat_mode = lib::Worker::CompatMode::MYSQL; + if (0 == user_name.case_compare(OB_ORA_AUDITOR_NAME)) { + if (OB_FAIL(ObCompatModeGetter::get_tenant_mode(tenant_id, compat_mode))) { + LOG_WARN("fail to get tenant mode in convert_oracle_object_name", K(ret)); + } else if (compat_mode == lib::Worker::CompatMode::MYSQL) { + omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id)); + if (tenant_config.is_valid()) { + ObString audit_mode(tenant_config->_audit_mode.get_value()); + if (0 != audit_mode.case_compare("ORACLE")) { + ret = OB_ERR_NO_PRIVILEGE; + LOG_WARN("audit user cannot login in because of the" + "audit_mode not be oracle", K(ret)); + } + } + } + } + return ret; +} +#endif int ObMPConnect::update_login_stat_in_trans(const uint64_t tenant_id, const bool is_login_succ, diff --git a/src/observer/mysql/obmp_connect.h b/src/observer/mysql/obmp_connect.h index a711a67aa..6b1b0dd59 100644 --- a/src/observer/mysql/obmp_connect.h +++ b/src/observer/mysql/obmp_connect.h @@ -115,6 +115,9 @@ private: int check_password_expired(const uint64_t tenant_id, share::schema::ObSchemaGetterGuard &schema_guard, sql::ObSQLSessionInfo &session); +#ifdef OB_BUILD_AUDIT_SECURITY + int check_audit_user(const uint64_t tenant_id, ObString &user_name); +#endif int set_proxy_version(ObSMConnection &conn); int update_charset_sys_vars(ObSMConnection &conn, sql::ObSQLSessionInfo &sess_info); diff --git a/src/observer/mysql/obmp_packet_sender.cpp b/src/observer/mysql/obmp_packet_sender.cpp index ba3ebf268..5a29fdd66 100644 --- a/src/observer/mysql/obmp_packet_sender.cpp +++ b/src/observer/mysql/obmp_packet_sender.cpp @@ -85,14 +85,6 @@ ObMPPacketSender::~ObMPPacketSender() void ObMPPacketSender::reset() { - if (conn_valid_) { - if (OB_NOT_NULL(req_)) { - if (rpc::ObRequest::TRANSPORT_PROTO_RDMA == nio_protocol_) { - SQL_REQ_OP.destroy(req_); - } - } - } - req_ = NULL; seq_ = 0; comp_context_.reset(); diff --git a/src/observer/mysql/obmp_query.cpp b/src/observer/mysql/obmp_query.cpp index 55d2ded07..9df6e2ee9 100644 --- a/src/observer/mysql/obmp_query.cpp +++ b/src/observer/mysql/obmp_query.cpp @@ -990,8 +990,23 @@ OB_INLINE int ObMPQuery::do_process(ObSQLSessionInfo &session, } bool is_need_retry = THIS_THWORKER.need_retry() || RETRY_TYPE_NONE != retry_ctrl_.get_retry_type(); +#ifdef OB_BUILD_SPM + if (!is_need_retry) { + (void)ObSQLUtils::handle_plan_baseline(audit_record, plan, ret, ctx_); + } +#endif (void)ObSQLUtils::handle_audit_record(is_need_retry, EXECUTE_LOCAL, session, ctx_.is_sensitive_); +#ifdef OB_BUILD_AUDIT_SECURITY + // 对于触发重试的语句不需要进行审计，以免一条语句被审计多次 + if (!retry_ctrl_.need_retry() && !async_resp_used) { + (void)ObSecurityAuditUtils::handle_security_audit(result, + ctx_.schema_guard_, + ctx_.cur_stmt_, + ObString::make_empty_string(), + ret); + } +#endif } return ret; } @@ -1238,7 +1253,13 @@ OB_INLINE int ObMPQuery::response_result(ObMySQLResultSet &result, ObSQLSessionInfo &session = result.get_session(); CHECK_COMPATIBILITY_MODE(&session); +#ifndef OB_BUILD_SPM bool need_trans_cb = result.need_end_trans_callback() && (!force_sync_resp); +#else + bool need_trans_cb = result.need_end_trans_callback() && + (!force_sync_resp) && + (!ctx_.spm_ctx_.check_execute_status_); +#endif // 通过判断 plan 是否为 null 来确定是 plan 还是 cmd // 针对 plan 和 cmd 分开处理，逻辑会较为清晰。 diff --git a/src/observer/mysql/obmp_stmt_execute.cpp b/src/observer/mysql/obmp_stmt_execute.cpp index 3f47949eb..0bf83b7cb 100644 --- a/src/observer/mysql/obmp_stmt_execute.cpp +++ b/src/observer/mysql/obmp_stmt_execute.cpp @@ -1397,6 +1397,11 @@ int ObMPStmtExecute::do_process(ObSQLSessionInfo &session, bool need_retry = (THIS_THWORKER.need_retry() || RETRY_TYPE_NONE != retry_ctrl_.get_retry_type()); if (!is_ps_cursor()) { +#ifdef OB_BUILD_SPM + if (!need_retry) { + (void)ObSQLUtils::handle_plan_baseline(audit_record, result.get_physical_plan(), ret, ctx_); + } +#endif // ps cursor has already record after inner_open in spi ObSQLUtils::handle_audit_record(need_retry, EXECUTE_PS_EXECUTE, session, ctx_.is_sensitive_); } @@ -1412,7 +1417,13 @@ int ObMPStmtExecute::response_result( bool &async_resp_used) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_SPM bool need_trans_cb = result.need_end_trans_callback() && (!force_sync_resp); +#else + bool need_trans_cb = result.need_end_trans_callback() && + (!force_sync_resp) && + (!ctx_.spm_ctx_.check_execute_status_); +#endif // NG_TRACE_EXT(exec_begin, ID(arg1), force_sync_resp, ID(end_trans_cb), need_trans_cb); @@ -2037,10 +2048,53 @@ int ObMPStmtExecute::get_pl_type_by_type_info(ObIAllocator &allocator, const pl::ObUserDefinedType *&pl_type) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSEDx(allocator, type_info, pl_type); ret = OB_NOT_SUPPORTED; LOG_WARN("not support", K(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "Get PL type by type info is not supported in CE version"); +#else + const share::schema::ObUDTTypeInfo *udt_info = NULL; + if (OB_ISNULL(type_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("type info is null", K(ret), K(type_info)); + } else if (!type_info->is_elem_type_) { + if (type_info->package_name_.empty()) { + OZ (get_udt_by_name(type_info->relation_name_, type_info->type_name_, udt_info)); + OZ (udt_info->transform_to_pl_type(allocator, pl_type)); + } else { + OZ (get_package_type_by_name(allocator, type_info, pl_type)); + } + } else { + void *ptr = NULL; + pl::ObNestedTableType *table_type = NULL; + pl::ObPLDataType elem_type; + const pl::ObUserDefinedType *elem_type_ptr = NULL; + if (type_info->elem_type_.get_obj_type() != ObExtendType) { + elem_type.set_data_type(type_info->elem_type_); + } else if (OB_FAIL(get_udt_by_name(type_info->relation_name_, type_info->type_name_, udt_info))) { + LOG_WARN("failed to get udt info", K(ret), K(type_info->relation_name_), K(type_info->type_name_)); + } else if (OB_FAIL(udt_info->transform_to_pl_type(allocator, elem_type_ptr))) { + LOG_WARN("failed to transform udt to pl type", K(ret)); + } else if (OB_ISNULL(elem_type_ptr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get elem type ptr", K(ret)); + } else { + elem_type = *(static_cast(elem_type_ptr)); + } + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(ptr = allocator.alloc(sizeof(pl::ObNestedTableType)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory for ObNestedTableType", K(ret)); + } else { + table_type = new(ptr)pl::ObNestedTableType(); + table_type->set_type_from(pl::ObPLTypeFrom::PL_TYPE_LOCAL); + table_type->set_element_type(elem_type); + pl_type = table_type; + } + } + CK (OB_NOT_NULL(pl_type)); +#endif return ret; } diff --git a/src/observer/mysql/obsm_utils.cpp b/src/observer/mysql/obsm_utils.cpp index d12deb3f8..d26ae62ab 100644 --- a/src/observer/mysql/obsm_utils.cpp +++ b/src/observer/mysql/obsm_utils.cpp @@ -243,6 +243,31 @@ int ObSMUtils::cell_str( K(ObString(len, buf)), K(ret)); } else { /*do nothing*/ } +#ifdef OB_BUILD_ORACLE_PL + } else if (obj.is_pl_extend() + && PL_NESTED_TABLE_TYPE == obj.get_meta().get_extend_type()) { + // anonymous collection + ObPLCollection *coll = reinterpret_cast(obj.get_ext()); + ObNestedTableType *nested_type = NULL; + ObPLDataType element_type; + if (OB_ISNULL(nested_type = + reinterpret_cast(allocator.alloc(sizeof(ObNestedTableType))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + OB_LOG(WARN, "failed to alloc memory for ObNestedTableType", K(ret)); + } else if (OB_ISNULL(coll)) { + ret = OB_ERR_UNEXPECTED; + OB_LOG(WARN, "coll is null", K(ret)); + } else if (FALSE_IT(new (nested_type) ObNestedTableType())) { + } else if (FALSE_IT(element_type.reset())) { + } else if (FALSE_IT(element_type.set_data_type(coll->get_element_type()))) { + } else if (FALSE_IT(nested_type->set_element_type(element_type))) { + } else if (OB_FAIL(nested_type->serialize( + *schema_guard, dtc_params.tz_info_, type, src, buf, len, pos))) { + OB_LOG(WARN, "failed to serialize anonymous collection", K(ret)); + } else { + OB_LOG(DEBUG, "success to serialize anonymous collection", K(ret)); + } +#endif } else { ret = OB_ERR_UNEXPECTED; OB_LOG(WARN, "type owner or type name is empty", KPC(field), K(ret)); diff --git a/src/observer/ob_heartbeat.cpp b/src/observer/ob_heartbeat.cpp index e67d3c785..d5ce11357 100644 --- a/src/observer/ob_heartbeat.cpp +++ b/src/observer/ob_heartbeat.cpp @@ -29,6 +29,9 @@ #include "observer/omt/ob_tenant_config_mgr.h" #include "common/ob_timeout_ctx.h" #include "storage/slog/ob_storage_logger_manager.h" +#ifdef OB_BUILD_TDE_SECURITY +#include "share/ob_master_key_getter.h" +#endif namespace oceanbase { @@ -93,6 +96,24 @@ void ObHeartBeatProcess::destroy() TG_DESTROY(lib::TGDefIDs::ObHeartbeat); } +#ifdef OB_BUILD_TDE_SECURITY +int ObHeartBeatProcess::set_lease_request_max_stored_versions( + share::ObLeaseRequest &lease_request, + const common::ObIArray > &max_stored_versions) +{ + int ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < max_stored_versions.count(); ++i) { + const std::pair &src = max_stored_versions.at(i); + std::pair dest; + dest.first = src.first; + dest.second.max_flushed_key_version_ = src.second; + if (OB_FAIL(lease_request.tenant_max_flushed_key_version_.push_back(dest))) { + LOG_WARN("fail to push back", KR(ret)); + } + } + return ret; +} +#endif int ObHeartBeatProcess::init_lease_request(ObLeaseRequest &lease_request) { @@ -107,6 +128,13 @@ int ObHeartBeatProcess::init_lease_request(ObLeaseRequest &lease_request) LOG_WARN("GCTX.ob_service_ is null", KR(ret), KP(GCTX.ob_service_)); } else if (OB_FAIL((GCTX.ob_service_->get_server_resource_info(lease_request.resource_info_)))) { LOG_WARN("fail to get server resource info", KR(ret)); +#ifdef OB_BUILD_TDE_SECURITY + } else if (OB_FAIL(ObMasterKeyGetter::instance().get_max_stored_versions(max_stored_versions))) { + LOG_WARN("fail to get max stored versions", KR(ret)); + } else if (OB_FAIL(set_lease_request_max_stored_versions(lease_request, max_stored_versions))) { + LOG_WARN("fail to set lease request max stored key versions", + KR(ret), K(lease_request), K(max_stored_versions)); +#endif } else { lease_request.request_lease_time_ = 0; // this is not a valid member lease_request.version_ = ObLeaseRequest::LEASE_VERSION; diff --git a/src/observer/ob_heartbeat.h b/src/observer/ob_heartbeat.h index 4d3aa6710..7bb78c11b 100644 --- a/src/observer/ob_heartbeat.h +++ b/src/observer/ob_heartbeat.h @@ -70,6 +70,11 @@ private: int try_reload_config(const int64_t config_version); int try_reload_time_zone_info(const int64_t time_zone_info_version); +#ifdef OB_BUILD_TDE_SECURITY + int set_lease_request_max_stored_versions( + share::ObLeaseRequest &lease_request, + const common::ObIArray > &max_stored_versions); +#endif private: void check_and_update_server_id_(const uint64_t server_id); bool inited_; diff --git a/src/observer/ob_inner_sql_connection.cpp b/src/observer/ob_inner_sql_connection.cpp index e6e80a49c..c69c514d0 100644 --- a/src/observer/ob_inner_sql_connection.cpp +++ b/src/observer/ob_inner_sql_connection.cpp @@ -45,6 +45,9 @@ #include "share/rc/ob_tenant_base.h" #include "storage/tablelock/ob_table_lock_service.h" #include "storage/tablelock/ob_table_lock_rpc_struct.h" +#ifdef OB_BUILD_AUDIT_SECURITY +#include "sql/monitor/ob_security_audit_utils.h" +#endif #include "sql/plan_cache/ob_ps_cache.h" #include "observer/mysql/obmp_stmt_execute.h" @@ -859,6 +862,14 @@ int ObInnerSQLConnection::query(sqlclient::ObIExecutor &executor, } if (res.is_inited()) { +#ifdef OB_BUILD_AUDIT_SECURITY + (void)ObSecurityAuditUtils::handle_security_audit(res.result_set(), + res.sql_ctx().schema_guard_, + res.sql_ctx().cur_stmt_, + ObString::make_string("inner sql"), + ret); + +#endif if (OB_SUCC(ret) && get_session().get_in_transaction()) { if (ObStmt::is_dml_write_stmt(res.result_set().get_stmt_type()) || ObStmt::is_savepoint_stmt(res.result_set().get_stmt_type())) { diff --git a/src/observer/ob_lease_state_mgr.cpp b/src/observer/ob_lease_state_mgr.cpp index 0a3c13288..fb5e97454 100644 --- a/src/observer/ob_lease_state_mgr.cpp +++ b/src/observer/ob_lease_state_mgr.cpp @@ -21,6 +21,9 @@ #include "observer/ob_server.h" #include "storage/compaction/ob_tenant_tablet_scheduler.h" #include "storage/tx_storage/ob_ls_service.h" +#ifdef OB_BUILD_TDE_SECURITY +#include "share/ob_master_key_getter.h" +#endif namespace oceanbase { @@ -306,6 +309,48 @@ int ObLeaseStateMgr::start_heartbeat() return ret; } +#ifdef OB_BUILD_TDE_SECURITY +int ObLeaseStateMgr::update_master_key_info( + const share::ObLeaseResponse &lease_response) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else { + const common::ObIArray > &master_key_array + = lease_response.tenant_max_key_version_; + common::ObArray > max_key_version_array; + common::ObArray > max_available_key_version_array; + for (int64_t i = 0; OB_SUCC(ret) && i < master_key_array.count(); ++i) { + const std::pair &key = master_key_array.at(i); + std::pair max_key_version + = std::pair(key.first, key.second.max_key_version_); + std::pair available_key_version + = std::pair(key.first, key.second.max_available_key_version_); + if (OB_FAIL(max_key_version_array.push_back(max_key_version))) { + LOG_WARN("fail to push back", KR(ret)); + } else { + if (available_key_version.second > 0) { + if (OB_FAIL(max_available_key_version_array.push_back(available_key_version))) { + LOG_WARN("fail to push back", KR(ret)); + } + } + } + } + if (OB_SUCC(ret)) { + if (OB_FAIL(ObMasterKeyGetter::instance().got_versions( + max_key_version_array))) { + LOG_WARN("fail to update got versions", KR(ret)); + } else if (OB_FAIL(ObMasterKeyGetter::instance().update_active_versions( + max_available_key_version_array))) { + LOG_WARN("fail to update active versions", KR(ret)); + } + } + } + return ret; +} +#endif int ObLeaseStateMgr::do_renew_lease() { @@ -347,6 +392,14 @@ int ObLeaseStateMgr::do_renew_lease() baseline_schema_version_ = lease_response.baseline_schema_version_; } } +#ifdef OB_BUILD_TDE_SECURITY + if (OB_SUCC(ret)) { + if (OB_SUCCESS != (tmp_ret = update_master_key_info(lease_response))) { + LOG_WARN("fail to update master key info", KR(ret), K(tmp_ret), K(lease_response)); + } + } + NG_TRACE_EXT(update_master_key_info, OB_ID(ret), tmp_ret); +#endif const int64_t now = ObTimeUtility::current_time(); if (OB_SUCC(ret) && lease_response.heartbeat_expire_time_ > now) { LOG_DEBUG("renew_lease from master_rs successfully", K(rs_addr)); diff --git a/src/observer/ob_lease_state_mgr.h b/src/observer/ob_lease_state_mgr.h index d00c86769..04175a51f 100644 --- a/src/observer/ob_lease_state_mgr.h +++ b/src/observer/ob_lease_state_mgr.h @@ -97,6 +97,9 @@ private: private: int start_heartbeat(); int do_renew_lease(); +#ifdef OB_BUILD_TDE_SECURITY + int update_master_key_info(const share::ObLeaseResponse &lease_response); +#endif int update_major_merge_info(const share::ObLeaseResponse &lease_response); private: bool inited_; diff --git a/src/observer/ob_rpc_processor_simple.cpp b/src/observer/ob_rpc_processor_simple.cpp index 9b3ba9e38..4d9681fbc 100644 --- a/src/observer/ob_rpc_processor_simple.cpp +++ b/src/observer/ob_rpc_processor_simple.cpp @@ -36,6 +36,9 @@ #include "observer/mysql/ob_mysql_request_manager.h" #include "observer/omt/ob_multi_tenant.h" #include "observer/omt/ob_tenant_node_balancer.h" +#ifdef OB_BUILD_TDE_SECURITY +#include "share/ob_master_key_getter.h" +#endif #include "share/stat/ob_opt_stat_manager.h" #include "share/stat/ob_opt_stat_monitor_manager.h" // for 4.0 @@ -61,6 +64,9 @@ #include "rootserver/ob_tenant_transfer_service.h" // ObTenantTransferService #include "storage/high_availability/ob_transfer_service.h" // ObTransferService #include "sql/udr/ob_udr_mgr.h" +#ifdef OB_BUILD_SPM +#include "sql/spm/ob_spm_controller.h" +#endif #include "sql/plan_cache/ob_ps_cache.h" #include "rootserver/ob_primary_ls_service.h" // for ObPrimaryLSService #include "sql/session/ob_sql_session_info.h" @@ -424,6 +430,67 @@ int ObRpcLSCheckDRTaskExistP::process() return ret; } +#ifdef OB_BUILD_ARBITRATION +int ObRpcAddArbP::process() +{ + int ret = OB_SUCCESS; + ObLSHandle handle; + ObLS *ls = nullptr; + uint64_t tenant_id = arg_.tenant_id_; + share::ObLSID ls_id = arg_.ls_id_; + ObLSService *ls_svr = nullptr; + if (tenant_id != MTL_ID()) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("ObRpcSetMemberListP::process tenant not match", K(ret), K(tenant_id)); + } + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(ls_svr = MTL(ObLSService*))) { + ret = OB_ERR_UNEXPECTED; + COMMON_LOG(ERROR, "mtl ObLSService should not be null", K(ret)); + } else if (OB_FAIL(ls_svr->get_ls(ls_id, handle, ObLSGetMod::OBSERVER_MOD))) { + COMMON_LOG(WARN, "get ls failed", K(ret), K(ls_id)); + } else if (OB_ISNULL(ls = handle.get_ls())) { + ret = OB_ERR_UNEXPECTED; + COMMON_LOG(ERROR, "ls should not be null", K(ret)); + } else if (OB_FAIL(ls->add_arbitration_member(arg_.arb_member_, arg_.timeout_us_))) { + COMMON_LOG(WARN, "failed to add_arbitration_member", KR(ret), K(arg_)); + } else { + COMMON_LOG(INFO, "success to add_arbitration_member", K_(arg)); + } + result_.set_result(ret); + return ret; +} + +int ObRpcRemoveArbP::process() +{ + int ret = OB_SUCCESS; + ObLSHandle handle; + ObLS *ls = nullptr; + uint64_t tenant_id = arg_.tenant_id_; + share::ObLSID ls_id = arg_.ls_id_; + ObLSService *ls_svr = nullptr; + if (tenant_id != MTL_ID()) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("ObRpcSetMemberListP::process tenant not match", K(ret), K(tenant_id)); + } + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(ls_svr = MTL(ObLSService*))) { + ret = OB_ERR_UNEXPECTED; + COMMON_LOG(ERROR, "mtl ObLSService should not be null", K(ret)); + } else if (OB_FAIL(ls_svr->get_ls(ls_id, handle, ObLSGetMod::OBSERVER_MOD))) { + COMMON_LOG(WARN, "get ls failed", K(ret), K(ls_id)); + } else if (OB_ISNULL(ls = handle.get_ls())) { + ret = OB_ERR_UNEXPECTED; + COMMON_LOG(ERROR, "ls should not be null", K(ret)); + } else if (OB_FAIL(ls->remove_arbitration_member(arg_.arb_member_, arg_.timeout_us_))) { + COMMON_LOG(WARN, "failed to remove_arbitration_member", KR(ret), K(arg_)); + } else { + COMMON_LOG(INFO, "success to remove_arbitration_member", K_(arg)); + } + result_.set_result(ret); + return ret; +} +#endif int ObRpcSetConfigP::process() { @@ -904,6 +971,19 @@ int ObRpcCheckDeploymentModeP::process() return ret; } +#ifdef OB_BUILD_TDE_SECURITY +int ObRpcWaitMasterKeyInSyncP::process() +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(gctx_.ob_service_)) { + ret = OB_INVALID_ARGUMENT; + LOG_ERROR("invalid argument", K(gctx_.ob_service_), K(ret)); + } else { + ret = gctx_.ob_service_->wait_master_key_in_sync(arg_); + } + return ret; +} +#endif int ObRpcSyncAutoincValueP::process() { @@ -1430,6 +1510,19 @@ int ObRpcCreateLSP::process() return ret; } +#ifdef OB_BUILD_ARBITRATION +int ObRpcCreateArbP::process() +{ + int ret = OB_SUCCESS; + return ret; +} + +int ObRpcDeleteArbP::process() +{ + int ret = OB_SUCCESS; + return ret; +} +#endif int ObRpcCheckLSCanOfflineP::process() { @@ -1589,6 +1682,17 @@ int ObRpcSetMemberListP::process() } else if (OB_ISNULL(ls = handle.get_ls())) { ret = OB_ERR_UNEXPECTED; COMMON_LOG(ERROR, "ls should not be null", K(ret)); +#ifdef OB_BUILD_ARBITRATION + } else if (arg_.get_arbitration_service().is_valid()) { + if (OB_FAIL(ls->set_initial_member_list(arg_.get_member_list(), + arg_.get_arbitration_service(), + arg_.get_paxos_replica_num(), + arg_.get_learner_list()))) { + COMMON_LOG(WARN, "failed to set member list and arbitration service", KR(ret), K(arg_)); + } else { + COMMON_LOG(INFO, "success to set initial member list and arbitration service"); + } +#endif } else if (OB_FAIL(ls->set_initial_member_list(arg_.get_member_list(), arg_.get_paxos_replica_num(), arg_.get_learner_list()))) { @@ -1980,6 +2084,40 @@ int ObPreProcessServerP::process() return ret; } +#ifdef OB_BUILD_TDE_SECURITY +int ObGetMasterKeyP::process() +{ + int ret = OB_SUCCESS; + // int64_t master_key_version = (int64_t)arg_; + // int64_t master_key_len = 0; + // if (OB_FAIL(share::ObMasterKeyGetter::get_master_key(master_key_version, + // result_.str_.ptr(), OB_MAX_MASTER_KEY_LENGTH, master_key_len))) { + // TRANS_LOG(WARN, "failed to get master key", K(ret)); + // } + // result_.str_.set_length(master_key_len); + ret = OB_NOT_SUPPORTED; + return ret; +} + +int ObRestoreKeyP::process() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ObMasterKeyUtil::restore_key(arg_.tenant_id_, + arg_.backup_dest_, arg_.encrypt_key_))) { + LOG_WARN("failed to restore key", K(ret)); + } + return ret; +} + +int ObSetRootKeyP::process() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ObMasterKeyGetter::instance().set_root_key(arg_, result_))) { + LOG_WARN("failed to set root key", K(ret)); + } + return ret; +} +#endif int ObHandlePartTransCtxP::process() { @@ -2037,6 +2175,13 @@ int ObRpcBatchSetTabletAutoincSeqP::process() return ret; } +#ifdef OB_BUILD_TDE_SECURITY +int ObDumpTenantCacheMasterKeyP::process() +{ + const uint64_t tenant_id = (uint64_t)arg_; + return ObMasterKeyGetter::instance().dump_tenant_cache_master_key(tenant_id, result_); +} +#endif int ObBatchBroadcastSchemaP::process() { @@ -2309,6 +2454,78 @@ int ObRpcBackupCleanLSResP::process() return ret; } +#ifdef OB_BUILD_SPM +int ObServerAcceptPlanBaselineP::process() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(sql::ObSpmController::accept_plan_baseline_by_user(arg_))) { + LOG_WARN("failed to accept plan baseline", K(ret), K(arg_)); + } + return ret; +} + +int ObServerCancelEvolveTaskP::process() +{ + int ret = OB_SUCCESS; + observer::ObReqTimeGuard req_timeinfo_guard; + ObPlanCache *plan_cache = nullptr; + obrpc::ObModifyPlanBaselineArg &arg = arg_; + bool evict_baseline = (ObModifyPlanBaselineArg::Action::EVICT_BASELINE == arg.action_ || + ObModifyPlanBaselineArg::Action::EVICT_ALL == arg.action_); + bool evict_plan = (ObModifyPlanBaselineArg::Action::EVICT_EVOLVE == arg.action_ || + ObModifyPlanBaselineArg::Action::EVICT_ALL == arg.action_); + MTL_SWITCH(arg.tenant_id_) { + plan_cache = MTL(ObPlanCache*); + if (evict_baseline && OB_FAIL(plan_cache-> + cache_evict_baseline_by_sql_id(arg.database_id_, arg.sql_id_))) { + LOG_WARN("failed to evict baseline by sql id", K(ret)); + } else if (evict_plan && OB_FAIL(plan_cache-> + cache_evict_plan_by_sql_id(arg.database_id_, arg.sql_id_))) { + LOG_WARN("failed to evict plan by sql id", K(ret)); + } + } + return ret; +} + +int ObLoadBaselineP::process() +{ + int ret = OB_SUCCESS; + + observer::ObReqTimeGuard req_timeinfo_guard; + MTL_SWITCH(arg_.tenant_id_) { + ObPlanCache *plan_cache = MTL(ObPlanCache*); + uint64_t dummy_count = 0; + if (OB_INVALID_ID == arg_.tenant_id_ || arg_.sql_id_.empty()) { // load appointed tenant cache + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K_(arg), K(ret)); + } else if (OB_FAIL(plan_cache->load_plan_baseline(arg_, dummy_count))) { + LOG_WARN("fail to load baseline from pc with arg", K_(arg)); + } + } + + return ret; +} + +int ObLoadBaselineV2P::process() +{ + int ret = OB_SUCCESS; + observer::ObReqTimeGuard req_timeinfo_guard; + MTL_SWITCH(arg_.tenant_id_) { + ObPlanCache *plan_cache = MTL(ObPlanCache*); + uint64_t load_count = 0; + if (OB_INVALID_ID == arg_.tenant_id_ || arg_.sql_id_.empty()) { // load appointed tenant cache + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K_(arg), K(ret)); + } else if (OB_FAIL(plan_cache->load_plan_baseline(arg_, load_count))) { + LOG_WARN("fail to load baseline from pc with arg", K_(arg)); + } else { + result_.load_count_ = load_count; + } + } + + return ret; +} +#endif int ObEstimateTabletBlockCountP::process() { diff --git a/src/observer/ob_rpc_processor_simple.h b/src/observer/ob_rpc_processor_simple.h index b788c7812..7f90da8e5 100644 --- a/src/observer/ob_rpc_processor_simple.h +++ b/src/observer/ob_rpc_processor_simple.h @@ -135,6 +135,9 @@ OB_DEFINE_PROCESSOR_S(Srv, OB_IS_EMPTY_SERVER, ObRpcIsEmptyServerP); OB_DEFINE_PROCESSOR_S(Srv, OB_CHECK_SERVER_FOR_ADDING_SERVER, ObRpcCheckServerForAddingServerP); OB_DEFINE_PROCESSOR_S(Srv, OB_CHECK_DEPLOYMENT_MODE, ObRpcCheckDeploymentModeP); OB_DEFINE_PROCESSOR_S(Srv, OB_NOTIFY_CREATE_TENANT_USER_LS, ObRpcCreateTenantUserLSP); +#ifdef OB_BUILD_TDE_SECURITY +OB_DEFINE_PROCESSOR_S(Srv, OB_WAIT_MASTER_KEY_IN_SYNC, ObRpcWaitMasterKeyInSyncP); +#endif OB_DEFINE_PROCESSOR_S(Srv, OB_REFRESH_SYNC_VALUE, ObRpcSyncAutoincValueP); OB_DEFINE_PROCESSOR_S(Srv, OB_CLEAR_AUTOINC_CACHE, ObRpcClearAutoincCacheP); OB_DEFINE_PROCESSOR_OBADMIN(Srv, OB_DUMP_MEMTABLE, ObDumpMemtableP); @@ -164,6 +167,10 @@ OB_DEFINE_PROCESSOR_S(Srv, OB_LS_REMOVE_PAXOS_REPLICA, ObRpcLSRemovePaxosReplica OB_DEFINE_PROCESSOR_S(Srv, OB_LS_REMOVE_NONPAXOS_REPLICA, ObRpcLSRemoveNonPaxosReplicaP); OB_DEFINE_PROCESSOR_S(Srv, OB_LS_MODIFY_PAXOS_REPLICA_NUMBER, ObRpcLSModifyPaxosReplicaNumberP); OB_DEFINE_PROCESSOR_S(Srv, OB_LS_CHECK_DR_TASK_EXIST, ObRpcLSCheckDRTaskExistP); +#ifdef OB_BUILD_ARBITRATION +OB_DEFINE_PROCESSOR_S(Srv, OB_ADD_ARB, ObRpcAddArbP); +OB_DEFINE_PROCESSOR_S(Srv, OB_REMOVE_ARB, ObRpcRemoveArbP); +#endif OB_DEFINE_PROCESSOR_S(Srv, OB_REPORT_REPLICA, ObReportReplicaP); OB_DEFINE_PROCESSOR_S(Srv, OB_RECYCLE_REPLICA, ObRecycleReplicaP); OB_DEFINE_PROCESSOR_S(Srv, OB_CLEAR_LOCATION_CACHE, ObClearLocationCacheP); @@ -198,10 +205,22 @@ OB_DEFINE_PROCESSOR_S(Srv, OB_GET_TENANT_REFRESHED_SCHEMA_VERSION, ObGetTenantSc OB_DEFINE_PROCESSOR_OBADMIN(Srv, OB_UPDATE_TENANT_MEMORY, ObUpdateTenantMemoryP); OB_DEFINE_PROCESSOR_S(Srv, OB_RENEW_IN_ZONE_HB, ObRenewInZoneHbP); OB_DEFINE_PROCESSOR_S(Srv, OB_PRE_PROCESS_SERVER, ObPreProcessServerP); +#ifdef OB_BUILD_TDE_SECURITY +OB_DEFINE_PROCESSOR_S(Srv, OB_GET_MASTER_KEY, ObGetMasterKeyP); +OB_DEFINE_PROCESSOR_S(Srv, OB_RESTORE_KEY, ObRestoreKeyP); +OB_DEFINE_PROCESSOR_S(Srv, OB_SET_ROOT_KEY, ObSetRootKeyP); +#endif OB_DEFINE_PROCESSOR_S(Srv, OB_HANDLE_PART_TRANS_CTX, ObHandlePartTransCtxP); OB_DEFINE_PROCESSOR_S(Srv, OB_SERVER_FLUSH_OPT_STAT_MONITORING_INFO, ObFlushLocalOptStatMonitoringInfoP); +#ifdef OB_BUILD_TDE_SECURITY +OB_DEFINE_PROCESSOR_S(Srv, OB_DUMP_TENANT_CACHE_MASTER_KEY, ObDumpTenantCacheMasterKeyP); +#endif OB_DEFINE_PROCESSOR_S(Srv, OB_SET_MEMBER_LIST, ObRpcSetMemberListP); OB_DEFINE_PROCESSOR_S(Srv, OB_CREATE_LS, ObRpcCreateLSP); +#ifdef OB_BUILD_ARBITRATION +OB_DEFINE_PROCESSOR_S(Srv, OB_CREATE_ARB, ObRpcCreateArbP); +OB_DEFINE_PROCESSOR_S(Srv, OB_DELETE_ARB, ObRpcDeleteArbP); +#endif OB_DEFINE_PROCESSOR_S(Srv, OB_BATCH_BROADCAST_SCHEMA, ObBatchBroadcastSchemaP); OB_DEFINE_PROCESSOR_S(Srv, OB_REMOTE_WRITE_DDL_REDO_LOG, ObRpcRemoteWriteDDLRedoLogP); OB_DEFINE_PROCESSOR_S(Srv, OB_REMOTE_WRITE_DDL_COMMIT_LOG, ObRpcRemoteWriteDDLCommitLogP); @@ -215,6 +234,12 @@ OB_DEFINE_PROCESSOR_S(Srv, OB_GET_LS_ACCESS_MODE, ObRpcGetLSAccessModeP); OB_DEFINE_PROCESSOR_S(Srv, OB_CHANGE_LS_ACCESS_MODE, ObRpcChangeLSAccessModeP); OB_DEFINE_PROCESSOR_S(Srv, OB_GET_LS_SYNC_SCN, ObRpcGetLSSyncScnP); OB_DEFINE_PROCESSOR_S(Srv, OB_LOG_FORCE_SET_LS_AS_SINGLE_REPLICA, ObForceSetLSAsSingleReplicaP); +#ifdef OB_BUILD_SPM +OB_DEFINE_PROCESSOR_S(Srv, OB_SERVER_ACCEPT_PLAN_BASELINE, ObServerAcceptPlanBaselineP); +OB_DEFINE_PROCESSOR_S(Srv, OB_SERVER_CANCEL_EVOLVE_TASK, ObServerCancelEvolveTaskP); +OB_DEFINE_PROCESSOR_S(Srv, OB_LOAD_BASELINE, ObLoadBaselineP); +OB_DEFINE_PROCESSOR_S(Srv, OB_LOAD_BASELINE_V2, ObLoadBaselineV2P); +#endif OB_DEFINE_PROCESSOR_S(Srv, OB_ESTIMATE_TABLET_BLOCK_COUNT, ObEstimateTabletBlockCountP); OB_DEFINE_PROCESSOR_S(Srv, OB_GEN_UNIQUE_ID, ObRpcGenUniqueIDP); OB_DEFINE_PROCESSOR_S(Srv, OB_START_TRANSFER_TASK, ObRpcStartTransferTaskP); diff --git a/src/observer/ob_server.cpp b/src/observer/ob_server.cpp index 5ed0c326d..8a7c18fe2 100644 --- a/src/observer/ob_server.cpp +++ b/src/observer/ob_server.cpp @@ -106,6 +106,18 @@ #include "observer/virtual_table/ob_mds_event_buffer.h" #include "observer/ob_server_startup_task_handler.h" #include "share/detect/ob_detect_manager.h" +#ifdef OB_BUILD_ARBITRATION +#include "logservice/arbserver/palf_env_lite_mgr.h" +#include "logservice/arbserver/ob_arb_srv_network_frame.h" +#include "logservice/arbserver/ob_arb_cluster_white_list.h" +#include "logservice/arbserver/ob_arb_server_config.h" +#endif +#ifdef OB_BUILD_TDE_SECURITY +#include "share/ob_master_key_getter.h" +#endif +#ifdef OB_BUILD_ORACLE_XML +#include "lib/xml/ob_libxml2_sax_handler.h" +#endif using namespace oceanbase::lib; using namespace oceanbase::common; @@ -188,6 +200,10 @@ ObServer::ObServer() unix_domain_listener_(), disk_usage_report_task_(), log_block_mgr_() +#ifdef OB_BUILD_ARBITRATION + ,arb_gcs_(), + arb_timer_() +#endif { memset(&gctx_, 0, sizeof (gctx_)); } @@ -206,6 +222,11 @@ int ObServer::parse_mode() if (0 == STRCASECMP(mode_str, NORMAL_MODE_STR)) { gctx_.startup_mode_ = NORMAL_MODE; LOG_INFO("set normal mode"); +#ifdef OB_BUILD_ARBITRATION + } else if (0 == STRCASECMP(mode_str, ARBITRATION_MODE_STR)) { + gctx_.startup_mode_ = ARBITRATION_MODE; + LOG_INFO("set arbitration mode"); +#endif } else if (0 == STRCASECMP(mode_str, FLASHBACK_MODE_STR)) { gctx_.startup_mode_ = PHY_FLASHBACK_MODE; LOG_INFO("set physical_flashback mode"); @@ -244,6 +265,25 @@ int ObServer::init(const ObServerOptions &opts, const ObPLogWriterCfg &log_cfg) ObLargePageHelper::set_param(config_.use_large_pages); if (is_arbitration_mode()) { +#ifdef OB_BUILD_ARBITRATION + FLOG_INFO("begin init observer in arbitration mode", KR(ret)); + if (FAILEDx(OB_LOGGER.init(log_cfg, true))) { + LOG_ERROR("async log init error.", KR(ret)); + ret = OB_ELECTION_ASYNC_LOG_WARN_INIT; + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(init_pre_setting())) { + LOG_ERROR("init pre setting failed", KR(ret)); + } else if (OB_FAIL(ObClockGenerator::init())) { + LOG_ERROR("init create clock generator failed", KR(ret)); + } else if (OB_FAIL(init_version())) { + LOG_ERROR("init version failed", KR(ret)); + } else if (OB_FAIL(init_server_in_arb_mode())) { + LOG_ERROR("init_server_in_arb_mode failed", KR(ret)); + } + lib::g_runtime_enabled = true; + FLOG_INFO("end init observer in arbitration mode", KR(ret)); +#endif } else { if (FAILEDx(OB_LOGGER.init(log_cfg, false))) { LOG_ERROR("async log init error.", KR(ret)); @@ -406,6 +446,10 @@ int ObServer::init(const ObServerOptions &opts, const ObPLogWriterCfg &log_cfg) LOG_ERROR("init get compat mode server failed",KR(ret)); } else if (OB_FAIL(table_service_.init())) { LOG_ERROR("init table service failed", KR(ret)); +#ifdef OB_BUILD_TDE_SECURITY + } else if (OB_FAIL(ObMasterKeyGetter::instance().init(&sql_proxy_))) { + LOG_ERROR("init get master key server failed", KR(ret)); +#endif } else if (OB_FAIL(ObTimerMonitor::get_instance().init())) { LOG_ERROR("init timer monitor failed", KR(ret)); } else if (OB_FAIL(ObBGThreadMonitor::get_instance().init())) { @@ -439,6 +483,13 @@ int ObServer::init(const ObServerOptions &opts, const ObPLogWriterCfg &log_cfg) } else if (OB_FAIL(ObDDLRedoLogWriter::get_instance().init())) { LOG_WARN("init DDL redo log writer failed", KR(ret)); } +#ifdef OB_BUILD_ARBITRATION + else if (OB_FAIL(arb_gcs_.init(GCTX.self_addr(), + lib::TGDefIDs::ArbGCSTh, + GCTX.srv_rpc_proxy_, GCTX.sql_proxy_))) { + LOG_ERROR("init arb_gcs_ failed", KR(ret)); + } +#endif else if (OB_FAIL(ObDetectManagerThread::instance().init(GCTX.self_addr(), net_frame_.get_req_transport()))) { LOG_WARN("init ObDetectManagerThread failed", KR(ret)); } else { @@ -477,6 +528,9 @@ void ObServer::destroy() FLOG_INFO("destroy config manager success"); if (is_arbitration_mode()) { +#ifdef OB_BUILD_ARBITRATION + destroy_server_in_arb_mode(); +#endif } else if (!has_destroy_ && has_stopped_) { FLOG_INFO("begin to destroy OB_LOGGER"); @@ -519,6 +573,11 @@ void ObServer::destroy() ObTableStoreStatMgr::get_instance().destroy(); FLOG_INFO("table store stat mgr destroyed"); +#ifdef OB_BUILD_TDE_SECURITY + FLOG_INFO("begin to destroy master key getter"); + ObMasterKeyGetter::instance().destroy(); + FLOG_INFO("master key getter destroyed"); +#endif FLOG_INFO("begin to destroy unix domain listener"); unix_domain_listener_.destroy(); @@ -588,6 +647,11 @@ void ObServer::destroy() sql_engine_.destroy(); FLOG_INFO("sql engine destroyed"); +#ifdef OB_BUILD_ORACLE_XML + FLOG_INFO("begin to destroy xml ctx"); + ObLibXml2SaxHandler::destroy(); + FLOG_INFO("xml ctx destroyed"); +#endif FLOG_INFO("begin to destroy pl engine"); pl_engine_.destory(); @@ -717,6 +781,10 @@ void ObServer::destroy() ObClockGenerator::destroy(); FLOG_INFO("clock generator destroyed"); +#ifdef OB_BUILD_ARBITRATION + arb_gcs_.destroy(); + FLOG_INFO("ArbGarbageCollectSerivce destroyed"); +#endif has_destroy_ = true; FLOG_INFO("[OBSERVER_NOTICE] destroy observer end"); @@ -745,6 +813,18 @@ int ObServer::start() FLOG_INFO("[OBSERVER_NOTICE] start observer begin"); if (is_arbitration_mode()) { +#ifdef OB_BUILD_ARBITRATION + if (OB_FAIL(start_sig_worker_and_handle())) { + LOG_ERROR("fail to start signal worker and handle", KR(ret)); + } else if (OB_FAIL(start_server_in_arb_mode())) { + LOG_ERROR("start_server_in_arb_mode failed", K(ret)); + } else { + FLOG_INFO("[OBSERVER_NOTICE] server instance start succeed"); + prepare_stop_ = false; + stop_ = false; + has_stopped_ = false; + } +#endif } else { if (OB_FAIL(start_sig_worker_and_handle())) { LOG_ERROR("fail to start signal worker", KR(ret)); @@ -899,6 +979,13 @@ int ObServer::start() FLOG_INFO("success to start location service"); } +#ifdef OB_BUILD_ARBITRATION + if (FAILEDx(arb_gcs_.start())) { + LOG_ERROR("start arb_gcs_ failed", KR(ret)); + } else { + FLOG_INFO("success to start ArbGarbageCollectSerivce"); + } +#endif if (OB_SUCC(ret)) { FLOG_INFO("[OBSERVER_NOTICE] server instance start succeed"); @@ -1069,6 +1156,9 @@ int ObServer::stop() FLOG_INFO("stop config manager success"); if (is_arbitration_mode()) { +#ifdef OB_BUILD_ARBITRATION + (void) stop_server_in_arb_mode(); +#endif } else { #ifdef ENABLE_IMC FLOG_INFO("begin to stop imc tasks", KR(ret)); @@ -1124,6 +1214,11 @@ int ObServer::stop() ObTableStoreStatMgr::get_instance().stop(); FLOG_INFO("table store stat mgr stopped"); +#ifdef OB_BUILD_TDE_SECURITY + FLOG_INFO("begin to stop master key getter"); + ObMasterKeyGetter::instance().stop(); + FLOG_INFO("master key getter stopped"); +#endif FLOG_INFO("begin to stop unix domain listener"); unix_domain_listener_.stop(); @@ -1289,6 +1384,10 @@ int ObServer::stop() rl_mgr_.stop(); FLOG_INFO("ratelimit manager stopped"); +#ifdef OB_BUILD_ARBITRATION + arb_gcs_.stop(); + FLOG_INFO("ArbGarbageCollectSerivce stopped"); +#endif FLOG_INFO("begin to shutdown rpc network"); if (OB_FAIL(net_frame_.rpc_shutdown())) { @@ -1386,6 +1485,9 @@ int ObServer::wait() FLOG_INFO("wait config manager success"); if (is_arbitration_mode()) { +#ifdef OB_BUILD_ARBITRATION + (void) wait_server_in_arb_mode(); +#endif } else { FLOG_INFO("begin to wait OB_LOGGER"); @@ -1416,6 +1518,11 @@ int ObServer::wait() ObTableStoreStatMgr::get_instance().wait(); FLOG_INFO("wait table store stat mgr success"); +#ifdef OB_BUILD_TDE_SECURITY + FLOG_INFO("begin to wait master key getter"); + ObMasterKeyGetter::instance().wait(); + FLOG_INFO("wait master key getter success"); +#endif FLOG_INFO("begin to wait unix domain listener"); unix_domain_listener_.wait(); @@ -1586,6 +1693,10 @@ int ObServer::wait() palf::election::GLOBAL_REPORT_TIMER.wait(); FLOG_INFO("wait global election report timer success"); + #ifdef OB_BUILD_ARBITRATION + arb_gcs_.wait(); + FLOG_INFO("wait ArbGarbageCollectSerivce success"); + #endif FLOG_INFO("begin to wait rootservice event history"); ROOTSERVICE_EVENT_INSTANCE.wait(); @@ -2327,6 +2438,11 @@ int ObServer::init_sql() } } +#ifdef OB_BUILD_ORACLE_XML + if (OB_SUCC(ret)) { + ObLibXml2SaxHandler::init(); + } +#endif if (OB_SUCC(ret)) { LOG_INFO("init sql done"); @@ -2432,6 +2548,9 @@ int ObServer::init_global_context() gctx_.locality_manager_ = &locality_manager_; gctx_.disk_reporter_ = &disk_usage_report_task_; gctx_.log_block_mgr_ = &log_block_mgr_; +#ifdef OB_BUILD_ARBITRATION + gctx_.arb_gcs_ = &arb_gcs_; +#endif (void)gctx_.set_upgrade_stage(obrpc::OB_UPGRADE_STAGE_INVALID); gctx_.flashback_scn_ = opts_.flashback_scn_; @@ -3376,10 +3495,205 @@ int ObServer::clean_up_invalid_tables_by_tenant( } // ---------------------------------- arb server start ------------------------------- +#ifdef OB_BUILD_ARBITRATION +int ObServer::init_server_in_arb_mode() +{ + int ret = OB_SUCCESS; + // init GCTX's config members. + gctx_.config_ = &config_; + gctx_.config_mgr_ = &config_mgr_; + // set_extra_payload() call is necessary, or dest may return -4007 when deserialie rpc packet. + obrpc::ObIRpcExtraPayload::set_extra_payload(ObRpcExtraPayload::extra_payload_instance()); + palflite::PalfEnvLiteMgr &palf_env_mgr = palflite::PalfEnvLiteMgr::get_instance(); + arbserver::ObArbSrvNetworkFrame &net_work_farme = arbserver::ObArbSrvNetworkFrame::get_instance(); + rpc::frame::ObNetOptions opts; + arbserver::ArbNetOptions arb_opts; + const uint32_t rpc_port = static_cast(GCONF.rpc_port); + + int rpc_io_cnt = static_cast(GCONF.net_thread_count); + // make net thread count adaptive + if (0 == rpc_io_cnt) { + rpc_io_cnt = get_default_net_thread_count(); + } + const int hp_io_cnt = static_cast(GCONF.high_priority_net_thread_count); + opts.rpc_io_cnt_ = rpc_io_cnt; // rpc io thread count + opts.high_prio_rpc_io_cnt_ = hp_io_cnt; + // Do not need set mysql/batch io cnt for arb server + // opts.mysql_io_cnt_ = io_cnt; + // opts.batch_rpc_io_cnt_ = io_cnt; + opts.use_ipv6_ = GCONF.use_ipv6; + //TODO(tony.wzh): fix opts.tcp_keepidle negative + opts.tcp_user_timeout_ = static_cast(GCONF.dead_socket_detection_timeout); + opts.tcp_keepidle_ = static_cast(GCONF.tcp_keepidle); + opts.tcp_keepintvl_ = static_cast(GCONF.tcp_keepintvl); + opts.tcp_keepcnt_ = static_cast(GCONF.tcp_keepcnt); + + if (GCONF.enable_tcp_keepalive) { + opts.enable_tcp_keepalive_ = 1; + } else { + opts.enable_tcp_keepalive_ = 0; + } + + arb_opts.opts_ = opts; + arb_opts.self_ = self_addr_; + + LOG_INFO("io thread connection negotiation enabled!"); + arb_opts.negotiation_enable_ = 1; // enable negotiation + arb_opts.rpc_port_ = rpc_port; + + if (OB_FAIL(net_work_farme.init(arb_opts, &palf_env_mgr))) { + LOG_ERROR("init ObArbSrvNetworkFrame failed", K(ret), K(arb_opts)); + } else if (OB_FAIL(palf_env_mgr.init(GCONF.data_dir, self_addr_, net_work_farme.get_req_transport()))) { + LOG_ERROR("init PalfEnvLiteMgr failed", K(ret), K(arb_opts)); + } else if (OB_FAIL(arb_timer_.init(lib::TGDefIDs::ArbServerTimer, &palf_env_mgr))) { + LOG_ERROR("init ArbServerTimer failed", K(ret)); + #ifndef OB_USE_ASAN + } else if (OB_FAIL(ObMemoryDump::get_instance().init())) { + LOG_ERROR("init memory dumper failed", KR(ret)); + #endif + } else if (OB_FAIL(arbserver::ObArbWhiteList::get_instance().init())) { + LOG_ERROR("init ObArbWhiteList failed", K(ret)); + } else if (OB_FAIL(ASCONF.init())){ + LOG_ERROR("init ObArbServerConfig failed", K(ret)); + } else if (OB_FAIL(ASCONF.init_config_with_file())) { + LOG_ERROR("init config with file failed", K(ret)); + } else { + LOG_INFO("init_server_in_arb_mode success", K(ret), K(arb_opts)); + } + + return ret; +} + +int ObServer::start_server_in_arb_mode() +{ + int ret = OB_SUCCESS; + arbserver::ObArbSrvNetworkFrame &net_work_farme = arbserver::ObArbSrvNetworkFrame::get_instance(); + if (OB_FAIL(net_work_farme.start())) { + LOG_ERROR("start ObArbSrvNetworkFrame failed", K(ret)); + } else if (OB_FAIL(arb_timer_.start())) { + LOG_ERROR("start ObArbServerTimer failed", K(ret)); + } else { + LOG_INFO("start_server_in_arb_mode success", K(ret)); + } + return ret; +} + +int ObServer::stop_server_in_arb_mode() +{ + int ret = OB_SUCCESS; + palf::election::GLOBAL_REPORT_TIMER.stop(); + FLOG_INFO("global election report timer stopped"); + + arbserver::ObArbSrvNetworkFrame &net_work_farme = arbserver::ObArbSrvNetworkFrame::get_instance(); + FLOG_INFO("begin to stop net_frame"); + if (OB_FAIL(net_work_farme.rpc_shutdown())) { + FLOG_WARN("rpc_shutdown failed"); + } else if (OB_FAIL(net_work_farme.stop())) { + FLOG_WARN("stop net_frame failed"); + } else if (OB_FAIL(arb_timer_.stop())) { + LOG_ERROR("stop ObArbServerTimer failed", K(ret)); + } else { + FLOG_INFO("stop net_frame success"); + FLOG_INFO("begin to stop OB_LOGGER"); + OB_LOGGER.stop(); + FLOG_INFO("stop OB_LOGGER success"); + + FLOG_INFO("begin to stop task controller"); + ObTaskController::get().stop(); + FLOG_INFO("stop task controller success"); + + FLOG_INFO("begin stop arbserver config"); + ASCONF.stop(); + FLOG_INFO("stop arbserver config success"); + + FLOG_INFO("begin stop signal worker"); + sig_worker_->stop(); + FLOG_INFO("stop signal worker success"); + + FLOG_INFO("begin stop signal handle"); + signal_handle_->stop(); + FLOG_INFO("stop signal handle success"); + + FLOG_INFO("begin to stop memory dump"); + ObMemoryDump::get_instance().stop(); + FLOG_INFO("memory dump stopped"); + + } + + FLOG_INFO("stop_server_in_arb_mode success", K(ret)); + return ret; +} + +int ObServer::wait_server_in_arb_mode() +{ + int ret = OB_SUCCESS; + palf::election::GLOBAL_REPORT_TIMER.wait(); + FLOG_INFO("wait global election report timer success"); + + FLOG_INFO("begin to wait OB_LOGGER"); + OB_LOGGER.wait(); + FLOG_INFO("wait OB_LOGGER success"); + + FLOG_INFO("begin to wait task controller"); + ObTaskController::get().wait(); + FLOG_INFO("wait task controller success"); + + arbserver::ObArbSrvNetworkFrame &net_work_farme = arbserver::ObArbSrvNetworkFrame::get_instance(); + FLOG_INFO("begin to wait net_frame"); + net_work_farme.wait(); + FLOG_INFO("wait net_frame success"); + + FLOG_INFO("begin to wait arb_server"); + arb_timer_.wait(); + FLOG_INFO("wait arb_server success"); + + FLOG_INFO("begin wait arbserver config"); + ASCONF.wait(); + FLOG_INFO("wait arbserver config success"); + + FLOG_INFO("begin wait signal worker"); + sig_worker_->wait(); + FLOG_INFO("wait signal worker success"); + + FLOG_INFO("begin wait signal handle"); + signal_handle_->wait(); + FLOG_INFO("wait signal handle success"); + + FLOG_INFO("begin to wait memory dump"); + ObMemoryDump::get_instance().wait(); + FLOG_INFO("wait memory dump success"); + + FLOG_INFO("wait_server_in_arb_mode success", K(ret)); + return ret; +} + +int ObServer::destroy_server_in_arb_mode() +{ + int ret = OB_SUCCESS; + OB_LOGGER.destroy(); + ObTaskController::get().destroy(); + sig_worker_->destroy(); + signal_handle_->destroy(); + palflite::PalfEnvLiteMgr &palf_env_mgr = palflite::PalfEnvLiteMgr::get_instance(); + arbserver::ObArbSrvNetworkFrame &net_work_farme = arbserver::ObArbSrvNetworkFrame::get_instance(); + palf_env_mgr.destroy(); + net_work_farme.destroy(); + arb_timer_.destroy(); + ObMemoryDump::get_instance().destroy(); + ASCONF.destroy(); + palf::election::GLOBAL_REPORT_TIMER.destroy(); + LOG_WARN("destroy_server_in_arb_mode success", K(ret)); + return ret; +} +#endif bool ObServer::is_arbitration_mode() const { +#ifdef OB_BUILD_ARBITRATION + return (ARBITRATION_MODE == gctx_.startup_mode_) ? true : false; +#else return false; +#endif } diff --git a/src/observer/ob_server.h b/src/observer/ob_server.h index eb501a777..61881b61c 100644 --- a/src/observer/ob_server.h +++ b/src/observer/ob_server.h @@ -69,6 +69,10 @@ #include "storage/ob_disk_usage_reporter.h" #include "observer/dbms_scheduler/ob_dbms_sched_job_rpc_proxy.h" #include "logservice/ob_server_log_block_mgr.h" +#ifdef OB_BUILD_ARBITRATION +#include "logservice/arbserver/ob_arb_srv_garbage_collect_service.h" +#include "logservice/arbserver/ob_arb_server_timer.h" +#endif #include "share/table/ob_table_rpc_proxy.h" @@ -446,6 +450,10 @@ private: ObDiskUsageReportTask disk_usage_report_task_; logservice::ObServerLogBlockMgr log_block_mgr_; +#ifdef OB_BUILD_ARBITRATION + arbserver::ObArbGarbageCollectService arb_gcs_; + arbserver::ObArbServerTimer arb_timer_; +#endif }; // end of class ObServer inline ObServer &ObServer::get_instance() diff --git a/src/observer/ob_server_struct.h b/src/observer/ob_server_struct.h index 835127746..bbe64d596 100644 --- a/src/observer/ob_server_struct.h +++ b/src/observer/ob_server_struct.h @@ -107,6 +107,12 @@ namespace logservice class ObServerLogBlockMgr; } +#ifdef OB_BUILD_ARBITRATION +namespace arbserver +{ +class ObArbGarbageCollectService; +} +#endif namespace observer { @@ -265,6 +271,9 @@ struct ObGlobalContext share::ObAliveServerTracer *server_tracer_; ObIDiskReport *disk_reporter_; logservice::ObServerLogBlockMgr *log_block_mgr_; +#ifdef OB_BUILD_ARBITRATION + arbserver::ObArbGarbageCollectService *arb_gcs_; +#endif bool inited_; transaction::ObIWeakReadService *weak_read_service_; diff --git a/src/observer/ob_service.cpp b/src/observer/ob_service.cpp index 57711dca4..f4af30cdc 100644 --- a/src/observer/ob_service.cpp +++ b/src/observer/ob_service.cpp @@ -75,6 +75,9 @@ #include "observer/report/ob_tenant_meta_checker.h"//ObTenantMetaChecker #include "rootserver/backup/ob_backup_task_scheduler.h" // ObBackupTaskScheduler #include "rootserver/backup/ob_backup_schedule_task.h" // ObBackupScheduleTask +#ifdef OB_BUILD_TDE_SECURITY +#include "share/ob_master_key_getter.h" +#endif #include "storage/compaction/ob_tenant_tablet_scheduler.h" #include "share/ob_cluster_event_history_table_operator.h"//CLUSTER_EVENT_INSTANCE #include "storage/ddl/ob_tablet_ddl_kv_mgr.h" @@ -1654,6 +1657,125 @@ int ObService::get_partition_count(obrpc::ObGetPartitionCountResult &result) return ret; } +#ifdef OB_BUILD_TDE_SECURITY +int ObService::convert_tenant_max_key_version( + const ObIArray > &max_key_version, + ObIArray > &got_version_array) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < max_key_version.count(); ++i) { + const std::pair &key_version + = max_key_version.at(i); + std::pair got_version; + got_version.first = key_version.first; + got_version.second = key_version.second.max_key_version_; + if (OB_FAIL(got_version_array.push_back(got_version))) { + LOG_WARN("fail to push back", KR(ret)); + } + } + } + return ret; +} + +int ObService::do_wait_master_key_in_sync( + const common::ObIArray > &got_version_array) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else { + const int64_t abs_time = THIS_WORKER.get_timeout_ts(); + int64_t finished_idx = -1; + const int64_t SLEEP_INTERVAL = 50L * 1000L; + LOG_INFO("do wait master key in sync", K(abs_time)); + while (ObTimeUtility::current_time() < abs_time && OB_SUCC(ret)) { + for (int64_t i = finished_idx + 1; OB_SUCC(ret) && i < got_version_array.count(); ++i) { + uint64_t max_stored_key_version = 0; + const std::pair &got_version = got_version_array.at(i); + if (OB_FAIL(ObMasterKeyGetter::instance().get_max_stored_version( + got_version.first, max_stored_key_version))) { + LOG_WARN("fail to get max active version", KR(ret), + "tenant_id", got_version.first); + } else if (max_stored_key_version >= got_version.second) { + finished_idx = i; + } else { + // TODO: wenduo + // remove got_versions after renqing make got_versions has a retry logic in it + (void)ObMasterKeyGetter::instance().got_versions(got_version_array); + ob_usleep(std::min(SLEEP_INTERVAL, abs_time - ObTimeUtility::current_time())); + break; + } + } + if (OB_FAIL(ret)) { + // failed + } else if (finished_idx >= got_version_array.count() - 1) { + break; // succ + LOG_INFO("wait master key in sync succ"); + } else if (ObTimeUtility::current_time() >= abs_time) { + ret = OB_TIMEOUT; + LOG_WARN("fail master key in sync timeout", KR(ret), K(finished_idx)); + } else { + // still need wait + } + } + } + return ret; +} + +int ObService::trigger_tenant_config( + const obrpc::ObWaitMasterKeyInSyncArg &wms_in_sync_arg) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), K(ret)); + } else { + // ignore ret in for condition + for (int64_t i = 0; i < wms_in_sync_arg.tenant_config_version_.count(); ++i) { + const uint64_t tenant_id = wms_in_sync_arg.tenant_config_version_.at(i).first; + const int64_t version = wms_in_sync_arg.tenant_config_version_.at(i).second; + OTC_MGR.add_tenant_config(tenant_id); // ignore ret + OTC_MGR.got_version(tenant_id, version); // ignore ret + } + } + return ret; +} + +int ObService::wait_master_key_in_sync( + const obrpc::ObWaitMasterKeyInSyncArg &wms_in_sync_arg) +{ + int ret = OB_SUCCESS; + common::ObArray > got_version_array; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (OB_FAIL(broadcast_rs_list(wms_in_sync_arg.rs_list_arg_))) { + LOG_WARN("fail to broadcast rs list", KR(ret), "rs_list_arg", wms_in_sync_arg.rs_list_arg_); + } else if (wms_in_sync_arg.tenant_max_key_version_.count() <= 0) { + // bypass, since tenant max key version is empty + } else if (OB_FAIL(convert_tenant_max_key_version( + wms_in_sync_arg.tenant_max_key_version_, got_version_array))) { + LOG_WARN("fail to convert tenant max key version", KR(ret), K(wms_in_sync_arg)); + } else { + ObRefreshSchemaInfo schema_info; + if (OB_FAIL(schema_updater_.try_reload_schema(schema_info))) { + LOG_WARN("fail to try reload schema", KR(ret)); + } else if (OB_FAIL(trigger_tenant_config(wms_in_sync_arg))) { + LOG_WARN("fail to got versions", KR(ret)); + } else if (OB_FAIL(ObMasterKeyGetter::instance().got_versions(got_version_array))) { + LOG_WARN("fail to get versions", KR(ret)); + } else if (OB_FAIL(do_wait_master_key_in_sync(got_version_array))) { + LOG_WARN("fail to do wait master key in sync", KR(ret)); + } + } + return ret; +} +#endif int ObService::check_server_empty(bool &is_empty) { @@ -1676,6 +1798,14 @@ int ObService::check_server_empty(bool &is_empty) is_empty = false; } } +#ifdef OB_BUILD_TDE_SECURITY + if (is_empty) { + if (ObMasterKeyGetter::instance().is_wallet_exist()) { + FLOG_WARN("[CHECK_SERVER_EMPTY] master_key file exists"); + is_empty = false; + } + } +#endif } return ret; } diff --git a/src/observer/ob_service.h b/src/observer/ob_service.h index c1d3025b0..0191bf68c 100644 --- a/src/observer/ob_service.h +++ b/src/observer/ob_service.h @@ -209,6 +209,15 @@ public: obrpc::ObBatchBroadcastSchemaResult &result); //////////////////////////////////////////////////////////////// +#ifdef OB_BUILD_TDE_SECURITY + int wait_master_key_in_sync(const obrpc::ObWaitMasterKeyInSyncArg &wms_in_sync_arg); + int trigger_tenant_config(const obrpc::ObWaitMasterKeyInSyncArg &wms_in_sync_arg); + int do_wait_master_key_in_sync( + const common::ObIArray > &got_version_array); + int convert_tenant_max_key_version( + const common::ObIArray > &, + common::ObIArray > &); +#endif // ObReportReplicaP @RS::admin to report replicas int report_replica(); int load_leader_cluster_login_info(); diff --git a/src/observer/ob_srv_deliver.cpp b/src/observer/ob_srv_deliver.cpp index e6f0af800..79af8116e 100644 --- a/src/observer/ob_srv_deliver.cpp +++ b/src/observer/ob_srv_deliver.cpp @@ -183,8 +183,7 @@ int check_easy_memory_limit(ObRequest &req) int ret = OB_SUCCESS; easy_mod_stat_t *stat = NULL; - if (req.get_nio_protocol() == ObRequest::TRANSPORT_PROTO_POC - || req.get_nio_protocol() == ObRequest::TRANSPORT_PROTO_RDMA) { + if (req.get_nio_protocol() == ObRequest::TRANSPORT_PROTO_POC) { // Todo: return ret; } @@ -517,7 +516,7 @@ int ObSrvDeliver::deliver_mysql_request(ObRequest &req) EVENT_INC(MYSQL_PACKET_IN); EVENT_ADD(MYSQL_PACKET_IN_BYTES, pkt.get_clen() + OB_MYSQL_HEADER_LENGTH); } - if (OB_UNLIKELY((ObRequest::TRANSPORT_PROTO_RDMA != req.get_nio_protocol()) && NULL != diagnose_queue_ && SQL_REQ_OP.get_peer(&req).get_port() <= 0)) { + if (OB_UNLIKELY(NULL != diagnose_queue_ && SQL_REQ_OP.get_peer(&req).get_port() <= 0)) { LOG_INFO("receive login request from unix domain socket"); if (!diagnose_queue_->queue_.push(&req, MAX_QUEUE_LEN)) { ret = OB_QUEUE_OVERFLOW; diff --git a/src/observer/ob_srv_network_frame.cpp b/src/observer/ob_srv_network_frame.cpp index 82827279e..d104aa2b0 100644 --- a/src/observer/ob_srv_network_frame.cpp +++ b/src/observer/ob_srv_network_frame.cpp @@ -31,6 +31,9 @@ extern "C" { #include #include #include "storage/ob_locality_manager.h" +#ifdef OB_USE_BABASSL +#include "share/ob_encrypt_kms.h" +#endif using namespace oceanbase::rpc::frame; using namespace oceanbase::common; @@ -155,6 +158,11 @@ int ObSrvNetworkFrame::init() } else if (ingress_service_.init(GCONF.cluster_id)) { LOG_ERROR("endpoint ingress service init fail", K(ret)); } +#ifdef OB_USE_BABASSL + else if (OB_FAIL(net_.add_rpc_unix_listen(rpc_unix_path, rpc_handler_))) { + LOG_ERROR("listen rpc unix path fail"); + } +#endif else { if (OB_FAIL(obrpc::global_poc_server.start(rpc_port, io_cnt, &deliver_))) { LOG_ERROR("poc rpc server start fail", K(ret)); @@ -398,6 +406,24 @@ int ObSrvNetworkFrame::reload_ssl_config() const char *enc_cert = NULL; const char *enc_private_key = NULL; +#ifdef OB_USE_BABASSL + share::ObSSLClient client; + if (!ssl_config.empty()) { + if (OB_FAIL(client.init(ssl_config.ptr(), ssl_config.length()))) { + OB_LOG(WARN, "kms client init", K(ret), K(ssl_config)); + } else if (OB_FAIL(client.check_param_valid())) { + OB_LOG(WARN, "kms client param is not valid", K(ret)); + } else { + use_bkmi = client.is_bkmi_mode(); + use_sm = client.is_sm_scene(); + ca_cert = client.get_root_ca().ptr(); + public_cert = client.public_cert_.content_.ptr(); + private_key = client.private_key_.content_.ptr(); + enc_cert = client.public_cert_for_enc_.content_.ptr(); + enc_private_key = client.private_key_for_enc_.content_.ptr(); + } + } +#endif if (! use_bkmi) { if (!use_sm) { @@ -411,6 +437,23 @@ int ObSrvNetworkFrame::reload_ssl_config() public_cert = OB_SSL_CERT_FILE; private_key = OB_SSL_KEY_FILE; } +#ifdef OB_USE_BABASSL + } else { + if (EASY_OK != easy_ssl_ob_config_check(OB_SSL_CA_FILE, + OB_SSL_SM_SIGN_CERT_FILE, OB_SSL_SM_SIGN_KEY_FILE, + OB_SSL_SM_ENC_CERT_FILE, OB_SSL_SM_ENC_KEY_FILE, + true, true)) { + ret = OB_INVALID_CONFIG; + LOG_WARN("key and cert not match", K(ret)); + LOG_USER_ERROR(OB_INVALID_CONFIG, "key and cert not match"); + } else { + ca_cert = OB_SSL_CA_FILE; + public_cert = OB_SSL_SM_SIGN_CERT_FILE; + private_key = OB_SSL_SM_SIGN_KEY_FILE; + enc_cert = OB_SSL_SM_ENC_CERT_FILE; + enc_private_key = OB_SSL_SM_ENC_KEY_FILE; + } +#endif } } @@ -419,6 +462,11 @@ int ObSrvNetworkFrame::reload_ssl_config() if (!use_bkmi && !use_sm &&OB_FAIL(extract_expired_time(OB_SSL_CERT_FILE, ssl_key_expired_time))) { OB_LOG(WARN, "extract_expired_time intl failed", K(ret), K(use_bkmi)); } + #ifdef OB_USE_BABASSL + else if (!use_bkmi && use_sm && OB_FAIL(extract_expired_time(OB_SSL_SM_ENC_CERT_FILE, ssl_key_expired_time))) { + OB_LOG(WARN, "extract_expired_time sm failed", K(ret), K(use_bkmi)); + } + #endif else if (OB_FAIL(net_.load_ssl_config(use_bkmi, use_sm, ca_cert, public_cert, private_key, enc_cert, enc_private_key))) { OB_LOG(WARN, "load_ssl_config failed", K(ret), K(use_bkmi), K(use_sm)); @@ -427,7 +475,15 @@ int ObSrvNetworkFrame::reload_ssl_config() mysql_handler_.ez_handler()->is_ssl_opt = 1; rpc_handler_.ez_handler()->is_ssl = 1; rpc_handler_.ez_handler()->is_ssl_opt = 0; +#ifndef OB_USE_BABASSL GCTX.ssl_key_expired_time_ = ssl_key_expired_time; +#else + if (use_bkmi) { + GCTX.ssl_key_expired_time_ = client.public_cert_.key_expired_time_; + } else { + GCTX.ssl_key_expired_time_ = ssl_key_expired_time; + } +#endif last_ssl_info_hash_ = new_hash_value; LOG_INFO("finish reload_ssl_config", K(use_bkmi), K(use_bkmi), K(use_sm), "ssl_key_expired_time", GCTX.ssl_key_expired_time_, K(new_hash_value)); diff --git a/src/observer/ob_srv_network_frame.h b/src/observer/ob_srv_network_frame.h index 4b60ab5a3..7f8f54802 100644 --- a/src/observer/ob_srv_network_frame.h +++ b/src/observer/ob_srv_network_frame.h @@ -96,7 +96,6 @@ private: ObSMHandler mysql_handler_; rootserver::ObIngressBWAllocService ingress_service_; - rpc::frame::ObNetEasy net_; rpc::frame::ObReqTransport *rpc_transport_; rpc::frame::ObReqTransport *high_prio_rpc_transport_; diff --git a/src/observer/ob_srv_xlator.cpp b/src/observer/ob_srv_xlator.cpp index 77323baea..bff9cf980 100644 --- a/src/observer/ob_srv_xlator.cpp +++ b/src/observer/ob_srv_xlator.cpp @@ -354,15 +354,6 @@ int ObSrvXlator::release(ObReqProcessor *processor) } else if (reinterpret_cast(processor) == cpbuf) { processor->destroy(); ObRequest::TransportProto nio_protocol = (ObRequest::TransportProto)processor->get_nio_protocol(); - if (ObRequest::TRANSPORT_PROTO_RDMA == nio_protocol) { - ObRequest *req = const_cast(processor->get_ob_request()); - bool need_retry = processor->get_need_retry(); - bool async_resp_used = processor->get_async_resp_used(); - if (req && !need_retry && !async_resp_used) { - SQL_REQ_OP.destroy(req); - // ob_free(req); - } - } processor->~ObReqProcessor(); } else { processor->destroy(); @@ -387,12 +378,6 @@ int ObSrvXlator::release(ObReqProcessor *processor) ObSqlTaskFactory::get_instance().free(static_cast(req)); req = NULL; } else { - if ((ObRequest::OB_MYSQL == req_type) && (ObRequest::TRANSPORT_PROTO_RDMA == nio_protocol)) { - if (req && !need_retry && !async_resp_used) { - SQL_REQ_OP.destroy(req); - // ob_free(req); - } - } worker_allocator_delete(processor); processor = NULL; } diff --git a/src/observer/ob_srv_xlator_partition.cpp b/src/observer/ob_srv_xlator_partition.cpp index 92ac27341..d1dd440c9 100644 --- a/src/observer/ob_srv_xlator_partition.cpp +++ b/src/observer/ob_srv_xlator_partition.cpp @@ -96,9 +96,21 @@ void oceanbase::observer::init_srv_xlator_for_partition(ObSrvRpcXlator *xlator) RPC_PROCESSOR(ObAddDiskP, gctx_); RPC_PROCESSOR(ObDropDiskP, gctx_); RPC_PROCESSOR(ObForceSetServerListP, gctx_); +#ifdef OB_BUILD_TDE_SECURITY + RPC_PROCESSOR(ObGetMasterKeyP, gctx_); + RPC_PROCESSOR(ObRestoreKeyP, gctx_); + RPC_PROCESSOR(ObSetRootKeyP, gctx_); +#endif RPC_PROCESSOR(ObHandlePartTransCtxP, gctx_); +#ifdef OB_BUILD_TDE_SECURITY + RPC_PROCESSOR(ObDumpTenantCacheMasterKeyP, gctx_); +#endif RPC_PROCESSOR(ObRpcSetMemberListP, gctx_); RPC_PROCESSOR(ObRpcCreateLSP, gctx_); +#ifdef OB_BUILD_ARBITRATION + RPC_PROCESSOR(ObRpcCreateArbP, gctx_); + RPC_PROCESSOR(ObRpcDeleteArbP, gctx_); +#endif RPC_PROCESSOR(ObRpcCheckLSCanOfflineP, gctx_); RPC_PROCESSOR(ObCleanSequenceCacheP, gctx_); RPC_PROCESSOR(ObRegisterTxDataP, gctx_); @@ -256,6 +268,13 @@ void oceanbase::observer::init_srv_xlator_for_others(ObSrvRpcXlator *xlator) { RPC_PROCESSOR(ObGAISPushAutoIncP); RPC_PROCESSOR(ObGAISClearAutoIncCacheP); +#ifdef OB_BUILD_SPM + // sql plan baseline + RPC_PROCESSOR(ObServerAcceptPlanBaselineP, gctx_); + RPC_PROCESSOR(ObServerCancelEvolveTaskP, gctx_); + RPC_PROCESSOR(ObLoadBaselineP, gctx_); + RPC_PROCESSOR(ObLoadBaselineV2P, gctx_); +#endif //sql optimizer estimate tablet block count RPC_PROCESSOR(ObEstimateTabletBlockCountP, gctx_); diff --git a/src/observer/ob_srv_xlator_primary.cpp b/src/observer/ob_srv_xlator_primary.cpp index 7292b47d2..09832f128 100644 --- a/src/observer/ob_srv_xlator_primary.cpp +++ b/src/observer/ob_srv_xlator_primary.cpp @@ -189,6 +189,9 @@ void oceanbase::observer::init_srv_xlator_for_logservice(ObSrvRpcXlator *xlator) { RPC_PROCESSOR(logservice::LogMembershipChangeP); RPC_PROCESSOR(logservice::LogGetPalfStatReqP); +#ifdef OB_BUILD_ARBITRATION + RPC_PROCESSOR(logservice::LogServerProbeP); +#endif RPC_PROCESSOR(logservice::LogChangeAccessModeP); RPC_PROCESSOR(logservice::LogFlashbackMsgP); } diff --git a/src/observer/ob_srv_xlator_rootserver.cpp b/src/observer/ob_srv_xlator_rootserver.cpp index 9e1409a77..eda3694ac 100644 --- a/src/observer/ob_srv_xlator_rootserver.cpp +++ b/src/observer/ob_srv_xlator_rootserver.cpp @@ -256,10 +256,25 @@ void oceanbase::observer::init_srv_xlator_for_rootserver(ObSrvRpcXlator *xlator) RPC_PROCESSOR(rootserver::ObRpcDropDirectoryP, *gctx_.root_service_); RPC_PROCESSOR(rootserver::ObRpcDisasterRecoveryTaskReplyP, *gctx_.root_service_); +#ifdef OB_BUILD_SPM + // sql plan baseline + RPC_PROCESSOR(rootserver::ObRpcAcceptPlanBaselineP, *gctx_.root_service_); + RPC_PROCESSOR(rootserver::ObRpcCancelEvolveTaskP, *gctx_.root_service_) + RPC_PROCESSOR(rootserver::ObRpcAdminLoadBaselineP, *gctx_.root_service_); + RPC_PROCESSOR(rootserver::ObRpcAdminLoadBaselineV2P, *gctx_.root_service_); + // arb service related + RPC_PROCESSOR(rootserver::ObRpcAdminAddArbitrationServiceP, *gctx_.root_service_); + RPC_PROCESSOR(rootserver::ObRpcAdminRemoveArbitrationServiceP, *gctx_.root_service_); + RPC_PROCESSOR(rootserver::ObRpcAdminReplaceArbitrationServiceP, *gctx_.root_service_); + RPC_PROCESSOR(rootserver::ObRpcRemoveClusterInfoFromArbServerP, *gctx_.root_service_); +#endif RPC_PROCESSOR(rootserver::ObRpcAdminSyncRewriteRulesP, *gctx_.root_service_); RPC_PROCESSOR(rootserver::ObRpcHandleRlsPolicyDDLP, *gctx_.root_service_); RPC_PROCESSOR(rootserver::ObRpcHandleRlsGroupDDLP, *gctx_.root_service_); RPC_PROCESSOR(rootserver::ObRpcHandleRlsContextDDLP, *gctx_.root_service_); +#ifdef OB_BUILD_TDE_SECURITY + RPC_PROCESSOR(rootserver::ObGetRootKeyP, *gctx_.root_service_); +#endif } diff --git a/src/observer/ob_srv_xlator_storage.cpp b/src/observer/ob_srv_xlator_storage.cpp index 0a89f6b7f..368e89b46 100644 --- a/src/observer/ob_srv_xlator_storage.cpp +++ b/src/observer/ob_srv_xlator_storage.cpp @@ -70,6 +70,9 @@ void oceanbase::observer::init_srv_xlator_for_storage(ObSrvRpcXlator *xlator) { RPC_PROCESSOR(ObRpcIsEmptyServerP, gctx_); RPC_PROCESSOR(ObRpcCheckServerForAddingServerP, gctx_); RPC_PROCESSOR(ObRpcCheckDeploymentModeP, gctx_); +#ifdef OB_BUILD_TDE_SECURITY + RPC_PROCESSOR(ObRpcWaitMasterKeyInSyncP, gctx_); +#endif RPC_PROCESSOR(ObRpcSyncAutoincValueP, gctx_); RPC_PROCESSOR(ObRpcClearAutoincCacheP, gctx_); RPC_PROCESSOR(ObReportReplicaP, gctx_); @@ -111,6 +114,10 @@ void oceanbase::observer::init_srv_xlator_for_storage(ObSrvRpcXlator *xlator) { RPC_PROCESSOR(ObRpcGenUniqueIDP, gctx_); RPC_PROCESSOR(ObRpcStartTransferTaskP, gctx_); RPC_PROCESSOR(ObRpcFinishTransferTaskP, gctx_); +#ifdef OB_BUILD_ARBITRATION + RPC_PROCESSOR(ObRpcAddArbP, gctx_); + RPC_PROCESSOR(ObRpcRemoveArbP, gctx_); +#endif RPC_PROCESSOR(ObRpcDDLCheckTabletMergeStatusP, gctx_); RPC_PROCESSOR(ObRpcCreateDuplicateLSP, gctx_); } diff --git a/src/observer/omt/ob_multi_tenant.cpp b/src/observer/omt/ob_multi_tenant.cpp index 397571e09..9b6ca5d9f 100644 --- a/src/observer/omt/ob_multi_tenant.cpp +++ b/src/observer/omt/ob_multi_tenant.cpp @@ -108,6 +108,15 @@ #include "logservice/leader_coordinator/ob_leader_coordinator.h" #include "storage/lob/ob_lob_manager.h" #include "share/deadlock/ob_deadlock_detector_mgr.h" +#ifdef OB_BUILD_SPM +#include "sql/spm/ob_plan_baseline_mgr.h" +#endif +#ifdef OB_BUILD_ARBITRATION +#include "rootserver/ob_arbitration_service.h" +#endif +#ifdef OB_BUILD_DBLINK +#include "lib/oracleclient/ob_oci_environment.h" +#endif #include "lib/mysqlclient/ob_tenant_oci_envs.h" #include "sql/udr/ob_udr_mgr.h" #include "storage/blocksstable/ob_shared_macro_block_manager.h" @@ -437,6 +446,9 @@ int ObMultiTenant::init(ObAddr myaddr, MTL_BIND2(mtl_new_default, rootserver::ObCreateStandbyFromNetActor::mtl_init, nullptr, rootserver::ObCreateStandbyFromNetActor::mtl_stop, rootserver::ObCreateStandbyFromNetActor::mtl_wait, mtl_destroy_default); MTL_BIND2(mtl_new_default, rootserver::ObPrimaryLSService::mtl_init, nullptr, rootserver::ObPrimaryLSService::mtl_stop, rootserver::ObPrimaryLSService::mtl_wait, mtl_destroy_default); MTL_BIND2(mtl_new_default, rootserver::ObCommonLSService::mtl_init, nullptr, rootserver::ObCommonLSService::mtl_stop, rootserver::ObCommonLSService::mtl_wait, mtl_destroy_default); +#ifdef OB_BUILD_ARBITRATION + MTL_BIND2(mtl_new_default, rootserver::ObArbitrationService::mtl_init, mtl_start_default, rootserver::ObArbitrationService::mtl_stop, rootserver::ObArbitrationService::mtl_wait, mtl_destroy_default); +#endif MTL_BIND2(mtl_new_default, rootserver::ObBalanceTaskExecuteService::mtl_init, nullptr, rootserver::ObBalanceTaskExecuteService::mtl_stop, rootserver::ObBalanceTaskExecuteService::mtl_wait, mtl_destroy_default); MTL_BIND2(mtl_new_default, rootserver::ObTenantBalanceService::mtl_init, nullptr, rootserver::ObTenantBalanceService::mtl_stop, rootserver::ObTenantBalanceService::mtl_wait, mtl_destroy_default); MTL_BIND2(mtl_new_default, rootserver::ObRecoveryLSService::mtl_init, nullptr, rootserver::ObRecoveryLSService::mtl_stop, rootserver::ObRecoveryLSService::mtl_wait, mtl_destroy_default); @@ -451,6 +463,9 @@ int ObMultiTenant::init(ObAddr myaddr, MTL_BIND2(mtl_new_default, rootserver::ObArchiveSchedulerService::mtl_init, nullptr, rootserver::ObArchiveSchedulerService::mtl_stop, rootserver::ObArchiveSchedulerService::mtl_wait, mtl_destroy_default); MTL_BIND2(mtl_new_default, ObGlobalAutoIncService::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); MTL_BIND2(mtl_new_default, share::detector::ObDeadLockDetectorMgr::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); +#ifdef OB_BUILD_ARBITRATION + MTL_BIND2(mtl_new_default, ObPlanBaselineMgr::mtl_init, nullptr, ObPlanBaselineMgr::mtl_stop, ObPlanBaselineMgr::mtl_wait, mtl_destroy_default); +#endif MTL_BIND2(mtl_new_default, ObTenantSchemaService::mtl_init, nullptr, nullptr, nullptr, mtl_destroy_default); MTL_BIND2(mtl_new_default, ObTimestampService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); MTL_BIND2(mtl_new_default, ObStandbyTimestampService::mtl_init, mtl_start_default, mtl_stop_default, mtl_wait_default, mtl_destroy_default); diff --git a/src/observer/virtual_table/ob_all_virtual_arbitration_member_info.cpp b/src/observer/virtual_table/ob_all_virtual_arbitration_member_info.cpp index 46bcfa10c..f87b9c20e 100644 --- a/src/observer/virtual_table/ob_all_virtual_arbitration_member_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_arbitration_member_info.cpp @@ -67,7 +67,53 @@ int ObAllVirtualArbMemberInfo::inner_get_next_row(common::ObNewRow *&row) { int ret = OB_SUCCESS; if (false == start_to_read_) { +#ifdef OB_BUILD_ARBITRATION + auto func_iterate_palf = [&](const palf::PalfHandle &palf_handle) -> int { + int ret = OB_SUCCESS; + palf::ArbMemberInfo arb_member_info; + int64_t palf_id = -1; + palf_handle.get_palf_id(palf_id); + if (OB_FAIL(palf_handle.get_remote_arb_member_info(arb_member_info))) { + if (OB_NOT_MASTER != ret + && OB_STATE_NOT_MATCH != ret) { + SERVER_LOG(WARN, "PalfHandle get_remote_arb_member_info failed", K(ret), K(palf_id)); + } else { + // rewrite ret to OB_SUCCESS. + ret = OB_SUCCESS; + } + } else if (!arb_member_info.is_valid()) { + // info is invalid, skip + } else if (OB_FAIL(insert_arb_member_info_(arb_member_info, &cur_row_))){ + SERVER_LOG(WARN, "ObAllVirtualArbMemberInfo insert_arb_member_info_ failed", K(ret), K(palf_id), K(arb_member_info)); + } else { + SERVER_LOG(TRACE, "iterate this log_stream success", K(palf_id), K(arb_member_info)); + scanner_.add_row(cur_row_); + } + return ret; + }; + auto func_iterate_tenant = [&func_iterate_palf]() -> int + { + int ret = OB_SUCCESS; + logservice::ObLogService *log_service = MTL(logservice::ObLogService*); + if (NULL == log_service) { + SERVER_LOG(INFO, "tenant has no ObLogService", K(MTL_ID())); + } else if (OB_FAIL(log_service->iterate_palf(func_iterate_palf))) { + SERVER_LOG(WARN, "ObLogService iterate_palf failed", K(ret)); + } else { + SERVER_LOG(TRACE, "itearte this tenant success", K(MTL_ID())); + } + return ret; + }; + if (OB_FAIL(omt_->operate_each_tenant_for_sys_or_self(func_iterate_tenant))) { + SERVER_LOG(WARN, "ObMultiTenant operate_each_tenant_for_sys_or_self failed", K(ret)); + ret = OB_ARBITRATION_INFO_QUERY_FAILED; + } else { + scanner_it_ = scanner_.begin(); + start_to_read_ = true; + } +#else ret = OB_ITER_END; +#endif } if (OB_SUCC(ret) && true == start_to_read_) { if (OB_FAIL(scanner_it_.get_next_row(cur_row_))) { @@ -81,6 +127,169 @@ int ObAllVirtualArbMemberInfo::inner_get_next_row(common::ObNewRow *&row) return ret; } +#ifdef OB_BUILD_ARBITRATION +int ObAllVirtualArbMemberInfo::insert_arb_member_info_( + const palf::ArbMemberInfo &arb_member_info, + common::ObNewRow *row) +{ + int ret = OB_SUCCESS; + const int64_t count = output_column_ids_.count(); + for (int64_t i = 0; OB_SUCC(ret) && i < count; ++i) { + uint64_t col_id = output_column_ids_.at(i); + switch (col_id) { + case OB_APP_MIN_COLUMN_ID: { + cur_row_.cells_[i].set_int(MTL_ID()); + break; + } + case OB_APP_MIN_COLUMN_ID + 1: { + cur_row_.cells_[i].set_int(arb_member_info.palf_id_); + break; + } + case OB_APP_MIN_COLUMN_ID + 2: { + if (false == arb_member_info.arb_server_.ip_to_string(ip_, common::OB_IP_PORT_STR_BUFF)) { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "ip_to_string failed", K(ret), K(arb_member_info)); + } else { + cur_row_.cells_[i].set_varchar(ObString::make_string(ip_)); + cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( + ObCharset::get_default_charset())); + } + break; + } + case OB_APP_MIN_COLUMN_ID + 3: { + cur_row_.cells_[i].set_int(arb_member_info.arb_server_.get_port()); + break; + } + case OB_APP_MIN_COLUMN_ID + 4: { + cur_row_.cells_[i].set_int(arb_member_info.log_proposal_id_); + cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( + ObCharset::get_default_charset())); + break; + } + case OB_APP_MIN_COLUMN_ID + 5: { + if (0 >= arb_member_info.config_version_.to_string(config_version_buf_, VARCHAR_128)) { + SERVER_LOG(WARN, "config_version_ to_string failed", K(ret), K(arb_member_info)); + } else { + cur_row_.cells_[i].set_varchar(ObString::make_string(config_version_buf_)); + cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( + ObCharset::get_default_charset())); + } + break; + } + case OB_APP_MIN_COLUMN_ID + 6: { + if (OB_FAIL(palf::access_mode_to_string(arb_member_info.access_mode_, + access_mode_str_, sizeof(access_mode_str_)))) { + SERVER_LOG(WARN, "access_mode_to_string failed", K(ret), K(arb_member_info)); + } else { + cur_row_.cells_[i].set_varchar(ObString::make_string(access_mode_str_)); + cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( + ObCharset::get_default_charset())); + } + break; + } + case OB_APP_MIN_COLUMN_ID + 7: { + if (OB_FAIL(member_list_to_string_(arb_member_info.paxos_member_list_))) { + SERVER_LOG(WARN, "memberlist to_string failed", K(ret), K(arb_member_info)); + } else { + cur_row_.cells_[i].set_varchar(member_list_buf_.string()); + cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( + ObCharset::get_default_charset())); + } + break; + } + case OB_APP_MIN_COLUMN_ID + 8: { + cur_row_.cells_[i].set_int(arb_member_info.paxos_replica_num_); + break; + } + case OB_APP_MIN_COLUMN_ID + 9: { + const ObAddr arb_server = arb_member_info.arbitration_member_.get_server(); + if (arb_server.is_valid() + && OB_FAIL(arb_server.ip_port_to_string(arbitration_member_buf_, MAX_SINGLE_MEMBER_LENGTH))) { + SERVER_LOG(WARN, "ip_port_to_string failed", K(ret), K(arb_member_info)); + } else { + if (!arb_server.is_valid()) { + memset(arbitration_member_buf_, 0, MAX_SINGLE_MEMBER_LENGTH); + } + cur_row_.cells_[i].set_varchar(ObString::make_string(arbitration_member_buf_)); + cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( + ObCharset::get_default_charset())); + } + break; + } + case OB_APP_MIN_COLUMN_ID + 10: { + if (OB_FAIL(learner_list_to_string_(arb_member_info.degraded_list_))) { + SERVER_LOG(WARN, "learner list to_string failed", K(ret), K(arb_member_info)); + } else { + cur_row_.cells_[i].set_varchar(ObString::make_string(degraded_list_buf_)); + cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( + ObCharset::get_default_charset())); + } + break; + } + } + } + return ret; +} + +int ObAllVirtualArbMemberInfo::member_list_to_string_( + const common::ObMemberList &member_list) +{ + int ret = OB_SUCCESS; + share::ObLSReplica::MemberList tmp_member_list; + if (OB_FAIL(share::ObLSReplica::transform_ob_member_list( + member_list, + tmp_member_list))) { + SERVER_LOG(WARN, "fail to transform member_list", KR(ret), K(member_list)); + } else if (OB_FAIL(share::ObLSReplica::member_list2text( + tmp_member_list, + member_list_buf_))) { + SERVER_LOG(WARN, "member_list2text failed", KR(ret), + K(member_list), K(tmp_member_list), K_(member_list_buf)); + } + return ret; +} + +int ObAllVirtualArbMemberInfo::learner_list_to_string_( + const common::GlobalLearnerList &learner_list) +{ + int ret = OB_SUCCESS; + int64_t pos = 0; + char buf[MAX_IP_PORT_LENGTH]; + if (learner_list.get_member_number() == 0) { + memset(degraded_list_buf_, 0, MAX_LEARNER_LIST_LENGTH); + } else { + const int64_t count = learner_list.get_member_number(); + ObMember tmp_learner; + for (int64_t i = 0; i < count && (OB_SUCCESS == ret); ++i) { + if (OB_FAIL(learner_list.get_learner(i, tmp_learner))) { + SERVER_LOG(WARN, "get_learner failed", KR(ret), K(i)); + } + if (0 != pos) { + if (pos + 1 < MAX_LEARNER_LIST_LENGTH) { + degraded_list_buf_[pos++] = ','; + } else { + ret = OB_BUF_NOT_ENOUGH; + SERVER_LOG(WARN, "buffer not enough", KR(ret), K(pos)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(tmp_learner.get_server().ip_port_to_string(buf, sizeof(buf)))) { + SERVER_LOG(WARN, "convert server to string failed", KR(ret), K(tmp_learner)); + } else { + int n = snprintf(degraded_list_buf_ + pos, MAX_LEARNER_LIST_LENGTH - pos, \ + "%s:%ld", buf, tmp_learner.get_timestamp()); + if (n < 0 || n >= MAX_LEARNER_LIST_LENGTH - pos) { + ret = OB_BUF_NOT_ENOUGH; + SERVER_LOG(WARN, "snprintf error or buf not enough", KR(ret), K(n), K(pos)); + } else { + pos += n; + } + } + } + } + return ret; +} +#endif }//namespace observer }//namespace oceanbase diff --git a/src/observer/virtual_table/ob_all_virtual_arbitration_member_info.h b/src/observer/virtual_table/ob_all_virtual_arbitration_member_info.h index da25d9e56..3d385dbd2 100644 --- a/src/observer/virtual_table/ob_all_virtual_arbitration_member_info.h +++ b/src/observer/virtual_table/ob_all_virtual_arbitration_member_info.h @@ -21,6 +21,9 @@ namespace oceanbase { namespace palf { +#ifdef OB_BUILD_ARBITRATION +class ArbMemberInfo; +#endif } namespace observer @@ -35,6 +38,11 @@ public: virtual int inner_get_next_row(common::ObNewRow *&row); void destroy(); private: +#ifdef OB_BUILD_ARBITRATION + int insert_arb_member_info_(const palf::ArbMemberInfo &arb_member_info, common::ObNewRow *row); + int member_list_to_string_(const common::ObMemberList &member_list); + int learner_list_to_string_(const common::GlobalLearnerList &learner_list); +#endif private: static const int64_t VARCHAR_32 = 32; static const int64_t VARCHAR_64 = 64; diff --git a/src/observer/virtual_table/ob_all_virtual_arbitration_service_status.cpp b/src/observer/virtual_table/ob_all_virtual_arbitration_service_status.cpp index e5405ebc8..5afeafafb 100644 --- a/src/observer/virtual_table/ob_all_virtual_arbitration_service_status.cpp +++ b/src/observer/virtual_table/ob_all_virtual_arbitration_service_status.cpp @@ -42,7 +42,43 @@ int ObAllVirtualArbServiceStatus::inner_get_next_row(common::ObNewRow *&row) { int ret = OB_SUCCESS; if (false == start_to_read_) { +#ifdef OB_BUILD_ARBITRATION + ObAddr self = GCTX.self_addr(); + bool is_in_blacklist = false; + obrpc::ObNetKeepAliveData alive_data; + storage::ObLocalityManager *locality_manager = GCTX.locality_manager_; + ObAddr arb_service_addr; + int tmp_ret = OB_SUCCESS; + if (OB_ISNULL(locality_manager)) { + ret = OB_ITER_END; + SERVER_LOG(WARN, "locality_manager ptr is NULL", K(ret), KP(locality_manager)); + } else if (OB_FAIL(locality_manager->get_arb_service_addr(arb_service_addr))) { + SERVER_LOG(WARN, "get_arb_service_addr failed", K(ret), K(self)); + } else if (!arb_service_addr.is_valid()) { + // no row, return OB_ITER_END + ret = OB_ITER_END; + if (REACH_TIME_INTERVAL(10 * 1000 * 1000)) { + SERVER_LOG(WARN, "arb_service_addr is invalid", K(ret), K(self)); + } + } else { + if (OB_SUCCESS != (tmp_ret = obrpc::ObNetKeepAlive::get_instance().in_black(arb_service_addr, \ + is_in_blacklist, &alive_data))) { + // in_black may return err code(-4016) when arg server has not been detected. + // we can ignore err code here, and keep default value for is_in_blacklist. + SERVER_LOG(WARN, "ObNetKeepAlive in_black failed", K(tmp_ret), K(arb_service_addr)); + } + if (OB_FAIL(insert_row_(self, arb_service_addr, is_in_blacklist, &cur_row_))){ + SERVER_LOG(WARN, "insert_row_ failed", K(ret), K(self)); + } else { + SERVER_LOG(TRACE, "insert_row_ success"); + scanner_.add_row(cur_row_); + scanner_it_ = scanner_.begin(); + start_to_read_ = true; + } + } +#else ret = OB_ITER_END; +#endif } if (OB_SUCC(ret) && true == start_to_read_) { if (OB_FAIL(scanner_it_.get_next_row(cur_row_))) { @@ -56,6 +92,69 @@ int ObAllVirtualArbServiceStatus::inner_get_next_row(common::ObNewRow *&row) return ret; } +#ifdef OB_BUILD_ARBITRATION +void ObAllVirtualArbServiceStatus::reset_buf_() +{ + memset(ip_, '\0', common::OB_IP_PORT_STR_BUFF); + memset(arb_service_addr_buf_, '\0', VARCHAR_128); + memset(arb_service_status_buf_, '\0', VARCHAR_32); +} + +int ObAllVirtualArbServiceStatus::insert_row_( + const common::ObAddr &self, + const common::ObAddr &arb_service_addr, + const bool is_in_blacklist, + common::ObNewRow *row) +{ + int ret = OB_SUCCESS; + const int64_t count = output_column_ids_.count(); + // clear buf members + reset_buf_(); + for (int64_t i = 0; OB_SUCC(ret) && i < count; ++i) { + uint64_t col_id = output_column_ids_.at(i); + switch (col_id) { + case OB_APP_MIN_COLUMN_ID: { + if (false == self.ip_to_string(ip_, common::OB_IP_PORT_STR_BUFF)) { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "ip_to_string failed", K(ret), K(self)); + } else { + cur_row_.cells_[i].set_varchar(ObString::make_string(ip_)); + cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( + ObCharset::get_default_charset())); + } + break; + } + case OB_APP_MIN_COLUMN_ID + 1: { + cur_row_.cells_[i].set_int(self.get_port()); + break; + } + case OB_APP_MIN_COLUMN_ID + 2: { + int32_t out_len = 0; + if (OB_FAIL(arb_service_addr.addr_to_buffer(arb_service_addr_buf_, VARCHAR_128, out_len))) { + SERVER_LOG(WARN, "arb_service_addr to_string failed", K(ret), K(arb_service_addr)); + } else { + cur_row_.cells_[i].set_varchar(ObString::make_string(arb_service_addr_buf_)); + cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( + ObCharset::get_default_charset())); + } + break; + } + case OB_APP_MIN_COLUMN_ID + 3: { + if (is_in_blacklist) { + strncpy(arb_service_status_buf_, ARB_STATUS_INACTIVE, VARCHAR_32); + } else { + strncpy(arb_service_status_buf_, ARB_STATUS_ACTIVE, VARCHAR_32); + } + cur_row_.cells_[i].set_varchar(ObString::make_string(arb_service_status_buf_)); + cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( + ObCharset::get_default_charset())); + break; + } + } + } + return ret; +} +#endif }//namespace observer }//namespace oceanbase diff --git a/src/observer/virtual_table/ob_all_virtual_arbitration_service_status.h b/src/observer/virtual_table/ob_all_virtual_arbitration_service_status.h index aa145aa39..fbeeda54e 100644 --- a/src/observer/virtual_table/ob_all_virtual_arbitration_service_status.h +++ b/src/observer/virtual_table/ob_all_virtual_arbitration_service_status.h @@ -29,10 +29,22 @@ public: virtual int inner_get_next_row(common::ObNewRow *&row); void destroy(); private: +#ifdef OB_BUILD_ARBITRATION + int insert_row_(const common::ObAddr &self, + const common::ObAddr &arb_service_addr, + const bool is_in_blacklist, + common::ObNewRow *row); + void reset_buf_(); +#endif private: static const int64_t VARCHAR_32 = 32; static const int64_t VARCHAR_128 = 128; private: +#ifdef OB_BUILD_ARBITRATION + char ip_[common::OB_IP_PORT_STR_BUFF] = {'\0'}; + char arb_service_addr_buf_[VARCHAR_128] = {'\0'}; + char arb_service_status_buf_[VARCHAR_32] = {'\0'}; +#endif }; }//namespace observer }//namespace oceanbase diff --git a/src/observer/virtual_table/ob_all_virtual_audit_action.h b/src/observer/virtual_table/ob_all_virtual_audit_action.h index 5271df7eb..469e8fa9f 100644 --- a/src/observer/virtual_table/ob_all_virtual_audit_action.h +++ b/src/observer/virtual_table/ob_all_virtual_audit_action.h @@ -21,10 +21,24 @@ namespace observer { class ObAllVirtualAuditActionTable : public common::ObVirtualTableScannerIterator { +#ifndef OB_BUILD_AUDIT_SECURITY public: ObAllVirtualAuditActionTable() {} virtual ~ObAllVirtualAuditActionTable() {} virtual int inner_get_next_row(common::ObNewRow *&row) { return OB_ITER_END; } +#else + static const int32_t AUDIT_ACTION_COLUMN_COUNT = 2; + enum COLUMN_NAME { + ACTION_ID = common::OB_APP_MIN_COLUMN_ID, + ACTION_NAME, + }; + +public: + ObAllVirtualAuditActionTable(); + virtual ~ObAllVirtualAuditActionTable(); + + virtual int inner_get_next_row(common::ObNewRow *&row); +#endif private: DISALLOW_COPY_AND_ASSIGN(ObAllVirtualAuditActionTable); diff --git a/src/observer/virtual_table/ob_all_virtual_audit_operation.h b/src/observer/virtual_table/ob_all_virtual_audit_operation.h index edf74e9e0..f1e7a6b66 100644 --- a/src/observer/virtual_table/ob_all_virtual_audit_operation.h +++ b/src/observer/virtual_table/ob_all_virtual_audit_operation.h @@ -21,10 +21,25 @@ namespace observer { class ObAllVirtualAuditOperationTable : public common::ObVirtualTableScannerIterator { +#ifndef OB_BUILD_AUDIT_SECURITY public: ObAllVirtualAuditOperationTable() {} virtual ~ObAllVirtualAuditOperationTable() {} virtual int inner_get_next_row(common::ObNewRow *&row) { return OB_ITER_END; } +#else + static const int32_t AUDIT_OPERATION_COLUMN_COUNT = 3; + enum COLUMN_NAME { + OPERATION_TYPE = common::OB_APP_MIN_COLUMN_ID, + OPERATION_NAME, + AUDIT_TYPE_NAME, + }; + +public: + ObAllVirtualAuditOperationTable(); + virtual ~ObAllVirtualAuditOperationTable(); + + virtual int inner_get_next_row(common::ObNewRow *&row); +#endif private: DISALLOW_COPY_AND_ASSIGN(ObAllVirtualAuditOperationTable); diff --git a/src/observer/virtual_table/ob_all_virtual_dblink_info.cpp b/src/observer/virtual_table/ob_all_virtual_dblink_info.cpp index 7387f8645..71c0870bd 100644 --- a/src/observer/virtual_table/ob_all_virtual_dblink_info.cpp +++ b/src/observer/virtual_table/ob_all_virtual_dblink_info.cpp @@ -11,6 +11,10 @@ */ #include "ob_all_virtual_dblink_info.h" +#ifdef OB_BUILD_DBLINK +#include "lib/oracleclient/ob_oci_environment.h" +#include "lib/oracleclient/ob_oracle_oci_connection.h" +#endif #include "observer/ob_server_struct.h" using namespace oceanbase::common; @@ -72,6 +76,10 @@ int ObAllVirtualDblinkInfo::inner_open() // do nothing } else if (OB_ISNULL(link_pool = proxy->get_dblink_conn_pool())) { // do nothing +#ifdef OB_BUILD_DBLINK + } else if (OB_FAIL(link_pool->get_oci_pool().get_dblink_status(link_status_, *get_allocator()))) { + SERVER_LOG(ERROR, "failed to get link status", K(ret)); +#endif } else { row_cnt_ = 0; } diff --git a/src/observer/virtual_table/ob_all_virtual_dblink_info.h b/src/observer/virtual_table/ob_all_virtual_dblink_info.h index 5c699566e..029b9e28f 100644 --- a/src/observer/virtual_table/ob_all_virtual_dblink_info.h +++ b/src/observer/virtual_table/ob_all_virtual_dblink_info.h @@ -16,6 +16,7 @@ namespace oceanbase { +#ifndef OB_BUILD_DBLINK namespace common { struct Dblink_status { uint64_t link_id; @@ -55,6 +56,7 @@ namespace common { K(extra_info)); }; } +#endif namespace observer { diff --git a/src/observer/virtual_table/ob_all_virtual_ha_diagnose.cpp b/src/observer/virtual_table/ob_all_virtual_ha_diagnose.cpp index eee654026..ac21d4200 100644 --- a/src/observer/virtual_table/ob_all_virtual_ha_diagnose.cpp +++ b/src/observer/virtual_table/ob_all_virtual_ha_diagnose.cpp @@ -249,6 +249,9 @@ int ObAllVirtualHADiagnose::insert_stat_(storage::DiagnoseInfo &diagnose_info) break; case ARB_SRV_INFO: cur_row_.cells_[i].set_varchar(ObString("")); +#ifdef OB_BUILD_ARBITRATION + cur_row_.cells_[i].set_varchar(diagnose_info.arb_srv_diagnose_info_.diagnose_str_.string()); +#endif cur_row_.cells_[i].set_collation_type(ObCharset::get_default_collation( ObCharset::get_default_charset())); break; diff --git a/src/observer/virtual_table/ob_all_virtual_master_key_version_info.h b/src/observer/virtual_table/ob_all_virtual_master_key_version_info.h index ba5b032a4..6e7643751 100644 --- a/src/observer/virtual_table/ob_all_virtual_master_key_version_info.h +++ b/src/observer/virtual_table/ob_all_virtual_master_key_version_info.h @@ -23,9 +23,33 @@ class ObAllVirtualMasterKeyVersionInfo : public common::ObVirtualTableScannerIte { public: +#ifndef OB_BUILD_TDE_SECURITY ObAllVirtualMasterKeyVersionInfo() {} virtual ~ObAllVirtualMasterKeyVersionInfo() {} virtual int inner_get_next_row(common::ObNewRow *&row) { return OB_ITER_END; } +#else + enum COLUMN_ID_LIST + { + SVR_IP = common::OB_APP_MIN_COLUMN_ID, + SVR_PORT, + TENANT_ID, + MAX_ACTIVE_VERSION, + MAX_STORED_VERSION, + EXPECT_VERSION + }; + ObAllVirtualMasterKeyVersionInfo(); + virtual ~ObAllVirtualMasterKeyVersionInfo(); + virtual void reset(); + virtual int inner_open(); + virtual int inner_get_next_row(common::ObNewRow *&row); +private: + int gen_row(common::ObNewRow *&row); + int fill_tenant_ids(); + + char ip_buf_[common::OB_IP_STR_BUFF]; + ObSEArray tenant_ids_; + int64_t tenant_ids_idx_; +#endif private: DISALLOW_COPY_AND_ASSIGN(ObAllVirtualMasterKeyVersionInfo); }; diff --git a/src/observer/virtual_table/ob_all_virtual_tablet_encrypt_info.h b/src/observer/virtual_table/ob_all_virtual_tablet_encrypt_info.h index 3b6ede4b0..d570d7fee 100644 --- a/src/observer/virtual_table/ob_all_virtual_tablet_encrypt_info.h +++ b/src/observer/virtual_table/ob_all_virtual_tablet_encrypt_info.h @@ -16,10 +16,29 @@ #include "lib/guard/ob_shared_guard.h" #include "observer/omt/ob_multi_tenant.h" #include "observer/omt/ob_multi_tenant_operator.h" +#ifdef OB_BUILD_TDE_SECURITY +#include "share/ob_scanner.h" +#include "share/ob_virtual_table_scanner_iterator.h" +#include "share/rc/ob_tenant_base.h" +#include "storage/blocksstable/ob_index_block_macro_iterator.h" +#include "storage/blocksstable/ob_sstable_meta.h" +#include "storage/meta_mem/ob_tablet_handle.h" +#include "storage/meta_mem/ob_tablet_handle.h" +#endif #include "storage/ob_i_table.h" namespace oceanbase { +#ifdef OB_BUILD_TDE_SECURITY +namespace storage +{ +class ObTenantTabletIterator; +} +namespace blocksstable +{ +class ObSSTable; +} +#endif namespace observer { @@ -28,12 +47,67 @@ class ObAllVirtualTabletEncryptInfo : public common::ObVirtualTableScannerIterat { public: +#ifndef OB_BUILD_TDE_SECURITY ObAllVirtualTabletEncryptInfo() {} virtual ~ObAllVirtualTabletEncryptInfo() {} int init(common::ObIAllocator *allocator, common::ObAddr &addr) { return OB_SUCCESS; } int inner_get_next_row(common::ObNewRow *&row) override { return OB_ITER_END; } int process_curr_tenant(common::ObNewRow *&row) override { return OB_ITER_END; } void release_last_tenant() override {} +#else + enum COLUMN_ID_LIST + { + TENANT_ID = common::OB_APP_MIN_COLUMN_ID, + TABLET_ID, + SVR_IP, + SVR_PORT, + MACRO_BLOCK_COUNT, + ENCRYPTED_MACRO_BLOCK_COUNT, + }; + struct TabletEncryptInfo + { + uint64_t tenant_id_; + int64_t tablet_id_; + int64_t macro_block_count_; + int64_t encrypted_macro_block_count_; + TO_STRING_KV(K_(tenant_id), K_(tablet_id), + K_(macro_block_count), K_(encrypted_macro_block_count)); + }; + ObAllVirtualTabletEncryptInfo(); + virtual ~ObAllVirtualTabletEncryptInfo(); + int init(common::ObIAllocator *allocator, common::ObAddr &addr); + virtual int inner_get_next_row(common::ObNewRow *&row); + virtual void reset(); +private: + // ObMultiTenantOperator interface + virtual int process_curr_tenant(common::ObNewRow *&row) override; + virtual void release_last_tenant() override; + virtual bool is_need_process(uint64_t tenant_id) override; + int get_next_tablet(); + int gen_row(common::ObNewRow *&row); + void clean_cur_sstable(); + int gen_encrypt_info(); + int check_need_ignore(bool &need_ignore); + int gen_tablet_range(const uint64_t tablet_id, common::ObNewRange &range); +private: + common::ObAddr addr_; + ObTenantTabletIterator *tablet_iter_; + common::ObArenaAllocator tablet_allocator_; + ObTabletHandle tablet_handle_; + uint64_t tenant_id_; + char ip_buf_[common::OB_IP_STR_BUFF]; + char range_buf_[common::OB_MAX_RANGE_LENGTH + 1]; // extra byte for '\0' + storage::ObTableStoreIterator table_store_iter_; + blocksstable::ObSSTable *curr_sstable_; + blocksstable::ObIMacroBlockIterator *macro_iter_; + blocksstable::ObMacroBlockDesc macro_desc_; + blocksstable::ObDataMacroBlockMeta macro_meta_; + ObArenaAllocator iter_allocator_; + blocksstable::ObDatumRange curr_range_; + common::ObObj objs_[common::OB_MAX_ROWKEY_COLUMN_NUMBER]; + void *iter_buf_; + TabletEncryptInfo tablet_encrypt_info_; +#endif private: DISALLOW_COPY_AND_ASSIGN(ObAllVirtualTabletEncryptInfo); }; diff --git a/src/pl/CMakeLists.txt b/src/pl/CMakeLists.txt index b68d3390e..88cf20391 100644 --- a/src/pl/CMakeLists.txt +++ b/src/pl/CMakeLists.txt @@ -36,7 +36,7 @@ ob_set_subtarget(ob_pl common ob_pl_user_type.cpp ) -ob_set_subtarget(ob_pl common_mixed +ob_set_subtarget(ob_pl pl_cache pl_cache/ob_pl_cache.cpp pl_cache/ob_pl_cache_mgr.cpp pl_cache/ob_pl_cache_object.cpp @@ -51,7 +51,6 @@ ob_set_subtarget(ob_pl sys_package sys_package/ob_dbms_upgrade.cpp sys_package/ob_dbms_sql.cpp sys_package/ob_dbms_user_define_rule.cpp - sys_package/ob_pl_dbms_resource_manager.cpp ) ob_server_add_target(ob_pl) diff --git a/src/pl/ob_pl.cpp b/src/pl/ob_pl.cpp index 3169abe04..4d42702d8 100644 --- a/src/pl/ob_pl.cpp +++ b/src/pl/ob_pl.cpp @@ -42,6 +42,11 @@ #include "observer/ob_req_time_service.h" #include "sql/privilege_check/ob_ora_priv_check.h" #include "sql/engine/expr/ob_expr_pl_integer_checker.h" +#ifdef OB_BUILD_ORACLE_PL +#include "pl/ob_pl_udt_object_manager.h" +#include "pl/debug/ob_pl_debugger.h" +#include "pl/debug/ob_pl_debugger_manager.h" +#endif #include "pl/pl_cache/ob_pl_cache_mgr.h" #include "src/sql/engine/dml/ob_trigger_handler.h" namespace oceanbase @@ -416,28 +421,135 @@ int ObPLContext::check_debug_priv(ObSchemaGetterGuard *guard, ObPLFunction *func) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSEDx(guard, sess_info, func); +#else + if (OB_ISNULL(guard) || OB_ISNULL(sess_info) || OB_ISNULL(func)) { + // do thing + } else if (func->has_debug_priv()) { + // do nothing + // according to oracle, if a routine has debug priv, before debug stop, it will hold it + // even if it's debug priv is revoked during the debug process. + } else { + uint64_t package_id = func->get_package_id(); + uint64_t routine_id = func->get_routine_id(); + uint64_t obj_owner_id = func->get_owner(); + uint64_t db_id = func->get_database_id(); + uint64_t obj_id = OB_INVALID_ID; + ObObjectType obj_type = ObObjectType::INVALID; + const share::schema::ObDatabaseSchema *db_schema = NULL; + const uint64_t tenant_id = func->get_tenant_id(); + OZ (guard->get_database_schema(tenant_id, db_id, db_schema), db_id); + if (OB_SUCC(ret)) { + if (OB_INVALID_ID != package_id) { + if (ObUDTObjectType::is_object_id_masked(package_id)) { + obj_type = ObObjectType::TYPE; + } else { + obj_type = ObObjectType::PACKAGE; + } + obj_id = package_id; + } else if (OB_INVALID_ID != routine_id) { + if (func->is_function()) { + obj_type = ObObjectType::FUNCTION; + } else { + obj_type = ObObjectType::PROCEDURE; + } + obj_id = routine_id; + } + if (OB_INVALID_ID != obj_id) { + uint64_t tenant_id = OB_INVALID_ID; + uint64_t user_id = OB_INVALID_ID; + CK (OB_NOT_NULL(sess_info)); + OX (tenant_id = sess_info->get_effective_tenant_id()); + OX (user_id = sess_info->get_user_id()); + // first check debug any procedure + OZ (ObDBMSDebug::check_debug_sys_priv_impl(*guard, *sess_info, PRIV_ID_DEBUG_ANY_PROC)); + if (OB_ERR_NO_PRIVILEGE == ret || OB_ERR_NO_SYS_PRIVILEGE == ret) { + ret = OB_SUCCESS; + OZ (sql::ObOraSysChecker::check_ora_obj_priv(*guard, + tenant_id, + user_id, + db_schema->get_database_name_str(), + obj_id, + OBJ_LEVEL_FOR_TAB_PRIV, + static_cast(obj_type), + OBJ_PRIV_ID_DEBUG, + CHECK_FLAG_NORMAL, + obj_owner_id, + sess_info->get_enable_role_array()), + user_id, obj_id, obj_type, obj_owner_id); + } + if (OB_ERR_NO_PRIVILEGE == ret || OB_ERR_NO_SYS_PRIVILEGE == ret) { + func->clean_debug_priv(); + } else if (OB_SUCC(ret)) { + func->set_debug_priv(); + } else { + // do nothing + } + } + } + } +#endif return ret; } int ObPLContext::debug_start(ObSQLSessionInfo *sql_session) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSED(sql_session); +#else + pl::debugger::ObPLDebugger *pl_debugger = NULL; + if (sql_session->is_pl_debug_on()) { + CK (OB_NOT_NULL(pl_debugger = sql_session->get_pl_debugger())); + if (OB_SUCC(ret) && !pl_debugger->is_debug_thread_running()) { + OZ (pl_debugger->debug_start()); + } + } +#endif return ret; } int ObPLContext::debug_stop(ObSQLSessionInfo *sql_session) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSED(sql_session); +#else + pl::debugger::ObPLDebugger *pl_debugger = NULL; + OX (pl_debugger = sql_session->get_pl_debugger()); + if (OB_SUCC(ret) + && sql_session->is_pl_debug_on() + && NULL == sql_session->get_pl_context() // we only stop debugger on top pl/sql + && pl_debugger->is_debug_thread_running()) { + OZ (pl_debugger->debug_stop()); + if (OB_SUCC(ret) && pl_debugger->need_debug_off()) { + OZ (sql_session->free_pl_debugger()); + } + } +#endif return ret; } int ObPLContext::notify(ObSQLSessionInfo *sql_session) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSED(sql_session); +#else + pl::debugger::ObPLDebugger *pl_debugger = NULL; + OX (pl_debugger = sql_session->get_pl_debugger()); + if (OB_SUCC(ret) + && sql_session->is_pl_debug_on() + && pl_debugger->is_debug_thread_running()) { + OZ (pl_debugger->notify()); + } else if (pl_debugger != NULL) { + LOG_INFO("[TARGET THREAD] No Need To Notify!", + K(ret), + K(sql_session->is_pl_debug_on()), + K(pl_debugger->is_debug_thread_running())); + } +#endif return ret; } @@ -713,6 +825,16 @@ void ObPLContext::destory( } else if (lib::is_mysql_mode()) { session_info.set_pl_can_retry(false); } +#ifdef OB_BUILD_ORACLE_PL + } else if (session_info.associated_xa()) { + if (OB_TRANS_XA_BRANCH_FAIL != ret) { + tmp_ret = ObDbmsXA::xa_rollback_savepoint(ctx); + if (OB_SUCCESS != tmp_ret) { + LOG_WARN("xa trans roll back to save point failed", + K(tmp_ret), KPC(session_info.get_tx_desc())); + } + } +#endif } else if (!in_nested_sql_ctrl() && session_info.get_in_transaction()) { // 如果没有隐式的检查点且不再嵌套事务中, 说明当前事务中仅包含该PL, 直接回滚事务 // 嵌套语句中的PL会随着顶层的语句一起回滚, 不需要单独回滚 @@ -787,6 +909,16 @@ void ObPLContext::destory( if (OB_SUCCESS != (tmp_ret = ObSqlTransControl::rollback_savepoint(ctx, PL_IMPLICIT_SAVEPOINT))) { LOG_WARN("failed to rollback current pl to implicit savepoint", K(ret), K(tmp_ret)); } +#ifdef OB_BUILD_ORACLE_PL + } else if (session_info.associated_xa()) { + if (OB_TRANS_XA_BRANCH_FAIL != ret) { + tmp_ret = ObDbmsXA::xa_rollback_savepoint(ctx); + if (OB_SUCCESS != tmp_ret) { + LOG_WARN("xa trans roll back to save point failed", + K(tmp_ret), KPC(session_info.get_tx_desc())); + } + } +#endif } else if (!in_nested_sql_ctrl() && session_info.get_in_transaction()) { tmp_ret = implicit_end_trans(session_info, ctx, true); } @@ -1211,6 +1343,44 @@ int ObPLContext::set_subprogram_var_from_local( return ret; } +#ifdef OB_BUILD_ORACLE_PL +int ObPLContext::get_exact_error_msg(ObIArray &error_trace, + ObIArray &call_stack, + common::ObSqlString &err_msg) +{ + int ret = OB_SUCCESS; + err_msg.reuse(); + if (0 == call_stack.count() || 0 == error_trace.count()) { + LOG_INFO("error message is incomplete. ", K(call_stack.count()), K(error_trace.count())); + } else { + int64_t i = call_stack.count() - 1; + for (; OB_SUCC(ret) && i>=0 && OB_NOT_NULL(call_stack.at(i)); i--) { + int64_t line = call_stack.at(i)->get_line_number(); + int64_t col = call_stack.at(i)->get_column_number(); + for (int64_t j = 0 ; j < error_trace.count(); j++) { + if (OB_NOT_NULL(error_trace.at(j)) + && call_stack.at(i)->handler == error_trace.at(j)->handler) { + line = error_trace.at(j)->get_line_number(); + col = error_trace.at(j)->get_column_number(); + break; + } + } + if (OB_FAIL(err_msg.append_fmt("\nat "))) { + LOG_WARN("fail to get call stack name.", K(ret)); + } else if (OB_FAIL(err_msg.append_fmt("%.*s , line : %ld, col : %ld", + call_stack.at(i)->object_name.length(), + call_stack.at(i)->object_name.ptr(), line, col))) { + LOG_WARN("fail to get call stack name.", K(ret), + K(call_stack.at(i)->object_name), K(line), K(col)); + } else { + LOG_DEBUG("exact error msg: ", K(call_stack.at(i)->object_name), + K(line), K(col)); + } + } + } + return ret; +} +#endif // for common execute routine. int ObPL::execute(ObExecContext &ctx, @@ -2005,11 +2175,24 @@ int ObPL::get_pl_function(ObExecContext &ctx, package_guard, *ctx.get_sql_proxy(), false /*PS MODE*/); +#ifdef OB_BUILD_ORACLE_PL + if (ObUDTObjectType::is_object_id_masked(package_id)) { + OX (package_id = ObUDTObjectType::clear_object_id_mask(package_id)); + OZ (ObPLUDTObjectManager::get_udt_function(pl_ctx, + ctx, + package_id, + routine_id, + local_routine)); + } else { +#endif OZ (package_manager_.get_package_routine(pl_ctx, ctx, package_id, routine_id, local_routine)); +#ifdef OB_BUILD_ORACLE_PL + } +#endif CK (OB_NOT_NULL(local_routine)); } else { // standalone routine static const ObString PLSQL = ObString("PL/SQL"); @@ -2288,6 +2471,11 @@ int ObPL::check_trigger_arg(const ParamStore ¶ms, const ObPLFunction &func) ObPLExecState::~ObPLExecState() { +#ifdef OB_BUILD_ORACLE_PL + if (dwarf_helper_ != NULL) { + dwarf_helper_->~ObDWARFHelper(); + } +#endif } int ObPLExecState::get_var(int64_t var_idx, ObObjParam& result) @@ -3057,8 +3245,18 @@ do { \ && (func_.get_variables().at(i).is_nested_table_type() || func_.get_variables().at(i).is_varray_type()) && params->at(i).is_ext()) { +#ifndef OB_BUILD_ORACLE_PL ret = OB_NOT_SUPPORTED; LOG_WARN("not support nested table type", K(ret)); +#else + // anonymous out collection only in ps mode. + ObPLNestedTable *table = NULL; + CK (OB_NOT_NULL(table = reinterpret_cast(params->at(i).get_ext()))); + for (int64_t i = 0; OB_SUCC(ret) && i < table->get_count(); ++i) { + OZ (table->delete_collection_elem(i), K(i), KPC(table)); + } + OX (get_params().at(i) = params->at(i)); +#endif } else { // 纯OUT参数, 对于复杂类型需要重新初始化值; 如果传入的复杂类型值为NULL(PS协议), 则初始化一个新的复杂类型 // 这里先copy入参的值, 由init_complex_obj函数判断是否重新分配内存 @@ -3391,6 +3589,9 @@ int ObPL::simple_execute(ObPLExecCtx *ctx, int64_t argc, int64_t *argv) ret = OB_SUCCESS; } + #ifdef OB_BUILD_ORACLE_PL + ObPLEH::eh_adjust_call_stack(func, ctx->pl_ctx_, sql_infos.at(i).loc_, ret); + #endif } } @@ -3623,7 +3824,26 @@ int ObPLExecState::execute() if (OB_SUCC(ret)) { ObSQLSessionInfo *session_info = ctx_.exec_ctx_->get_my_session(); +#ifdef OB_BUILD_ORACLE_PL + int64_t call_cnt = top_context_->get_call_stack().count(); + int64_t err_cnt = top_context_->get_error_trace().count(); + if (OB_SUCC(ret) && err_cnt > 0) { + DbmsUtilityHelper::BtInfo* top_error = top_context_->get_error_trace().at(err_cnt - 1); + if (func_.get_action() == top_error->handler) { + OX (top_context_->get_error_trace().pop_back()); + } + } + if (OB_SUCC(ret) && call_cnt > 0) { + DbmsUtilityHelper::BtInfo* top_call = top_context_->get_call_stack().at(call_cnt - 1); + if (func_.get_action() == top_call->handler) { + OX (top_context_->get_call_stack().pop_back()); + } + } } else if (!inner_call_) { + ObPLContext::get_exact_error_msg(top_context_->get_error_trace(), + top_context_->get_call_stack(), + ctx_.exec_ctx_->get_my_session()->get_pl_exact_err_msg()); +#endif } } return ret; @@ -4005,6 +4225,52 @@ int ObPLINS::init_complex_obj(ObIAllocator &allocator, } } +#ifdef OB_BUILD_ORACLE_PL + if (OB_SUCC(ret) && user_type->is_opaque_type()) { + const ObOpaqueType *opaque_type = static_cast(user_type); + CK (OB_NOT_NULL(opaque_type)); + OX (new(ptr)ObPLOpaque()); + } + + if (OB_SUCC(ret) && user_type->is_nested_table_type()) { + const ObNestedTableType* nested_type = static_cast(user_type); + CK (OB_NOT_NULL(nested_type)); + OX (new(ptr)ObPLNestedTable(user_type->get_user_type_id())); + } + if (OB_SUCC(ret) && user_type->is_varray_type()) { + const ObVArrayType* varray_type = static_cast(user_type); + ObPLVArray *varray = reinterpret_cast(ptr); + CK (OB_NOT_NULL(varray_type)); + CK (OB_NOT_NULL(varray)); + OX (new(varray)ObPLVArray(varray_type->get_user_type_id())); + OX (varray->set_capacity(varray_type->get_capacity())); + } + if (OB_SUCC(ret) && user_type->is_associative_array_type()) { + const ObAssocArrayType *assoc_type = static_cast(user_type); + CK (OB_NOT_NULL(assoc_type)); + CK (OB_NOT_NULL(ptr)); + OX (new(ptr)ObPLAssocArray(assoc_type->get_user_type_id())); + } + if (OB_SUCC(ret) && user_type->is_collection_type()) { + int64_t field_cnt = OB_INVALID_COUNT; + ObPLCollection *coll = reinterpret_cast(ptr); + ObElemDesc elem_desc; + bool not_null; + const ObCollectionType *coll_type = static_cast(user_type); + CK (OB_NOT_NULL(coll)); + OX (set_allocator ? coll->set_allocator(&allocator) : coll->set_allocator(NULL)); + OX ((obj.is_ext() && obj.get_ext() != 0) ? (void)NULL : coll->set_inited()); + OX (coll->set_type(pl_type.get_type())); + OZ (get_element_data_type(pl_type, elem_desc, &allocator)); + OZ (get_not_null(pl_type, not_null, &allocator)); + OZ (coll_type->get_element_type().get_field_count(*this, field_cnt)); + OX (elem_desc.set_pl_type(coll_type->get_element_type().get_type())); + OX (elem_desc.set_field_count(field_cnt)); + OX (elem_desc.set_not_null(not_null)); + OX (elem_desc.set_udt_id(coll_type->get_element_type().get_user_type_id())); + OX (coll->set_element_desc(elem_desc)); + } +#endif OX (obj.set_extend(reinterpret_cast(ptr), user_type->get_type(), init_size)); OX (obj.set_param_meta()); OX (obj.set_udt_id(pl_type.get_user_type_id())); @@ -4030,7 +4296,28 @@ int ObPLINS::get_element_data_type(const ObPLDataType &pl_type, ObIAllocator *allocator) const { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSEDx(pl_type, elem_type, allocator); +#else + const ObUserDefinedType *user_type = NULL; + const ObCollectionType *collection_type = NULL; + CK (pl_type.is_composite_type()); + CK (pl_type.is_collection_type()); + OZ (get_user_type(pl_type.get_user_type_id(), user_type, allocator)); + CK (OB_NOT_NULL(user_type)); + CK (user_type->is_collection_type()); + CK (OB_NOT_NULL(collection_type = static_cast(user_type))); + if (OB_SUCC(ret)) { + if (collection_type->get_element_type().is_obj_type()) { + CK (OB_NOT_NULL(collection_type->get_element_type().get_data_type())); + if (OB_SUCC(ret)) { + elem_type = *(collection_type->get_element_type().get_data_type()); + } + } else { + elem_type.set_obj_type(common::ObExtendType); + } + } +#endif return ret; } @@ -4039,7 +4326,21 @@ int ObPLINS::get_not_null(const ObPLDataType &pl_type, ObIAllocator *allocator) const { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSEDx(pl_type, not_null, allocator); +#else + const ObUserDefinedType *user_type = NULL; + const ObCollectionType *collection_type = NULL; + CK (pl_type.is_composite_type()); + CK (pl_type.is_collection_type()); + OZ (get_user_type(pl_type.get_user_type_id(), user_type, allocator)); + CK (OB_NOT_NULL(user_type)); + CK (user_type->is_collection_type()); + CK (OB_NOT_NULL(collection_type = static_cast(user_type))); + if (OB_SUCC(ret)) { + not_null = collection_type->get_element_type().get_not_null(); + } +#endif return ret; } diff --git a/src/pl/ob_pl.h b/src/pl/ob_pl.h index ed6126296..7cbc71f44 100644 --- a/src/pl/ob_pl.h +++ b/src/pl/ob_pl.h @@ -867,6 +867,9 @@ public: old_priv_user_id_ = OB_INVALID_ID; old_in_definer_ = false; has_output_arguments_ = false; +#ifdef OB_BUILD_ORACLE_PL + call_trace_.reset(); +#endif old_worker_timeout_ts_ = 0; old_phy_plan_timeout_ts_ = 0; parent_stack_ctx_ = nullptr; @@ -942,6 +945,12 @@ public: void reset_role_id_array(int &ret); ObIArray &get_exec_stack() { return exec_stack_; } +#ifdef OB_BUILD_ORACLE_PL + ObIArray &get_error_trace() { return call_trace_.error_trace; } + ObIArray &get_call_stack() { return call_trace_.call_stack; } + void set_call_trace_error_code(int errcode) { call_trace_.err_code = errcode; } + int get_call_trace_error_code() const { return call_trace_.err_code; } +#endif ObPLExecState *get_current_state() { return exec_stack_.empty() ? NULL : exec_stack_.at(exec_stack_.count() - 1); @@ -954,6 +963,11 @@ public: { return NULL == get_current_state() ? NULL : &get_current_state()->get_exec_ctx(); } +#ifdef OB_BUILD_ORACLE_PL + static int get_exact_error_msg(ObIArray &error_trace, + ObIArray &call_stack, + common::ObSqlString &err_msg); +#endif bool has_output_arguments() { return has_output_arguments_; } void set_has_output_arguments(bool has_output_arguments) { @@ -1022,6 +1036,9 @@ private: common::ObString cur_query_; common::ObSEArray exec_stack_; +#ifdef OB_BUILD_ORACLE_PL + DbmsUtilityHelper::BackTrace call_trace_; +#endif ObPLContext *parent_stack_ctx_; ObPLContext *top_stack_ctx_; sql::ObExecContext *my_exec_ctx_; //my exec context diff --git a/src/pl/ob_pl_allocator.cpp b/src/pl/ob_pl_allocator.cpp index 6a16155cd..d9b13b64f 100644 --- a/src/pl/ob_pl_allocator.cpp +++ b/src/pl/ob_pl_allocator.cpp @@ -87,9 +87,73 @@ int ObPLCollAllocator::free_child_coll(ObPLCollection &dest) int ObPLCollAllocator::copy_all_element_with_new_allocator(ObIAllocator *allocator) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSED(allocator); ret = OB_NOT_SUPPORTED; LOG_WARN("not support", K(ret)); +#else + if (OB_ISNULL(allocator)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("copy allocator is null", K(ret), K(allocator)); + } else if (OB_ISNULL(coll_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("collection is null", K(ret), K(coll_)); + } else { + +#define DEEP_COPY_COLLECTION(type, class) \ + case type: { \ + if (OB_ISNULL(dest = reinterpret_cast(allocator->alloc(sizeof(class))))) { \ + ret = OB_ALLOCATE_MEMORY_FAILED; \ + LOG_WARN("failed to alloc memory for collection", K(ret), K(allocator), KPC(coll_), KPC(dest)); \ + } else if (FALSE_IT(dest = new (dest) class(coll_->get_id()))) { \ + } else if (OB_FAIL((reinterpret_cast(dest))->deep_copy(coll_, allocator))) { \ + LOG_WARN("failed to deep copy", K(ret), K(allocator), KPC(coll_), KPC(dest)); \ + int tmp = ObPLCollAllocator::free_child_coll(reinterpret_cast(*dest)); \ + if (OB_SUCCESS != tmp) { \ + LOG_WARN("failed to free child memory", K(tmp)); \ + } \ + } \ + break; \ + } + ObPLCollection* dest = NULL; + switch (coll_->get_type()) { + DEEP_COPY_COLLECTION(PL_NESTED_TABLE_TYPE, ObPLNestedTable); + DEEP_COPY_COLLECTION(PL_ASSOCIATIVE_ARRAY_TYPE, ObPLAssocArray); + DEEP_COPY_COLLECTION(PL_VARRAY_TYPE, ObPLVArray); + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unknow collection type", K(ret), K(coll_->get_type()), KPC(coll_)); + break; + } + } +#undef DEEP_COPY_COLLECTION + + if (OB_SUCC(ret)) { + dest->set_allocator(this); + for (int64_t i = 0; OB_SUCC(ret) && i < coll_->get_count(); ++i) { + if (OB_FAIL(ObUserDefinedType::destruct_obj(coll_->get_data()[i], NULL))) { + LOG_WARN("failed to destruct collection", K(ret), K(coll_->get_data()[i]), K(i)); + } + } + if (OB_SUCC(ret)) { + coll_->set_allocator(dest->get_allocator()); + coll_->set_data(dest->get_data()); + if (PL_ASSOCIATIVE_ARRAY_TYPE == coll_->get_type()) { + ObPLAssocArray* src = static_cast(coll_); + ObPLAssocArray* dst = static_cast(dest); + if (OB_ISNULL(src) || OB_ISNULL(dst)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected associative array pointer", K(ret), KPC(coll_), KPC(dst)); + } else { + src->set_key(dst->get_key()); + src->set_sort(dst->get_sort()); + } + } + } + } + LOG_INFO("copy all element with new allocator in collection", K(ret), KPC(coll_), KPC(dest), K(lbt())); + } +#endif return ret; } diff --git a/src/pl/ob_pl_code_generator.cpp b/src/pl/ob_pl_code_generator.cpp index f0185fe96..c7ca7580b 100644 --- a/src/pl/ob_pl_code_generator.cpp +++ b/src/pl/ob_pl_code_generator.cpp @@ -3232,6 +3232,40 @@ int ObPLCodeGenerator::generate_user_type(const ObUserDefinedType &type) OZ (build_record_type(record_type, elem_type_array)); } break; +#ifdef OB_BUILD_ORACLE_PL + case PL_NESTED_TABLE_TYPE: { + const ObNestedTableType &table_type = static_cast(type); + OZ (build_nested_table_type(table_type, elem_type_array)); + } + break; + case PL_ASSOCIATIVE_ARRAY_TYPE: { + const ObAssocArrayType &assoc_array_type = static_cast(type); + OZ (build_assoc_array_type(assoc_array_type, elem_type_array)); + } + break; + case PL_VARRAY_TYPE: { + const ObVArrayType &varray_type = static_cast(type); + OZ (build_varray_type(varray_type, elem_type_array)); + } + break; + case PL_SUBTYPE: { + const ObUserDefinedSubType &subtype = static_cast(type); + ObLLVMType base_type; + CK (OB_NOT_NULL(subtype.get_base_type())); + if (OB_FAIL(ret)) { + } else if (!subtype.get_base_type()->is_cursor_type()) { + OZ (build_subtype(subtype, elem_type_array)); + } else { + is_cursor_type = true; + } + } + break; + case PL_OPAQUE_TYPE: { + const ObOpaqueType &opaque_type = static_cast(type); + OZ (build_opaque_type(opaque_type, elem_type_array)); + } + break; +#endif case PL_CURSOR_TYPE: case PL_REF_CURSOR_TYPE: { is_cursor_type = true; @@ -3262,6 +3296,98 @@ int ObPLCodeGenerator::generate_user_type(const ObUserDefinedType &type) return ret; } +#ifdef OB_BUILD_ORACLE_PL +int ObPLCodeGenerator::build_nested_table_type(const ObNestedTableType &table_type, + ObIArray &elem_type_array) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(build_collection_type(table_type, elem_type_array))) { + LOG_WARN("declare collection super type failed", K(ret)); + } + return ret; +} + +int ObPLCodeGenerator::build_assoc_array_type(const ObAssocArrayType &array_type, + ObIArray &elem_type_array) +{ + int ret = OB_SUCCESS; + //ObObj* + ObLLVMType obj_type; + ObLLVMType key_type; + //int64* + ObLLVMType int_type; + ObLLVMType sort_type; + if (OB_FAIL(build_collection_type(array_type, elem_type_array))) { + LOG_WARN("declare collection super type failed", K(ret)); + } else if (OB_FAIL(adt_service_.get_obj(obj_type))) { + LOG_WARN("failed to get obj type", K(ret)); + } else if (OB_FAIL(obj_type.get_pointer_to(key_type))) { + LOG_WARN("failed to get_pointer_to", K(ret)); + } else if (OB_FAIL(helper_.get_llvm_type(ObIntType, int_type))) { + LOG_WARN("failed to get int type", K(ret)); + } else if (OB_FAIL(int_type.get_pointer_to(sort_type))) { + LOG_WARN("failed to get_pointer_to", K(ret)); + } else if (OB_FAIL(elem_type_array.push_back(key_type))) { + LOG_WARN("push_back error", K(ret)); + } else if (OB_FAIL(elem_type_array.push_back(sort_type))) { + LOG_WARN("push_back error", K(ret)); + } else { /*do nothing*/ } + return ret; +} + +int ObPLCodeGenerator::build_varray_type(const ObVArrayType &array_type, + ObIArray &elem_type_array) +{ + int ret = OB_SUCCESS; + ObLLVMType int_type; + if (OB_FAIL(build_collection_type(array_type, elem_type_array))) { + LOG_WARN("declare collection super type failed", K(ret)); + } else if (OB_FAIL(helper_.get_llvm_type(ObIntType, int_type))) { + LOG_WARN("failed to get int type", K(ret)); + } else if (OB_FAIL(elem_type_array.push_back(int_type))) { + LOG_WARN("push_back error", K(ret)); + } else { /*do nothing*/ } + return ret; +} + +int ObPLCodeGenerator::build_collection_type(const ObCollectionType &collection_type, + ObIArray &elem_type_array) +{ + int ret = OB_SUCCESS; + + //common::ObIAllocator *allocator_; + ObLLVMType allocator_type; + //common::ObDataType element_type_; + ObLLVMType element_type; + //int64_t count_; + ObLLVMType count_type; + //int64_t first_; + ObLLVMType first_type; + //int64_t last_; + ObLLVMType last_type; + + ObLLVMType element_ir_type; + ObLLVMType data_array_type; + ObLLVMType data_array_pointer_type; + + OZ (build_composite(elem_type_array)); + OZ (helper_.get_llvm_type(ObIntType, allocator_type)); + OZ (adt_service_.get_elem_desc(element_type)); + OZ (helper_.get_llvm_type(ObIntType, count_type)); + OZ (helper_.get_llvm_type(ObIntType, first_type)); + OZ (helper_.get_llvm_type(ObIntType, last_type)); + OZ (elem_type_array.push_back(allocator_type)); + OZ (elem_type_array.push_back(element_type)); + OZ (elem_type_array.push_back(count_type)); + OZ (elem_type_array.push_back(first_type)); + OZ (elem_type_array.push_back(last_type)); + OZ (get_datum_type(collection_type.get_element_type(), element_ir_type)); + OZ (ObLLVMHelper::get_array_type(element_ir_type, 0, data_array_type)); + OZ (data_array_type.get_pointer_to(data_array_pointer_type)); + OZ (elem_type_array.push_back(data_array_pointer_type)); + return ret; +} +#endif int ObPLCodeGenerator::build_record_type(const ObRecordType &record_type, ObIArray &elem_type_array) @@ -3318,6 +3444,29 @@ int ObPLCodeGenerator::build_composite(ObIArray &elem_type_arra return ret; } +#ifdef OB_BUILD_ORACLE_PL +int ObPLCodeGenerator::build_subtype(const ObUserDefinedSubType &subtype, + ObIArray &elem_type_array) +{ + int ret = OB_SUCCESS; + ObLLVMType base_type; + CK (OB_NOT_NULL(subtype.get_base_type())); + OZ (get_datum_type(*subtype.get_base_type(), base_type)); + OZ (elem_type_array.push_back(base_type)); + return ret; +} + +int ObPLCodeGenerator::build_opaque_type(const ObUserDefinedType &opaque_type, + ObIArray &elem_type_array) +{ + int ret = OB_SUCCESS; + UNUSED(opaque_type); + ObLLVMType type; + OZ (helper_.get_llvm_type(ObInt32Type, type)); + OZ (elem_type_array.push_back(type)); + return ret; +} +#endif int ObPLCodeGenerator::init() { @@ -7258,6 +7407,13 @@ int ObPLCodeGenerator::generate_di_user_type(const ObUserDefinedType &type, uint if (OB_FAIL(generate_di_record_type(record_type, line))) { LOG_WARN("failed to generate di record type", K(ret)); } +#ifdef OB_BUILD_ORACLE_PL + } else if (type.is_nested_table_type()) { + const ObNestedTableType &table_type = static_cast(type); + if (OB_FAIL(generate_di_table_type(table_type, line))) { + LOG_WARN("failed to generate di table type", K(ret)); + } +#endif } else { ret = OB_ERR_UNEXPECTED; LOG_WARN("user defined type is invalid", K(type.get_type())); @@ -7266,6 +7422,27 @@ int ObPLCodeGenerator::generate_di_user_type(const ObUserDefinedType &type, uint return ret; } +#ifdef OB_BUILD_ORACLE_PL +int ObPLCodeGenerator::generate_di_table_type(const ObNestedTableType &table_type, uint32_t line) +{ + int ret = OB_SUCCESS; + if (debug_mode_) { + const ObString &type_name = table_type.get_name(); + const ObPLDataType &data_type = table_type.get_element_type(); + ObLLVMDIType element_type; + ObLLVMDIType di_table_type; + uint64_t type_id = table_type.get_user_type_id(); + if (OB_FAIL(get_di_datum_type(data_type, element_type))) { + LOG_WARN("failed to get di datum type", K(ret)); + } else if (OB_FAIL(di_adt_service_.get_table_type(type_name, line, element_type, di_table_type))) { + LOG_WARN("failed to get array type", K(ret)); + } else if (OB_FAIL(di_user_type_map_.set_refactored(type_id, di_table_type))) { + LOG_WARN("failed to set di table type to user type map", K(ret), K(table_type)); + } + } + return ret; +} +#endif int ObPLCodeGenerator::generate_di_record_type(const ObRecordType &record_type, uint32_t line) { diff --git a/src/pl/ob_pl_code_generator.h b/src/pl/ob_pl_code_generator.h index 794978a9e..dd0b10425 100644 --- a/src/pl/ob_pl_code_generator.h +++ b/src/pl/ob_pl_code_generator.h @@ -665,6 +665,14 @@ private: int64_t param_count, const common::ObString &func_name, bool for_write); +#ifdef OB_BUILD_ORACLE_PL + int build_nested_table_type(const ObNestedTableType &table_type, ObIArray &elem_type_array); + int build_assoc_array_type(const ObAssocArrayType &table_type, ObIArray &elem_type_array); + int build_varray_type(const ObVArrayType &array_type, ObIArray &elem_type_array); + int build_collection_type(const ObCollectionType &collection_type, ObIArray &elem_type_array); + int build_subtype(const ObUserDefinedSubType &subtype, ObIArray &elem_type_array); + int build_opaque_type(const ObUserDefinedType &opaque_type, ObIArray &elem_type_array); +#endif int build_record_type(const ObRecordType &record_type, ObIArray &elem_type_array); int build_composite(ObIArray &elem_type_array); int generate_spi_calc(int64_t expr_idx, @@ -727,6 +735,9 @@ public: int get_di_llvm_type(const ObPLDataType &pl_type, jit::ObLLVMDIType &di_type); int get_di_datum_type(const ObPLDataType &pl_type, jit::ObLLVMDIType &di_type); int generate_di_user_type(const ObUserDefinedType &type, uint32_t line); +#ifdef OB_BUILD_ORACLE_PL + int generate_di_table_type(const ObNestedTableType &table_type, uint32_t line); +#endif int generate_di_record_type(const ObRecordType &record_type, uint32_t line); int generate_di_argument(); // generate debug info for variables which have a llvm::Value, diff --git a/src/pl/ob_pl_compile.cpp b/src/pl/ob_pl_compile.cpp index cfd5a35e0..a06f0b8bc 100644 --- a/src/pl/ob_pl_compile.cpp +++ b/src/pl/ob_pl_compile.cpp @@ -66,6 +66,31 @@ int ObPLCompiler::init_anonymous_ast( CK (OB_NOT_NULL(user_type)); OX (pl_type.reset()); OX (pl_type = *user_type); +#ifdef OB_BUILD_ORACLE_PL + } else if (PL_REF_CURSOR_TYPE == param.get_meta().get_extend_type()) { + pl_type.reset(); + pl_type.set_type(pl::PL_REF_CURSOR_TYPE); + pl_type.set_type_from(pl::PL_TYPE_SYS_REFCURSOR); + } else if (PL_NESTED_TABLE_TYPE == param.get_meta().get_extend_type()) { + ObPLCollection *coll = reinterpret_cast(param.get_ext()); + ObNestedTableType *nested_type = NULL; + ObPLDataType element_type; + if (OB_ISNULL(nested_type = + reinterpret_cast(allocator.alloc(sizeof(ObNestedTableType))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for ObNestedTableType", K(ret)); + } + CK (OB_NOT_NULL(coll)); + OX (new(nested_type)ObNestedTableType()); + OX (element_type.reset()); + OX (element_type.set_data_type(coll->get_element_type())); + OX (nested_type->set_element_type(element_type)); + OX (nested_type->set_user_type_id( + func_ast.get_user_type_table().generate_user_type_id(OB_INVALID_ID))); + OZ (func_ast.get_user_type_table().add_type(nested_type)); + OZ (func_ast.get_user_type_table().add_external_type(nested_type)); + OX (pl_type = *nested_type); +#endif } else { ret = OB_NOT_SUPPORTED; LOG_WARN( @@ -1038,6 +1063,88 @@ int ObPLCompiler::compile_types(const ObIArray &types, } } break; +#ifdef OB_BUILD_ORACLE_PL + case PL_NESTED_TABLE_TYPE: { + ObNestedTableType *table_type = static_cast(alloc.alloc(sizeof(ObNestedTableType))); + if (OB_ISNULL(table_type)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate memory failed", K(ret), KPC(ast_type), K(i)); + } else { + new (table_type) ObNestedTableType(); + if (OB_FAIL(table_type->deep_copy(alloc, *(static_cast(ast_type))))) { + LOG_WARN("pl user type deep copy failed", K(ret), KPC(ast_type), K(i)); + } else { + user_type = table_type; + } + } + } + break; + case PL_ASSOCIATIVE_ARRAY_TYPE: { + ObAssocArrayType *assoc_array_type = static_cast(alloc.alloc(sizeof(ObAssocArrayType))); + if (OB_ISNULL(assoc_array_type)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate memory failed", K(ret), KPC(ast_type), K(i)); + } else { + new (assoc_array_type) ObAssocArrayType(); + if (OB_FAIL(assoc_array_type->deep_copy(alloc, *(static_cast(ast_type))))) { + LOG_WARN("pl user type deep copy failed", K(ret), KPC(ast_type), K(i)); + } else { + user_type = assoc_array_type; + } + } + } + break; + case PL_VARRAY_TYPE: { + ObVArrayType *varray_type = static_cast(alloc.alloc(sizeof(ObVArrayType))); + if (OB_ISNULL(varray_type)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate memory for varray type failed", K(ret), K(varray_type), KPC(ast_type), K(i)); + } else { + new (varray_type) ObVArrayType(); + if (OB_FAIL(varray_type->deep_copy(alloc, *(static_cast(ast_type))))) { + LOG_WARN("deep copy varray type failed", K(ret), KPC(ast_type), K(i)); + } else { + user_type = varray_type; + } + } + } + break; + case PL_SUBTYPE: { + ObUserDefinedSubType *subtype = + static_cast(alloc.alloc(sizeof(ObUserDefinedSubType))); + if (OB_ISNULL(subtype)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory for user define sub type", + K(ret), K(subtype), KPC(ast_type), K(i)); + } else { + new(subtype)ObUserDefinedSubType(); + OZ (subtype->deep_copy(alloc, + *(static_cast(ast_type))), + ast_type, i); + OX (user_type = subtype); + } + } + break; + case PL_OPAQUE_TYPE: { + ObOpaqueType *opaqua_type = static_cast(alloc.alloc(sizeof(ObOpaqueType))); + if (OB_ISNULL(opaqua_type)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate memory for cursor type failed", + K(ret), + K(opaqua_type), + KPC(ast_type), + K(i)); + } else { + new (opaqua_type) ObOpaqueType(); + if (OB_FAIL(opaqua_type->deep_copy(alloc, *(static_cast(ast_type))))) { + LOG_WARN("deep copy ref cursor type failed", K(ret), KPC(ast_type), K(i)); + } else { + user_type = opaqua_type; + } + } + } + break; +#endif default: { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid pl type", K(ret), KPC(ast_type), K(i)); diff --git a/src/pl/ob_pl_di_adt_service.cpp b/src/pl/ob_pl_di_adt_service.cpp index a0ae807fb..60264a647 100644 --- a/src/pl/ob_pl_di_adt_service.cpp +++ b/src/pl/ob_pl_di_adt_service.cpp @@ -642,7 +642,33 @@ int ObPLDIADTService::get_table_type(const ObString &type_name, uint32_t line, ObLLVMDIType &element_type, ObLLVMDIType &table_type) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSEDx(type_name, line, element_type, table_type); +#else + ObLLVMDIType element_array_type; + ObLLVMDIType element_array_ptr_type; + ObLLVMDIType member_type; + ObSEArray member_types; + if (OB_FAIL(get_collection(line, member_types))) { + LOG_WARN("failed to get collation", K(ret)); + } else if (OB_FAIL(helper_.create_array_type(element_type, 0, element_array_type))) { + LOG_WARN("failed to create array type", K(ret)); + } else if (OB_FAIL(helper_.create_pointer_type(element_array_type, element_array_ptr_type))) { + LOG_WARN("failed to create pointer type", K(ret)); + } else if (OB_FAIL(helper_.create_member_type("data_", ObPLNestedTable::data_offset_bits(), + line, element_array_ptr_type, member_type))) { + LOG_WARN("failed to create member type", K(ret)); + } else if (OB_FAIL(member_types.push_back(member_type))) { + LOG_WARN("failed to push back member type", K(ret)); + } else if (OB_FAIL(helper_.create_struct_type(type_name, line, + sizeof(ObPLNestedTable) * 8, 32, + member_types, table_type))) { + LOG_WARN("failed to create di struct type", K(ret)); + } else if (OB_ISNULL(table_type.get_v())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to create di table type", K(ret)); + } +#endif return ret; } diff --git a/src/pl/ob_pl_exception_handling.cpp b/src/pl/ob_pl_exception_handling.cpp index 645d5dd9c..55888a244 100644 --- a/src/pl/ob_pl_exception_handling.cpp +++ b/src/pl/ob_pl_exception_handling.cpp @@ -120,6 +120,67 @@ ObPLException::ObPLException(int64_t error_code) new(&type_)ObPLConditionValue(ERROR_CODE, error_code); } +#ifdef OB_BUILD_ORACLE_PL +int ObPLEH::eh_adjust_call_stack( + ObPLFunction *pl_func, ObPLContext *pl_ctx, uint64_t loc, int error_code) +{ + int ret = OB_SUCCESS; + + if (error_code != OB_SUCCESS + && OB_ALLOCATE_MEMORY_FAILED != error_code + && OB_TENANT_OUT_OF_MEM != error_code + && OB_EXCEED_MEM_LIMIT != error_code + && OB_NOT_NULL(pl_ctx) + && lib::is_oracle_mode()) { + + ObIAllocator *allocator = NULL; + CK (OB_NOT_NULL(pl_ctx->get_exec_stack().at(0))); + CK (OB_NOT_NULL(pl_ctx->get_exec_stack().at(0)->get_exec_ctx().status_)); + OX (*(pl_ctx->get_exec_stack().at(0)->get_exec_ctx().status_) = error_code); + OX (allocator = pl_ctx->get_exec_stack().at(0)->get_exec_ctx().allocator_); + CK (OB_NOT_NULL(allocator)); + if (0 == pl_ctx->get_call_stack().count()) { + OZ (DbmsUtilityHelper::get_backtrace(*allocator, *pl_ctx, pl_ctx->get_call_stack())); + } + if (OB_SUCC(ret) && OB_NOT_NULL(pl_func)) { + bool is_exist = false; + DbmsUtilityHelper::BtInfo *tbi = NULL; + DbmsUtilityHelper::BtInfo *bi = NULL; + // check if same error code exist, this is possible + for (int64_t i = 0; + OB_SUCC(ret) && !is_exist && i < pl_ctx->get_error_trace().count(); ++i) { + tbi = pl_ctx->get_error_trace().at(i); + CK (OB_NOT_NULL(tbi)); + OX (is_exist = (tbi->error_code == error_code)); + } + if (OB_SUCC(ret) && OB_NOT_NULL(allocator)) { + bi = static_cast( + allocator->alloc(sizeof(DbmsUtilityHelper::BtInfo))); + } + if (OB_SUCC(ret) && !is_exist && OB_NOT_NULL(bi)) { + bi->init(); + bi->error_type = DbmsUtilityHelper::BTErrorType::ERROR_INFO; // 0: stack info, 1 : error info + bi->loc = loc + (1LL << 32); + bi->handler = pl_func->get_action(); + bi->error_code = error_code; + ObSqlString exception_msg; + OZ (pl_ctx->get_error_trace().push_back(bi)); + for (int64_t i = 0; OB_SUCC(ret) && i < pl_ctx->get_error_trace().count(); ++i) { + if (pl_ctx->get_error_trace().at(i)->handler == bi->handler) { + pl_ctx->get_error_trace().at(i)->loc = loc + (1LL << 32); + } + } + OZ (pl_ctx->get_exact_error_msg(pl_ctx->get_error_trace(), + pl_ctx->get_call_stack(), + exception_msg)); + LOG_WARN("got exception! ", KR(error_code), K(exception_msg)); + } + } + pl_ctx->set_call_trace_error_code(ret); + } + return ret; +} +#endif ObUnwindException *ObPLEH::eh_create_exception(int64_t pl_context, int64_t pl_function, @@ -173,6 +234,10 @@ ObUnwindException *ObPLEH::eh_create_exception(int64_t pl_context, } tl_eptr = unwind; +#ifdef OB_BUILD_ORACLE_PL + OZ (eh_adjust_call_stack( + reinterpret_cast(pl_function), pl_ctx, loc, value->error_code_)); +#endif } return unwind; diff --git a/src/pl/ob_pl_exception_handling.h b/src/pl/ob_pl_exception_handling.h index 994d96ae7..2180646c0 100644 --- a/src/pl/ob_pl_exception_handling.h +++ b/src/pl/ob_pl_exception_handling.h @@ -65,6 +65,10 @@ public: static int eh_convert_exception(bool oracle_mode, int oberr, ObPLConditionType *type, int64_t *error_code, const char **sql_state, int64_t *str_len); static ObPLConditionType eh_classify_exception(const char *sql_state); +#ifdef OB_BUILD_ORACLE_PL + static int eh_adjust_call_stack( + ObPLFunction *pl_func, ObPLContext *pl_ctx, uint64_t loc, int error_code); +#endif public: static void eh_debug_int64(const char *name_ptr, int64_t name_len, int64_t object); diff --git a/src/pl/ob_pl_interface_pragma.h b/src/pl/ob_pl_interface_pragma.h index 817fc506f..2c109f122 100644 --- a/src/pl/ob_pl_interface_pragma.h +++ b/src/pl/ob_pl_interface_pragma.h @@ -21,7 +21,48 @@ #include "pl/sys_package/ob_dbms_monitor.h" #include "pl/sys_package/ob_dbms_sql.h" #include "pl/sys_package/ob_dbms_user_define_rule.h" +#ifdef OB_BUILD_ORACLE_PL +#include "pl/sys_package/ob_dbms_utl_encode.h" +#include "pl/ob_pl_warning.h" +#include "pl/ob_pl_lock.h" +#include "pl/ob_pl_error_log.h" +#include "pl/sys_package/ob_dbms_debug.h" +#include "pl/sys_package/ob_dbms_xa.h" +#include "pl/sys_package/ob_dbms_lob.h" +#include "pl/sys_package/ob_utl_raw.h" +#include "pl/sys_package/ob_utl_i18n.h" +#include "pl/sys_package/ob_pl_dbms_utility_helper.h" +#include "pl/sys_package/ob_dbms_anytype.h" +#include "pl/sys_package/ob_dbms_anydata.h" +#include "pl/sys_package/ob_dbms_xmlgen.h" +#include "pl/sys_package/ob_dbms_scheduler.h" +#include "pl/sys_package/ob_dbms_scheduler_mysql.h" +#include "pl/sys_package/ob_dbms_crypto.h" +#include "pl/sys_package/ob_dbms_job.h" +#include "pl/sys_package/ob_pl_utl_file.h" +#include "pl/sys_package/ob_dbms_plan_cache.h" +#include "pl/sys_package/ob_dbms_sys_error.h" +#include "pl/sys_package/ob_dbms_preprocessor.h" +#include "pl/sys_package/ob_utl_inaddr.h" +#include "pl/sys_package/ob_pl_sa_sysdba.h" +#include "pl/sys_package/ob_pl_sa_session.h" +#include "pl/sys_package/ob_pl_sa_user_admin.h" +#include "pl/sys_package/ob_pl_sa_policy_admin.h" +#include "pl/sys_package/ob_pl_sa_label_admin.h" +#include "pl/sys_package/ob_pl_sa_components.h" +#include "pl/sys_package/ob_dbms_audit_mgmt.h" +#include "pl/sys_package/ob_dbms_application.h" +#include "pl/sys_package/ob_dbms_monitor.h" +#include "pl/sys_package/ob_dbms_xplan.h" +#include "pl/sys_package/ob_dbms_spm.h" +#include "pl/sys_package/ob_dbms_rls.h" +#include "pl/sys_package/ob_json_object_type.h" +#include "pl/sys_package/ob_json_element_type.h" #include "pl/sys_package/ob_pl_dbms_resource_manager.h" +#endif +#ifdef OB_BUILD_ORACLE_XML +#include "pl/sys_package/ob_xml_type.h" +#endif #include "pl/sys_package/ob_dbms_session.h" #ifdef INTERFACE_DEF @@ -33,6 +74,168 @@ INTERFACE_DEF(INTERFACE_DBMS_UPGRADE_ALL, "UPGRADE_ALL", (ObDBMSUpgrade::upgrade_all)) // end of __dbms_upgrade +#ifdef OB_BUILD_ORACLE_PL + // start utl_encode package + INTERFACE_DEF(INTERFACE_DBMS_BASE64_ENCODE, "BASE64_ENCODE", (ObDBMSUtlEncode::base64_encode)) + INTERFACE_DEF(INTERFACE_DBMS_BASE64_DECODE, "BASE64_DECODE", (ObDBMSUtlEncode::base64_decode)) + INTERFACE_DEF(INTERFACE_DBMS_QUOTED_PRINTABLE_ENCODE, "QUOTED_PRINTABLE_ENCODE", + (ObDBMSUtlEncode::quoted_printable_encode)) + INTERFACE_DEF(INTERFACE_DBMS_QUOTED_PRINTABLE_DECODE, "QUOTED_PRINTABLE_DECODE", + (ObDBMSUtlEncode::quoted_printable_decode)) + INTERFACE_DEF(INTERFACE_DBMS_MIMEHEADER_ENCODE, "MIMEHEADER_ENCODE", + (ObDBMSUtlEncode::mimeheader_encode)) + INTERFACE_DEF(INTERFACE_DBMS_MIMEHEADER_DECODE, "MIMEHEADER_DECODE", + (ObDBMSUtlEncode::mimeheader_decode)) + INTERFACE_DEF(INTERFACE_DBMS_TEXT_ENCODE, "TEXT_ENCODE", (ObDBMSUtlEncode::text_encode)) + INTERFACE_DEF(INTERFACE_DBMS_TEXT_DECODE, "TEXT_DECODE", (ObDBMSUtlEncode::text_decode)) + INTERFACE_DEF(INTERFACE_DBMS_UU_ENCODE, "UU_ENCODE", (ObDBMSUtlEncode::uu_encode)) + INTERFACE_DEF(INTERFACE_DBMS_UU_DECODE, "UU_DECODE", (ObDBMSUtlEncode::uu_decode)) + // end utl_encode package + + // dbms_warning + INTERFACE_DEF(INTERFACE_WARNING_ADD_CAT, "WARNING_ADD_CAT", (PlCompilerWarningCategory::add_pl_warning_setting_cat)) + INTERFACE_DEF(INTERFACE_WARNING_ADD_NUM, "WARNING_ADD_NUM", (PlCompilerWarningCategory::add_pl_warning_setting_num)) + INTERFACE_DEF(INTERFACE_WARNING_SET, "WARNING_SET", (PlCompilerWarningCategory::set_pl_warning_setting)) + INTERFACE_DEF(INTERFACE_WARNING_GET_CAT, "WARNING_GET_CAT", (PlCompilerWarningCategory::get_category)) + INTERFACE_DEF(INTERFACE_WARNING_GET_SETTING_CAT, "WARNING_GET_SETTING_CAT", (PlCompilerWarningCategory::get_warning_setting_cat)) + INTERFACE_DEF(INTERFACE_WARNING_GET_SETTING_NUM, "WARNING_GET_SETTING_NUM", (PlCompilerWarningCategory::get_warning_setting_num)) + INTERFACE_DEF(INTERFACE_WARNING_GET_SETTING_STR, "WARNING_GET_SETTING_STR", (PlCompilerWarningCategory::get_warning_setting_string)) + // end dbms_warning + + // dbms_lock + INTERFACE_DEF(INTERFACE_LOCK_SLEEP_LOCK, "SLEEP_LOCK", (PlPackageLock::sleep)) + // end dbms_lock + + // DBMS_ERRLOG + INTERFACE_DEF(INTERFACE_CREATE_ERROR_LOG, "CREATE_ERROR_LOG", (PlPackageErrorLog::create_error_log)) + // end DBMS_ERRLOG + + // start of dbms_debug +#define DEFINE_DEBUG_INTERFACE(symbol, func) \ + INTERFACE_DEF(INTERFACE_DBMS_DEBUG_##symbol, #symbol, (func)) + + DEFINE_DEBUG_INTERFACE(GET_VALUES, ObDBMSDebug::get_values) + DEFINE_DEBUG_INTERFACE(TARGET_PROGRAM_RUNNING, ObDBMSDebug::target_program_running) + DEFINE_DEBUG_INTERFACE(PING, ObDBMSDebug::ping) + DEFINE_DEBUG_INTERFACE(GET_RUNTIME_INFO, ObDBMSDebug::get_runtime_info) + DEFINE_DEBUG_INTERFACE(DEBUG_ON, ObDBMSDebug::on) + DEFINE_DEBUG_INTERFACE(DEBUG_OFF, ObDBMSDebug::off) + DEFINE_DEBUG_INTERFACE(INITIALIZE, ObDBMSDebug::initialize) + DEFINE_DEBUG_INTERFACE(DETACH_SESSION, ObDBMSDebug::detach_session) + DEFINE_DEBUG_INTERFACE(PRINT_BACKTRACE, ObDBMSDebug::print_backtrace) + DEFINE_DEBUG_INTERFACE(PRINT_VARCHAR_BACKTRACE, ObDBMSDebug::print_varchar_backtrace) + DEFINE_DEBUG_INTERFACE(SHOW_BREAKPOINTS, ObDBMSDebug::show_breakpoints) + DEFINE_DEBUG_INTERFACE(GET_TIMEOUT_BEHAVIOUR, ObDBMSDebug::get_timeout_behaviour) + DEFINE_DEBUG_INTERFACE(ATTACH_SESSION, ObDBMSDebug::attach_session) + DEFINE_DEBUG_INTERFACE(DEBUG_CONTINUE, ObDBMSDebug::debug_continue) + DEFINE_DEBUG_INTERFACE(DEL_BREAKPOINT, ObDBMSDebug::del_breakpoint) + DEFINE_DEBUG_INTERFACE(ENABLE_BREAKPOINT, ObDBMSDebug::enable_breakpoint) + DEFINE_DEBUG_INTERFACE(DISABLE_BREAKPOINT, ObDBMSDebug::disable_breakpoint) + DEFINE_DEBUG_INTERFACE(SET_TIMEOUT, ObDBMSDebug::set_timeout) + DEFINE_DEBUG_INTERFACE(SET_TIMEOUT_BEHAVIOUR, ObDBMSDebug::set_timeout_behaviour) + DEFINE_DEBUG_INTERFACE(SET_BREAKPOINT, ObDBMSDebug::set_breakpoint) + DEFINE_DEBUG_INTERFACE(GET_VALUE, ObDBMSDebug::get_value) + DEFINE_DEBUG_INTERFACE(SYNCHRONIZE, ObDBMSDebug::synchronize) + DEFINE_DEBUG_INTERFACE(GET_TARGET_DEBUG_ID, ObDBMSDebug::get_target_debug_id) +#undef DEFINE_DEBUG_INTERFACE + // end of dbms_debug + + // start of dbms_sql + INTERFACE_DEF(INTERFACE_DBMS_SQL_OPEN_CURSOR, "DBMS_SQL_OPEN_CURSOR", (ObPLDbmsSql::open_cursor)) + INTERFACE_DEF(INTERFACE_DBMS_SQL_PARSE, "DBMS_SQL_PARSE", (ObPLDbmsSql::parse)) + INTERFACE_DEF(INTERFACE_DBMS_SQL_BIND_VARIABLE, "DBMS_SQL_BIND_VARIABLE", (ObPLDbmsSql::bind_variable)) + INTERFACE_DEF(INTERFACE_DBMS_SQL_EXECUTE, "DBMS_SQL_EXECUTE", (ObPLDbmsSql::execute)) + INTERFACE_DEF(INTERFACE_DBMS_SQL_DEFINE_COLUMN, "DBMS_SQL_DEFINE_COLUMN", (ObPLDbmsSql::define_column)) + INTERFACE_DEF(INTERFACE_DBMS_SQL_DEFINE_ARRAY, "DBMS_SQL_DEFINE_ARRAY", (ObPLDbmsSql::define_array)) + INTERFACE_DEF(INTERFACE_DBMS_SQL_COLUMN_VALUE, "DBMS_SQL_COLUMN_VALUE", (ObPLDbmsSql::column_value)) + INTERFACE_DEF(INTERFACE_DBMS_SQL_FETCH_ROWS, "DBMS_SQL_FETCH_ROWS", (ObPLDbmsSql::fetch_rows)) + INTERFACE_DEF(INTERFACE_DBMS_SQL_CLOSE_CURSOR, "DBMS_SQL_CLOSE_CURSOR", (ObPLDbmsSql::close_cursor)) + INTERFACE_DEF(INTERFACE_DBMS_SQL_DESCRIBE_COLUMNS, "DBMS_SQL_DESCRIBE_COLUMNS", (ObPLDbmsSql::describe_columns)) + INTERFACE_DEF(INTERFACE_DBMS_SQL_DESCRIBE_COLUMNS2, "DBMS_SQL_DESCRIBE_COLUMNS2", (ObPLDbmsSql::describe_columns2)) + INTERFACE_DEF(INTERFACE_DBMS_SQL_DESCRIBE_COLUMNS3, "DBMS_SQL_DESCRIBE_COLUMNS3", (ObPLDbmsSql::describe_columns3)) + INTERFACE_DEF(INTERFACE_DBMS_SQL_IS_OPEN, "DBMS_SQL_IS_OPEN", (ObPLDbmsSql::is_open)) + INTERFACE_DEF(INTERFACE_DBMS_SQL_EXECUTE_AND_FETCH, "DBMS_SQL_EXECUTE_AND_FETCH", (ObPLDbmsSql::execute_and_fetch)) + INTERFACE_DEF(INTERFACE_DBMS_SQL_TO_CURSOR_NUMBER, "DBMS_SQL_TO_CURSOR_NUMBER", (ObPLDbmsSql::to_cursor_number)) + INTERFACE_DEF(INTERFACE_DBMS_SQL_DEFINE_COLUMN_LONG, "DBMS_SQL_DEFINE_COLUMN_LONG", (ObPLDbmsSql::define_column_long)) + INTERFACE_DEF(INTERFACE_DBMS_SQL_COLUMN_VALUE_LONG, "DBMS_SQL_COLUMN_VALUE_LONG", (ObPLDbmsSql::column_value_long)) + INTERFACE_DEF(INTERFACE_DBMS_SQL_VARIABLE_VALUE, "DBMS_SQL_VARIABLE_VALUE", (ObPLDbmsSql::variable_value)) + INTERFACE_DEF(INTERFACE_DBMS_SQL_LAST_ERROR_POSITION, "DBMS_SQL_LAST_ERROR_POSITION", (ObPLDbmsSql::last_error_position)) + // end of dbms_sql + + //start of DBMS_XA + INTERFACE_DEF(INTERFACE_DBMS_XA_XID, "INNER_XA_XID", (ObDbmsXA::xa_xid)) + INTERFACE_DEF(INTERFACE_DBMS_XA_START, "INNER_XA_START", (ObDbmsXA::xa_start)) + INTERFACE_DEF(INTERFACE_DBMS_XA_END, "INNER_XA_END", (ObDbmsXA::xa_end)) + INTERFACE_DEF(INTERFACE_DBMS_XA_PREPARE, "INNER_XA_PREPARE", (ObDbmsXA::xa_prepare)) + INTERFACE_DEF(INTERFACE_DBMS_XA_COMMIT, "INNER_XA_COMMIT", (ObDbmsXA::xa_commit)) + INTERFACE_DEF(INTERFACE_DBMS_XA_COMMIT_WITH_FLAGS, "INNER_XA_COMMIT_WITH_FLAGS", (ObDbmsXA::xa_commit_with_flags)) + INTERFACE_DEF(INTERFACE_DBMS_XA_FORGET, "INNER_XA_FORGET", (ObDbmsXA::xa_forget)) + INTERFACE_DEF(INTERFACE_DBMS_XA_GETLASTOER, "INNER_XA_GETLASTOER", (ObDbmsXA::xa_getlastoer)) + INTERFACE_DEF(INTERFACE_DBMS_XA_RECOVER, "INNER_XA_RECOVER", (ObDbmsXA::xa_recover)) + INTERFACE_DEF(INTERFACE_DBMS_XA_ROLLBACK, "INNER_XA_ROLLBACK", (ObDbmsXA::xa_rollback)) + INTERFACE_DEF(INTERFACE_DBMS_XA_SETTIMEOUT, "INNER_XA_SETTIMEOUT", (ObDbmsXA::xa_settimeout)) + INTERFACE_DEF(INTERFACE_DBMS_DIST_TXN_SYNC, "INNER_DIST_TXN_SYNC", (ObDbmsXA::dist_txn_sync)) + //end of DBMS_XA + + // start of dbms_lob + INTERFACE_DEF(INTERFACE_DBMS_LOB_TRIM, "DBMS_LOB_TRIM", (ObDbmsLob::trim)) + INTERFACE_DEF(INTERFACE_DBMS_LOB_APPEND, "DBMS_LOB_APPEND", (ObDbmsLob::append)) + INTERFACE_DEF(INTERFACE_DBMS_LOB_WRITE, "DBMS_LOB_WRITE", (ObDbmsLob::write)) + INTERFACE_DEF(INTERFACE_DBMS_LOB_OPEN, "DBMS_LOB_OPEN", (ObDbmsLob::open)) + INTERFACE_DEF(INTERFACE_DBMS_LOB_CLOSE, "DBMS_LOB_CLOSE", (ObDbmsLob::close)) + INTERFACE_DEF(INTERFACE_DBMS_LOB_ISOPEN, "DBMS_LOB_ISOPEN", (ObDbmsLob::isopen)) + INTERFACE_DEF(INTERFACE_DBMS_LOB_GETLENGTH, "DBMS_LOB_GETLENGTH", (ObDbmsLob::getlength)) + INTERFACE_DEF(INTERFACE_DBMS_LOB_READ, "DBMS_LOB_READ", (ObDbmsLob::read)) + INTERFACE_DEF(INTERFACE_DBMS_LOB_SUBSTR, "DBMS_LOB_SUBSTR", (ObDbmsLob::substr)) + INTERFACE_DEF(INTERFACE_DBMS_LOB_INSTR, "DBMS_LOB_INSTR", (ObDbmsLob::instr)) + INTERFACE_DEF(INTERFACE_DBMS_LOB_WRITEAPPEND, "DBMS_LOB_WRITEAPPEND", (ObDbmsLob::writeappend)) + INTERFACE_DEF(INTERFACE_DBMS_LOB_ERASE, "DBMS_LOB_ERASE", (ObDbmsLob::erase)) + INTERFACE_DEF(INTERFACE_DBMS_LOB_COPY, "DBMS_LOB_COPY", (ObDbmsLob::copy)) + INTERFACE_DEF(INTERFACE_DBMS_LOB_CREATETEMPORARY, "DBMS_LOB_CREATETEMPORARY", (ObDbmsLob::createtemporary)) + INTERFACE_DEF(INTERFACE_DBMS_LOB_FREETEMPORARY, "DBMS_LOB_FREETEMPORARY", (ObDbmsLob::freetemporary)) + INTERFACE_DEF(INTERFACE_DBMS_LOB_CONVERTTOBLOB, "DBMS_LOB_CONVERTTOBLOB", (ObDbmsLob::converttoblob)) + INTERFACE_DEF(INTERFACE_DBMS_LOB_OBCI_WRITE, "DBMS_LOB_OBCI_WRITE", (ObDbmsLob::obci_write)) + INTERFACE_DEF(INTERFACE_DBMS_LOB_COMPARE, "DBMS_LOB_COMPARE", (ObDbmsLob::compare)) + // end of dbms_lob + + // start of utl_raw + INTERFACE_DEF(INTERFACE_UTL_RAW_CAST_TO_RAW, "UTL_RAW_CAST_TO_RAW", (ObUtlRaw::cast_to_raw)) + INTERFACE_DEF(INTERFACE_UTL_RAW_CAST_TO_VARCHAR2, "UTL_RAW_CAST_TO_VARCHAR2", (ObUtlRaw::cast_to_varchar2)) + INTERFACE_DEF(INTERFACE_UTL_RAW_CAST_TO_NVARCHAR2, "UTL_RAW_CAST_TO_NVARCHAR2", (ObUtlRaw::cast_to_nvarchar2)) + INTERFACE_DEF(INTERFACE_UTL_RAW_LENGTH, "UTL_RAW_LENGTH", (ObUtlRaw::length)) + INTERFACE_DEF(INTERFACE_UTL_RAW_BIT_COMPLEMENT, "UTL_RAW_BIT_COMPLEMENT", (ObUtlRaw::bit_complement)) + INTERFACE_DEF(INTERFACE_UTL_RAW_BIT_AND, "UTL_RAW_BIT_AND", (ObUtlRaw::bit_and)) + INTERFACE_DEF(INTERFACE_UTL_RAW_BIT_OR, "UTL_RAW_BIT_OR", (ObUtlRaw::bit_or)) + INTERFACE_DEF(INTERFACE_UTL_RAW_BIT_XOR, "UTL_RAW_BIT_XOR", (ObUtlRaw::bit_xor)) + INTERFACE_DEF(INTERFACE_UTL_RAW_COPIES, "UTL_RAW_COPIES", (ObUtlRaw::copies)) + INTERFACE_DEF(INTERFACE_UTL_RAW_COMPARE, "UTL_RAW_COMPARE", (ObUtlRaw::compare)) + INTERFACE_DEF(INTERFACE_UTL_RAW_CONCAT, "UTL_RAW_CONCAT", (ObUtlRaw::concat)) + INTERFACE_DEF(INTERFACE_UTL_RAW_SUBSTR, "UTL_RAW_SUBSTR", (ObUtlRaw::substr)) + INTERFACE_DEF(INTERFACE_UTL_RAW_REVERSE, "UTL_RAW_REVERSE", (ObUtlRaw::reverse)) + INTERFACE_DEF(INTERFACE_UTL_RAW_CAST_FROM_BINARY_INTEGER, "UTL_RAW_CAST_FROM_BINARY_INTEGER", (ObUtlRaw::cast_from_binary_integer)) + INTERFACE_DEF(INTERFACE_UTL_RAW_CAST_TO_BINARY_INTEGER, "UTL_RAW_CAST_TO_BINARY_INTEGER", (ObUtlRaw::cast_to_binary_integer)) + INTERFACE_DEF(INTERFACE_UTL_RAW_CAST_FROM_BINARY_FLOAT, "UTL_RAW_CAST_FROM_BINARY_FLOAT", (ObUtlRaw::cast_from_binary_float)) + INTERFACE_DEF(INTERFACE_UTL_RAW_CAST_TO_BINARY_FLOAT, "UTL_RAW_CAST_TO_BINARY_FLOAT", (ObUtlRaw::cast_to_binary_float)) + INTERFACE_DEF(INTERFACE_UTL_RAW_CAST_FROM_BINARY_DOUBLE, "UTL_RAW_CAST_FROM_BINARY_DOUBLE", (ObUtlRaw::cast_from_binary_double)) + INTERFACE_DEF(INTERFACE_UTL_RAW_CAST_TO_BINARY_DOUBLE, "UTL_RAW_CAST_TO_BINARY_DOUBLE", (ObUtlRaw::cast_to_binary_double)) + INTERFACE_DEF(INTERFACE_UTL_RAW_CAST_FROM_NUMBER, "UTL_RAW_CAST_FROM_NUMBER", (ObUtlRaw::cast_from_number)) + INTERFACE_DEF(INTERFACE_UTL_RAW_CAST_TO_NUMBER, "UTL_RAW_CAST_TO_NUMBER", (ObUtlRaw::cast_to_number)) + // end of utl_raw + + // start of utl_i18n + INTERFACE_DEF(INTERFACE_UTL_I18N_STRING_TO_RAW, "UTL_I18N_STRING_TO_RAW", (ObUtlI18n::string_to_raw)) + INTERFACE_DEF(INTERFACE_UTL_I18N_RAW_TO_CHAR, "UTL_I18N_RAW_TO_CHAR", (ObUtlI18n::raw_to_char)) + // end of utl_i18n + + // start of utl_inaddr + INTERFACE_DEF(INTERFACE_UTL_INADDR_GET_HOST_NAME, "UTL_INADDR_GET_HOST_NAME", (ObUtlInaddr::get_host_name)) + INTERFACE_DEF(INTERFACE_UTL_INADDR_GET_HOST_ADDERSS, "UTL_INADDR_GET_HOST_ADDRESS", (ObUtlInaddr::get_host_address)) + // end of utl_inaddr + + // start of dbms_plan_cache + INTERFACE_DEF(INTERFACE_DBMS_PLAN_CACHE_PURGE, "PLAN_CACHE_PURGE", (ObDBMSPlanCache::purge_single_plan_by_sql_id)) + // end of dbms_plan_cache +#endif // start of dbms_application_info INTERFACE_DEF(INTERFACE_DBMS_READ_CLIENT_INFO, "READ_CLIENT_INFO", (ObDBMSAppInfo::read_client_info)) @@ -53,6 +256,25 @@ INTERFACE_DEF(INTERFACE_DBMS_TENANT_TRACE_DISABLE, "OB_TENANT_TRACE_DISABLE", (ObDBMSMonitor::tenant_trace_disable)) // end of dbms_monitor +#ifdef OB_BUILD_ORACLE_PL + // dbms_utility + INTERFACE_DEF(INTERFACE_CALL_STACK, "FORMAT_CALL_STACK", (DbmsUtilityHelper::format_call_stack)) + INTERFACE_DEF(INTERFACE_ERROR_STACK, "FORMAT_ERROR_STACK", (DbmsUtilityHelper::format_error_stack)) + INTERFACE_DEF(INTERFACE_ERROR_BACKTRACE, "FORMAT_ERROR_BACKTRACE", (DbmsUtilityHelper::format_error_backtrace)) + INTERFACE_DEF(INTERFACE_NAME_TOKENIZE, "NAME_TOKENIZE", (DbmsUtilityHelper::name_tokenize)) + INTERFACE_DEF(INTERFACE_NAME_RESOLVE, "NAME_RESOLVE", (DbmsUtilityHelper::name_resolve)) + INTERFACE_DEF(INTERFACE_GET_PARAM_VALUE, "GET_PARAM_VALUE", (DbmsUtilityHelper::get_param_value)) + INTERFACE_DEF(INTERFACE_GET_SQL_HASH, "GETSQLHASH", (DbmsUtilityHelper::get_sql_hash)) + INTERFACE_DEF(INTERFACE_IS_BIT_SET, "IS_BIT_SET", (DbmsUtilityHelper::is_bit_set)) + INTERFACE_DEF(INTERFACE_CANONICALIZE, "CANONICALIZE", (DbmsUtilityHelper::canonicalize)) + INTERFACE_DEF(INTERFACE_GET_ENDIANESS, "GET_ENDIAN", (DbmsUtilityHelper::get_endian)) + INTERFACE_DEF(INTERFACE_GET_HASH_VALUE, "GET_HASH_VALUE", (DbmsUtilityHelper::get_hash_value)) + INTERFACE_DEF(INTERFACE_GET_DB_VERSION, "GET_DB_VERSION", (DbmsUtilityHelper::get_db_version)) + INTERFACE_DEF(INTERFACE_GET_PORT_STRING, "GET_PORT_STRING", (DbmsUtilityHelper::get_port_string)) + INTERFACE_DEF(INTERFACE_GET_TIME, "GET_TIME", (DbmsUtilityHelper::get_time)) + INTERFACE_DEF(INTERFACE_INVALIDATE, "INVALIDATE", (DbmsUtilityHelper::invalidate)) + INTERFACE_DEF(INTERFACE_VALIDATE, "VALIDATE", (DbmsUtilityHelper::validate)) + // end dbms_utility //start of resource manager INTERFACE_DEF(INTERFACE_DBMS_RESOURCE_MANAGER_CREATE_PLAN, "CREATE_PLAN_INNER", (ObPlDBMSResourceManager::create_plan)) @@ -65,6 +287,8 @@ INTERFACE_DEF(INTERFACE_DBMS_RESOURCE_MANAGER_SET_CONSUMER_GROUP_MAPPING, "SET_CONSUMER_GROUP_MAPPING_INNER", (ObPlDBMSResourceManager::set_consumer_group_mapping)) //end of resource manager +#endif + //start of dbms_stat INTERFACE_DEF(INTERFACE_DBMS_STATS_GATHER_TABLE_STATS, "GATHER_TABLE_STATS", (ObDbmsStats::gather_table_stats)) INTERFACE_DEF(INTERFACE_DBMS_STATS_GATHER_SCHEMA_STATS, "GATHER_SCHEMA_STATS", (ObDbmsStats::gather_schema_stats)) @@ -109,6 +333,166 @@ INTERFACE_DEF(INTERFACE_DBMS_STATS_IMPORT_INDEX_STATS, "IMPORT_INDEX_STATS", (ObDbmsStats::import_index_stats)) //end of dbms_stat +#ifdef OB_BUILD_ORACLE_PL + //start of dbms xplan + INTERFACE_DEF(INTERFACE_DBMS_XPLAN_ENABLE_OPT_TRACE, "ENABLE_OPT_TRACE", (ObDbmsXplan::enable_opt_trace)) + INTERFACE_DEF(INTERFACE_DBMS_XPLAN_DISABLE_OPT_TRACE, "DISABLE_OPT_TRACE", (ObDbmsXplan::disable_opt_trace)) + INTERFACE_DEF(INTERFACE_DBMS_XPLAN_SET_OPT_TRACE_PARAMETER, "SET_OPT_TRACE_PARAMETER", (ObDbmsXplan::set_opt_trace_parameter)) + INTERFACE_DEF(INTERFACE_DBMS_XPLAN_DISPLAY, "DISPLAY", (ObDbmsXplan::display)) + INTERFACE_DEF(INTERFACE_DBMS_XPLAN_DISPLAY_CURSOR, "DISPLAY_CURSOR", (ObDbmsXplan::display_cursor)) + INTERFACE_DEF(INTERFACE_DBMS_XPLAN_DISPLAY_SQL_PLAN_BASELINE, "DISPLAY_SQL_PLAN_BASELINE", (ObDbmsXplan::display_sql_plan_baseline)) + INTERFACE_DEF(INTERFACE_DBMS_XPLAN_DISPLAY_ACTIVE_SESSION_PLAN, "DISPLAY_ACTIVE_SESSION_PLAN", (ObDbmsXplan::display_active_session_plan)) + //end of dbms xplan + + // start of oracle label security + // sysdba + INTERFACE_DEF(INTERFACE_SA_SYSDBA_ALTER_POLICY, "SA_SYSDBA_ALTER_POLICY", (ObSASysdba::alter_policy)) + INTERFACE_DEF(INTERFACE_SA_SYSDBA_CREATE_POLICY, "SA_SYSDBA_CREATE_POLICY", (ObSASysdba::create_policy)) + INTERFACE_DEF(INTERFACE_SA_SYSDBA_ENABLE_POLICY, "SA_SYSDBA_ENABLE_POLICY", (ObSASysdba::enable_policy)) + INTERFACE_DEF(INTERFACE_SA_SYSDBA_DROP_POLICY, "SA_SYSDBA_DROP_POLICY", (ObSASysdba::drop_policy)) + INTERFACE_DEF(INTERFACE_SA_SYSDBA_DISABLE_POLICY, "SA_SYSDBA_DISABLE_POLICY", (ObSASysdba::disable_policy)) + + //session + //INTERFACE_DEF(INTERFACE_SA_SESSION_LABEL, "SA_SESSION_LABEL", (ObSASession::label)) + INTERFACE_DEF(INTERFACE_SA_SESSION_RESTORE_DEFAULT_LABELS, "SA_SESSION_RESTORE_DEFAULT_LABELS", (ObSASession::restore_default_labels)) + //INTERFACE_DEF(INTERFACE_SA_SESSION_ROW_LABEL, "SA_SESSION_ROW_LABEL", (ObSASession::row_label)) + INTERFACE_DEF(INTERFACE_SA_SESSION_SET_LABEL, "SA_SESSION_SET_LABEL", (ObSASession::set_label)) + INTERFACE_DEF(INTERFACE_SA_SESSION_SET_ROW_LABEL, "SA_SESSION_SET_ROW_LABEL", (ObSASession::set_row_label)) + + //user admin + INTERFACE_DEF(INTERFACE_SA_USER_ADMIN_SET_LEVELS, "SA_USER_ADMIN_SET_LEVELS", (ObSAUserAdmin::set_levels)) + + //policy admin + INTERFACE_DEF(INTERFACE_SA_POLICY_ADMIN_APPLY_TABLE_POLICY, "SA_POLICY_ADMIN_APPLY_TABLE_POLICY", (ObSAPolicyAdmin::apply_table_policy)) + INTERFACE_DEF(INTERFACE_SA_POLICY_ADMIN_DISABLE_TABLE_POLICY, "SA_POLICY_ADMIN_DISABLE_TABLE_POLICY", (ObSAPolicyAdmin::disable_table_policy)) + INTERFACE_DEF(INTERFACE_SA_POLICY_ADMIN_REMOVE_TABLE_POLICY, "SA_POLICY_ADMIN_REMOVE_TABLE_POLICY", (ObSAPolicyAdmin::remove_table_policy)) + INTERFACE_DEF(INTERFACE_SA_POLICY_ADMIN_ENABLE_TABLE_POLICY, "SA_POLICY_ADMIN_ENABLE_TABLE_POLICY", (ObSAPolicyAdmin::enable_table_policy)) + + //label admin + INTERFACE_DEF(INTERFACE_SA_LABEL_ADMIN_CRETAE_LABEL, "SA_LABEL_ADMIN_CRETAE_LABEL", (ObSALabelAdmin::create_label)) + INTERFACE_DEF(INTERFACE_SA_LABEL_ADMIN_ALTER_LABEL, "SA_LABEL_ADMIN_ALTER_LABEL", (ObSALabelAdmin::alter_label)) + INTERFACE_DEF(INTERFACE_SA_LABEL_ADMIN_DROP_LABEL, "SA_LABEL_ADMIN_DROP_LABEL", (ObSALabelAdmin::drop_label)) + + //components + INTERFACE_DEF(INTERFACE_SA_COMPONENTS_CREATE_LEVEL, "SA_COMPONENTS_CREATE_LEVEL", (ObSAComponents::create_level)) + INTERFACE_DEF(INTERFACE_SA_COMPONENTS_ALTER_LEVEL, "SA_COMPONENTS_ALTER_LEVEL", (ObSAComponents::alter_level)) + INTERFACE_DEF(INTERFACE_SA_COMPONENTS_DROP_LEVEL, "SA_COMPONENTS_DROP_LEVEL", (ObSAComponents::drop_level)) + // end of oracle label security + + //start of anytype +#define DEFINE_ANYTYPE_INTERFACE(symbol, func) \ + INTERFACE_DEF(INTERFACE_DBMS_ANYTYPE_##symbol, #symbol, (func)) + + DEFINE_ANYTYPE_INTERFACE(BEGINCREATE_ANYTYPE, ObDBMSAnyType::begincreate) + DEFINE_ANYTYPE_INTERFACE(SETINFO, ObDBMSAnyType::setinfo) + DEFINE_ANYTYPE_INTERFACE(ADDATTR, ObDBMSAnyType::addattr) + DEFINE_ANYTYPE_INTERFACE(ENDCREATE_ANYTYPE, ObDBMSAnyType::endcreate) + DEFINE_ANYTYPE_INTERFACE(GETPERSISTENT, ObDBMSAnyType::getpersistent) + DEFINE_ANYTYPE_INTERFACE(GETINFO, ObDBMSAnyType::getinfo) + DEFINE_ANYTYPE_INTERFACE(GETATTRELEMINFO, ObDBMSAnyType::getattreleminfo) + +#undef DEFINE_ANYTYPE_INTERFACE + //end of anytype + + //start of anydata +#define PREFIX INTERFACE_DBMS_ANYDATA_ +#define DEFINE_ANYDATA_INTERFACE(symbol, func) \ + INTERFACE_DEF(PREFIX##symbol, #symbol, (func)) +#define DEFINE_ANYDATA_INTERFACE_WITH_TYPE(symbol, func) \ + DEFINE_ANYDATA_INTERFACE(symbol##NUMBER, func##number) \ + DEFINE_ANYDATA_INTERFACE(symbol##DATE, func##date) \ + DEFINE_ANYDATA_INTERFACE(symbol##CHAR, func##char) \ + DEFINE_ANYDATA_INTERFACE(symbol##VARCHAR, func##varchar) \ + DEFINE_ANYDATA_INTERFACE(symbol##VARCHAR2, func##varchar2) \ + DEFINE_ANYDATA_INTERFACE(symbol##RAW, func##raw) \ + DEFINE_ANYDATA_INTERFACE(symbol##BLOB, func##blob) \ + DEFINE_ANYDATA_INTERFACE(symbol##CLOB, func##clob) \ + DEFINE_ANYDATA_INTERFACE(symbol##BFILE, func##bfile) \ + DEFINE_ANYDATA_INTERFACE(symbol##OBJECT, func##object) \ + DEFINE_ANYDATA_INTERFACE(symbol##COLLECTION, func##collection) \ + DEFINE_ANYDATA_INTERFACE(symbol##TIMESTAMP, func##timestamp) \ + DEFINE_ANYDATA_INTERFACE(symbol##TIMESTAMPTZ, func##timestamptz) \ + DEFINE_ANYDATA_INTERFACE(symbol##TIMESTAMPLTZ, func##timestampltz) \ + DEFINE_ANYDATA_INTERFACE(symbol##INTERVALYM, func##intervalym) \ + DEFINE_ANYDATA_INTERFACE(symbol##INTERVALDS, func##intervalds) \ + DEFINE_ANYDATA_INTERFACE(symbol##NCHAR, func##nchar) \ + DEFINE_ANYDATA_INTERFACE(symbol##NVARCHAR2, func##nvarchar2) \ + DEFINE_ANYDATA_INTERFACE(symbol##NCLOB, func##nclob) \ + DEFINE_ANYDATA_INTERFACE(symbol##BFLOAT, func##bfloat) \ + DEFINE_ANYDATA_INTERFACE(symbol##BDOUBLE, func##bdouble) \ + DEFINE_ANYDATA_INTERFACE(symbol##UROWID, func##urowid) + + DEFINE_ANYDATA_INTERFACE_WITH_TYPE(CONVERT, ObDBMSAnyData::convert) + DEFINE_ANYDATA_INTERFACE_WITH_TYPE(ACCESS, ObDBMSAnyData::access) + DEFINE_ANYDATA_INTERFACE_WITH_TYPE(GET, ObDBMSAnyData::get) + DEFINE_ANYDATA_INTERFACE_WITH_TYPE(SET, ObDBMSAnyData::set) + DEFINE_ANYDATA_INTERFACE(BEGINCREATE_ANYDATA, ObDBMSAnyData::begincreate) + DEFINE_ANYDATA_INTERFACE(ENDCREATE_ANYDATA, ObDBMSAnyData::endcreate) + DEFINE_ANYDATA_INTERFACE(GETTYPE, ObDBMSAnyData::gettype) + DEFINE_ANYDATA_INTERFACE(GETTYPENAME, ObDBMSAnyData::gettypename) + DEFINE_ANYDATA_INTERFACE(PIECEWISE, ObDBMSAnyData::piecewise) + +#undef DEFINE_ANYDATA_INTERFACE_WITH_TYPE +#undef DEFINE_ANYDATA_INTERFACE +#undef PREFIX + //end of anydata + +#ifdef OB_BUILD_ORACLE_XML + //start of xmltype + INTERFACE_DEF(INTERFACE_XML_TYPE_TRANSFORM, "XML_TYPE_TRANSFORM", (ObXmlType::transform)) + INTERFACE_DEF(INTERFACE_XML_TYPE_GETCLOBVAL, "XML_TYPE_GETCLOBVAL", (ObXmlType::getclobval)) + INTERFACE_DEF(INTERFACE_XML_TYPE_GETSTRINGVAL, "XML_TYPE_GETSTRINGVAL", (ObXmlType::getstringval)) + INTERFACE_DEF(INTERFACE_XML_TYPE_CREATEXML, "XML_TYPE_CREATEXML", (ObXmlType::createxml)) + INTERFACE_DEF(INTERFACE_XML_TYPE_CONSTRUCTOR, "XML_TYPE_CONSTRUCTOR", (ObXmlType::constructor)) + //end of xmltype + + //start of dbms_xmlgen + INTERFACE_DEF(INTERFACE_DBMS_XMLGEN_CONVERT, "DBMS_XMLGEN_CONVERT", (ObDbmsXmlGen::convert)) + //end of dbms_xmlgen +#endif + + //start of dbms_crypto + INTERFACE_DEF(INTERFACE_DBMS_CRYPTO_ENCRYPT, "DBMS_CRYPTO_ENCRYPT", (ObDbmsCrypto::encrypt)) + INTERFACE_DEF(INTERFACE_DBMS_CRYPTO_DECRYPT, "DBMS_CRYPTO_DECRYPT", (ObDbmsCrypto::decrypt)) + INTERFACE_DEF(INTERFACE_DBMS_CRYPTO_HASH, "DBMS_CRYPTO_HASH", (ObDbmsCrypto::hash)) + //end of dbms_crypto + + //start of dbms_job +#define DEFINE_DBMS_JOB_INTERFACE(symbol, func) \ + INTERFACE_DEF(INTERFACE_##symbol, #symbol, (func)) + + DEFINE_DBMS_JOB_INTERFACE(DBMS_JOB_SUBMIT, ObDBMSJob::submit) + DEFINE_DBMS_JOB_INTERFACE(DBMS_JOB_REMOVE, ObDBMSJob::remove) + DEFINE_DBMS_JOB_INTERFACE(DBMS_JOB_RUN, ObDBMSJob::run) + DEFINE_DBMS_JOB_INTERFACE(DBMS_JOB_SET_AFFINITY, ObDBMSJob::set_affinity) + DEFINE_DBMS_JOB_INTERFACE(DBMS_JOB_SET_WHAT, ObDBMSJob::set_what) + DEFINE_DBMS_JOB_INTERFACE(DBMS_JOB_SET_NEXTDATE, ObDBMSJob::set_nextdate) + DEFINE_DBMS_JOB_INTERFACE(DBMS_JOB_SET_INTERVAL, ObDBMSJob::set_interval) + DEFINE_DBMS_JOB_INTERFACE(DBMS_JOB_SET_FLAG, ObDBMSJob::set_flag) + DEFINE_DBMS_JOB_INTERFACE(DBMS_JOB_ZONE_CHECK, ObDBMSJob::zone_check) + +#undef DEFINE_DBMS_JOB_INTERFACE + //end of dbms_job + + //start of dbms_scheduler +#define DEFINE_DBMS_SCHEDULER_INTERFACE(symbol, func) \ + INTERFACE_DEF(INTERFACE_##symbol, #symbol, (func)) + + DEFINE_DBMS_SCHEDULER_INTERFACE(DBMS_SCHEDULER_CREATE_JOB, ObDBMSScheduler::create_job) + DEFINE_DBMS_SCHEDULER_INTERFACE(DBMS_SCHEDULER_DROP_JOB, ObDBMSScheduler::drop_job) + DEFINE_DBMS_SCHEDULER_INTERFACE(DBMS_SCHEDULER_RUN_JOB, ObDBMSScheduler::run_job) + DEFINE_DBMS_SCHEDULER_INTERFACE(DBMS_SCHEDULER_STOP_JOB, ObDBMSScheduler::stop_job) + DEFINE_DBMS_SCHEDULER_INTERFACE(DBMS_SCHEDULER_GENERATE_JOB_NAME, ObDBMSScheduler::generate_job_name) + DEFINE_DBMS_SCHEDULER_INTERFACE(DBMS_SCHEDULER_SET_ATTRIBUTE, ObDBMSScheduler::set_attribute) + DEFINE_DBMS_SCHEDULER_INTERFACE(DBMS_SCHEDULER_DISABLE, ObDBMSScheduler::disable) + DEFINE_DBMS_SCHEDULER_INTERFACE(DBMS_SCHEDULER_CREATE_PROGRAM, ObDBMSScheduler::create_program) + DEFINE_DBMS_SCHEDULER_INTERFACE(DBMS_SCHEDULER_DEFINE_PROGRAM_ARGUMENT, ObDBMSScheduler::define_program_argument) + DEFINE_DBMS_SCHEDULER_INTERFACE(DBMS_SCHEDULER_ENABLE, ObDBMSScheduler::enable) + DEFINE_DBMS_SCHEDULER_INTERFACE(DBMS_SCHEDULER_DROP_PROGRAM, ObDBMSScheduler::drop_program) + +#undef DEFINE_DBMS_SCHEDULER_INTERFACE + //end of dbms_scheduler +#endif //start of dbms_scheduler_mysql #define DEFINE_DBMS_SCHEDULER_MYSQL_INTERFACE(symbol, func) \ @@ -122,6 +506,67 @@ #undef DEFINE_DBMS_SCHEDULER_MYSQL_INTERFACE //end of dbms_scheduler_mysql +#ifdef OB_BUILD_ORACLE_PL + //start of utl_file + INTERFACE_DEF(INTERFACE_UTL_FILE_FOPEN, "UTL_FILE_FOPEN", (ObPLUtlFile::fopen)) + INTERFACE_DEF(INTERFACE_UTL_FILE_FCLOSE, "UTL_FILE_FCLOSE", (ObPLUtlFile::fclose)) + INTERFACE_DEF(INTERFACE_UTL_FILE_GET_LINE, "UTL_FILE_GET_LINE", (ObPLUtlFile::get_line)) + INTERFACE_DEF(INTERFACE_UTL_FILE_GET_LINE_RAW, "UTL_FILE_GET_LINE_RAW", (ObPLUtlFile::get_line_raw)) + INTERFACE_DEF(INTERFACE_UTL_FILE_PUT_BUFFER, "UTL_FILE_PUT_BUFFER", (ObPLUtlFile::put_buffer)) + INTERFACE_DEF(INTERFACE_UTL_FILE_FFLUSH, "UTL_FILE_FFLUSH", (ObPLUtlFile::fflush)) + INTERFACE_DEF(INTERFACE_UTL_FILE_FOPEN_NCHAR, "UTL_FILE_FOPEN_NCHAR", (ObPLUtlFile::fopen_nchar)) + INTERFACE_DEF(INTERFACE_UTL_FILE_PUT_RAW, "UTL_FILE_PUT_RAW", (ObPLUtlFile::put_raw)) + INTERFACE_DEF(INTERFACE_UTL_FILE_FSEEK, "UTL_FILE_FSEEK", (ObPLUtlFile::fseek)) + INTERFACE_DEF(INTERFACE_UTL_FILE_FREMOVE, "UTL_FILE_FREMOVE", (ObPLUtlFile::fremove)) + INTERFACE_DEF(INTERFACE_UTL_FILE_FCOPY, "UTL_FILE_FCOPY", (ObPLUtlFile::fcopy)) + INTERFACE_DEF(INTERFACE_UTL_FILE_FGETATTR, "UTL_FILE_FGETATTR", (ObPLUtlFile::fgetattr)) + INTERFACE_DEF(INTERFACE_UTL_FILE_FGETPOS, "UTL_FILE_FGETPOS", (ObPLUtlFile::fgetpos)) + INTERFACE_DEF(INTERFACE_UTL_FILE_FRENAME, "UTL_FILE_FRENAME", (ObPLUtlFile::frename)) + INTERFACE_DEF(INTERFACE_UTL_FILE_FIS_OPEN, "UTL_FILE_FIS_OPEN", (ObPLUtlFile::fis_open)) + //end of utl_file + + //start of dbms_sys_error + INTERFACE_DEF(INTERFACE_SYS_ERROR_KKXERE0, "KKXERE0", (ObDBMSSysError::ere0)) + INTERFACE_DEF(INTERFACE_SYS_ERROR_KKXERE1, "KKXERE1", (ObDBMSSysError::ere1)) + INTERFACE_DEF(INTERFACE_SYS_ERROR_KKXERE2, "KKXERE2", (ObDBMSSysError::ere2)) + INTERFACE_DEF(INTERFACE_SYS_ERROR_KKXERE3, "KKXERE3", (ObDBMSSysError::ere3)) + INTERFACE_DEF(INTERFACE_SYS_ERROR_KKXERE4, "KKXERE4", (ObDBMSSysError::ere4)) + INTERFACE_DEF(INTERFACE_SYS_ERROR_KKXERE5, "KKXERE5", (ObDBMSSysError::ere5)) + INTERFACE_DEF(INTERFACE_SYS_ERROR_KKXERE6, "KKXERE6", (ObDBMSSysError::ere6)) + INTERFACE_DEF(INTERFACE_SYS_ERROR_KKXERE7, "KKXERE7", (ObDBMSSysError::ere7)) + INTERFACE_DEF(INTERFACE_SYS_ERROR_KKXERE8, "KKXERE8", (ObDBMSSysError::ere8)) + //end of dbms_sys_error + + // start of dbms_preprocessor + INTERFACE_DEF(INTERFACE_PREPROCESSOR_KKXCCG1, "KKXCCG1", (ObDBMSPreprocessor::kkxccg1)) + INTERFACE_DEF(INTERFACE_PREPROCESSOR_KKXCCG2, "KKXCCG2", (ObDBMSPreprocessor::kkxccg2)) + INTERFACE_DEF(INTERFACE_PREPROCESSOR_KKXCCG3, "KKXCCG3", (ObDBMSPreprocessor::kkxccg3)) + // end of dbms_preprocessor +#endif + +#ifdef OB_BUILD_SPM + // start of dbms_spm + INTERFACE_DEF(INTERFACE_DBMS_SPM_ACCEPT_SQL_PLAN_BASELINE, "ACCEPT_SQL_PLAN_BASELINE", (ObDBMSSpm::accept_baseline)) + INTERFACE_DEF(INTERFACE_DBMS_SPM_ALTER_SQL_PLAN_BASELINE, "ALTER_SQL_PLAN_BASELINE", (ObDBMSSpm::alter_baseline)) + INTERFACE_DEF(INTERFACE_DBMS_SPM_CANCEL_EVOLVE_TASK, "CANCEL_EVOLVE_TASK", (ObDBMSSpm::cancel_evolve_task)) + INTERFACE_DEF(INTERFACE_DBMS_SPM_CONFIGURE, "CONFIGURE", (ObDBMSSpm::configure)) + INTERFACE_DEF(INTERFACE_DBMS_SPM_DROP_SQL_PLAN_BASELINE, "DROP_SQL_PLAN_BASELINE", (ObDBMSSpm::drop_baseline)) + INTERFACE_DEF(INTERFACE_DBMS_SPM_LOAD_PLANS_FROM_CURSOR_CACHE, "LOAD_PLANS_FROM_CURSOR_CACHE", (ObDBMSSpm::load_plans_from_cursor_cache)) + INTERFACE_DEF(INTERFACE_DBMS_SPM_AUTO_PURGE_SQL_PLAN_BASELINE, "AUTO_PURGE_SQL_PLAN_BASELINE", (ObDBMSSpm::auto_purge_sql_plan_baseline)) + // end of dbms_spm +#endif + +#ifdef OB_BUILD_ORACLE_PL + // start of dbms_audit_mgmt + INTERFACE_DEF(INTERFACE_DBMS_AUDIT_MGMT_CLEAN_TRAIL, "CLEAN_AUDIT_TRAIL", (ObDbmsAuditMgmt::clean_audit_trail)) + INTERFACE_DEF(INTERFACE_DBMS_AUDIT_MGMT_SET_ARCH_TS, "SET_LAST_ARCH_TS", (ObDbmsAuditMgmt::update_last_arch_ts_info)) + INTERFACE_DEF(INTERFACE_DBMS_AUDIT_MGMT_DEL_ARCH_TS, "DEL_LAST_ARCH_TS", (ObDbmsAuditMgmt::delete_last_arch_ts_info)) + INTERFACE_DEF(INTERFACE_DBMS_AUDIT_MGMT_UPDATE_JOB_INFO, "UPDATE_JOB_INFO", (ObDbmsAuditMgmt::update_clean_job_info)) + INTERFACE_DEF(INTERFACE_DBMS_AUDIT_MGMT_UPDATE_JOB_INTERVAL, "UPDATE_JOB_INTERVAL", (ObDbmsAuditMgmt::update_job_interval_oracle)) + INTERFACE_DEF(INTERFACE_DBMS_AUDIT_MGMT_UPDATE_JOB_STATUS, "UPDATE_JOB_STATUS", (ObDbmsAuditMgmt::set_purge_job_status_oracle)) + INTERFACE_DEF(INTERFACE_DBMS_AUDIT_MGMT_DROP_CLEANUP_JOB, "DROP_PURGE_JOB", (ObDbmsAuditMgmt::drop_purge_job_oracle)) + // end of dbms_audit_mgmt +#endif // start of dbms_session INTERFACE_DEF(INTERFACE_DBMS_SESSION_CLEAR_ALL_CONTEXT, "CLEAR_ALL_CONTEXT", (ObDBMSSession::clear_all_context)) @@ -132,6 +577,80 @@ INTERFACE_DEF(INTERFACE_DBMS_SESSION_RESET_PACKAGE, "RESET_PACKAGE", (ObDBMSSession::reset_package)) // end of dbms_session +#ifdef OB_BUILD_ORACLE_PL + // start of dbms_rls + INTERFACE_DEF(INTERFACE_RLS_ADD_GROUPED_POLICY, "RLS_ADD_GROUPED_POLICY", (ObDBMSRls::add_grouped_policy)) + INTERFACE_DEF(INTERFACE_RLS_ADD_POLICY, "RLS_ADD_POLICY", (ObDBMSRls::add_policy)) + INTERFACE_DEF(INTERFACE_RLS_ADD_POLICY_CONTEXT, "RLS_ADD_POLICY_CONTEXT", (ObDBMSRls::add_policy_context)) + INTERFACE_DEF(INTERFACE_RLS_ALTER_POLICY, "RLS_ALTER_POLICY", (ObDBMSRls::alter_policy)) + INTERFACE_DEF(INTERFACE_RLS_ALTER_GROUPED_POLICY, "RLS_ALTER_GROUPED_POLICY", (ObDBMSRls::alter_grouped_policy)) + INTERFACE_DEF(INTERFACE_RLS_CREATE_POLICY_GROUP, "RLS_CREATE_POLICY_GROUP", (ObDBMSRls::create_policy_group)) + INTERFACE_DEF(INTERFACE_RLS_DELETE_POLICY_GROUP, "RLS_DELETE_POLICY_GROUP", (ObDBMSRls::delete_policy_group)) + INTERFACE_DEF(INTERFACE_RLS_DISABLE_GROUPED_POLICY, "RLS_DISABLE_GROUPED_POLICY", (ObDBMSRls::disable_grouped_policy)) + INTERFACE_DEF(INTERFACE_RLS_DROP_GROUPED_POLICY, "RLS_DROP_GROUPED_POLICY", (ObDBMSRls::drop_grouped_policy)) + INTERFACE_DEF(INTERFACE_RLS_DROP_POLICY, "RLS_DROP_POLICY", (ObDBMSRls::drop_policy)) + INTERFACE_DEF(INTERFACE_RLS_DROP_POLICY_CONTEXT, "RLS_DROP_POLICY_CONTEXT", (ObDBMSRls::drop_policy_context)) + INTERFACE_DEF(INTERFACE_RLS_ENABLE_GROUPED_POLICY, "RLS_ENABLE_GROUPED_POLICY", (ObDBMSRls::enable_grouped_policy)) + INTERFACE_DEF(INTERFACE_RLS_ENABLE_POLICY, "RLS_ENABLE_POLICY", (ObDBMSRls::enable_policy)) + INTERFACE_DEF(INTERFACE_RLS_REFRESH_GROUPED_POLICY, "RLS_REFRESH_GROUPED_POLICY", (ObDBMSRls::refresh_grouped_policy)) + INTERFACE_DEF(INTERFACE_RLS_REFRESH_POLICY, "RLS_REFRESH_POLICY", (ObDBMSRls::refresh_policy)) + // end of dbms_rls + + // start of json_element_t + INTERFACE_DEF(INTERFACE_JSON_ELEMENT_ON_ERROR, "JSON_ELEMENT_ON_ERROR", (ObPlJsonElement::set_on_error)) + INTERFACE_DEF(INTERFACE_JSON_ELEMENT_PARSE, "JSON_ELEMENT_PARSE", (ObPlJsonElement::parse)) + INTERFACE_DEF(INTERFACE_JSON_IS_ARRAY, "JSON_IS_ARRAY", (ObPlJsonElement::is_array)) + INTERFACE_DEF(INTERFACE_JSON_IS_OBJECT, "JSON_IS_OBJECT", (ObPlJsonElement::is_object)) + INTERFACE_DEF(INTERFACE_JSON_IS_SCALAR, "JSON_IS_SCALAR", (ObPlJsonElement::is_scalar)) + INTERFACE_DEF(INTERFACE_JSON_IS_STRING, "JSON_IS_STRING", (ObPlJsonElement::is_string)) + INTERFACE_DEF(INTERFACE_JSON_IS_NUMBER, "JSON_IS_NUMBER", (ObPlJsonElement::is_number)) + INTERFACE_DEF(INTERFACE_JSON_IS_BOOLEAN, "JSON_IS_BOOLEAN", (ObPlJsonElement::is_boolean)) + INTERFACE_DEF(INTERFACE_JSON_IS_TRUE, "JSON_IS_TRUE", (ObPlJsonElement::is_true)) + INTERFACE_DEF(INTERFACE_JSON_IS_FALSE, "JSON_IS_FALSE", (ObPlJsonElement::is_false)) + INTERFACE_DEF(INTERFACE_JSON_IS_NULL, "JSON_IS_NULL", (ObPlJsonElement::is_null)) + INTERFACE_DEF(INTERFACE_JSON_IS_DATE, "JSON_IS_DATE", (ObPlJsonElement::is_date)) + INTERFACE_DEF(INTERFACE_JSON_IS_TIMESTAMP, "JSON_IS_TIMESTAMP", (ObPlJsonElement::is_timestamp)) + INTERFACE_DEF(INTERFACE_JSON_TO_STRING, "JSON_TO_STRING", (ObPlJsonElement::to_string)) + INTERFACE_DEF(INTERFACE_JSON_TO_NUMBER, "JSON_TO_NUMBER", (ObPlJsonElement::to_number)) + INTERFACE_DEF(INTERFACE_JSON_TO_DATE, "JSON_TO_DATE", (ObPlJsonElement::to_date)) + INTERFACE_DEF(INTERFACE_JSON_TO_TIMESTAMP, "JSON_TO_TIMESTAMP", (ObPlJsonElement::to_timestamp)) + INTERFACE_DEF(INTERFACE_JSON_TO_BOOLEAN, "JSON_TO_BOOLEAN", (ObPlJsonElement::to_boolean)) + INTERFACE_DEF(INTERFACE_JSON_TO_CLOB, "JSON_TO_CLOB", (ObPlJsonElement::to_clob)) + INTERFACE_DEF(INTERFACE_JSON_TO_BLOB, "JSON_TO_BLOB", (ObPlJsonElement::to_blob)) + INTERFACE_DEF(INTERFACE_JSON_TO_CLOB_PROC, "JSON_TO_CLOB_PROC", (ObPlJsonElement::to_clob_proc)) + INTERFACE_DEF(INTERFACE_JSON_TO_BLOB_PROC, "JSON_TO_BLOB_PROC", (ObPlJsonElement::to_blob_proc)) + INTERFACE_DEF(INTERFACE_JSON_GET_SIZE, "JSON_GET_SIZE", (ObPlJsonElement::get_size)) + // end of json_element_t + + // start of json_object_t + INTERFACE_DEF(INTERFACE_JSON_OBJECT_ON_ERROR, "JSON_OBJECT_ON_ERROR", (ObPlJsonObject::set_on_error)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_PARSE, "JSON_OBJECT_PARSE", (ObPlJsonObject::parse)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_GETSTR, "JSON_OBJECT_GETSTR", (ObPlJsonObject::get_string)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_GET_NUMBER, "JSON_OBJECT_GET_NUMBER", (ObPlJsonObject::get_number)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_PUT, "JSON_OBJECT_PUT", (ObPlJsonObject::put)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_PUT_BLOB, "JSON_OBJECT_PUT_BLOB", (ObPlJsonObject::put_blob)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_PUT_CLOB, "JSON_OBJECT_PUT_CLOB", (ObPlJsonObject::put_clob)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_PUT_JSON, "JSON_OBJECT_PUT_JSON", (ObPlJsonObject::put_json)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_PUT_VARCHAR, "JSON_OBJECT_PUT_VARCHAR", (ObPlJsonObject::put_varchar)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_PUT_BOOL, "JSON_OBJECT_PUT_BOOL", (ObPlJsonObject::put_boolean)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_PUT_NULL, "JSON_OBJECT_PUT_NULL", (ObPlJsonObject::put_null)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_CONSTRUCTOR, "JSON_OBJECT_CONSTRUCTOR", (ObPlJsonObject::constructor)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_GET_DATE, "JSON_OBJECT_GET_DATE", (ObPlJsonObject::get_date)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_GET_TIMESTAMP, "JSON_OBJECT_GET_TIMESTAMP", (ObPlJsonObject::get_timestamp)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_GET_BOOLEAN, "JSON_OBJECT_GET_BOOLEAN", (ObPlJsonObject::get_boolean)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_GET_CLOB, "JSON_OBJECT_GET_CLOB", (ObPlJsonObject::get_clob)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_GET_BLOB, "JSON_OBJECT_GET_BLOB", (ObPlJsonObject::get_blob)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_GET_BLOB_PROC, "JSON_OBJECT_GET_BLOB_PROC", (ObPlJsonObject::get_blob_proc)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_GET_CLOB_PROC, "JSON_OBJECT_GET_CLOB_PROC", (ObPlJsonObject::get_clob_proc)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_GET_OBJECT, "JSON_OBJECT_GET_OBJECT", (ObPlJsonObject::get_object)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_GET_ELEMENT, "JSON_OBJECT_GET_ELEMENT", (ObPlJsonObject::get_element)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_HAS, "JSON_OBJECT_HAS", (ObPlJsonObject::exist)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_GET_TYPE, "JSON_OBJECT_GET_TYPE", (ObPlJsonObject::get_type)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_REMOVE, "JSON_OBJECT_REMOVE", (ObPlJsonObject::remove)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_RENAME_KEY, "JSON_OBJECT_RENAME_KEY", (ObPlJsonObject::rename_key)) + INTERFACE_DEF(INTERFACE_JSON_OBJECT_CLONE, "JSON_OBJECT_CLONE", (ObPlJsonObject::clone)) + // end of json_object_t +#endif // start of dbms_udr INTERFACE_DEF(INTERFACE_DBMS_UDR_CREATE_RULE, "CREATE_RULE", (ObDBMSUserDefineRule::create_rule)) INTERFACE_DEF(INTERFACE_DBMS_UDR_REMOVE_RULE, "REMOVE_RULE", (ObDBMSUserDefineRule::remove_rule)) diff --git a/src/pl/ob_pl_package_manager.cpp b/src/pl/ob_pl_package_manager.cpp index 674c4430d..28050252b 100644 --- a/src/pl/ob_pl_package_manager.cpp +++ b/src/pl/ob_pl_package_manager.cpp @@ -206,6 +206,59 @@ int ObPLPackageManager::load_sys_package( static const char* sys_package_dir = "admin"; static ObSysPackageFile oracle_sys_package_file_table[] = { +#ifdef OB_BUILD_ORACLE_PL + {"dbms_standard", "dbms_standard.sql", "dbms_standard_body.sql"}, + {"dbms_output", "dbms_output.sql", "dbms_output_body.sql"}, + {"dbms_metadata", "dbms_metadata.sql", "dbms_metadata_body.sql"}, + {"dbms_spm", "dbms_spm.sql", "dbms_spm_body.sql"}, + {"utl_raw", "utl_raw.sql", "utl_raw_body.sql"}, + {"dbms_lob", "dbms_lob.sql", "dbms_lob_body.sql"}, + {"sa_components", "sa_components.sql", "sa_components_body.sql"}, + {"sa_label_admin", "sa_label_admin.sql", "sa_label_admin_body.sql"}, + {"sa_policy_admin", "sa_policy_admin.sql", "sa_policy_admin_body.sql"}, + {"sa_session", "sa_session.sql", "sa_session_body.sql"}, + {"sa_sysdba", "sa_sysdba.sql", "sa_sysdba_body.sql"}, + {"sa_user_admin", "sa_user_admin.sql", "sa_user_admin_body.sql"}, + {"utl_i18n", "utl_i18n.sql", "utl_i18n_body.sql"}, + {"dbms_crypto", "dbms_crypto.sql","dbms_crypto_body.sql"}, + {"dbms_random", "dbms_random.sql", "dbms_random_body.sql"}, + {"dbms_debug", "dbms_debug.sql", "dbms_debug_body.sql"}, + {"utl_inaddr", "utl_inaddr.sql", "utl_inaddr_body.sql"}, + {"utl_encode", "dbms_utl_encode.sql", "dbms_utl_encode_body.sql"}, + {"dbms_warning", "dbms_warning.sql", "dbms_warning_body.sql"}, + {"dbms_errlog", "dbms_errlog.sql", "dbms_errlog_body.sql"}, + {"dbms_lock", "dbms_lock.sql", "dbms_lock_body.sql"}, + {"dbms_sql", "dbms_sql.sql", "dbms_sql_body.sql"}, + {"dbms_xa", "dbms_xa.sql", "dbms_xa_body.sql"}, + {"dbms_resource_manager", "dbms_resource_manager.sql", "dbms_resource_manager_body.sql"}, + {"dbms_utility", "dbms_utility.sql", "dbms_utility_body.sql"}, + {"odciconst", "odciconst.sql", "odciconst_body.sql"}, + {"dbms_stats", "dbms_stats.sql", "dbms_stats_body.sql"}, + {"dbms_any", "dbms_any.sql", "dbms_any_body.sql"}, + {"xml_type", "xml_type.sql", "xml_type_body.sql"}, + {"dbms_crypto", "dbms_crypto.sql", "dbms_crypto_body.sql"}, + {"dbms_ijob", "dbms_ijob.sql", "dbms_ijob_body.sql"}, + {"dbms_job", "dbms_job.sql", "dbms_job_body.sql"}, + {"dbms_ischeduler", "dbms_ischeduler.sql", "dbms_ischeduler_body.sql"}, + {"dbms_scheduler", "dbms_scheduler.sql", "dbms_scheduler_body.sql"}, + {"catodci", "catodci.sql", "catodci_body.sql"}, + {"dbms_describe", "dbms_describe.sql", "dbms_describe_body.sql"}, + {"utl_file", "utl_file.sql", "utl_file_body.sql"}, + {"dbms_plan_cache", "dbms_plancache.sql", "dbms_plancache_body.sql"}, + {"dbms_sys_error", "dbms_sys_error.sql", "dbms_sys_error_body.sql"}, + {"dbms_preprocessor", "dbms_preprocessor.sql", "dbms_preprocessor_body.sql"}, + {"dbms_audit_mgmt", "dbms_audit_mgmt.sql", "dbms_audit_mgmt_body.sql"}, + {"dbms_application", "dbms_application.sql", "dbms_application_body.sql"}, + {"dbms_session", "dbms_session.sql", "dbms_session_body.sql"}, + {"dbms_monitor", "dbms_monitor.sql", "dbms_monitor_body.sql"}, + {"dbms_xplan", "dbms_xplan.sql", "dbms_xplan_body.sql"}, + {"dbms_workload_repository", "dbms_workload_repository.sql", "dbms_workload_repository_body.sql"}, + {"dbms_ash_internal", "dbms_ash_internal.sql", "dbms_ash_internal_body.sql"}, + {"dbms_rls", "dbms_rls.sql", "dbms_rls_body.sql"}, + {"dbms_udr", "dbms_udr.sql", "dbms_udr_body.sql"}, + {"json_element_t", "json_element_type.sql", "json_element_type_body.sql"}, + {"json_object_t", "json_object_type.sql", "json_object_type_body.sql"}, +#endif }; static ObSysPackageFile mysql_sys_package_file_table[] = { @@ -215,6 +268,10 @@ static ObSysPackageFile mysql_sys_package_file_table[] = { {"dbms_session", "dbms_session_mysql.sql", "dbms_session_body_mysql.sql"}, {"dbms_monitor", "dbms_monitor_mysql.sql", "dbms_monitor_body_mysql.sql"}, {"dbms_resource_manager", "dbms_resource_manager_mysql.sql", "dbms_resource_manager_body_mysql.sql"}, +#ifdef OB_BUILD_ORACLE_PL + {"dbms_xplan", "dbms_xplan_mysql.sql", "dbms_xplan_mysql_body.sql"}, + {"dbms_spm", "dbms_spm_mysql.sql", "dbms_spm_body_mysql.sql"}, +#endif {"dbms_udr", "dbms_udr_mysql.sql", "dbms_udr_body_mysql.sql"} }; @@ -345,6 +402,13 @@ int ObPLPackageManager::load_all_special_sys_package(ObMySQLProxy &sql_proxy) char package_spec_full_path[MAX_PATH_SIZE] = {}; char package_body_full_path[MAX_PATH_SIZE] = {}; +#ifdef OB_BUILD_ORACLE_PL + OZ (databuff_printf( + package_spec_full_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, "__dbms_upgrade.sql")); + OZ (databuff_printf( + package_body_full_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, "__dbms_upgrade_body.sql")); + OZ (load_sys_package(sql_proxy, package_spec_full_path, package_body_full_path, ObCompatibilityMode::ORACLE_MODE)); +#endif memset(package_spec_full_path, 0, sizeof(package_spec_full_path)); memset(package_body_full_path, 0, sizeof(package_body_full_path)); diff --git a/src/pl/ob_pl_resolver.cpp b/src/pl/ob_pl_resolver.cpp index d3dfd3b8e..a58459294 100644 --- a/src/pl/ob_pl_resolver.cpp +++ b/src/pl/ob_pl_resolver.cpp @@ -33,6 +33,11 @@ #include "sql/rewrite/ob_transform_pre_process.h" #include "share/schema/ob_trigger_info.h" #include "sql/resolver/expr/ob_raw_expr_copier.h" +#ifdef OB_BUILD_ORACLE_PL +#include "pl/ob_pl_warning.h" +#include "pl/ob_pl_udt_object_manager.h" +#include "pl/sys_package/ob_json_pl_utils.h" +#endif namespace oceanbase { using namespace common; @@ -328,10 +333,23 @@ int ObPLResolver::resolve(const ObStmtNodeTree *parse_tree, ObPLFunctionAST &fun if (!data_type.is_cursor_type() || data_type.is_rowtype_type()) { RESOLVE_STMT(PL_VAR, resolve_declare_var, ObPLDeclareVarStmt); } else { +#ifdef OB_BUILD_ORACLE_PL + RESOLVE_STMT(PL_CURSOR, resolve_declare_ref_cursor, ObPLDeclareCursorStmt); + func.set_modifies_sql_data(); +#endif } } } break; +#ifdef OB_BUILD_ORACLE_PL + case T_SP_DECL_USER_TYPE: { + RESOLVE_STMT(PL_USER_TYPE, resolve_declare_user_type, ObPLDeclareUserTypeStmt); + } + break; + case T_SP_DECL_USER_SUBTYPE: { + RESOLVE_STMT(PL_USER_SUBTYPE, resolve_declare_user_subtype, ObPLDeclareUserTypeStmt); + } +#endif break; case T_VARIABLE_SET: { RESOLVE_STMT(PL_ASSIGN, resolve_assign, ObPLAssignStmt); @@ -943,6 +961,16 @@ int ObPLResolver::resolve(const ObStmtNodeTree *parse_tree, ObPLPackageAST &pack OZ (resolve_declare_var(parse_tree, package_ast)); break; } +#ifdef OB_BUILD_ORACLE_PL + case T_SP_DECL_USER_TYPE: { + OZ (resolve_declare_user_type(parse_tree, package_ast)); + break; + } + case T_SP_DECL_USER_SUBTYPE: { + OZ (resolve_declare_user_subtype(parse_tree, NULL, package_ast)); + break; + } +#endif case T_SP_INIT_PRAGMA: case T_SP_DECL_COND: { OZ (resolve_declare_cond(parse_tree, package_ast)); @@ -1013,6 +1041,21 @@ int ObPLResolver::resolve(const ObStmtNodeTree *parse_tree, ObPLPackageAST &pack } } break; +#ifdef OB_BUILD_ORACLE_PL + case T_SP_OBJ_ELEMENT_SPEC_LIST: { + OZ (resolve_object_elem_spec_list(parse_tree, package_ast)); + } + break; + case T_SP_OBJ_ELEMENT_SPEC: + case T_SP_OBJ_ELEM_CONSTRUCTOR : { + OZ (resolve_object_elem_spec_def(parse_tree, package_ast)); + } + break; + case T_SP_CREATE_TYPE_SRC : { + OZ (resolve_object_def(parse_tree, package_ast)); + } + break; +#endif case T_SP_OBJ_CONSTR_IMPL : { CK (OB_NOT_NULL(parse_tree)); OX (const_cast(parse_tree)->type_ = T_SUB_FUNC_DEF); @@ -1035,6 +1078,30 @@ int ObPLResolver::resolve(const ObStmtNodeTree *parse_tree, ObPLPackageAST &pack OZ (resolve_routine_def(parse_tree->children_[0], package_ast, true)); } break; +#ifdef OB_BUILD_ORACLE_PL + case T_SP_PRE_STMTS: { + const ParseNode *new_node = NULL; + OZ (ObPLResolver::resolve_condition_compile( + resolve_ctx_.allocator_, + &(resolve_ctx_.session_info_), + &(resolve_ctx_.schema_guard_), + &(resolve_ctx_.package_guard_), + &(resolve_ctx_.sql_proxy_), + NULL, + parse_tree, + new_node, + false, + false, + false, + NULL, + &(package_ast.get_dependency_table()))); + CK (OB_NOT_NULL(new_node)); + OZ (ObPLUDTObjectManager::make_self_node( + resolve_ctx_.allocator_, const_cast(new_node), package_ast.get_name())); + OZ (SMART_CALL(resolve(new_node, package_ast))); + } + break; +#endif default: { ret = OB_NOT_SUPPORTED; LOG_WARN("Not support parser node", K(get_type_name(parse_tree->type_)), K(ret)); @@ -1049,6 +1116,170 @@ int ObPLResolver::resolve(const ObStmtNodeTree *parse_tree, ObPLPackageAST &pack return ret; } +#ifdef OB_BUILD_ORACLE_PL +int ObPLResolver::resolve_object_def(const ParseNode *parse_tree, ObPLCompileUnitAST &package_ast) +{ + int ret = OB_SUCCESS; + CK (OB_LIKELY(T_SP_CREATE_TYPE_SRC == parse_tree->type_)); + const ParseNode *spec_body_node = parse_tree; + CK (OB_NOT_NULL(spec_body_node)); + if (OB_SUCC(ret)) { + // bool replace = static_cast(parse_tree->int32_values_[0]); + // bool noneditioable = static_cast(parse_tree->int32_values_[1]); + const ParseNode *name_node = spec_body_node->children_[0]; + const ParseNode *oid_node = spec_body_node->children_[1]; + const ParseNode *object_type_node = spec_body_node->children_[2]; + + UNUSED(oid_node); + CK (OB_NOT_NULL(name_node)); + CK (OB_NOT_NULL(object_type_node)); + if (OB_SUCC(ret)) { + ObString db_name, udt_name; + if (OB_FAIL(ObResolverUtils::resolve_sp_name(get_resolve_ctx().session_info_, + *name_node, + db_name, + udt_name))) { + LOG_WARN("failed to resolve type name", K(ret)); + } else { + CK (T_SP_OBJECT_DEF == object_type_node->type_ + || T_SP_OPAQUE_DEF == object_type_node->type_); + CK (OB_LIKELY(5 == object_type_node->num_child_)); + CK (OB_NOT_NULL(object_type_node->children_[3])); + CK (OB_NOT_NULL(object_type_node->children_[1])); + if (OB_SUCC(ret)) { + // const ParseNode* invoke_accessby_node = object_type_node->children_[0]; + const ParseNode* object_or_under = object_type_node->children_[1]; + // const ParseNode* sqlj_node = object_type_node->children_[2]; + const ParseNode* attrs_node = object_type_node->children_[3]; + // const ParseNode* final_node = object_type_node->children_[4]; + CK (OB_NOT_NULL(object_or_under)); + CK (OB_NOT_NULL(attrs_node)); + CK (OB_LIKELY(T_SP_OBJ_ATTR_AND_ELEMENT_SPEC == attrs_node->type_)); + const ParseNode *elem_spec_list = attrs_node->children_[1]; + if (OB_SUCC(ret)) { + if (T_SP_IS_UNDER_OBJECT == object_or_under->type_) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not support yet", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "under object"); + } else if (OB_NOT_NULL(elem_spec_list)) { + if (OB_FAIL(resolve_object_elem_spec_list(elem_spec_list, package_ast))){ + LOG_WARN("failed to resolve elem spec list", K(ret)); + } + } + } + } + } + } + } + + return ret; +} + +int ObPLResolver::resolve_object_elem_spec_list(const ParseNode *parse_tree, + ObPLCompileUnitAST &package_ast) +{ + int ret = OB_SUCCESS; + const ParseNode* element_spec_list = parse_tree; + if (OB_ISNULL(element_spec_list)) { + ret = OB_ERR_UNEXPECTED; + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < element_spec_list->num_child_; ++i) { + const ParseNode *element_spec = element_spec_list->children_[i]; + if (OB_FAIL(resolve_object_elem_spec_def(element_spec, package_ast))) { + LOG_WARN("failed to resolve type spec", K(ret)); + } + } + } + return ret; +} + +int ObPLResolver::resolve_object_elem_spec_def(const ParseNode *parse_tree, + ObPLCompileUnitAST &package_ast) +{ + int ret = OB_SUCCESS; + + CK (OB_NOT_NULL(parse_tree)); + if (OB_SUCC(ret)) { + if (T_SP_OBJ_ELEMENT_SPEC == parse_tree->type_) { + CK (3 == parse_tree->num_child_); + // const ParseNode *inherit_node = parse_tree->children_[0]; + const ParseNode *subprog_node = parse_tree->children_[1]; + // const ParseNode *restrict_ref_node = parse_tree->children_[2]; + if (OB_NOT_NULL(subprog_node)) { + if (T_SP_OBJ_ELEM_SUBPROG_SPEC == subprog_node->type_) { + const ParseNode *routine_node = subprog_node->children_[0]; + CK (OB_NOT_NULL(routine_node)); + if (OB_SUCC(ret)) { + int64_t member_or_static = static_cast(subprog_node->int16_values_[0]); + ObPLRoutineInfo *routine_info = NULL; + + if (OB_FAIL(resolve_routine_decl(routine_node, package_ast, routine_info, true))) { + LOG_WARN("failed to resolve object function", K(ret)); + } else if (OB_ISNULL(routine_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("udt routine info is null", K(ret)); + } else { + if (UdtUdfType::UDT_UDF_STATIC == member_or_static) { + routine_info->get_compile_flag().add_static(); + } + routine_info->set_is_udt_routine(); + // check static and self + if (routine_info->is_udt_static_routine() && routine_info->has_self_param()) { + ret = OB_ERR_STATIC_METHOD_HAS_SELF; + LOG_WARN("static member routine should not have self param"); + } + } + } + } else if (T_SP_OBJ_ELEM_CONSTRUCTOR == subprog_node->type_) { + ObPLRoutineInfo *routine_info = NULL; + // 0: name, 1 : param, 2 : attr + OZ (resolve_object_constructor(subprog_node, package_ast, routine_info)); + CK (OB_NOT_NULL(routine_info)); + if (OB_SUCC(ret)) { + routine_info->set_is_udt_routine(); + routine_info->set_is_udt_cons(); + } + } else if (T_SP_OBJ_MAP_ORDER == subprog_node->type_) { + ObPLRoutineInfo *routine_info = NULL; + const ParseNode *routine_node = subprog_node->children_[0]; + CK (OB_NOT_NULL(routine_node)); + OZ (resolve_routine_decl(routine_node, package_ast, routine_info, true)); + CK (OB_NOT_NULL(routine_info)); + OX (routine_info->set_is_udt_routine()); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unknown type subprogram type", K(ret)); + } + } + } else if (T_SP_OBJ_ELEM_CONSTRUCTOR == parse_tree->type_) { + ObPLRoutineInfo *routine_info = NULL; + OZ (resolve_object_constructor(parse_tree, package_ast, routine_info)); + OV (OB_NOT_NULL(routine_info), OB_ERR_UNEXPECTED); + if (OB_SUCC(ret)) { + routine_info->set_is_udt_routine(); + routine_info->set_is_udt_cons(); + } + } + // inherit and restrict ref not support yet + } + + return ret; +} + +int ObPLResolver::resolve_object_constructor(const ParseNode *parse_tree, + ObPLCompileUnitAST &package_ast, + ObPLRoutineInfo *&routine_info) +{ + int ret = OB_SUCCESS; + ParseNode *subp = const_cast(parse_tree); + CK (OB_NOT_NULL(parse_tree)); + CK (5 == parse_tree->num_child_); + CK (T_SP_OBJ_ELEM_CONSTRUCTOR == parse_tree->type_); + OX (subp->type_ = T_SUB_FUNC_DECL); + OZ (resolve_routine_decl(subp, package_ast, routine_info, true)); + return ret; +} +#endif int ObPLResolver::check_declare_order(ObPLStmtType type) { @@ -1085,6 +1316,23 @@ int ObPLResolver::check_declare_order(ObPLStmtType type) return ret; } +#ifdef OB_BUILD_ORACLE_PL +int ObPLResolver::resolve_declare_user_type(const ObStmtNodeTree *parse_tree, ObPLDeclareUserTypeStmt *stmt, ObPLFunctionAST &func) +{ + int ret = OB_SUCCESS; + ObPLCompileUnitAST &unit_ast = static_cast(func); + ret = resolve_declare_user_type_comm(parse_tree, stmt, unit_ast); + return ret; +} + +int ObPLResolver::resolve_declare_user_type(const ObStmtNodeTree *parse_tree, ObPLPackageAST &package_ast) +{ + int ret = OB_SUCCESS; + ObPLCompileUnitAST &unit_ast = static_cast(package_ast); + ret = resolve_declare_user_type_comm(parse_tree, NULL, unit_ast); + return ret; +} +#endif int ObPLResolver::get_number_literal_value(ObRawExpr *expr, int64_t &result) { @@ -1187,6 +1435,262 @@ int ObPLResolver::calc_subtype_range_bound(const ObStmtNodeTree *bound_node, return ret; } +#ifdef OB_BUILD_ORACLE_PL +int ObPLResolver::resolve_subtype_precision(const ObStmtNodeTree *precision_node, + ObPLDataType &base_type) +{ + int ret = OB_SUCCESS; + int16_t precision = -1; + int16_t scale = 0; + CK (OB_NOT_NULL(precision_node)); + CK (T_LINK_NODE == precision_node->type_); + CK (1 == precision_node->param_num_ || 2 == precision_node->param_num_); + OX (precision = precision_node->int16_values_[0]); + OX (2 == precision_node->param_num_ ? scale = precision_node->int16_values_[1] : int16_t(0)); + CK (OB_NOT_NULL(base_type.get_data_type())); + if (OB_SUCC(ret)) { + ObObjTypeClass tclass = base_type.get_data_type()->get_type_class(); + if (2 == precision_node->param_num_) { + if (ObNumberTC != tclass) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not supported yet", K(ret), K(tclass)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "precision of subtype is non-number type"); + } + } else { + if (ObFloatTC != tclass && ObDoubleTC != tclass && ObStringTC != tclass + && ObNumberTC != tclass && ObOTimestampTC != tclass) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not supported yet", K(ret), K(tclass)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "the base type of subtype is not float/double/string/number/timestamp class"); + } + } + if (OB_SUCC(ret)) { + if (ObOTimestampTC == tclass) { + if (precision > OB_MAX_TIMESTAMP_TZ_PRECISION) { + ret = OB_ERR_DATETIME_INTERVAL_PRECISION_OUT_OF_RANGE; + } else { + base_type.get_data_type()->set_precision(precision); + base_type.get_data_type()->set_scale(scale); + } + } else if (ObNumberTC == tclass) { + if (base_type.get_data_type()->get_meta_type().is_number_float()) { + if ((OB_UNLIKELY(precision < OB_MIN_NUMBER_FLOAT_PRECISION) + || OB_UNLIKELY(precision > OB_MAX_NUMBER_FLOAT_PRECISION))) { + ret = OB_FLOAT_PRECISION_OUT_RANGE; + LOG_WARN("precision of float out of range", K(ret), K(precision)); + } + } else if ((OB_UNLIKELY(precision < OB_MIN_NUMBER_PRECISION) + || OB_UNLIKELY(precision > OB_MAX_NUMBER_PRECISION))) { + ret = OB_NUMERIC_PRECISION_OUT_RANGE; + LOG_WARN("precision of number overflow", K(ret), K(scale), K(precision)); + } else if (OB_UNLIKELY(scale < OB_MIN_NUMBER_SCALE) + || OB_UNLIKELY(scale > OB_MAX_NUMBER_SCALE)) { + ret = OB_NUMERIC_SCALE_OUT_RANGE; + LOG_WARN("scale of number out of range", K(ret), K(scale)); + } else { + base_type.get_data_type()->set_precision(precision); + base_type.get_data_type()->set_scale(scale); + } + } else if (ObFloatTC == tclass || ObDoubleTC == tclass) { + base_type.get_data_type()->set_precision(precision); + base_type.get_data_type()->set_scale(scale); + } else if (ObStringTC == tclass) { + if (0 == precision) { + ret = OB_ERR_ZERO_LEN_COL; + LOG_WARN("Oracle not allowed zero length", K(ret)); + // @FIXME comparison is always false due to limited range of data type +_Pragma("GCC diagnostic push") +_Pragma("GCC diagnostic ignored \"-Wtype-limits\"") + } else if (OB_MAX_ORACLE_VARCHAR_LENGTH < precision || precision < 0) { +_Pragma("GCC diagnostic pop") + ret = OB_ERR_TOO_LONG_COLUMN_LENGTH; + LOG_WARN("column data length is invalid", K(ret), K(precision), K(base_type)); + } else { + base_type.get_data_type()->set_length(precision); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("not supported yet", K(ret), K(base_type)); + LOG_USER_ERROR(OB_ERR_UNEXPECTED, "type class"); + } + } + } + return ret; +} + +int ObPLResolver::resolve_declare_user_subtype(const ObStmtNodeTree *parse_tree, + ObPLDeclareUserTypeStmt *stmt, + ObPLCompileUnitAST &unit_ast) +{ + int ret = OB_SUCCESS; + ObString subtype_name; + bool not_null = false; + bool has_range = false; + int32_t lower = -1; + int32_t upper = -1; + ObPLDataType base_type; + const ObStmtNodeTree *name_node = NULL; + const ObStmtNodeTree *type_node = NULL; + const ObStmtNodeTree *constraint_node = NULL; + // 合法性检查 + CK (OB_NOT_NULL(parse_tree)); + CK (OB_NOT_NULL(current_block_)); + CK (OB_LIKELY(T_SP_DECL_USER_SUBTYPE == parse_tree->type_)); + CK (OB_LIKELY(2 == parse_tree->num_child_)); + CK (OB_NOT_NULL(parse_tree->children_[0]) + && OB_LIKELY(T_IDENT == parse_tree->children_[0]->type_)); + CK (OB_NOT_NULL(name_node = parse_tree->children_[0])); + CK (OB_NOT_NULL(type_node = parse_tree->children_[1])); + CK (T_SP_USER_SUBTYPE_BASETYPE == type_node->type_); + CK (2 == type_node->num_child_); + OX (constraint_node = type_node->children_[1]); + CK (OB_NOT_NULL(type_node = type_node->children_[0])); + // 解析子类型名称 + OX (subtype_name = ObString(name_node->str_len_, name_node->str_value_)); + // 解析子类型的基础类型 + CK (OB_NOT_NULL(type_node)); + OZ (resolve_sp_data_type(type_node, subtype_name, unit_ast, base_type, NULL)); + // 解析子类型的RANGE子句 + if (OB_SUCC(ret) && OB_NOT_NULL(constraint_node)) { + if (T_SP_USER_SUBTYPE_RANGE == constraint_node->type_) { + CK (OB_LIKELY(2 == constraint_node->num_child_)); + CK (OB_NOT_NULL(constraint_node->children_[0])); + CK (OB_NOT_NULL(constraint_node->children_[1])); + OZ (calc_subtype_range_bound(constraint_node->children_[0], unit_ast, lower)); + OZ (calc_subtype_range_bound(constraint_node->children_[1], unit_ast, upper)); + if (OB_SUCC(ret)) { + if (!base_type.is_pl_integer_type() || upper < lower) { + ret = OB_ERR_IMPROPER_CONSTRAINT_FORM; + LOG_WARN("PLS-00572: improper constraint form used", K(ret), K(base_type)); + } + } + OX (has_range = true); + } else { + CK (T_LINK_NODE == constraint_node->type_); + OZ (resolve_subtype_precision(constraint_node, base_type)); + } + } + // 解析NOT NULL子句 + OX (not_null = (-1 == parse_tree->value_) ? false : static_cast(parse_tree->value_)); + // 构造并加入NameSpace + if (OB_SUCC(ret)) { + ObUserDefinedSubType *subtype = NULL; + if (OB_ISNULL(subtype = static_cast + (resolve_ctx_.allocator_.alloc(sizeof(ObUserDefinedSubType))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for user defiend sub type", K(ret)); + } + OX (subtype = new(subtype)ObUserDefinedSubType()); + OX (subtype->set_name(subtype_name)); + CK (!base_type.is_subtype()); + + if (OB_SUCC(ret)) { + if (base_type.is_not_null() && -1 == parse_tree->value_) { + ret = OB_ERR_SUBTYPE_NOTNULL_MISMATCH; + LOG_WARN("PLS-00366: subtype of a not null type must also be not null", K(ret)); + } else { + OX (base_type.set_not_null(not_null || base_type.get_not_null())); + } + } + OX (has_range ? base_type.set_range(lower, upper) : void(NULL)); + OX (subtype->set_base_type(base_type)); + OZ (current_block_->get_namespace().add_type(subtype)); + OX (NULL != stmt ? stmt->set_user_type(subtype) : (void)NULL); + } + return ret; +} + +int ObPLResolver::resolve_declare_user_type_comm(const ObStmtNodeTree *parse_tree, + ObPLDeclareUserTypeStmt *stmt, + ObPLCompileUnitAST &unit_ast) +{ + int ret = OB_SUCCESS; + const ObStmtNodeTree *type_definition_node = NULL; + if (OB_ISNULL(parse_tree) || OB_ISNULL(current_block_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(parse_tree), K(stmt)); + } else if (OB_ISNULL(type_definition_node = parse_tree->children_[0])) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("type definition node is null"); + } else { + switch (type_definition_node->type_) { + case T_SP_REF_CURSOR_TYPE: { + if (OB_FAIL(resolve_ref_cursor_type(type_definition_node, stmt, unit_ast))) { + LOG_WARN("resolve declare ref cursor type failed", K(ret)); + } + } + break; + case T_SP_RECORD_TYPE: { + //record type + if (OB_FAIL(resolve_declare_record_type(type_definition_node, stmt, unit_ast))) { + LOG_WARN("resolve declare record type failed", K(ret)); + } + } + break; + case T_SP_COLLECTION_TYPE: { + //collection type + if (OB_FAIL(resolve_declare_collection_type(type_definition_node, stmt, unit_ast))) { + LOG_WARN("resolve declare collection type failed", K(ret)); + } + } + break; + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid type node", K_(type_definition_node->type)); + } + break; + } + } + return ret; +} + +int ObPLResolver::resolve_ref_cursor_type(const ParseNode *node, + ObPLDeclareUserTypeStmt *stmt, + ObPLCompileUnitAST &unit_ast) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(node) || OB_ISNULL(current_block_)) { //如果是package，stmt为空，所以这里不检查stmt + ret = OB_INVALID_ARGUMENT; + LOG_WARN("parse_tree is NULL", K(node), K(stmt), K(current_block_), K(ret)); + } else { + //解析name + const ObStmtNodeTree *name_node = node->children_[0]; + const ObStmtNodeTree *type_node = node->children_[1]; + if (OB_ISNULL(name_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("parse_tree is NULL", K(name_node), K(ret)); + } else { + ObString name; + ObPLDataType return_type; + if (OB_FAIL(resolve_cursor_common(name_node, type_node, unit_ast, name, return_type))) { + LOG_WARN("failed to resolve cursor comm", K(ret), K(name)); + } else if (NULL != type_node && !return_type.is_record_type()) { + ret = OB_ERR_INVALID_CURSOR_RETURN_TYPE; + LOG_WARN("ref cursor must return record type", K(return_type), K(ret)); + } else { + ObRefCursorType *ref_cursor_type = NULL; + if (OB_ISNULL(ref_cursor_type = static_cast(resolve_ctx_.allocator_.alloc(sizeof(ObRefCursorType))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory", K(ret)); + } else { + ref_cursor_type = new(ref_cursor_type)ObRefCursorType(); + ref_cursor_type->set_name(name); + if (unit_ast.is_package()) { + ref_cursor_type->set_type_from(PL_TYPE_PACKAGE); + } + NULL == type_node ? (void)NULL : ref_cursor_type->set_return_type_id(return_type.get_user_type_id()); + if (OB_FAIL(current_block_->get_namespace().add_type(ref_cursor_type))) { + LOG_WARN("failed to add record type to type table", K(*ref_cursor_type), K(ret)); + } else if (!OB_ISNULL(stmt)) { + stmt->set_user_type(static_cast(ref_cursor_type)); + } else { /*do nothing*/ } + } + } + } + } + return ret; +} +#endif int ObPLResolver::resolve_declare_record_type(const ParseNode *type_node, ObPLDeclareUserTypeStmt *stmt, @@ -1344,6 +1848,163 @@ int ObPLResolver::resolve_declare_record_type(const ParseNode *type_node, return ret; } +#ifdef OB_BUILD_ORACLE_PL +int ObPLResolver::resolve_sp_subtype_precision(ObSQLSessionInfo &sesion_info, + ObIArray& params, + const ObUserDefinedType *user_type, + ObPLDataType &pl_type) +{ + int ret = OB_SUCCESS; + int64_t precision = -1; + int64_t scale = -1; + const ObUserDefinedSubType *sub_type = NULL; + + CK (OB_NOT_NULL(user_type)); + CK (params.count() > 0); + + if (OB_FAIL(ret)) { + } else if (!user_type->is_subtype()) { + ret = OB_ERR_TYPE_CANT_CONSTRAINED; + LOG_WARN("PLS-00566: type name \"string\" cannot be constrained", K(ret)); + } else if (params.count() > 2) { + ret = OB_ERR_IMPROPER_CONSTRAINT_FORM; + LOG_WARN("PLS-00572: improper constraint form used", K(ret), K(params.count())); + } + for (int64_t i = 0; OB_SUCC(ret) && i < params.count(); ++i) { + int64_t result = -1; + OZ (get_number_literal_value(params.at(i), result)); + OX (0==i ? precision = result : scale = result); + } + CK (OB_NOT_NULL(sub_type = static_cast(user_type))); + OX (pl_type = *(sub_type->get_base_type())); + if (OB_SUCC(ret)) { + if (OB_ISNULL(pl_type.get_data_type())) { + ret = OB_ERR_TYPE_CANT_CONSTRAINED; + LOG_WARN("PLS-00566: type name \"string\" cannot be constrained", K(ret), K(pl_type)); + } else if (2 == params.count() + && pl_type.get_data_type()->get_type_class() != ObNumberTC) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("only can specific scale for number type class", + K(ret), KPC(pl_type.get_data_type())); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "specific scale for non number type class"); + } + } + + if (OB_SUCC(ret)) { + //reference by ObResolverUtils::resolve_data_type + ObDataType &data_type = (*pl_type.get_data_type()); + const ObAccuracy &default_accuracy + = ObAccuracy::DDL_DEFAULT_ACCURACY2[true][data_type.get_obj_type()]; + switch (data_type.get_type_class()) { + case ObIntTC: + case ObUIntTC: { + if (precision <= 0) { + precision = default_accuracy.get_precision(); + } + if (precision > OB_MAX_INTEGER_DISPLAY_WIDTH) { + ret = OB_ERR_TOO_BIG_DISPLAYWIDTH; + } else { + data_type.set_precision(precision); + data_type.set_scale(0); + } + } break; + case ObFloatTC: + case ObDoubleTC: { + data_type.set_precision(precision); + data_type.set_scale(scale); + break; + } + case ObNumberTC: { + if (data_type.get_meta_type().is_number_float()) { + if (precision != PRECISION_UNKNOWN_YET + && (OB_UNLIKELY(precision < OB_MIN_NUMBER_FLOAT_PRECISION) + || OB_UNLIKELY(precision > OB_MAX_NUMBER_FLOAT_PRECISION))) { + ret = OB_FLOAT_PRECISION_OUT_RANGE; + LOG_WARN("precision of float out of range", K(ret), K(precision)); + } else { + data_type.set_precision(precision); + data_type.set_scale(ORA_NUMBER_SCALE_UNKNOWN_YET); + } + } else if (OB_UNLIKELY(precision < OB_MIN_NUMBER_PRECISION) + || OB_UNLIKELY(precision > OB_MAX_NUMBER_PRECISION)) { + ret = OB_NUMERIC_PRECISION_OUT_RANGE; + LOG_WARN("precision of number overflow", K(ret), K(scale), K(precision)); + } else if (2 == params.count() + && (OB_UNLIKELY(scale < OB_MIN_NUMBER_SCALE) + || OB_UNLIKELY(scale > OB_MAX_NUMBER_SCALE))) { + ret = OB_NUMERIC_SCALE_OUT_RANGE; + LOG_WARN("scale of number out of range", K(ret), K(scale)); + } else { + data_type.set_precision(precision); + if (2 == params.count()) { + data_type.set_scale(scale); + } else { + data_type.set_scale(0); + } + } + } break; + case ObOTimestampTC: { + if (OB_UNLIKELY(precision > OB_MAX_TIMESTAMP_TZ_PRECISION)) { + ret = OB_ERR_DATETIME_INTERVAL_PRECISION_OUT_OF_RANGE; + } else { + data_type.set_precision( + static_cast(default_accuracy.get_precision() + precision)); + data_type.set_scale(precision); + } + } break; + case ObStringTC: { + int32_t length = precision; + int64_t nchar_mbminlen = 0; + const ObSessionNLSParams &nls_session_param = sesion_info.get_session_nls_params(); + data_type.set_length(length); + if (0 == length) { + ret = OB_ERR_ZERO_LEN_COL; + LOG_WARN("Oracle not allowed zero length", K(ret)); + } else if (OB_FAIL(ObCharset::get_mbminlen_by_coll( + nls_session_param.nls_nation_collation_, nchar_mbminlen))) { + LOG_WARN("fail to get mbminlen of nchar", K(ret), K(nls_session_param)); + } else if (((ObVarcharType == data_type.get_obj_type() + || ObNVarchar2Type == data_type.get_obj_type()) + && OB_MAX_ORACLE_VARCHAR_LENGTH < length) + || (ObCharType == data_type.get_obj_type() + && OB_MAX_ORACLE_CHAR_LENGTH_BYTE < length) + || (ObNCharType == data_type.get_obj_type() + && OB_MAX_ORACLE_CHAR_LENGTH_BYTE < length * nchar_mbminlen)) { + ret = OB_ERR_TOO_LONG_COLUMN_LENGTH; + LOG_WARN("column data length is invalid", + K(ret), K(length), K(data_type), K(nchar_mbminlen)); + } + } break; + case ObRawTC: + case ObLobTC: + case ObTextTC: + case ObJsonTC: + case ObGeometryTC: { + data_type.set_length(precision); + } break; + case ObRowIDTC: { + if (ob_is_urowid(data_type.get_obj_type())) { + if (precision > OB_MAX_USER_ROW_KEY_LENGTH) { + ret = OB_ERR_TOO_LONG_COLUMN_LENGTH; + LOG_WARN("column data length is invalid", K(ret), K(precision), K(data_type)); + } else { + data_type.set_length(precision); + } + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("only support urowid type for now", K(ret), K(data_type)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "non-urowid type"); + } + } break; + default: { + ret = OB_ERR_ILLEGAL_TYPE; + LOG_WARN("Unsupported data type for subtype basetype", K(ret), K(data_type)); + } break; + } + } + return ret; +} +#endif int ObPLResolver::resolve_sp_composite_type(const ParseNode *sp_data_type_node, ObPLCompileUnitAST &func, @@ -1387,6 +2048,19 @@ int ObPLResolver::resolve_sp_composite_type(const ParseNode *sp_data_type_node, obj_access_idents.at(obj_access_idents.count()-1).access_name_.ptr()); } } +#ifdef OB_BUILD_ORACLE_PL + //所有Type表都没有找到, 看下是否是SYS_REFCURSOR + if (OB_FAIL(ret) + && 1 == obj_access_idents.count() + && ObCharset::case_insensitive_equal(obj_access_idents.at(0).access_name_, "SYS_REFCURSOR")) { + ret = OB_SUCCESS; + ob_reset_tsi_warning_buffer(); + CK (OB_NOT_NULL(current_block_->get_namespace().get_type_table())); + OX (user_type = &(current_block_->get_namespace().get_type_table()->get_sys_refcursor_type())); + OX (data_type = *user_type); + + } else +#endif if (OB_SUCC(ret)) { if (!ObObjAccessIdx::is_type(access_idxs)) { ret = OB_ERR_SP_UNDECLARED_TYPE; @@ -1400,6 +2074,9 @@ int ObPLResolver::resolve_sp_composite_type(const ParseNode *sp_data_type_node, if (!obj_access_idents.at(obj_access_idents.count() - 1).params_.empty()) { ObArray params; OZ (obj_access_idents.at(obj_access_idents.count() -1 ).extract_params(0, params)); +#ifdef OB_BUILD_ORACLE_PL + OZ (resolve_sp_subtype_precision(resolve_ctx_.session_info_, params, user_type, data_type)); +#endif } else { OX (data_type = *user_type); } @@ -1420,6 +2097,11 @@ int ObPLResolver::resolve_sp_composite_type(const ParseNode *sp_data_type_node, obj_access_idents, extern_type_info)); } else if (OB_FAIL(ret) || data_type.is_obj_type()) { +#ifdef OB_BUILD_ORACLE_PL + } else if (user_type->is_sys_refcursor_type()) { + OX (extern_type_info->flag_ = ObParamExternType::SP_EXTERN_SYS_REFCURSOR); + OX (extern_type_info->type_name_ = ObString("SYS_REFCURSOR")); +#endif } else if (ObObjAccessIdx::is_local_type(access_idxs)) { OX (extern_type_info->flag_ = ObParamExternType::SP_EXTERN_LOCAL_VAR); OX (extern_type_info->type_name_ = access_idxs.at(access_idxs.count()-1).var_name_); @@ -2565,6 +3247,9 @@ int ObPLResolver::resolve_sp_data_type(const ParseNode *sp_data_type_node, } OX (need_adjust_type = data_type.is_subtype() || data_type.is_type_type()); +#ifdef OB_BUILD_ORACLE_PL + OZ (current_block_->get_namespace().get_subtype_actually_basetype(data_type)); +#endif /*! * for number(38,0), always adjust presicion and scale. example: * declare @@ -2598,6 +3283,159 @@ int ObPLResolver::resolve_sp_data_type(const ParseNode *sp_data_type_node, return ret; } +#ifdef OB_BUILD_ORACLE_PL +int ObPLResolver::resolve_declare_collection_type(const ParseNode *type_node, + ObPLDeclareUserTypeStmt *stmt, + ObPLCompileUnitAST &unit_ast) +{ + UNUSED(unit_ast); + int ret = OB_SUCCESS; + if (OB_ISNULL(type_node)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("type node is null"); + } else if (OB_UNLIKELY(type_node->num_child_ != 2)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("type node is invalid", K(type_node->num_child_)); + } else { + const ParseNode *name_node = type_node->children_[0]; + const ParseNode *coll_type_def = type_node->children_[1]; + ObString table_name; + ObPLDataType element_type; + ObCollectionType *collection_type = NULL; + void *ptr = NULL; + if (OB_ISNULL(name_node) || OB_ISNULL(coll_type_def)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("parse node is invalid", K(name_node), K(coll_type_def)); + } else { + table_name.assign_ptr(name_node->str_value_, static_cast(name_node->str_len_)); + const ParseNode *elem_type_node = coll_type_def->children_[0]; + if (OB_FAIL(resolve_sp_data_type(elem_type_node, table_name, unit_ast, element_type))) { + LOG_WARN("resolve sp data type failed", K(ret)); + } else if (element_type.is_cursor_type()) { + ret = OB_ERR_INDEX_TABLE_OF_CURSOR; + LOG_WARN("Index Tables of Cursor Variables are disallowed", K(ret), K(element_type)); + } + +#define ALLOC_COLLECTION_TYPE(TYPE) \ + do { \ + if (OB_SUCC(ret)) { \ + TYPE *table_type = NULL; \ + if (OB_ISNULL(ptr = resolve_ctx_.allocator_.alloc(sizeof(TYPE)))) { \ + ret = OB_ALLOCATE_MEMORY_FAILED; \ + LOG_WARN("no memory to allocate ObTableType", "size", sizeof(TYPE)); \ + } else { \ + table_type = new(ptr) TYPE(); \ + collection_type = table_type; \ + } \ + } \ + } while (0) + + switch (coll_type_def->type_) { + case T_SP_NESTED_TABLE_TYPE: { + + ALLOC_COLLECTION_TYPE(ObNestedTableType); + + } + break; + case T_SP_ASSOC_ARRAY_TYPE: { + + ALLOC_COLLECTION_TYPE(ObAssocArrayType); + + if (OB_SUCC(ret)) { + ObPLDataType index_type; + const ParseNode *index_type_node = coll_type_def->children_[1]; + if (OB_FAIL(resolve_sp_data_type(index_type_node, table_name, unit_ast, index_type))) { + LOG_WARN("resolve sp data type failed", K(ret)); + } else if (!index_type.is_pl_integer_type() + && index_type.get_obj_type() != ObVarcharType) { + ret = OB_ERR_UNSUPPORTED_TABLE_INDEX_TYPE; + LOG_WARN("PLS-00315: Implementation restriction: unsupported table index type", + K(index_type), K(ret)); + } else { + static_cast(collection_type)->set_index_type(index_type); + } + } + } + break; + case T_SP_VARRAY_TYPE: { + + ALLOC_COLLECTION_TYPE(ObVArrayType); + + if (OB_SUCC(ret)) { + const ParseNode *size_node = coll_type_def->children_[1]; + CK (OB_NOT_NULL(size_node)); + CK (T_INT == size_node->type_); + if (OB_SUCC(ret)) { + if (size_node->value_ > 0) { + static_cast(collection_type)->set_capacity(size_node->value_); + } else { + ret = OB_ERR_ARRAY_MUST_HAVE_POSITIVE_LIMIT; + LOG_WARN("PLS-00537: A VARRAY must have a positive limit", K(size_node->value_), K(ret)); + } + } + } + } + break; + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("collection type is invalid", K_(coll_type_def->type)); + } + break; + } + + if (OB_SUCC(ret)) { + if (element_type.is_not_null() && -1 == coll_type_def->value_) { + ret = OB_ERR_SUBTYPE_NOTNULL_MISMATCH; + LOG_WARN("PLS-00366: subtype of a not null type must also be not null", K(ret)); + } else { + element_type.set_not_null(coll_type_def->value_ == 1 || element_type.get_not_null()); + } + } + + if (OB_SUCC(ret)) { + collection_type->set_element_type(element_type); + collection_type->set_name(table_name); + if (unit_ast.is_package()) { + collection_type->set_type_from(PL_TYPE_PACKAGE); + } + if (OB_FAIL(current_block_->get_namespace().add_type(collection_type))) { + LOG_WARN("failed to add table type to type table", K(*collection_type), K(ret)); + } else { + if (!OB_ISNULL(stmt)) { + stmt->set_user_type(collection_type); + } + } + } + } + } + return ret; +} + +int ObPLResolver::check_collection_constructor(const ParseNode *node, const ObString &type_name, bool &is_constructor) +{ + int ret = OB_SUCCESS; + is_constructor = false; + if (OB_NOT_NULL(node) + && T_OBJ_ACCESS_REF == node->type_ + && 2 == node->num_child_ + && OB_NOT_NULL(node->children_[0]) + && OB_ISNULL(node->children_[1]) + && T_FUN_SYS == node->children_[0]->type_ + && 1 == node->children_[0]->num_child_) { + const ParseNode *name_node = node->children_[0]->children_[0]; + CK (OB_NOT_NULL(name_node)); + if (OB_SUCC(ret)) { + ObString func_name(static_cast(name_node->str_len_), name_node->str_value_); + is_constructor = ObCharset::case_insensitive_equal(func_name, type_name); + if (is_constructor && node->children_[1] != NULL) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "Collection constructor with obj access"); + } + } + } + return ret; +} +#endif int ObPLResolver::resolve_declare_var(const ObStmtNodeTree *parse_tree, ObPLDeclareVarStmt *stmt, ObPLFunctionAST &func) { @@ -4030,6 +4868,28 @@ int ObPLResolver::resolve_forall_collection_and_check(const ObStmtNodeTree *coll ObObjAccessIdx::get_final_type(access_idxs).get_user_type_id(), user_type)); CK (OB_NOT_NULL(user_type)); } +#ifdef OB_BUILD_ORACLE_PL + if (OB_SUCC(ret)) { + const ObCollectionType *coll_type = static_cast(user_type); + CK (OB_NOT_NULL(coll_type)); + if (OB_FAIL(ret)) { + } else if (stmt->is_values_bound() && !coll_type->get_element_type().is_pl_integer_type()) { + ret = OB_ERR_ASSOC_ELEM_TYPE; + LOG_WARN( + "PLS-00667: Element type of associative array should be pls_integer or binary_integer", + K(ret), K(coll_type->get_element_type())); + } else if (coll_type->is_associative_array_type()) { + const ObAssocArrayType *assoc_type = static_cast(user_type); + CK (OB_NOT_NULL(assoc_type)); + if (OB_FAIL(ret)) { + } else if (!assoc_type->get_index_type().is_pl_integer_type()) { + ret = OB_ERR_UNSUPPORTED_TABLE_INDEX_TYPE; + LOG_WARN("PLS-00315: Implementation restriction: unsupported table index type", + K(ret), K(assoc_type->get_index_type())); + } + } + } +#endif return ret; } @@ -4417,6 +5277,18 @@ int ObPLResolver::resolve_return(const ObStmtNodeTree *parse_tree, ObPLReturnStm if (OB_SUCC(ret)) { ObRawExpr *expr = NULL; ObStmtNodeTree *expr_node = parse_tree->children_[0]; +#ifdef OB_BUILD_ORACLE_PL + if (func.is_udt_cons()) { + if (OB_NOT_NULL(expr_node)) { + ret = OB_ERR_CONS_HAS_RET_NODE; + LOG_WARN("udt constructor should not have return expr", K(ret)); + LOG_USER_ERROR(OB_ERR_CONS_HAS_RET_NODE); + } else if (OB_FAIL(ObPLUDTObjectManager::make_return_node(get_resolve_ctx().allocator_, + expr_node))) { + LOG_WARN("failed to make udt construtor's ret node", K(ret)); + } + } +#endif if (OB_FAIL(ret)) { // do nothing } else if (OB_ISNULL(expr_node)) { @@ -6868,6 +7740,91 @@ int ObPLResolver::resolve_cursor_formal_param( return ret; } +#ifdef OB_BUILD_ORACLE_PL +int ObPLResolver::resolve_declare_ref_cursor( + const ObStmtNodeTree *parse_tree, ObPLDeclareCursorStmt *stmt, ObPLFunctionAST &func) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(parse_tree) || OB_ISNULL(stmt) || OB_ISNULL(current_block_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("parse_tree is NULL", K(parse_tree), K(stmt), K(current_block_), K(ret)); + } else { + const ObStmtNodeTree *name_node = parse_tree->children_[0]; + const ObStmtNodeTree *type_node = parse_tree->children_[1]; + ObPLDataType data_type; + ObString ident_name; + CK (OB_NOT_NULL(type_node)); + OZ (resolve_sp_data_type(type_node, ident_name, func, data_type)); + CK (data_type.is_cursor_type()); + CK (OB_NOT_NULL(name_node)); + if (OB_SUCC(ret)) { + if (1 != name_node->num_child_) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("name list for ref cursor is not supported yet", K(name_node->num_child_), K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "name list for ref cursor"); + } else { + OZ (resolve_ident(name_node->children_[0], ident_name)); + } + } + + if (OB_SUCC(ret)) { + ObArenaAllocator allocator; + const ObUserDefinedType *cursor_type = NULL; + const ObUserDefinedType *return_type = NULL; + int64_t index = OB_INVALID_INDEX; + if (OB_FAIL(current_block_->get_namespace().get_user_type(data_type.get_user_type_id(), + cursor_type, &allocator))) { + LOG_WARN("failed to get user type", K(data_type), K(ret)); + } else if (OB_ISNULL(cursor_type)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get cursor type", K(data_type), K(ret)); + } else { + uint64_t return_type_id = static_cast(cursor_type)->get_return_type_id(); + if (OB_INVALID_ID != return_type_id) { + if (OB_FAIL(current_block_->get_namespace().get_user_type(return_type_id, return_type))) { + LOG_WARN("failed to get user type", K(*cursor_type), K(ret)); + } else if (OB_ISNULL(return_type)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get cursor type", K(*cursor_type), K(ret)); + } else { /*do nothing*/ } + } + } + + if (OB_SUCC(ret)) { + ObString dummy_sql; + ObArray dummy_params; + sql::stmt::StmtType dummy_stmt_type = sql::stmt::T_NONE; + bool dummy_for_update = false; + bool dummy_hidden_rowid = false; + common::ObArray dummy_ref_objects; + const ObPLDataType dummy_return_type; + const ObArray dummy_formal_params; + if (OB_FAIL(current_block_->get_namespace().add_cursor(ident_name, + data_type, + dummy_sql, + dummy_params, + dummy_sql, + dummy_stmt_type, + dummy_for_update, + dummy_hidden_rowid, + OB_INVALID_ID, + dummy_ref_objects, + NULL, /*ref cursor的row desc不确定*/ + NULL == return_type ? dummy_return_type : *return_type, + dummy_formal_params, + ObPLCursor::DECLARED, + false, + index))) { + LOG_WARN("failed to add cursor to symbol table", K(ident_name), K(return_type), K(index), K(ret)); + } else { + stmt->set_cursor_index(index); + } + } + } + } + return ret; +} +#endif int ObPLResolver::resolve_open( const ObStmtNodeTree *parse_tree, ObPLOpenStmt *stmt, ObPLFunctionAST &func) @@ -7368,7 +8325,38 @@ int ObPLResolver::resolve_pipe_row( const ObStmtNodeTree *parse_tree, ObPLPipeRowStmt *stmt, ObPLFunctionAST &func) { int ret = OB_SUCCESS; +#ifdef OB_BUILD_ORACLE_PL + ObRawExpr *row_expr = NULL; + CK (OB_NOT_NULL(parse_tree)); + CK (T_SP_PIPE_ROW == parse_tree->type_); + CK (1 == parse_tree->num_child_); + CK (OB_NOT_NULL(parse_tree->children_[0])); + if (OB_SUCC(ret) && !func.get_pipelined()) { + ret = OB_ERR_PIPE_STMT_IN_NON_PIPELINED_FUNC; + LOG_WARN("PLS-00629: PIPE statement cannot be used in non-pipelined functions", K(ret)); + } + const ObPLDataType *return_type = &(func.get_ret_type()); + const ObUserDefinedType* user_type = NULL; + const ObCollectionType* coll_type = NULL; + CK (OB_NOT_NULL(return_type)); + CK (OB_NOT_NULL(current_block_)); + OZ (current_block_-> + get_namespace().get_pl_data_type_by_id(return_type->get_user_type_id(), user_type)); + CK (OB_NOT_NULL(user_type)); + CK (user_type->is_collection_type()); + CK (OB_NOT_NULL(coll_type = static_cast(user_type))); + CK (return_type->is_collection_type()); + OZ (resolve_expr( + parse_tree->children_[0], func, row_expr, + combine_line_and_col(parse_tree->children_[0]->stmt_loc_), + true, &(coll_type->get_element_type()))); + CK (OB_NOT_NULL(row_expr)); + CK (OB_NOT_NULL(stmt)); + OX (stmt->set_row(func.get_expr_count() - 1)); + OX (stmt->set_type(func.get_ret_type())); +#else UNUSEDx(parse_tree, stmt, func); +#endif return ret; } @@ -8257,7 +9245,12 @@ int ObPLResolver::build_raw_expr(const ParseNode *node, bool ObPLResolver::is_json_type_compatible(const ObUserDefinedType *left, const ObUserDefinedType *right) { +#ifdef OB_BUILD_ORACLE_PL + return (ObPlJsonUtil::is_pl_json_element_type(left->get_user_type_id()) + && ObPlJsonUtil::is_pl_json_object_type(right->get_user_type_id())) ; +#else return false; +#endif } int ObPLResolver::check_composite_compatible(const ObPLINS &ns, @@ -8457,6 +9450,9 @@ int ObPLResolver::resolve_expr(const ParseNode *node, } CK (OB_NOT_NULL(current_block_)); +#ifdef OB_BUILD_ORACLE_PL + OZ (current_block_->get_namespace().get_subtype_actually_basetype(expected_type, expected_type)); +#endif // check op illegal if (OB_SUCC(ret) && !OB_ISNULL(expr) && (T_OP_MULTISET == expr->get_expr_type())) { @@ -8525,6 +9521,13 @@ int ObPLResolver::resolve_expr(const ParseNode *node, if (OB_FAIL(ret)) { } else if (!is_compatible) { ret = OB_ERR_INVALID_TYPE_FOR_OP; +#ifdef OB_BUILD_ORACLE_PL + // error code compiltable with oracle + if (ObPlJsonUtil::is_pl_json_object_type(expected_type->get_user_type_id()) + && ObPlJsonUtil::is_pl_json_element_type(expr->get_result_type().get_udt_id())) { + ret = OB_ERR_EXPRESSION_WRONG_TYPE; + } +#endif LOG_WARN("PLS-00382: expression is of wrong type", K(ret), K(expected_type->is_obj_type()), KPC(expr), K(expr->get_result_type().get_obj_meta().get_type()), @@ -9996,7 +10999,65 @@ int ObPLResolver::resolve_collection_construct(const ObQualifiedName &q_name, ObRawExpr *&expr) { int ret = OB_SUCCESS; +#ifdef OB_BUILD_ORACLE_PL + const ObCollectionType *coll_type = NULL; + ObCollectionConstructRawExpr *coll_expr = NULL; + ObExprResType res_type; + const ObUDTTypeInfo *udt_info = NULL; + uint64_t tenant_id = OB_INVALID_ID; + bool is_udt_type = false; + CK (OB_NOT_NULL(user_type)); + OX (is_udt_type = user_type->is_udt_type()); + OZ (expr_factory_.create_raw_expr(T_FUN_PL_COLLECTION_CONSTRUCT, coll_expr)); + CK (OB_NOT_NULL(coll_expr)); + if (OB_SUCC(ret) && udf_info.param_names_.count() > 0) { // 构造函数不允许使用=>赋值 + ret = OB_ERR_CALL_WRONG_ARG; + LOG_WARN("PLS-00306: wrong number or types of arguments in call to", K(ret)); + } + CK (OB_NOT_NULL(coll_type = static_cast(user_type))); + OX (coll_expr->set_type(user_type->get_type())); + OX (coll_expr->set_capacity(user_type->is_varray_type() ? static_cast(user_type)->get_capacity() : OB_INVALID_SIZE)); + OX (coll_expr->set_udt_id(user_type->get_user_type_id())); + OX (coll_expr->set_elem_type(coll_type->get_element_type())); + OZ (coll_expr->set_access_names(q_name.access_idents_)); + OX (coll_expr->set_func_name(coll_type->get_name())); + if (is_udt_type) { + OX (tenant_id = get_tenant_id_by_object_id(user_type->get_user_type_id())); + OZ (resolve_ctx_.schema_guard_.get_udt_info( + tenant_id, user_type->get_user_type_id(), udt_info)); + CK (OB_NOT_NULL(udt_info)); + OX (coll_expr->set_database_id(udt_info->get_database_id())); + OX (coll_expr->set_coll_schema_version(udt_info->get_schema_version())); + } + OX (res_type.set_type(ObExtendType)); + OX (res_type.set_extend_type(user_type->get_type())); + OX (res_type.set_udt_id(user_type->get_user_type_id())); + OX (coll_expr->set_result_type(res_type)); + CK (OB_NOT_NULL(udf_info.ref_expr_)); + for (int64_t i = 0; OB_SUCC(ret) && i < udf_info.ref_expr_->get_param_exprs().count(); ++i) { + if (coll_type->get_element_type().is_obj_type()) { + ObRawExpr *child = udf_info.ref_expr_->get_param_exprs().at(i); + const ObDataType *data_type = coll_type->get_element_type().get_data_type(); + CK (OB_NOT_NULL(data_type)); + OZ (ObRawExprUtils::build_column_conv_expr(&resolve_ctx_.session_info_, + expr_factory_, + data_type->get_obj_type(), + data_type->get_collation_type(), + data_type->get_accuracy_value(), + true, + NULL, + NULL, + child, + true)); + OZ (coll_expr->add_param_expr(child)); + } else { + OZ (coll_expr->add_param_expr(udf_info.ref_expr_->get_param_exprs().at(i))); + } + } + OX (expr = coll_expr); +#else UNUSEDx(q_name, udf_info, user_type, expr); +#endif return ret; } @@ -10231,6 +11292,73 @@ int ObPLResolver::make_self_symbol_expr(ObPLCompileUnitAST &func, ObRawExpr *&ex return ret; } +#ifdef OB_BUILD_ORACLE_PL +int ObPLResolver::add_udt_self_argument(const ObIRoutineInfo *routine_info, + ObIArray &expr_params, + ObIArray &access_idxs, + ObUDFInfo &udf_info, + ObPLCompileUnitAST &func) +{ + int ret = OB_SUCCESS; + if (OB_NOT_NULL(routine_info) + && routine_info->is_udt_routine() // Add Self Argument For UDT Routine. + && !routine_info->is_udt_static_routine() // Static Routine Has Not Self Argument. + && expr_params.count() < routine_info->get_param_count() + && (0 == expr_params.count() + || (expr_params.count() >= 1 + && !expr_params.at(0)->has_flag(IS_UDT_UDF_SELF_PARAM) + && !(access_idxs.count() > 0 + && ObObjAccessIdx::IS_UDT_NS == access_idxs.at(access_idxs.count() - 1).access_type_ + && expr_params.at(0)->get_result_type().get_expr_udt_id() + == access_idxs.at(access_idxs.count() - 1).var_index_)))) { + ObRawExpr *self_argument = NULL; + CK (OB_NOT_NULL(udf_info.ref_expr_)); + if (OB_FAIL(ret)) { + } else if (routine_info->is_udt_cons()) { // UDT Construct Self Argument. + OZ (ObPLUDTObjectManager::make_constructor_self_expr( + resolve_ctx_, + udf_info.udf_database_, + udf_info.udf_name_, + resolve_ctx_.session_info_.get_effective_tenant_id(), + expr_factory_, + current_block_->get_namespace(), + self_argument)); + OZ (self_argument->formalize(&resolve_ctx_.session_info_)); + OX (udf_info.set_is_udf_udt_cons()); + OZ (func.add_expr(self_argument)); + } else if (access_idxs.count() > 0) { // Member Self Argument With Prefix. + if (access_idxs.at(access_idxs.count() - 1).is_udf_type()) { + OX (self_argument = access_idxs.at(access_idxs.count() - 1).get_sysfunc_); + OX (access_idxs.reset()); + } else { + OZ (make_var_from_access(access_idxs, + expr_factory_, + &(resolve_ctx_.session_info_), + &resolve_ctx_.schema_guard_, + current_block_->get_namespace(), + self_argument)); + OX (access_idxs.reset()); // Erase Pre Access. Start New Begin with UDF. + } + } else { // Member Self Argument Without Prefix. + OZ (make_self_symbol_expr(func, self_argument)); + } + CK (OB_NOT_NULL(self_argument)); + OZ (self_argument->add_flag(IS_UDT_UDF_SELF_PARAM)); + if (OB_SUCC(ret) && self_argument->is_obj_access_expr()) { + OZ (func.add_obj_access_expr(self_argument)); + } + OZ (udf_info.ref_expr_->add_param_expr(self_argument)); + OX (udf_info.udf_param_num_++); + OZ (expr_params.push_back(self_argument)); + for(int64_t i = udf_info.ref_expr_->get_children_count() - 1; OB_SUCC(ret) && i > 0; --i) { + OZ (udf_info.ref_expr_->replace_param_expr(i, expr_params.at(i - 1))); + } + OZ (udf_info.ref_expr_->replace_param_expr(0, self_argument)); + OX (udf_info.is_contain_self_param_ = true); + } + return ret; +} +#endif int ObPLResolver::resolve_udf_info( ObUDFInfo &udf_info, ObIArray &access_idxs, ObPLCompileUnitAST &func) @@ -10261,6 +11389,10 @@ int ObPLResolver::resolve_udf_info( routine_info), K(udf_info)); } +#ifdef OB_BUILD_ORACLE_PL + OZ (add_udt_self_argument(routine_info, expr_params, access_idxs, udf_info, func), + K(access_idxs), K(expr_params)); +#endif // adjust routine database name, will set to ObUDFRawExpr later. if (OB_SUCC(ret) && db_name.empty() && OB_NOT_NULL(routine_info)) { @@ -10307,6 +11439,22 @@ int ObPLResolver::resolve_udf_info( LOG_USER_ERROR(OB_ERR_PRIVATE_UDF_USE_IN_SQL, udf_name.length(), udf_name.ptr()); } +#ifdef OB_BUILD_ORACLE_PL + if (OB_SUCC(ret) && package_routine_info->is_udt_cons()) { + bool is_overloaded = false; + const ObUDTTypeInfo *udt_info = NULL; + const uint64_t tenant_id = package_routine_info->get_tenant_id(); + OZ (resolve_ctx_.schema_guard_.get_udt_info( + tenant_id, package_routine_info->get_pkg_id(), udt_info)); + CK (OB_NOT_NULL(udt_info)); + OZ (ObPLUDTObjectManager::check_overload_default_cons(package_routine_info, + udt_info, + is_overloaded)); + if (is_overloaded) { + OX (udf_info.set_is_udt_overload_default_cons()); + } + } +#endif OX (package_name = package_name.empty() ? current_block_->get_namespace().get_package_name() : package_name); @@ -10372,6 +11520,25 @@ int ObPLResolver::resolve_udf_info( CK (OB_NOT_NULL(package_info)); OX (schema_version = package_info->get_schema_version()); } +#ifdef OB_BUILD_ORACLE_PL + else { + OZ (resolve_ctx_.schema_guard_.get_udt_info( + schema_routine_info->get_tenant_id(), schema_routine_info->get_package_id(), udt_info)); + CK (OB_NOT_NULL(udt_info)); + OX (schema_version = udt_info->get_schema_version()); + + // to check is this overload the default constructor + if (OB_SUCC(ret) && schema_routine_info->is_udt_cons()) { + bool is_overloaded = false; + OZ (ObPLUDTObjectManager::check_overload_default_cons(schema_routine_info, + udt_info, + is_overloaded)); + if (is_overloaded) { + OX (udf_info.set_is_udt_overload_default_cons()); + } + } + } +#endif } OZ (ObRawExprUtils::resolve_udf_common_info(db_name, package_name, @@ -11983,6 +13150,9 @@ int ObPLResolver::resolve_routine(ObObjAccessIdent &access_ident, } } else { // find A routine, resolve it. CK (OB_NOT_NULL(routine_info)); +#ifdef OB_BUILD_ORACLE_PL + OZ (check_routine_callable(ns, access_idxs, expr_params, *routine_info)); +#endif if (OB_FAIL(ret)) { } else if (OB_NOT_NULL(routine_info->get_ret_info())) { CK (access_ident.is_pl_udf()); @@ -11995,6 +13165,50 @@ int ObPLResolver::resolve_routine(ObObjAccessIdent &access_ident, return ret; } +#ifdef OB_BUILD_ORACLE_PL +int ObPLResolver::check_routine_callable(const ObPLBlockNS &ns, + ObIArray &access_idxs, + ObIArray &expr_params, + const ObIRoutineInfo &routine_info) +{ + int ret = OB_SUCCESS; + if (routine_info.is_udt_static_routine()) { + if (access_idxs.empty()) { + uint64_t udt_id = OB_INVALID_ID; + OZ (ObPLUDTObjectManager::check_routine_callable(ns, true, udt_id)); + } else { + if (access_idxs.at(access_idxs.count() - 1).access_type_ != ObObjAccessIdx::IS_UDT_NS) { + ret = OB_ERR_INVOKE_STATIC_BY_INSTANCE; + LOG_WARN("invoke static udt function with instance", K(ret), K(access_idxs)); + LOG_USER_ERROR(OB_ERR_INVOKE_STATIC_BY_INSTANCE); + } + } + } else if (routine_info.is_udt_routine() + && !routine_info.is_udt_static_routine() + && !routine_info.is_udt_cons() + && !access_idxs.empty()) { + if (ObObjAccessIdx::IS_UDT_NS == access_idxs.at(access_idxs.count() - 1).access_type_) { + if (expr_params.count() > 0 + && expr_params.at(0)->get_result_type().get_udt_id() + == access_idxs.at(access_idxs.count() - 1).var_index_) { + expr_params.at(0)->clear_flag(IS_UDT_UDF_SELF_PARAM); + } else if (expr_params.count() > 0 + && expr_params.at(0)->get_result_type().is_xml_sql_type() + && (T_OBJ_XML == access_idxs.at(access_idxs.count() - 1).var_index_)) { + // select 'head' || xmlparse(document '123').getclobval() into a from dual; + expr_params.at(0)->clear_flag(IS_UDT_UDF_SELF_PARAM); + } /*else if (expr_params.count() > 0 + && expr_params.at(0)->get_expr_type() == T_QUESTIONMARK) { + // do nothing ... + } */ else { + ret = OB_ERR_SP_WRONG_ARG_NUM; + LOG_WARN("Incorrect number of arguments", K(ret)); + } + } + } + return ret; +} +#endif int ObPLResolver::resolve_function(ObObjAccessIdent &access_ident, ObIArray &access_idxs, @@ -12560,7 +13774,228 @@ int ObPLResolver::build_collection_attribute_access(ObRawExprFactory &expr_facto ObObjAccessIdx &access_idx) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSEDx(expr_factory, session_info, ns, func, user_type, access_idxs, attr_index, func_expr, access_idx); +#else + if (user_type.is_nested_table_type() || user_type.is_varray_type()) { + const ObCollectionType &table_type = static_cast(user_type); + ObString empty_name; + new(&access_idx)ObObjAccessIdx(table_type.get_element_type(), + NULL == func_expr ? + ObObjAccessIdx::AccessType::IS_CONST : ObObjAccessIdx::AccessType::IS_EXPR, + empty_name, + table_type.get_element_type(), + NULL == func_expr ? attr_index : reinterpret_cast(func_expr)); + if (NULL != func_expr && func_expr->is_const_raw_expr()) { + //如果是NestedTable，且括号里的参数是Const，那么不把这个Const作为param_expr处理，这是一个优化 + const ObConstRawExpr* const_expr = static_cast(func_expr); + CK (OB_NOT_NULL(const_expr)); + if (OB_FAIL(ret)) { + } else if (func_expr->get_expr_type() != T_QUESTIONMARK) { + const ObObj &const_obj = const_expr->get_value(); + int64_t var_index = OB_INVALID_INDEX; + if (const_obj.is_integer_type()) { + var_index = const_obj.get_int(); + } else if (const_obj.is_number()) { + int64_t int_value = OB_INVALID_INDEX; + if (!const_obj.get_number().is_valid_int64(int_value)) { + number::ObNumber number = const_obj.get_number(); + if (OB_FAIL(number.round(0))) { + LOG_WARN("failed to round number value", K(ret), K(number)); + } else if (!number.is_valid_int64(int_value)) { + ret = OB_ARRAY_OUT_OF_RANGE; + LOG_WARN("invalid const type for array index", K(const_obj), K(ret)); + } else { + var_index = int_value; + } + } else { + var_index = int_value; + } + } + if (OB_SUCC(ret) && var_index > 0) { + access_idx.access_type_ = ObObjAccessIdx::AccessType::IS_CONST; + access_idx.var_index_ = var_index; + access_idx.get_sysfunc_ = NULL; + } + } else { + access_idx.access_type_ = ObObjAccessIdx::AccessType::IS_LOCAL; + access_idx.var_index_ = const_expr->get_value().get_unknown(); + CK (OB_NOT_NULL(ns.get_symbol_table())); + CK (OB_NOT_NULL(ns.get_symbol_table()->get_symbol(access_idx.var_index_))); + if (OB_SUCC(ret)) { + access_idx.var_name_ = ns.get_symbol_table()->get_symbol(access_idx.var_index_)->get_name(); + } + } + } + if (OB_SUCC(ret) && OB_NOT_NULL(access_idx.get_sysfunc_)) { + OZ (formalize_expr(*(access_idx.get_sysfunc_))); + if (OB_FAIL(ret)) { + } else if (access_idx.get_sysfunc_->get_result_type().is_ext()) { + ret = OB_ERR_EXPRESSION_WRONG_TYPE; + LOG_WARN("PLS-00382: expression is of wrong type", + K(ret), K(access_idx.get_sysfunc_->get_result_type())); + } else { + OZ (ObRawExprUtils::build_column_conv_expr( + &resolve_ctx_.session_info_, + expr_factory_, + ObInt32Type, + CS_TYPE_INVALID, + (ObAccuracy::DDL_DEFAULT_ACCURACY2[lib::is_oracle_mode()][ObInt32Type]).accuracy_, + true, + NULL, /*"db_name."tb_name"."col_name"*/ + NULL, + access_idx.get_sysfunc_, + true/*+is_in_pl*/)); + CK (OB_NOT_NULL(access_idx.get_sysfunc_)); + OZ (formalize_expr(*(access_idx.get_sysfunc_))); + } + } + } else if (user_type.is_associative_array_type()) { + const ObAssocArrayType &assoc_type = static_cast(user_type); + ObPLAssocIndexRawExpr *result_expr = NULL; + if (OB_FAIL(expr_factory.create_raw_expr(T_FUN_PL_ASSOCIATIVE_INDEX, result_expr))) { + LOG_WARN("create ObOpRawExpr failed", K(ret)); + } else if (OB_ISNULL(result_expr)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate memory failed", K(ret)); + } else { + ObRawExpr *collection_expr = NULL; + if (OB_FAIL(make_var_from_access(access_idxs, expr_factory, session_info, + &resolve_ctx_.schema_guard_, ns, collection_expr))) { + LOG_WARN("failed to make var from access", K(access_idxs), K(ret)); + } else if (OB_ISNULL(collection_expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to make_var_from_access", K(user_type), K(access_idxs), K(ret)); + } else if (!collection_expr->is_obj_access_expr()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid collection expr", K(*collection_expr), K(user_type), K(access_idxs), K(ret)); + } else if (OB_FAIL(func.add_obj_access_expr(collection_expr))) { + LOG_WARN("push back error", K(*collection_expr)); + } else if (OB_FAIL(result_expr->add_param_expr(collection_expr))) { + LOG_WARN("failed to add param expr", K(ret)); + } else { + ObRawExpr *key_expr = NULL; + int64_t idx_value = OB_INVALID_INDEX; + if (NULL == func_expr) { + idx_value = attr_index; + } else if (func_expr->is_const_raw_expr()) { + const ObObj &const_obj = static_cast(func_expr)->get_value(); + if (const_obj.is_integer_type()) { + idx_value = const_obj.get_int(); + } else if (const_obj.is_number()) { + if (!const_obj.get_number().is_valid_int64(idx_value)) { + idx_value = OB_INVALID_INDEX; + } + } else { /*do nothing*/ } + } else { /*do nothing*/ } + + if (OB_SUCC(ret)) { + if (OB_INVALID_INDEX == idx_value) { + key_expr = func_expr; + } else { + ObConstRawExpr *const_expr = NULL; + if (OB_FAIL(ObRawExprUtils::build_const_int_expr(expr_factory, ObIntType, idx_value, const_expr))) { + LOG_WARN("failed to build_const_int_expr", K(ret)); + } else { + key_expr = const_expr; + } + } + if (OB_SUCC(ret)) { + CK (OB_NOT_NULL(key_expr)); + CK (OB_NOT_NULL(assoc_type.get_index_type().get_data_type())); + CK (OB_NOT_NULL(assoc_type.get_index_type().get_meta_type())); + if (OB_FAIL(ret)) { + } else if (OB_FAIL(key_expr->formalize(&(resolve_ctx_.session_info_)))) { + LOG_WARN("failed to formalize key expr", K(ret), KPC(key_expr)); + } else if (T_NULL == key_expr->get_expr_type()) { + // do nothing + } else if (key_expr->get_result_type().get_obj_meta() + == *assoc_type.get_index_type().get_meta_type() + && key_expr->get_result_type().get_accuracy() + == assoc_type.get_index_type().get_data_type()->get_accuracy()) { + //do nothing + } else if (key_expr->get_result_type().get_obj_meta() + == *assoc_type.get_index_type().get_meta_type() + && assoc_type.get_index_type().get_meta_type()->is_integer_type()) { + //如果是integer类型，不做类型转换，只检查值域 + if (lib::is_oracle_mode() + && assoc_type.get_index_type().is_pl_integer_type() + && is_need_add_checker( + assoc_type.get_index_type().get_pl_integer_type(), + key_expr) + && OB_FAIL(add_pl_integer_checker_expr( + expr_factory_, + PL_SIMPLE_INTEGER == assoc_type.get_index_type().get_pl_integer_type() + ? PL_PLS_INTEGER : assoc_type.get_index_type().get_pl_integer_type(), + assoc_type.get_index_type().get_lower(), + assoc_type.get_index_type().get_upper(), + key_expr))) { + LOG_WARN("failed to add pl integer checker expr", K(ret), KPC(key_expr)); + } else if (OB_FAIL(key_expr->formalize(&(resolve_ctx_.session_info_)))) { + LOG_WARN("failed to formalize expr", K(ret)); + } + } else if (!cast_supported(key_expr->get_result_type().get_type(), + key_expr->get_result_type().get_collation_type(), + assoc_type.get_index_type().get_data_type()->get_obj_type(), + assoc_type.get_index_type().get_data_type()->get_collation_type())) { + ret = OB_ERR_INVALID_TYPE_FOR_OP; + LOG_WARN("transition does not support", + K(ret), + K(key_expr->get_result_type()), + K(ob_obj_type_str(assoc_type.get_index_type().get_obj_type())), + KPC(key_expr)); + } else { + if (OB_FAIL(ObRawExprUtils::build_column_conv_expr( + &resolve_ctx_.session_info_, + expr_factory_, + assoc_type.get_index_type().get_obj_type(), + assoc_type.get_index_type().get_data_type()->get_collation_type(), + assoc_type.get_index_type().get_data_type()->get_accuracy_value(), + true, + NULL, /*"db_name"."tb_name"."col_name"*/ + &assoc_type.get_index_type().get_type_info(), + key_expr, + true /*is_in_pl*/))) { + LOG_WARN("fail to build column conv expr", K(ret)); + } else if (lib::is_oracle_mode() + && assoc_type.get_index_type().is_pl_integer_type() + && OB_FAIL(add_pl_integer_checker_expr( + expr_factory_, + assoc_type.get_index_type().get_pl_integer_type(), + assoc_type.get_index_type().get_lower(), + assoc_type.get_index_type().get_upper(), + key_expr))) { + LOG_WARN("failed to add pl integer checker expr", K(ret)); + } else if (OB_FAIL(key_expr->formalize(&(resolve_ctx_.session_info_)))) { + LOG_WARN("failed to formalize expr", K(ret)); + } else { /*do nothing*/ } + } + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(result_expr->add_param_expr(key_expr))) { + LOG_WARN("failed to add param expr", K(ret)); + } else if (assoc_type.get_index_type().get_meta_type()->is_string_type()) { + result_expr->set_is_index_by_varchar(true); + } + } + } + } + if (OB_SUCC(ret)) { + ObString empty_name; + new(&access_idx)ObObjAccessIdx(assoc_type.get_element_type(), + ObObjAccessIdx::AccessType::IS_EXPR, + empty_name, + assoc_type.get_element_type(), + reinterpret_cast(result_expr)); + access_idx.var_index_ = attr_index; + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Invalid user type", K(user_type), K(ret)); + } +#endif return ret; } @@ -12570,7 +14005,51 @@ int ObPLResolver::build_collection_access(ObObjAccessIdent &access_ident, int64_t start_level) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSEDx(access_ident, access_idxs, func, start_level); +#else + CK (OB_NOT_NULL(current_block_)); + if (OB_SUCC(ret)) { + const ObSQLSessionInfo &session_info = resolve_ctx_.session_info_; + ObPLBlockNS &ns = current_block_->get_namespace(); + ObObjAccessIdx access_idx; + ObPLDataType parent_type; + int64_t param_level = start_level - 1; + OX (parent_type = ObObjAccessIdx::get_final_type(access_idxs)); + for (int64_t i = 0; OB_SUCC(ret) && i < access_ident.params_.count(); ++i) { + if (access_ident.params_.at(i).second >= start_level) { + if (param_level == access_ident.params_.at(i).second) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("cannot access Collection by more than 1 index", K(access_ident), K(ret)); + } else if (!parent_type.is_collection_type()) { + ret = OB_ERR_PL_COMMON; + LOG_WARN("cannot access non collection with parameter", K(ret)); + LOG_USER_ERROR(OB_ERR_PL_COMMON, "can not access non collection with paremeter"); + } else { + CK (param_level + 1 == access_ident.params_.at(i).second); + if (OB_SUCC(ret)) { + param_level = access_ident.params_.at(i).second; + const ObUserDefinedType *user_type = NULL; + OZ (ns.get_user_type(parent_type.get_user_type_id(), user_type)); + CK (OB_NOT_NULL(user_type)); + OZ (user_type->get_all_depended_user_type(resolve_ctx_, ns)); + OX (parent_type = static_cast(user_type)->get_element_type()); + OZ (build_collection_attribute_access(expr_factory_, + &session_info, + ns, + func, + *user_type, + access_idxs, + OB_INVALID_INDEX, + access_ident.params_.at(i).first, + access_idx)); + OZ (access_idxs.push_back(access_idx)); + } + } + } + } + } +#endif return ret; } diff --git a/src/pl/ob_pl_resolver.h b/src/pl/ob_pl_resolver.h index 46815ce8c..eed59f538 100644 --- a/src/pl/ob_pl_resolver.h +++ b/src/pl/ob_pl_resolver.h @@ -19,6 +19,9 @@ #include "sql/resolver/expr/ob_raw_expr.h" #include "sql/resolver/dml/ob_sequence_namespace_checker.h" #include "objit/common/ob_item_type.h" +#ifdef OB_BUILD_ORACLE_PL +#include "pl/ob_pl_warning.h" +#endif #ifndef LOG_IN_CHECK_MODE #define LOG_IN_CHECK_MODE(fmt, args...) \ @@ -552,6 +555,19 @@ private: int resolve_declare_var(const ObStmtNodeTree *parse_tree, ObPLPackageAST &package_ast); int resolve_declare_var_comm(const ObStmtNodeTree *parse_tree, ObPLDeclareVarStmt *stmt, ObPLCompileUnitAST &unit_ast); +#ifdef OB_BUILD_ORACLE_PL + int resolve_declare_user_type(const ObStmtNodeTree *parse_tree, ObPLDeclareUserTypeStmt *stmt, ObPLFunctionAST &func); + int resolve_declare_user_type(const ObStmtNodeTree *parse_tree, ObPLPackageAST &package_ast); + int resolve_declare_user_type_comm(const ObStmtNodeTree *parse_tree, ObPLDeclareUserTypeStmt *stmt, + ObPLCompileUnitAST &unit_ast); + int resolve_subtype_precision(const ObStmtNodeTree *precision_node, ObPLDataType &base_type); + int resolve_declare_user_subtype(const ObStmtNodeTree *parse_tree, + ObPLDeclareUserTypeStmt *stmt, + ObPLCompileUnitAST &unit_ast); + int resolve_ref_cursor_type(const ParseNode *node, ObPLDeclareUserTypeStmt *stmt, ObPLCompileUnitAST &unit_ast); + int resolve_declare_collection_type(const ParseNode *type_node, ObPLDeclareUserTypeStmt *stmt, ObPLCompileUnitAST &unit_ast); + int resolve_declare_ref_cursor(const ObStmtNodeTree *parse_tree, ObPLDeclareCursorStmt *stmt, ObPLFunctionAST &func); +#endif int resolve_declare_record_type(const ParseNode *type_node, ObPLDeclareUserTypeStmt *stmt, ObPLCompileUnitAST &unit_ast); int resolve_extern_type_info(share::schema::ObSchemaGetterGuard &schema_guard, const ObSQLSessionInfo &session_info, @@ -647,6 +663,17 @@ private: int resolve_do(const ObStmtNodeTree *parse_tree, ObPLDoStmt *stmt, ObPLFunctionAST &func); +#ifdef OB_BUILD_ORACLE_PL + int resolve_object_elem_spec_def(const ParseNode *parse_tree, + ObPLCompileUnitAST &package_ast); + int resolve_object_def(const ParseNode *parse_tree, + ObPLCompileUnitAST &package_ast); + int resolve_object_elem_spec_list(const ParseNode *parse_tree, + ObPLCompileUnitAST &package_ast); + int resolve_object_constructor(const ParseNode *parse_tree, + ObPLCompileUnitAST &package_ast, + ObPLRoutineInfo *&routine_info); +#endif private: int resolve_ident(const ParseNode *node, common::ObString &ident); int build_raw_expr(const ParseNode *node, ObPLCompileUnitAST &unit_ast, ObRawExpr *&expr, @@ -831,6 +858,9 @@ private: int check_duplicate_condition(const ObPLDeclareHandlerStmt &stmt, const ObPLConditionValue &value, bool &dup, ObPLDeclareHandlerStmt::DeclareHandler::HandlerDesc* cur_desc); int analyze_actual_condition_type(const ObPLConditionValue &value, ObPLConditionType &type); +#ifdef OB_BUILD_ORACLE_PL + int check_collection_constructor(const ParseNode *node, const common::ObString &type_name, bool &is_constructor); +#endif int check_subprogram_variable_read_only(ObPLBlockNS &ns, uint64_t subprogram_id, int64_t var_idx); int check_package_variable_read_only(uint64_t package_id, uint64_t var_idx); int check_local_variable_read_only( diff --git a/src/pl/ob_pl_stmt.cpp b/src/pl/ob_pl_stmt.cpp index 0718474e4..e6fcd53c2 100644 --- a/src/pl/ob_pl_stmt.cpp +++ b/src/pl/ob_pl_stmt.cpp @@ -22,6 +22,9 @@ #include "sql/resolver/ob_stmt_resolver.h" #include "pl/ob_pl_package.h" #include "pl/ob_pl_user_type.h" +#ifdef OB_BUILD_ORACLE_PL +#include "pl/ob_pl_udt_object_manager.h" +#endif namespace oceanbase { using namespace common; @@ -112,6 +115,11 @@ const ObUserDefinedType *ObPLUserTypeTable::get_type(const ObString &type_name) const ObUserDefinedType *ObPLUserTypeTable::get_type(uint64_t type_id) const { const ObUserDefinedType *user_defined_type = NULL; +#ifdef OB_BUILD_ORACLE_PL + if (type_id == sys_refcursor_type_.get_user_type_id()) { + user_defined_type = &sys_refcursor_type_; + } +#endif for (int64_t i = 0; NULL == user_defined_type && i < user_types_.count(); ++i) { if (user_types_.at(i) != NULL && user_types_.at(i)->get_user_type_id() == type_id) { user_defined_type = user_types_.at(i); @@ -128,6 +136,11 @@ const ObUserDefinedType *ObPLUserTypeTable::get_type(int64_t idx) const const ObUserDefinedType *ObPLUserTypeTable::get_external_type(uint64_t type_id) const { const ObUserDefinedType *user_defined_type = NULL; +#ifdef OB_BUILD_ORACLE_PL + if (type_id == sys_refcursor_type_.get_user_type_id()) { + user_defined_type = &sys_refcursor_type_; + } +#endif for (int64_t i = 0; NULL == user_defined_type && i < external_user_types_.count(); ++i) { if (external_user_types_.at(i) != NULL && external_user_types_.at(i)->get_user_type_id() == type_id) { user_defined_type = external_user_types_.at(i); @@ -507,6 +520,15 @@ int ObPLRoutineInfo::make_routine_param(ObIAllocator &allocator, OZ (ob_write_string(allocator, extern_type_info.type_subname_, const_cast(param->get_type_subname())), extern_type_info); +#ifdef OB_BUILD_ORACLE_PL + if (OB_SUCC(ret)) { + if (ObPLUDTObjectManager::is_self_param(param_name)) { + param->set_is_self_param(true); + } else { + param->set_is_self_param(false); + } + } +#endif LOG_DEBUG("make call routine param", K(ret), K(extern_type_info), K(param_type), K(lbt())); @@ -2165,6 +2187,77 @@ int ObPLExternalNS::check_routine_exists(const ObString &db_name, return ret; } +#ifdef OB_BUILD_ORACLE_PL +#define SET_ACCESS(idx, data_type) \ + do { \ + if (OB_SUCC(ret)) { \ + var_idx = idx +1; \ + ObPLDataType property_type; \ + property_type.set_data_type(data_type); \ + new(&access_idx)ObObjAccessIdx(property_type, ObObjAccessIdx::IS_PROPERTY, \ + attr_name, property_type, var_idx); \ + } \ + } while(0) + +#define SET_ACCESS_AA(idx, data_type) \ + do { \ + if (OB_SUCC(ret)) { \ + if (user_type.is_associative_array_type()) {\ + const ObAssocArrayType &assoc_type = static_cast(user_type);\ + const ObDataType *dt= assoc_type.get_index_type().get_data_type();\ + CK (OB_NOT_NULL(dt));\ + SET_ACCESS(IDX_COLLECTION_FIRST, *dt);\ + } else {\ + SET_ACCESS(IDX_COLLECTION_FIRST, data_type);\ + }\ + } \ + } while(0) + +int ObPLBlockNS::add_column_conv_for_coll_func( + ObSQLSessionInfo &session_info, + ObRawExprFactory &expr_factory, + const ObUserDefinedType *user_type, + const ObString &attr_name, + ObRawExpr *&expr) const +{ + int ret = OB_SUCCESS; + ObDataType expected_type; + CK (OB_NOT_NULL(user_type)); + CK (OB_NOT_NULL(expr)); + if (OB_FAIL(ret)) { + } else if (0 == attr_name.case_compare("prior") + || 0 == attr_name.case_compare("next") + || 0 == attr_name.case_compare("exists") + || 0 == attr_name.case_compare("delete")) { + if (user_type->is_associative_array_type()) { + const ObAssocArrayType *assoc_type = static_cast(user_type); + CK (OB_NOT_NULL(assoc_type)); + if (OB_SUCC(ret) + && assoc_type->get_index_type().get_data_type() != NULL) { + expected_type = *(assoc_type->get_index_type().get_data_type()); + } + } else { + expected_type.set_obj_type(ObInt32Type); + } + } else if (0 == attr_name.case_compare("extend") + || 0 == attr_name.case_compare("trim")) { + expected_type.set_obj_type(ObInt32Type); + } + if (OB_SUCC(ret) && expected_type.get_obj_type() != ObNullType) { + OZ (ObRawExprUtils::build_column_conv_expr(&session_info, + expr_factory, + expected_type.get_obj_type(), + expected_type.get_collation_type(), + expected_type.get_accuracy_value(), + true, + NULL, + NULL, + expr, + true)); + } + return ret; +} +#endif int ObPLBlockNS::find_sub_attr_by_name(const ObUserDefinedType &user_type, const ObObjAccessIdent &access_ident, @@ -2194,6 +2287,99 @@ int ObPLBlockNS::find_sub_attr_by_name(const ObUserDefinedType &user_type, ret = OB_ERR_SP_UNDECLARED_VAR; LOG_WARN("PLS-00302: component 'A' must be declared", K(ret), K(access_ident), K(user_type)); } +#ifdef OB_BUILD_ORACLE_PL + } else if (user_type.is_collection_type()) { + ObPLExternalNS::ExternalType type = ObPLExternalNS::INVALID_VAR; + // declare v vvv, last number; val number; begin val = v.last; end; vvv 是 varray + // 这种场景下，last会被识别为一个local，所以collection的类型需要设置type id， + // 确保在resolve symbol的时候不会找错。 + if (OB_INVALID_ID == package_id) { + package_id = user_type.get_user_type_id(); + } + if (OB_FAIL(resolve_symbol(attr_name, type, data_type, package_id, var_idx))) { + LOG_WARN("get var index by name failed", K(ret)); + } else if (ObPLExternalNS::INVALID_VAR == type) { + ObDataType data_type; + data_type.set_int(); + if (0 == attr_name.case_compare("count")) { // TODO: bug!!! @ryan.ly @yuchen.wyc + + SET_ACCESS(IDX_COLLECTION_COUNT, data_type); + + } else if (0 == attr_name.case_compare("first")) { + + SET_ACCESS_AA(IDX_COLLECTION_FIRST, data_type); + + } else if (0 == attr_name.case_compare("last")) { + + SET_ACCESS_AA(IDX_COLLECTION_LAST, data_type); + + } else if (0 == attr_name.case_compare("limit")){ + + SET_ACCESS(IDX_VARRAY_CAPACITY, data_type); + + } else if (0 == attr_name.case_compare("prior") + || 0 == attr_name.case_compare("next") + || 0 == attr_name.case_compare("exists")) { + if (0 == attr_name.case_compare("exists")) { + data_type.set_obj_type(ObTinyIntType); + SET_ACCESS(IDX_COLLECTION_PLACEHOLD, data_type); + } else { + SET_ACCESS_AA(IDX_COLLECTION_PLACEHOLD, data_type); + } + if (OB_SUCC(ret) && access_ident.params_.count() > 1) { + ret = OB_ERR_CALL_WRONG_ARG; + LOG_WARN("call collection method with wrong parameter", + K(ret), K(access_ident.params_), K(attr_name)); + LOG_USER_ERROR(OB_ERR_CALL_WRONG_ARG, attr_name.length(), attr_name.ptr()); + } + ARRAY_FOREACH(access_ident.params_, idx) { + ObRawExpr *param_expr = access_ident.params_.at(idx).first; + int64_t expr_idx = OB_INVALID_INDEX; + OZ (add_column_conv_for_coll_func( + session_info, expr_factory, &user_type, attr_name, param_expr)); + if (OB_FAIL(ret)) { + } else if (!has_exist_in_array(func.get_exprs(), param_expr, &expr_idx)) { + OZ (func.add_expr(param_expr)); + OX (expr_idx = func.get_exprs().count() - 1); + } + OZ (access_idx.type_method_params_.push_back(expr_idx)); + } + + } else if (0 == attr_name.case_compare("extend") + || 0 == attr_name.case_compare("delete") + || 0 == attr_name.case_compare("trim")) { + ObPLDataType invalid_pl_data_type; + new(&access_idx)ObObjAccessIdx( + invalid_pl_data_type, ObObjAccessIdx::IS_TYPE_METHOD, attr_name, invalid_pl_data_type); + if (access_ident.params_.count() > 2) { + ret = OB_ERR_CALL_WRONG_ARG; + LOG_USER_ERROR(OB_ERR_CALL_WRONG_ARG, attr_name.length(), attr_name.ptr()); + } + ARRAY_FOREACH(access_ident.params_, idx) { + ObRawExpr *param_expr = access_ident.params_.at(idx).first; + int64_t expr_idx = OB_INVALID_INDEX; + OZ (add_column_conv_for_coll_func( + session_info, expr_factory, &user_type, attr_name, param_expr)); + if (OB_FAIL(ret)) { + } else if (!has_exist_in_array(func.get_exprs(), param_expr, &expr_idx)) { + OZ (func.add_expr(param_expr)); + OX (expr_idx = func.get_exprs().count() - 1); + } + OZ (access_idx.type_method_params_.push_back(expr_idx)); + } + } else { + ret = OB_ERR_SP_UNDECLARED_VAR; + LOG_USER_ERROR(OB_ERR_SP_UNDECLARED_VAR, attr_name.length(), attr_name.ptr()); + } + } else { + const ObCollectionType &collection_type = static_cast(user_type); + new(&access_idx)ObObjAccessIdx(collection_type.get_element_type(), + static_cast(type), + attr_name, + data_type, + var_idx); + } +#endif } else { ret = OB_ERR_COMPONENT_UNDECLARED; LOG_USER_ERROR(OB_ERR_COMPONENT_UNDECLARED, attr_name.length(), attr_name.ptr()); @@ -2926,6 +3112,21 @@ int ObPLBlockNS::check_routine_exists(const ObString &db_name, int ObPLBlockNS::find_sub_attr_by_index(const ObUserDefinedType &user_type, int64_t attr_index, const ObRawExpr *func_expr, ObObjAccessIdx &access_idx) const { int ret = OB_SUCCESS; +#ifdef OB_BUILD_ORACLE_PL + if (user_type.is_nested_table_type()) { + const ObNestedTableType &table_type = static_cast(user_type); + ObString empty_name; + new(&access_idx)ObObjAccessIdx(table_type.get_element_type(), + NULL == func_expr ? ObObjAccessIdx::IS_CONST : ObObjAccessIdx::IS_EXPR, + empty_name, + table_type.get_element_type(), + reinterpret_cast(func_expr)); + access_idx.var_index_ = attr_index; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Invalid user type", K(user_type), K(ret)); + } +#endif return ret; } @@ -2978,6 +3179,15 @@ int ObPLBlockNS::get_pl_data_type_by_name(const ObPLResolveCtx &resolve_ctx, } else { /*do nothing*/ } } } +#ifdef OB_BUILD_ORACLE_PL + //所有的type表都找不到，看是否是SYS_REFCURSOR + if ((OB_SUCC(ret) && OB_ISNULL(user_type)) || OB_ERR_SP_UNDECLARED_TYPE == ret) { + if (db_name.empty() && package_name.empty() && ObCharset::case_insensitive_equal(type_name, "SYS_REFCURSOR")) { + user_type = &type_table_->get_sys_refcursor_type(); + ret = OB_SUCCESS; + } + } +#endif if (OB_SUCC(ret) && OB_NOT_NULL(user_type)) { if (OB_FAIL(user_type->get_all_depended_user_type(resolve_ctx, *this))) { LOG_WARN("get all depended user type failed", K(ret)); @@ -2988,6 +3198,18 @@ int ObPLBlockNS::get_pl_data_type_by_name(const ObPLResolveCtx &resolve_ctx, return ret; } +#ifdef OB_BUILD_ORACLE_PL +int ObPLBlockNS::get_subtype(uint64_t type_id, const ObUserDefinedSubType *&subtype) +{ + int ret = OB_SUCCESS; + const ObUserDefinedType *type = NULL; + OZ (get_pl_data_type_by_id(type_id, type)); + CK (OB_NOT_NULL(type)); + CK (type->is_subtype()); + CK (OB_NOT_NULL(subtype = static_cast(type))); + return ret; +} +#endif int ObPLBlockNS::get_subtype_actually_basetype(ObPLDataType &pl_type) { @@ -3008,8 +3230,19 @@ int ObPLBlockNS::get_subtype_actually_basetype(const ObPLDataType *pl_type, const ObPLDataType *&actually_type) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL ret = OB_NOT_SUPPORTED; LOG_WARN("get_subtype_actually_basetype is not supported in mysql mode", K(ret)); +#else + const ObUserDefinedSubType *subtype = NULL; + if (OB_NOT_NULL(pl_type) && pl_type->is_subtype()) { + OZ (get_subtype(pl_type->get_user_type_id(), subtype)); + } + if (OB_SUCC(ret) && OB_NOT_NULL(subtype)) { + CK (OB_NOT_NULL(actually_type = subtype->get_base_type())); + OZ (get_subtype_actually_basetype(actually_type, actually_type)); + } +#endif return ret; } @@ -3751,6 +3984,24 @@ int ObPLInto::check_into(ObPLFunctionAST &func, ObPLBlockNS &ns, bool is_bulk) ret = OB_ERR_MIX_SINGLE_MULTI; LOG_WARN("PLS-00497: cannot mix between single row and multi-row (BULK) in INTO list", K(ret), K(i)); +#ifdef OB_BUILD_ORACLE_PL + } else if (!is_bulk && type.is_collection_type()) { + //ret = OB_ERR_INTO_EXPR_ILLEGAL; + //LOG_WARN("PLS-00597: expression 'string' in the INTO list is of wrong type", K(ret), K(i)); + } else if (is_bulk && type.is_associative_array_type()) { + const ObUserDefinedType *user_type = NULL; + const ObAssocArrayType *assoc_type = NULL; + OZ (ns.get_pl_data_type_by_id(type.get_user_type_id(), user_type)); + CK (OB_NOT_NULL(assoc_type = static_cast(user_type))); + CK (OB_NOT_NULL(assoc_type->get_index_type().get_data_type())); + if (OB_FAIL(ret)) { + } else if (ObStringTC == assoc_type->get_index_type().get_data_type()->get_type_class()) { + ret = OB_ERR_BULK_SQL_RESTRICTION; + LOG_WARN("PLS-00657: Implementation restriction:" + " bulk SQL with associative arrays with VARCHAR2 key is not supported.", + K(ret), K(i), KPC(assoc_type)); + } +#endif } } } diff --git a/src/pl/ob_pl_stmt.h b/src/pl/ob_pl_stmt.h index 331c23a59..381e19303 100644 --- a/src/pl/ob_pl_stmt.h +++ b/src/pl/ob_pl_stmt.h @@ -312,12 +312,24 @@ private: class ObPLUserTypeTable { public: +#ifdef OB_BUILD_ORACLE_PL + ObPLUserTypeTable() : type_start_gen_id_(0), sys_refcursor_type_(), user_types_(), external_user_types_() + { + sys_refcursor_type_.set_name("SYS_REFCURSOR"); + sys_refcursor_type_.set_user_type_id(generate_user_type_id(OB_INVALID_ID)); + sys_refcursor_type_.set_type_from(PL_TYPE_SYS_REFCURSOR); + } +#else ObPLUserTypeTable() : type_start_gen_id_(0), user_types_(), external_user_types_() {} +#endif virtual ~ObPLUserTypeTable() {} inline void set_type_start_gen_id(uint64_t type_start_gen_id) { type_start_gen_id_ = type_start_gen_id; } inline uint64_t get_type_start_gen_id() const { return type_start_gen_id_; } inline int64_t get_count() const { return user_types_.count(); } +#ifdef OB_BUILD_ORACLE_PL + inline const ObRefCursorType &get_sys_refcursor_type() const { return sys_refcursor_type_; } +#endif const common::ObIArray &get_types() const { return user_types_; } int add_type(ObUserDefinedType *user_defined_type); const ObUserDefinedType *get_type(const common::ObString &type_name) const; @@ -331,6 +343,9 @@ public: inline uint64_t generate_user_type_id(uint64_t package_id) { return common::combine_pl_type_id(package_id, type_start_gen_id_++); } private: uint64_t type_start_gen_id_; +#ifdef OB_BUILD_ORACLE_PL + ObRefCursorType sys_refcursor_type_; +#endif common::ObSEArray user_types_; common::ObSEArray external_user_types_; }; @@ -1348,6 +1363,9 @@ public: const ObString &db_name, const ObString &package_name, const ObString &type_name, const ObUserDefinedType *&user_type) const; int get_pl_data_type_by_id(uint64_t type_id, const ObUserDefinedType *&user_type) const; +#ifdef OB_BUILD_ORACLE_PL + int get_subtype(uint64_t type_id, const ObUserDefinedSubType *&subtype); +#endif int get_subtype_actually_basetype(ObPLDataType &pl_type); int get_subtype_actually_basetype(const ObPLDataType *pl_type, const ObPLDataType *&actually_type); @@ -1435,6 +1453,13 @@ public: bool &exists, pl::ObProcType &proc_type, uint64_t udt_id) const; +#ifdef OB_BUILD_ORACLE_PL + int add_column_conv_for_coll_func(ObSQLSessionInfo &session_info, + ObRawExprFactory &expr_factory, + const ObUserDefinedType *user_type, + const ObString &attr_name, + ObRawExpr *&expr) const; +#endif int find_sub_attr_by_name(const ObUserDefinedType &user_type, const sql::ObObjAccessIdent &access_ident, ObSQLSessionInfo &session_info, diff --git a/src/pl/ob_pl_type.cpp b/src/pl/ob_pl_type.cpp index 9dbff78e9..fa37f8ee0 100644 --- a/src/pl/ob_pl_type.cpp +++ b/src/pl/ob_pl_type.cpp @@ -77,7 +77,14 @@ int ObPLDataType::get_udt_type_by_name(uint64_t tenant_id, LOG_WARN("udt not exist", K(ret), K(tenant_id), K(owner_id), K(udt)); LOG_USER_ERROR(OB_ERR_SP_UNDECLARED_TYPE, udt.length(), udt.ptr()); } +#ifdef OB_BUILD_ORACLE_PL + OX (type = udt_info->is_opaque() + ? PL_OPAQUE_TYPE : + udt_info->is_collection() + ? (udt_info->is_varray() ? PL_VARRAY_TYPE : PL_NESTED_TABLE_TYPE) : PL_RECORD_TYPE); +#else OX (type = PL_RECORD_TYPE); +#endif OX (pl_type.set_user_type_id(type, udt_info->get_type_id())); OX (pl_type.set_type_from(PL_TYPE_UDT)); if (OB_SUCC(ret) && OB_NOT_NULL(obj_version)) { @@ -86,6 +93,103 @@ int ObPLDataType::get_udt_type_by_name(uint64_t tenant_id, return ret; } +#ifdef OB_BUILD_ORACLE_PL +int ObPLDataType::get_pkg_type_by_name(uint64_t tenant_id, + uint64_t owner_id, + const ObString &pkg, + const ObString &type, + ObIAllocator &allocator, + sql::ObSQLSessionInfo &session_info, + share::schema::ObSchemaGetterGuard &schema_guard, + common::ObMySQLProxy &sql_proxy, + bool is_pkg_var, // pkg var or pkg type + ObPLDataType &pl_type, + ObSchemaObjVersion *obj_version) +{ + int ret = OB_SUCCESS; + const share::schema::ObPackageInfo *package_info = NULL; + int64_t compatible_mode = lib::is_oracle_mode() ? COMPATIBLE_ORACLE_MODE + : COMPATIBLE_MYSQL_MODE; + ObPLPackageManager *package_manager = NULL; + pl::ObPLPackageGuard package_guard(session_info.get_effective_tenant_id()); + pl::ObPLResolveCtx resolve_ctx(allocator, session_info, schema_guard, + package_guard, sql_proxy, false); + OZ (package_guard.init()); + CK (OB_NOT_NULL(session_info.get_pl_engine())); + OZ (schema_guard.get_package_info(tenant_id, owner_id, pkg, share::schema::PACKAGE_TYPE, + compatible_mode, package_info)); + if (OB_SUCC(ret) && OB_ISNULL(package_info)) { + ret = OB_ERR_PACKAGE_DOSE_NOT_EXIST; + LOG_WARN("package not exist", K(ret), K(tenant_id), K(owner_id), K(pkg), K(type)); + { + ObString db_name(""); + const ObDatabaseSchema *database_schema = NULL; + if (OB_SUCCESS == schema_guard.get_database_schema(tenant_id, owner_id, database_schema)) { + if (NULL != database_schema) { + db_name =database_schema->get_database_name_str(); + } + } + LOG_USER_ERROR(OB_ERR_PACKAGE_DOSE_NOT_EXIST, "PACKAGE OR TABLE", + db_name.length(), db_name.ptr(), pkg.length(), pkg.ptr()); + } + } + OX (package_manager = &(session_info.get_pl_engine()->get_package_manager())); + CK (OB_NOT_NULL(package_manager)); + if (is_pkg_var) { + const ObPLVar *pkg_var = NULL; + int64_t var_idx = OB_INVALID_ID; + CK (OB_NOT_NULL(package_info)); + OZ (package_manager->get_package_var(resolve_ctx, + package_info->get_package_id(), + type, + pkg_var, + var_idx)); + if (OB_SUCC(ret) && OB_ISNULL(pkg_var)) { + ret = OB_ERR_SP_UNDECLARED_TYPE; + LOG_WARN("package variable is not exist", K(ret), K(tenant_id), K(owner_id), K(pkg), K(type)); + LOG_USER_ERROR(OB_ERR_SP_UNDECLARED_TYPE, type.length(), type.ptr()); + } + if (OB_FAIL(ret) || OB_ISNULL(pkg_var)) { + } else if (pl::PL_CURSOR_TYPE == pkg_var->get_type().get_type()) { + OX (pl_type.set_user_type_id(PL_RECORD_TYPE, pkg_var->get_type().get_user_type_id())); + OX (pl_type.set_type_from(PL_TYPE_PACKAGE)); + } else { + OX (pl_type = pkg_var->get_type()); + OX (pl_type.set_type_from(PL_TYPE_ATTR_TYPE)); + } + } else { + const ObUserDefinedType *user_type = NULL; + CK (OB_NOT_NULL(package_info)); + OZ (package_manager->get_package_type(resolve_ctx, + package_info->get_package_id(), + type, + user_type)); + if (OB_SUCC(ret) && OB_ISNULL(user_type)) { + ret = OB_ERR_SP_UNDECLARED_TYPE; + LOG_WARN("package type is not exist", K(ret), K(tenant_id), K(owner_id), K(pkg), K(type)); + LOG_USER_ERROR(OB_ERR_SP_UNDECLARED_TYPE, type.length(), type.ptr()); + } + if (OB_SUCC(ret)) { + if (user_type->is_subtype()) { + const ObUserDefinedSubType* subtype = static_cast(user_type); + CK (OB_NOT_NULL(subtype)); + OX (pl_type = *(subtype->get_base_type())); + } else { + OX (pl_type = *user_type); + } + if (!pl_type.is_sys_refcursor_type()) { + OX (pl_type.set_type_from(PL_TYPE_PACKAGE)); + } + } + } + if (OB_SUCC(ret) && OB_NOT_NULL(obj_version)) { + new(obj_version)ObSchemaObjVersion(package_info->get_package_id(), + package_info->get_schema_version(), + DEPENDENCY_PACKAGE); + } + return ret; +} +#endif int ObPLDataType::get_table_type_by_name(uint64_t tenant_id, uint64_t owner_id, @@ -185,6 +289,84 @@ int ObPLDataType::transform_from_iparam(const ObRoutineParam *iparam, obj_version)); break; } +#ifdef OB_BUILD_ORACLE_PL + case SP_EXTERN_PKG: { + OZ (get_pkg_type_by_name(tenant_id, + iparam->get_type_owner(), + iparam->get_type_subname(), + iparam->get_type_name(), + allocator, + session_info, + schema_guard, + sql_proxy, + false, + pl_type, + obj_version)); + break; + } + case SP_EXTERN_PKG_VAR: { + OZ (get_pkg_type_by_name(tenant_id, + iparam->get_type_owner(), + iparam->get_type_subname(), + iparam->get_type_name(), + allocator, + session_info, + schema_guard, + sql_proxy, + true, + pl_type, + obj_version)); + break; + } + case SP_EXTERN_TAB_COL: { + OZ (get_table_type_by_name(tenant_id, + iparam->get_type_owner(), + iparam->get_type_subname(), + iparam->get_type_name(), + allocator, + session_info, + schema_guard, + false, + pl_type, + obj_version)); + if (OB_SUCC(ret) && iparam->is_in_param() && ob_is_numeric_type(pl_type.get_obj_type())) { + const ObAccuracy &default_accuracy = ObAccuracy::DDL_DEFAULT_ACCURACY2[lib::is_oracle_mode()][pl_type.get_obj_type()]; + pl_type.get_data_type()->set_accuracy(default_accuracy); + } + break; + } + case SP_EXTERN_PKGVAR_OR_TABCOL: { + OZ (get_table_type_by_name(tenant_id, + iparam->get_type_owner(), + iparam->get_type_subname(), + iparam->get_type_name(), + allocator, + session_info, + schema_guard, + false, + pl_type, + obj_version)); + if (OB_TABLE_NOT_EXIST == ret || OB_ERR_COLUMN_NOT_FOUND == ret) { + ret = OB_SUCCESS; + OZ (get_pkg_type_by_name(tenant_id, + iparam->get_type_owner(), + iparam->get_type_subname(), + iparam->get_type_name(), + allocator, + session_info, + schema_guard, + sql_proxy, + true, + pl_type, + obj_version)); + } + break; + } + case SP_EXTERN_SYS_REFCURSOR: { + pl_type.set_sys_refcursor_type(); + } + break; +#endif case SP_EXTERN_TAB: { OZ (get_table_type_by_name(tenant_id, iparam->get_type_owner(), @@ -1189,6 +1371,14 @@ case type: { \ DEEP_COPY_TYPE(PL_RECORD_TYPE, ObRecordType, COPY_COMMON); DEEP_COPY_TYPE(PL_CURSOR_TYPE, ObRefCursorType, COPY_COMMON); case PL_INTEGER_TYPE: /*do nothing*/ break; +#ifdef OB_BUILD_ORACLE_PL + DEEP_COPY_TYPE(PL_NESTED_TABLE_TYPE, ObNestedTableType, COPY_COMMON); + DEEP_COPY_TYPE(PL_ASSOCIATIVE_ARRAY_TYPE, ObAssocArrayType, COPY_COMMON); + DEEP_COPY_TYPE(PL_VARRAY_TYPE, ObVArrayType, COPY_COMMON); + DEEP_COPY_TYPE(PL_SUBTYPE, ObUserDefinedSubType, COPY_COMMON); + DEEP_COPY_TYPE(PL_REF_CURSOR_TYPE, ObRefCursorType, COPY_COMMON); + case PL_OPAQUE_TYPE: /*do nothing*/ break; +#endif default: { ret = OB_NOT_SUPPORTED; LOG_WARN("type for anytype is not supported", K(ret), K(src)); @@ -1769,6 +1959,13 @@ int ObPLCursorInfo::deep_copy(ObPLCursorInfo &src, common::ObIAllocator *allocat src_cursor->row_store_.get_mem_limit())); CK (OB_NOT_NULL(dest_cursor)); OZ (dest_cursor->row_desc_.assign(src_cursor->row_desc_)); +#ifdef OB_BUILD_ORACLE_PL + if (OB_SUCC(ret) && src_cursor->fields_.count() > 0) { + OZ (ObDbmsCursorInfo::deep_copy_field_columns(*copy_allocator, + &(src_cursor->fields_), + dest_cursor->fields_)); + } +#endif OX (dest_cursor->cur_ = src_cursor->cur_); while (OB_SUCC(ret) && cur < src_cursor->row_store_.get_row_cnt()) { @@ -1818,6 +2015,13 @@ int ObPLCursorInfo::close(sql::ObSQLSessionInfo &session, bool is_reuse) get_allocator()->free(get_spi_cursor()); } } +#ifdef OB_BUILD_ORACLE_PL + if (lib::is_oracle_mode()) { + /* unregiter snapshot whether ret is succ or not*/ + MTL(transaction::ObTransService*)->unregister_tx_snapshot_verify(get_snapshot()); + get_snapshot().reset(); + } +#endif } else { LOG_INFO("NOTICE: cursor is closed without openning", K(*this), K(ret)); } diff --git a/src/pl/ob_pl_type.h b/src/pl/ob_pl_type.h index f43c3bfc6..38f7b2735 100644 --- a/src/pl/ob_pl_type.h +++ b/src/pl/ob_pl_type.h @@ -532,6 +532,19 @@ public: share::schema::ObSchemaGetterGuard &schema_guard, ObPLDataType &pl_type, share::schema::ObSchemaObjVersion *obj_version); +#ifdef OB_BUILD_ORACLE_PL + static int get_pkg_type_by_name(uint64_t tenant_id, + uint64_t owner_id, + const common::ObString &pkg, + const common::ObString &type, + common::ObIAllocator &allocator, + sql::ObSQLSessionInfo &session_info, + share::schema::ObSchemaGetterGuard &schema_guard, + common::ObMySQLProxy &sql_proxy, + bool is_pkg_var, // pkg var or pkg type + ObPLDataType &pl_type, + share::schema::ObSchemaObjVersion *obj_version); +#endif static int get_table_type_by_name(uint64_t tenant_id, uint64_t owner_id, const ObString &table, diff --git a/src/pl/ob_pl_user_type.cpp b/src/pl/ob_pl_user_type.cpp index 8e9917058..61dac5a5d 100644 --- a/src/pl/ob_pl_user_type.cpp +++ b/src/pl/ob_pl_user_type.cpp @@ -290,6 +290,13 @@ int ObUserDefinedType::deep_copy_obj( OZ (ObRefCursorType::deep_copy_cursor(allocator, src, dst)); } break; +#ifdef OB_BUILD_ORACLE_PL + //all Composite can call copy_element + case PL_OPAQUE_TYPE: //fallthrough + case PL_NESTED_TABLE_TYPE: //fallthrough + case PL_ASSOCIATIVE_ARRAY_TYPE: //fallthrough + case PL_VARRAY_TYPE: //fallthrough +#endif case PL_RECORD_TYPE: { OZ (ObPLComposite::copy_element(src, dst, allocator, NULL, NULL, NULL, need_new_allocator, ignore_del_element)); } @@ -333,6 +340,47 @@ int ObUserDefinedType::destruct_obj(ObObj &src, ObSQLSessionInfo *session) OX (record->set_null()); } break; +#ifdef OB_BUILD_ORACLE_PL + case PL_NESTED_TABLE_TYPE: //fallthrough + case PL_ASSOCIATIVE_ARRAY_TYPE: //fallthrough + case PL_VARRAY_TYPE: { + ObPLCollection *collection = reinterpret_cast(src.get_ext()); + CK (OB_NOT_NULL(collection)); + if (OB_SUCC(ret) && OB_NOT_NULL(collection->get_allocator())) { + for (int64_t i = 0; OB_SUCC(ret) && i < collection->get_count(); ++i) { + OZ (destruct_obj(collection->get_data()[i], session)); + } + } + if (OB_SUCC(ret)) { + common::ObIAllocator *collection_allocator = collection->get_allocator(); + if (NULL == collection_allocator) { + //只定义过而没有用过的Collection的allocator为空，这是正常的，跳过即可 + LOG_DEBUG("Notice: a collection declared but not used", K(src), K(ret)); + } else { + if (NULL == dynamic_cast(collection_allocator)) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("here must be a bug!!!", K(collection_allocator), K(ret)); + } else { + collection_allocator->reset(); + collection->set_allocator(NULL); + collection->set_data(NULL); + collection->set_count(-1); + collection->set_first(OB_INVALID_INDEX); + collection->set_last(OB_INVALID_INDEX); + collection->set_null(); + } + } + } + } + break; + case PL_OPAQUE_TYPE: { + ObPLOpaque *opaque = reinterpret_cast(src.get_ext()); + CK (OB_NOT_NULL(opaque)); + OX (opaque->~ObPLOpaque()); + OX (src.set_null()); + } + break; +#endif default: { ret = OB_ERR_UNEXPECTED; LOG_WARN("Unexpected type to destruct", K(src), K(src.get_meta().get_extend_type()), K(ret)); @@ -358,6 +406,22 @@ int ObUserDefinedType::serialize_obj(const ObObj &obj, char* buf, const int64_t ret = OB_NOT_SUPPORTED; } break; +#ifdef OB_BUILD_ORACLE_PL +#define SERIALIZE_COLLECTION(type, class) \ + case type: { \ + class *collection = reinterpret_cast(obj.get_ext()); \ + OZ (collection->serialize(buf, len, pos)); \ + } \ + break; + + SERIALIZE_COLLECTION(PL_NESTED_TABLE_TYPE, ObPLNestedTable) + + SERIALIZE_COLLECTION(PL_ASSOCIATIVE_ARRAY_TYPE, ObPLAssocArray) + + SERIALIZE_COLLECTION(PL_VARRAY_TYPE, ObPLVArray) + +#undef SERIALIZE_COLLECTION +#endif default: { ret = OB_ERR_UNEXPECTED; LOG_WARN("Unexpected type to serialize", K(obj), K(ret)); @@ -384,6 +448,33 @@ int ObUserDefinedType::deserialize_obj(ObObj &obj, const char* buf, const int64_ ret = OB_NOT_SUPPORTED; } break; +#ifdef OB_BUILD_ORACLE_PL +#define DESERIALIZE_COLLECTION(type, class) \ + case type: { \ + if (OB_SUCC(ret)) { \ + class *new_coll = NULL; \ + ObIAllocator &allocator = CURRENT_CONTEXT->get_arena_allocator(); \ + if (OB_ISNULL(new_coll = reinterpret_cast(allocator.alloc(sizeof(class))))) { \ + ret = OB_ALLOCATE_MEMORY_FAILED; \ + LOG_WARN("failed to allocator memory for collection", K(ret)); \ + } else { \ + new(new_coll) class(id); \ + OX (new_coll->set_allocator(&allocator)); \ + OZ (new_coll->deserialize(allocator, buf, len, pos)); \ + OX (obj.set_extend(reinterpret_cast(new_coll), type)); \ + } \ + } \ + } \ + break; + + DESERIALIZE_COLLECTION(PL_NESTED_TABLE_TYPE, ObPLNestedTable) + + DESERIALIZE_COLLECTION(PL_ASSOCIATIVE_ARRAY_TYPE, ObPLAssocArray) + + DESERIALIZE_COLLECTION(PL_VARRAY_TYPE, ObPLVArray) + +#undef DESERIALIZE_COLLECTION +#endif default: { ret = OB_ERR_UNEXPECTED; LOG_WARN("Unexpected type to deserialize", K(obj), K(ret)); @@ -409,6 +500,22 @@ int64_t ObUserDefinedType::get_serialize_obj_size(const ObObj &obj) ret = OB_NOT_SUPPORTED; } break; +#ifdef OB_BUILD_ORACLE_PL +#define COLLECTION_SERIALIZE_SIZE(type, class) \ + case type: { \ + class *collection = reinterpret_cast(obj.get_ext()); \ + OZ (collection->get_serialize_size(size)); \ + } \ + break; + + COLLECTION_SERIALIZE_SIZE(PL_NESTED_TABLE_TYPE, ObPLNestedTable) + + COLLECTION_SERIALIZE_SIZE(PL_ASSOCIATIVE_ARRAY_TYPE, ObPLAssocArray) + + COLLECTION_SERIALIZE_SIZE(PL_VARRAY_TYPE, ObPLVArray) + +#undef COLLECTION_SERIALIZE_SIZE +#endif default: { ret = OB_ERR_UNEXPECTED; LOG_ERROR("Unexpected type to get serialize size", K(obj), K(ret)); @@ -419,6 +526,113 @@ int64_t ObUserDefinedType::get_serialize_obj_size(const ObObj &obj) return size; } +#ifdef OB_BUILD_ORACLE_PL +//---------- for ObUserDefinedSubType ---------- + +int ObUserDefinedSubType::deep_copy(common::ObIAllocator &alloc, const ObUserDefinedSubType &other) +{ + int ret = OB_SUCCESS; + OZ (base_type_.deep_copy(alloc, other.base_type_)); + OZ (ObUserDefinedType::deep_copy(alloc, other)); + return ret; +} + +int ObUserDefinedSubType::generate_copy(ObPLCodeGenerator &generator, + const ObPLBlockNS &ns, + jit::ObLLVMValue &allocator, + jit::ObLLVMValue &src, + jit::ObLLVMValue &dest, + bool in_notfound, + bool in_warning, + uint64_t package_id) const +{ + int ret = OB_SUCCESS; + OZ (SMART_CALL(base_type_.generate_copy( + generator, ns, allocator, src, dest, in_notfound, in_warning, package_id))); + return ret; +} + +int ObUserDefinedSubType::generate_construct(ObPLCodeGenerator &generator, + const ObPLINS &ns, + jit::ObLLVMValue &value, + const pl::ObPLStmt *stmt) const +{ + int ret = OB_SUCCESS; + OZ (SMART_CALL(base_type_.generate_construct(generator, ns, value, stmt))); + return ret; +} + +int ObUserDefinedSubType::generate_new(ObPLCodeGenerator &generator, + const ObPLINS &ns, + jit::ObLLVMValue &value, + const pl::ObPLStmt *s) const +{ + int ret = OB_NOT_SUPPORTED; + ret = ObUserDefinedType::generate_new(generator, ns, value, s); + return ret; +} + +int ObUserDefinedSubType::newx(common::ObIAllocator &allocator, + const ObPLINS *ns, + int64_t &ptr) const +{ + int ret = OB_NOT_SUPPORTED; + UNUSEDx(allocator, ns, ptr); + return ret; +} + +int ObUserDefinedSubType::get_size(const ObPLINS &ns, ObPLTypeSize type, int64_t &size) const +{ + int ret = OB_SUCCESS; + OZ (base_type_.get_size(ns, type, size)); + return ret; +} + +int ObUserDefinedSubType::get_all_depended_user_type(const ObPLResolveCtx &resolve_ctx, + const ObPLBlockNS ¤t_ns) const +{ + int ret = OB_SUCCESS; + OZ (base_type_.get_all_depended_user_type(resolve_ctx, current_ns), base_type_); + return ret; +} + +int ObUserDefinedSubType::serialize(share::schema::ObSchemaGetterGuard &schema_guard, + const common::ObTimeZoneInfo *tz_info, + obmysql::MYSQL_PROTOCOL_TYPE type, + char *&src, + char *dst, + const int64_t dst_len, + int64_t &dst_pos) const +{ + int ret = OB_SUCCESS; + OZ (base_type_.serialize(schema_guard, tz_info, type, src, dst, dst_len, dst_pos)); + return ret; +} + +int ObUserDefinedSubType::deserialize(share::schema::ObSchemaGetterGuard &schema_guard, + common::ObIAllocator &allocator, + const common::ObCharsetType charset, + const common::ObCollationType cs_type, + const common::ObCollationType ncs_type, + const common::ObTimeZoneInfo *tz_info, + const char *&src, + char *dst, + const int64_t dst_len, + int64_t &dst_pos) const +{ + int ret = OB_SUCCESS; + OZ (base_type_.deserialize( + schema_guard, allocator, charset, cs_type, ncs_type, tz_info, src, dst, dst_len, dst_pos)); + return ret; +} + +int ObUserDefinedSubType::convert(ObPLResolveCtx &ctx, ObObj *&src, ObObj *&dst) const +{ + int ret = OB_SUCCESS; + OZ (base_type_.convert(ctx, src, dst)); + return ret; +} +#endif //---------- for ObRefCursorType ---------- @@ -1292,6 +1506,19 @@ int ObRecordType::serialize(share::schema::ObSchemaGetterGuard &schema_guard, ObMySQLUtil::update_null_bitmap(bitmap, i); new_src += sizeof(ObObj); } else if (type->is_collection_type()) { +#ifdef OB_BUILD_ORACLE_PL + char *coll_src = reinterpret_cast(obj->get_ext()); + ObPLNestedTable *coll_table = reinterpret_cast(coll_src); + CK (obj->is_ext()); + CK (OB_NOT_NULL(coll_table)); + CK (OB_NOT_NULL(coll_src)); + if (OB_FAIL(ret)) { + } else if (!coll_table->is_inited()) { + ObMySQLUtil::update_null_bitmap(bitmap, i); + } else { + OZ (type->serialize(schema_guard, tz_info, protocl_type, new_src, dst, dst_len, dst_pos)); + } +#endif } else { OZ (type->serialize(schema_guard, tz_info, protocl_type, new_src, dst, dst_len, dst_pos), K(i), KPC(this)); @@ -1403,6 +1630,1309 @@ int ObRecordType::convert(ObPLResolveCtx &ctx, ObObj *&src, ObObj *&dst) const return ret; } +#ifdef OB_BUILD_ORACLE_PL +//---------- for ObOpaqueType ---------- + +int ObOpaqueType::get_size(const ObPLINS &ns, ObPLTypeSize type, int64_t &size) const +{ + int ret = OB_SUCCESS; + if (PL_TYPE_INIT_SIZE == type) { + ObPLOpaque opaque; + size += opaque.get_init_size(); + } else { + OZ (ObUserDefinedType::get_size(ns, type, size)); + } + return ret; +} + +int ObOpaqueType::generate_construct(ObPLCodeGenerator &generator, + const ObPLINS &ns, + jit::ObLLVMValue &value, + const pl::ObPLStmt *stmt) const +{ + UNUSEDx(generator, ns, value, stmt); + return OB_SUCCESS; +} + +int ObOpaqueType::newx(common::ObIAllocator &allocator, const ObPLINS *ns, int64_t &ptr) const +{ + int ret = OB_SUCCESS; + ObPLOpaque *opaque = NULL; + ObPLOpaque tmp; + int64_t init_size = tmp.get_init_size(); + UNUSED(ns); + OX (opaque = reinterpret_cast(allocator.alloc(init_size))); + CK (OB_NOT_NULL(opaque)); + OX (new (opaque) ObPLOpaque()); + OX (ptr = reinterpret_cast(opaque)); + return ret; +} + +int ObOpaqueType::init_session_var(const ObPLResolveCtx &resolve_ctx, + ObIAllocator &obj_allocator, + sql::ObExecContext &exec_ctx, + const sql::ObSqlExpression *default_expr, + bool default_construct, + ObObj &obj) const +{ + int ret = OB_SUCCESS; + char *data = NULL; + int64_t init_size = 0; + UNUSEDx(exec_ctx, default_construct); + if (OB_NOT_NULL(default_expr)) { + ObObj calc_obj; + OZ (ObSQLUtils::calc_sql_expression_without_row(exec_ctx, *default_expr, calc_obj)); + CK (calc_obj.is_null() || calc_obj.is_pl_extend()); + if (calc_obj.is_pl_extend()) { + OZ (ObUserDefinedType::deep_copy_obj(obj_allocator, calc_obj, obj)); + } + } + if (OB_FAIL(ret) || obj.is_pl_extend()) { + // do nothing ... + } else if (OB_FAIL(get_size(resolve_ctx, PL_TYPE_INIT_SIZE, init_size))) { + LOG_WARN("get init size failed", K(ret)); + } else if (OB_ISNULL(data = static_cast(obj_allocator.alloc(init_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for opaque type", K(ret), K(init_size)); + } else { + MEMSET(data, 0, init_size); + new (data) ObPLOpaque(); + obj.set_extend(reinterpret_cast(data), PL_OPAQUE_TYPE); + } + return ret; +} + +int ObOpaqueType::free_session_var(const ObPLResolveCtx &resolve_ctx, + ObIAllocator &obj_allocator, + ObObj &obj) const +{ + int ret = OB_SUCCESS; + char *data = reinterpret_cast(obj.get_ext()); + UNUSED(resolve_ctx); + if (OB_NOT_NULL(data)) { + obj_allocator.free(data); + } + obj.set_null(); + return ret; +} + +//---------- for ObCollectionType ---------- + +int ObCollectionType::deep_copy(common::ObIAllocator &alloc, const ObCollectionType &other) +{ + int ret = OB_SUCCESS; + OZ (ObUserDefinedType::deep_copy(alloc, other)); + OZ (element_type_.deep_copy(alloc, other.get_element_type())); + return ret; +} + +int ObCollectionType::generate_construct(ObPLCodeGenerator &generator, + const ObPLINS &ns, + jit::ObLLVMValue &value, + const pl::ObPLStmt *stmt) const +{ + int ret = OB_SUCCESS; + ObLLVMValue type_ptr; + ObLLVMValue id_ptr; + ObLLVMValue isnull_ptr; + ObLLVMValue element_type_ptr; + ObLLVMValue rowsize_ptr; + ObLLVMValue count_ptr; + ObLLVMValue first_ptr; + ObLLVMValue last_ptr; + ObLLVMValue notnull_ptr; + ObElemDesc elem_desc; + OZ (SMART_CALL(ObUserDefinedType::generate_construct(generator, ns, value, stmt))); + OZ (generator.extract_type_ptr_from_collection(value, type_ptr)); + OZ (generator.get_helper().create_istore(type_, type_ptr)); + OZ (generator.extract_id_ptr_from_collection(value, id_ptr)); + OZ (generator.get_helper().create_istore(user_type_id_, id_ptr)); + OZ (generator.extract_isnull_ptr_from_collection(value, isnull_ptr)); + OZ (generator.get_helper().create_istore(FALSE, isnull_ptr)); + OZ (generator.extract_element_ptr_from_collection(value, element_type_ptr)); + + if (NULL == element_type_.get_data_type()) { //复杂类型 + OX (elem_desc.set_obj_type(ObExtendType)); + const ObUserDefinedType *user_type = NULL; + OZ (ns.get_user_type(element_type_.get_user_type_id(), user_type, NULL)); + CK (OB_NOT_NULL(user_type)); + if (OB_SUCC(ret)) { + if (user_type->is_record_type()) { + OX (elem_desc.set_field_count( + static_cast(user_type)->get_member_count())); + } else { + OX (elem_desc.set_field_count(1)); + } + OX (elem_desc.set_udt_id(element_type_.get_user_type_id())); + } + } else { //基础类型 + OX (elem_desc.set_meta_type(element_type_.get_data_type()->get_meta_type())); + OX (elem_desc.set_accuracy(element_type_.get_data_type()->get_accuracy())); + OX (elem_desc.set_field_count(1)); + } + OX (elem_desc.set_pl_type(element_type_.get_type())); + OX (elem_desc.set_not_null(element_type_.get_not_null())); + OZ (generator.store_elem_desc(elem_desc, element_type_ptr)); + OZ (generator.extract_count_ptr_from_collection(value, count_ptr)); + OZ (generator.get_helper().create_istore(OB_INVALID_COUNT, count_ptr)); + OZ (generator.extract_first_ptr_from_collection(value, first_ptr)); + OZ (generator.get_helper().create_istore(OB_INVALID_INDEX, first_ptr)); + OZ (generator.extract_last_ptr_from_collection(value, last_ptr)); + OZ (generator.get_helper().create_istore(OB_INVALID_INDEX, last_ptr)); + return ret; +} + +int ObCollectionType::generate_new(ObPLCodeGenerator &generator, + const ObPLINS &ns, + jit::ObLLVMValue &value, + const pl::ObPLStmt *s) const +{ + int ret = OB_SUCCESS; + ret = ObUserDefinedType::generate_new(generator, ns, value, s); + return ret; +} + + +int ObCollectionType::newx(common::ObIAllocator &allocator, const ObPLINS *ns, int64_t &ptr) const +{ + +#define COLLECTION_NEWX(class) \ + do { \ + if (OB_SUCC(ret)) { \ + class *table = NULL; \ + ObIAllocator *collection_allocator = NULL; \ + OX (table = reinterpret_cast(allocator.alloc(sizeof(class)))); \ + OX (collection_allocator \ + = reinterpret_cast(allocator.alloc(sizeof(ObPLCollAllocator)))); \ + if (OB_ISNULL(table) || OB_ISNULL(collection_allocator)) { \ + ret = OB_ALLOCATE_MEMORY_FAILED; \ + LOG_WARN("failed to alloc memory", K(ret)); \ + } \ + OX (new (table)class(user_type_id_)); \ + OX (collection_allocator = new(collection_allocator)ObPLCollAllocator(table)); \ + OX (table->set_allocator(collection_allocator)); \ + if (OB_SUCC(ret)) { \ + ObElemDesc elem_desc; \ + elem_desc.set_pl_type(element_type_.get_type()); \ + elem_desc.set_not_null(element_type_.get_not_null()); \ + if (OB_ISNULL(element_type_.get_data_type())) { \ + int64_t field_cnt = OB_INVALID_COUNT; \ + elem_desc.set_obj_type(common::ObExtendType); \ + OZ (element_type_.get_field_count(*ns, field_cnt)); \ + OX (elem_desc.set_field_count(field_cnt)); \ + OX (elem_desc.set_udt_id(element_type_.get_user_type_id())); \ + } else { \ + elem_desc.set_data_type(*(element_type_.get_data_type())); \ + elem_desc.set_field_count(1); \ + } \ + OX (table->set_element_desc(elem_desc)); \ + } \ + OX (ptr = reinterpret_cast(table)); \ + } \ + } while (0) + + int ret = OB_SUCCESS; + switch (get_type()) { + case PL_NESTED_TABLE_TYPE: { + COLLECTION_NEWX(ObPLNestedTable); + } + break; + case PL_ASSOCIATIVE_ARRAY_TYPE: { + COLLECTION_NEWX(ObPLAssocArray); + } + break; + case PL_VARRAY_TYPE: { + COLLECTION_NEWX(ObPLVArray); + } + break; + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected composite to copy", K(get_type()), K(ret)); + } + break; + } + +#undef COLLECTION_NEWX + + //TODO:@ryan.ly + UNUSED(ns); + return ret; +} + +int ObCollectionType::get_init_size(int64_t &size) const +{ + int ret = OB_SUCCESS; + if (is_associative_array_type()) { + size += sizeof(ObPLAssocArray) + 8; + } else if (is_varray_type()) { + size += sizeof(ObPLVArray) + 8; + } else if (is_nested_table_type()) { + size += sizeof(ObPLNestedTable) + 8; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("not support collection type in get_size", K(ret), K(type_)); + } + return ret; +} + +int ObCollectionType::get_size(const ObPLINS &ns, ObPLTypeSize type, int64_t &size) const +{ + int ret = OB_SUCCESS; + if (PL_TYPE_ROW_SIZE == type) { + OZ (get_element_type().get_size(ns, type, size)); + } else if (PL_TYPE_INIT_SIZE == type) { + OZ (get_init_size(size)); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("not support pl type size", K(ret), K(type)); + } + return ret; +} + +int ObCollectionType::generate_assign_with_null(ObPLCodeGenerator &generator, + const ObPLBlockNS &ns, + jit::ObLLVMValue &allocator, + jit::ObLLVMValue &dest) const +{ + UNUSED(allocator); UNUSED(ns); + int ret = OB_SUCCESS; + + ObSEArray args; + ObLLVMValue isnull_ptr; + ObLLVMType int_type; + ObLLVMValue int_value; + + if (generator.get_helper().get_llvm_type(ObIntType, int_type)) { + LOG_WARN("failed to get_llvm_type", K(ret)); + } else if (OB_FAIL(generator.get_helper().create_ptr_to_int(ObString("cast_ptr_to_int64"), dest, + int_type, int_value))) { + LOG_WARN("failed to create ptr to int", K(ret)); + } else if (OB_FAIL(args.push_back(int_value))) { + LOG_WARN("push_back error", K(ret)); + } else { + jit::ObLLVMValue ret_err; + if (OB_FAIL(generator.get_helper().create_call(ObString("spi_reset_collection"), + generator.get_spi_service().spi_reset_collection_, args, ret_err))) { + LOG_WARN("failed to create call", K(ret)); + } else if (OB_FAIL(generator.check_success(ret_err))) { + LOG_WARN("failed to check success", K(ret)); + } else { /*do nothing*/ } + } + OZ (generator.extract_isnull_ptr_from_record(dest, isnull_ptr)); + OZ (generator.get_helper().create_istore(TRUE, isnull_ptr)); + return ret; +} + +/* +int ObCollectionType::set_row_size(ObPLCodeGenerator &generator, const ObPLINS &ns, ObLLVMValue &collection) const +{ + int ret = OB_SUCCESS; + int64_t rowsize = 0; + ObLLVMValue p_rowsize; + OZ (generator.extract_rowsize_ptr_from_collection(collection, p_rowsize)); + OZ (get_size(ns, PL_TYPE_ROW_SIZE, rowsize)); + OZ (generator.get_helper().create_istore(rowsize, p_rowsize)); + return ret; +} +*/ + +int ObCollectionType::init_session_var(const ObPLResolveCtx &resolve_ctx, + common::ObIAllocator &obj_allocator, + sql::ObExecContext &exec_ctx, + const sql::ObSqlExpression *default_expr, + bool default_construct, + ObObj &obj) const +{ + UNUSEDx(exec_ctx, default_expr); + int ret = OB_SUCCESS; + char *data = NULL; + int64_t init_size = 0; + int64_t row_size = 0; + obj.set_null(); + if (OB_NOT_NULL(default_expr) && !default_construct) { + ObObj calc_obj; + OZ (ObSQLUtils::calc_sql_expression_without_row(exec_ctx, *default_expr, calc_obj)); + CK (calc_obj.is_null() || calc_obj.is_pl_extend()); + if (calc_obj.is_pl_extend()) { + OZ (ObUserDefinedType::deep_copy_obj(obj_allocator, calc_obj, obj)); + } + } + if (OB_FAIL(ret) || obj.is_pl_extend()) { + // do nothing ... + } else if (OB_FAIL(get_size(resolve_ctx, PL_TYPE_INIT_SIZE, init_size))) { + LOG_WARN("get init size failed", K(ret)); + } else if (OB_FAIL(get_size(resolve_ctx, PL_TYPE_ROW_SIZE, row_size))) { + LOG_WARN("get row size failed", K(ret)); + } else if (OB_ISNULL(data = static_cast(obj_allocator.alloc(init_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("memory allocate failed", K(ret)); + } else { + MEMSET(data, 0, init_size); + ObPLCollection *coll = NULL; + if (is_associative_array_type()) { + coll = new(data) ObPLAssocArray(user_type_id_); + } else if (is_nested_table_type()) { + coll = new(data) ObPLNestedTable(user_type_id_); + } else if (is_varray_type()) { + coll = new(data) ObPLVArray(user_type_id_); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected collection type", K(ret)); + } + + if (OB_SUCC(ret)) { + ObElemDesc elem_desc; + elem_desc.set_pl_type(element_type_.get_type()); + elem_desc.set_not_null(element_type_.get_not_null()); + if (OB_ISNULL(element_type_.get_data_type())) { + int64_t field_cnt = OB_INVALID_COUNT; + elem_desc.set_obj_type(common::ObExtendType); + OZ (element_type_.get_field_count(resolve_ctx, field_cnt)); + OX (elem_desc.set_field_count(field_cnt)); + OX (elem_desc.set_udt_id(element_type_.get_user_type_id())); + } else { + elem_desc.set_data_type(*(element_type_.get_data_type())); + elem_desc.set_field_count(1); + } + CK (OB_NOT_NULL(coll)); + OX (coll->set_element_desc(elem_desc)); + } + + // coll->set_allocator(&obj_allocator);// package variable的初始化使用外层的allocator, 避免内存泄漏 + CK (OB_NOT_NULL(exec_ctx.get_my_session())); + OZ (ObSPIService::spi_set_collection(exec_ctx.get_my_session()->get_effective_tenant_id(), + &resolve_ctx, + obj_allocator, + *coll, + 0, + false)); + OX (default_construct ? coll->set_inited() : void(NULL)); + OX (obj.set_extend(reinterpret_cast(data), type_, init_size)); + } + return ret; +} + +int ObCollectionType::free_session_var(const ObPLResolveCtx &resolve_ctx, + ObIAllocator &obj_allocator, + ObObj &obj) const +{ + int ret = OB_SUCCESS; + char *data = reinterpret_cast(obj.get_ext()); + if (!OB_ISNULL(data)) { + ObPLNestedTable *table = reinterpret_cast(data); + int64_t element_init_size = 0; + if (OB_FAIL(element_type_.get_size(resolve_ctx, PL_TYPE_INIT_SIZE, element_init_size))) { + LOG_WARN("get table element type init size failed", K(ret)); + } else { + char *free_ptr = NULL; + for (int64_t i = 0; OB_SUCC(ret) && i < table->get_count() ;i++) { + free_ptr = reinterpret_cast(table->get_data())+i*element_init_size; + if (OB_FAIL(element_type_.free_data(resolve_ctx, obj_allocator, free_ptr))) { + LOG_WARN("free table element type failed", K(ret)); + } + } + } + if (OB_SUCC(ret)) { + if (!OB_ISNULL(table->get_allocator())) { + if (NULL == dynamic_cast(table->get_allocator())) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("here nust be a bug", K(ret)); + } else { + table->get_allocator()->reset(); + } + obj_allocator.free(table->get_allocator()); + } + obj_allocator.free(data); + } + } + if (OB_SUCC(ret)) { + obj.set_null(); + } + return ret; +} + +int ObCollectionType::free_data(const ObPLResolveCtx &resolve_ctx, + ObIAllocator &data_allocator, + void *data) const +{ + int ret = OB_SUCCESS; + if (!OB_ISNULL(data)) { + ObPLNestedTable *table = reinterpret_cast(data); + int64_t element_init_size = 0; + if (OB_FAIL(element_type_.get_size(resolve_ctx, PL_TYPE_INIT_SIZE, element_init_size))) { + LOG_WARN("get table element type init size failed", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < table->get_count() ;i++) { + if (OB_FAIL(element_type_.free_data(resolve_ctx, + data_allocator, + reinterpret_cast(table->get_data())+i*element_init_size ))) { + LOG_WARN("free table element type failed", K(ret)); + } + } + } + if (OB_SUCC(ret)) { + if (!OB_ISNULL(table->get_allocator())) { + if (NULL == dynamic_cast(table->get_allocator())) { + LOG_WARN("WOCAONIMA", K(ret)); + } else { + table->get_allocator()->reset(); + } + } + } + } + return ret; +} + +// --------- for session serialize/deserialize interface --------- +int ObCollectionType::get_serialize_size( + const ObPLResolveCtx &resolve_ctx, char *&src, int64_t &size) const +{ + int ret = OB_SUCCESS; + ObPLNestedTable *table = reinterpret_cast(src); + char *data = NULL; + CK (OB_NOT_NULL(table)); + + OX (size += static_cast(table)->get_serialize_size()); + OX (size += table->get_element_desc().get_serialize_size()); + OX (size += serialization::encoded_length(table->get_count())); + OX (size += serialization::encoded_length(table->get_first())); + OX (size += serialization::encoded_length(table->get_last())); + + OX (data = reinterpret_cast(table->get_data())); + for (int64_t i = 0; OB_SUCC(ret) && i < table->get_count(); ++i) { + ObObj* obj = reinterpret_cast(data); + CK (OB_NOT_NULL(obj)); + if (OB_FAIL(ret)) { + } else if (element_type_.is_composite_type() && ObMaxType == obj->get_type()) { + ObPLComposite composite; + OX (size += composite.get_serialize_size()); + OX (data += sizeof(ObObj)); + } else { + OZ (element_type_.get_serialize_size(resolve_ctx, data, size)); + } + } + return ret; +} + +int ObCollectionType::serialize( + const ObPLResolveCtx &resolve_ctx, + char *&src, char *dst, int64_t dst_len, int64_t &dst_pos) const +{ +#define ENCODE(v) \ + OZ (serialization::encode(dst, dst_len, dst_pos, v)); + + int ret = OB_SUCCESS; + ObPLNestedTable *table = reinterpret_cast(src); + + CK (OB_NOT_NULL(table)); + + OV (table->get_column_count() > 0, OB_ERR_UNEXPECTED, KPC(table)); + + OX (static_cast(table)->serialize(dst, dst_len, dst_pos)); + OX (table->get_element_desc().serialize(dst, dst_len, dst_pos)); + ENCODE(table->get_count()); + ENCODE(table->get_first()); + ENCODE(table->get_last()); + + if (OB_SUCC(ret)) { + char *data = reinterpret_cast(table->get_data()); + for (int64_t i = 0; OB_SUCC(ret) && i < table->get_count(); ++i) { + ObObj *obj = reinterpret_cast(data); + CK (OB_NOT_NULL(obj)); + if (OB_FAIL(ret)) { + } else if (element_type_.is_composite_type() && ObMaxType == obj->get_type()) { + // deleted element + ObPLComposite composite; + OZ (composite.serialize(dst, dst_len, dst_pos)); + OX (data += sizeof(ObObj)); + } else { + OZ (element_type_.serialize(resolve_ctx, data, dst, dst_len, dst_pos)); + } + } + } + return ret; + +#undef ENCODE +} + +int ObCollectionType::deserialize( + const ObPLResolveCtx &resolve_ctx, + common::ObIAllocator &allocator, + const char *src, const int64_t src_len, int64_t &src_pos, char *&dst) const +{ + int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL + UNUSEDx(resolve_ctx, allocator, src, src_len, src_pos, dst); + ret = OB_NOT_SUPPORTED; + LOG_WARN("not support", K(ret)); +#else +#define DECODE(v) \ + OZ (serialization::decode(src, src_len, src_pos, v)); + + ObPLCollection *table = reinterpret_cast(dst); + int64_t count = 0; + int64_t first = 0; + int64_t last = 0; + + CK (OB_NOT_NULL(table)); + + OZ (static_cast(table)->deserialize(src, src_len, src_pos)); + // delete element will deserialize to a invalid composite + // when see invalid composte, stop table deserialize + if (OB_SUCC(ret) && table->get_type() != PL_INVALID_TYPE) { + OZ (table->get_element_desc().deserialize(src, src_len, src_pos)); + DECODE(count); + DECODE(first); + DECODE(last); + + UNUSED(allocator); + CK (OB_NOT_NULL(table->get_allocator())); + if (OB_FAIL(ret)) { + } else if (count <= 0) { + table->set_count(count); + } else if (is_associative_array_type()) { + ObPLAssocArray *assoc_table = static_cast(table); + CK (OB_NOT_NULL(assoc_table)); + CK (OB_NOT_NULL(table->get_allocator())); + OZ (ObSPIService::spi_extend_assoc_array( + OB_INVALID_ID, &resolve_ctx, *(table->get_allocator()), *assoc_table, count)); + } else { + OZ (ObSPIService::spi_set_collection( + OB_INVALID_ID, &resolve_ctx, *table->get_allocator(), *table, count, true)); + } + // CK (OB_NOT_NULL(table->get_data())); + + if (OB_SUCC(ret)) { + char *table_data = reinterpret_cast(table->get_data()); + for (int64_t i = 0; OB_SUCC(ret) && i < count; ++i) { + ObObj* obj = reinterpret_cast(table_data); + CK (OB_NOT_NULL(table->get_allocator())); + OZ (element_type_.deserialize( + resolve_ctx, *(table->get_allocator()), src, src_len, src_pos, table_data)); + if (OB_SUCC(ret) && obj->is_ext()) { + ObPLComposite* composite = reinterpret_cast(obj->get_ext()); + CK (OB_NOT_NULL(composite)); + if (OB_SUCC(ret) && composite->get_type() == PL_INVALID_TYPE) { + obj->set_type(ObMaxType); + } + } + } + } + + OX (table->set_first(first)); + OX (table->set_last(last)); + } +#endif + return ret; + +#undef DECODE +} + +int ObCollectionType::add_package_routine_schema_param(const ObPLResolveCtx &resolve_ctx, + const ObPLBlockNS &block_ns, + const common::ObString &package_name, + const common::ObString ¶m_name, + int64_t mode, int64_t position, + int64_t level, int64_t &sequence, + share::schema::ObRoutineInfo &routine_info) const +{ + UNUSEDx(param_name, position); + int ret = OB_SUCCESS; + ObString empty_param_name; + if (OB_FAIL(element_type_.add_package_routine_schema_param(resolve_ctx, block_ns, package_name, empty_param_name, + mode, 1, level+1, sequence, routine_info))) { + LOG_WARN("failed to add routine schema param", K(*this), K(ret)); + } + return ret; +} + +int ObCollectionType::get_all_depended_user_type( + const ObPLResolveCtx &resolve_ctx, const ObPLBlockNS ¤t_ns) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(element_type_.get_all_depended_user_type(resolve_ctx, current_ns))) { + LOG_WARN("element type get depended user type failed", K(ret)); + } + return ret; +} + +int ObCollectionType::init_obj(ObSchemaGetterGuard &schema_guard, + ObIAllocator &allocator, + ObObj &obj, + int64_t &init_size) const +{ + int ret = OB_SUCCESS; + char *data = NULL; + init_size = 0; + if (OB_FAIL(get_size(ObPLUDTNS(schema_guard), PL_TYPE_INIT_SIZE, init_size))) { + LOG_WARN("get init size failed", K(ret)); + } else if (OB_ISNULL(data = static_cast(allocator.alloc(init_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("memory allocate failed", K(ret)); + } else { + MEMSET(data, 0, init_size); + if (is_varray_type()) { + new (data) ObPLVArray(get_user_type_id()); + } else if (is_associative_array_type()) { + new (data) ObPLAssocArray(get_user_type_id()); + } else { + new (data) ObPLCollection(get_type(), get_user_type_id()); + } + obj.set_extend(reinterpret_cast(data), type_, init_size); + LOG_DEBUG("success to init obj", K(*this), K(init_size), K(data)); + } + return ret; +} + +int ObCollectionType::serialize(share::schema::ObSchemaGetterGuard &schema_guard, + const ObTimeZoneInfo *tz_info, + MYSQL_PROTOCOL_TYPE type, + char *&src, + char *dst, + const int64_t dst_len, + int64_t &dst_pos) const +{ + int ret = OB_SUCCESS; + int64_t init_size = 0; + ObObj *src_obj = NULL; + ObPLNestedTable *table = NULL; + if (OB_ISNULL(src_obj = reinterpret_cast(src))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("src is null", K(ret), KP(src_obj), KPC(this)); + } else if (!src_obj->is_ext()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("src obj not pl extend", K(ret), KPC(src_obj), KPC(this)); + } else if (OB_ISNULL(table + = reinterpret_cast(src_obj->get_ext()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table is null", K(ret), KPC(table), KPC(this)); + } else if (!table->is_inited()) { + // table未初始化应该序列化为null, 空在空值位图中标识, 上层已经处理过空值位图, 这里什么都不做 + } else if (OB_FAIL(get_size(ObPLUDTNS(schema_guard), PL_TYPE_INIT_SIZE, init_size))) { + LOG_WARN("failed to type init size", K(ret), KPC(this), KPC(table)); + } else if (OB_FAIL(ObMySQLUtil::store_length(dst, dst_len, table->get_actual_count(), dst_pos))) { + LOG_WARN("failed to stroe_length for table count", K(ret), KPC(this), KPC(table), K(table->get_count())); + } else { + int64_t bitmap_bytes = (table->get_actual_count() + 7 + 2) / 8; + char* bitmap = NULL; + // 计算空值位图位置 + if ((dst_len - dst_pos) < bitmap_bytes) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("size overflow", K(ret), KPC(this), KPC(table), K(dst_len), K(dst_pos), K(bitmap_bytes)); + } else { + bitmap = dst + dst_pos; + MEMSET(dst + dst_pos, 0, bitmap_bytes); + dst_pos += bitmap_bytes; + } + // 序列化值并更新空值位图 + for (int64_t i = 0; OB_SUCC(ret) && i < table->get_count(); ++i) { + char *data = reinterpret_cast(table->get_data()) + (sizeof(ObObj) * i); + ObObj* obj = reinterpret_cast(data); + CK (OB_NOT_NULL(obj)); + if (OB_FAIL(ret)) { + } else if (obj->is_invalid_type()) { + // deleted element, do nothing... + } else if (obj->is_null()) { + ObMySQLUtil::update_null_bitmap(bitmap, i); + } else if (element_type_.is_collection_type()) { + char *coll_src = reinterpret_cast(obj->get_ext()); + ObPLNestedTable *coll_table = reinterpret_cast(coll_src); + OV (obj->is_ext(), OB_ERR_UNEXPECTED, KP(obj), KP(data), K(i)); + CK (OB_NOT_NULL(coll_src)); + CK (OB_NOT_NULL(coll_table)); + if (OB_FAIL(ret)) { + } else if (!coll_table->is_inited()) { + ObMySQLUtil::update_null_bitmap(bitmap, i); + } else { + OZ (element_type_.serialize(schema_guard, tz_info, type, data, dst, dst_len, dst_pos), KPC(this), K(i)); + } + } else { + OZ (element_type_.serialize(schema_guard, tz_info, type, data, dst, dst_len, dst_pos), KPC(this), K(i)); + } + } + LOG_DEBUG("serialize length", K(ret), KPC(table), KPC(this), K(reinterpret_cast(dst)), K(dst_len), K(dst_pos)); + if (OB_SUCC(ret)) { + src += sizeof(ObObj); + } + } + return ret; +} + +int ObCollectionType::deserialize(ObSchemaGetterGuard &schema_guard, + ObIAllocator &allocator, + const ObCharsetType charset, + const ObCollationType cs_type, + const ObCollationType ncs_type, + const common::ObTimeZoneInfo *tz_info, + const char *&src, + char *dst, + const int64_t dst_len, + int64_t &dst_pos) const +{ + int ret = OB_SUCCESS; + int64_t init_size = 0; + int64_t element_init_size = 0; + int64_t field_cnt = OB_INVALID_COUNT; + + if (OB_FAIL(get_size(ObPLUDTNS(schema_guard), PL_TYPE_INIT_SIZE, init_size))) { + LOG_WARN("get table type init size failed", K(ret), KPC(this)); + } else if (OB_ISNULL(dst) || (dst_len - dst_pos) < init_size) { + ret = OB_DESERIALIZE_ERROR; + LOG_WARN("data deserialize failed", K(ret), K(dst), K(init_size), K(dst_len), K(dst_pos)); + } else if (OB_FAIL(element_type_.get_size(ObPLUDTNS(schema_guard), PL_TYPE_INIT_SIZE, element_init_size))) { + LOG_WARN("get element init size failed", K(ret), KPC(this), K(init_size)); + } else if (OB_FAIL(element_type_.get_field_count(ObPLUDTNS(schema_guard), field_cnt))) { + LOG_WARN("get field count failed", K(ret)); + } else { + ObPLNestedTable *table = reinterpret_cast(dst + dst_pos); + common::ObIAllocator *collection_allocator = NULL; + if (OB_ISNULL(table)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table is null", K(ret), KPC(this), K(dst_pos), K(init_size), K(element_init_size)); + } else if (OB_NOT_NULL(table->get_allocator())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table allocator is not null", K(ret), KPC(this), KPC(table)); + } else if (OB_ISNULL(collection_allocator + = static_cast(allocator.alloc(sizeof(ObPLCollAllocator))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc collection allocator", K(ret), KPC(this), KPC(table)); + } else { + uint64_t max_count = OB_INVALID_SIZE; + uint64_t tab_count = OB_INVALID_SIZE; + uint64_t count = OB_INVALID_SIZE; + char *table_data = NULL; + OZ (ObMySQLUtil::get_length(src, count), K(*this), K(*table)); + OX (max_count = count >> 32); + OX (tab_count = count & 0xffffffff); + OX (max_count = (0 == max_count) ? tab_count : max_count); + OX (count = tab_count); + CK (max_count >= count); + OX (new (collection_allocator) ObPLCollAllocator(table)); + if (OB_SUCC(ret) && OB_LIKELY(0 != max_count)) { + int64_t bitmap_bytes = ((count + 7) / 8); + const char* bitmap = src; + src += bitmap_bytes; + ObObj null_value; + int64_t table_data_len = element_init_size * max_count; + int64_t table_data_pos = 0; + + if (OB_ISNULL(table_data = static_cast(collection_allocator->alloc(table_data_len)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate memory failed", + K(ret), KPC(this), KPC(table), K(element_init_size), K(count), K(max_count)); + } + if (OB_SUCC(ret) && element_type_.is_record_type()) { + int table_data_pos_tmp = table_data_pos; + for (int i = 0; OB_SUCC(ret) && i < count; ++i) { + ObObj *value = reinterpret_cast(table_data + table_data_pos_tmp); + ObPLRecord *new_record = reinterpret_cast( + collection_allocator->alloc(ObRecordType::get_init_size(field_cnt))); + CK (OB_NOT_NULL(value)); + CK (OB_NOT_NULL(new_record)); + OX (new (new_record) ObPLRecord(element_type_.get_user_type_id(), field_cnt)); + OX (value->set_extend(reinterpret_cast(new_record), + PL_RECORD_TYPE, + ObRecordType::get_init_size(field_cnt))); + OX (table_data_pos_tmp += sizeof(ObObj)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < count; ++i) { + if (ObSMUtils::update_from_bitmap(null_value, bitmap, i)) { // null value + ObObj* value = reinterpret_cast(table_data + table_data_pos); + if (element_type_.is_obj_type()) { + value->set_null(); + } else { + ObPLComposite *composite = reinterpret_cast(value->get_ext()); + if (OB_ISNULL(composite)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error, got null composite", K(ret), KPC(value)); + } else { + composite->set_null(); + } + } + table_data_pos += sizeof(ObObj); + } else { + if (OB_FAIL(element_type_.deserialize(schema_guard, allocator, charset, cs_type, ncs_type, + tz_info, src, table_data, table_data_len, table_data_pos))) { + LOG_WARN("deserialize element failed", K(ret), K(i), K(element_init_size), K(count)); + } + } + LOG_DEBUG("deserialize element done", K(ret), KPC(this), K(i), K(element_init_size), K(count), + K(src), K(table_data), K(table_data_len), K(table_data_pos)); + } + } + if (OB_SUCC(ret)) { + ObElemDesc elem_desc; + table->set_type(PL_NESTED_TABLE_TYPE); + table->set_allocator(collection_allocator); + table->set_count(max_count); + table->set_first(1); + table->set_last(count); + table->set_not_null(element_type_.get_not_null()); + table->set_data(reinterpret_cast(table_data)); + table->set_column_count(field_cnt); + elem_desc.set_pl_type(element_type_.get_type()); + elem_desc.set_not_null(element_type_.get_not_null()); + if (OB_ISNULL(element_type_.get_data_type())) { + OX (elem_desc.set_obj_type(common::ObExtendType)); + OX (elem_desc.set_field_count(field_cnt)); + OX (elem_desc.set_udt_id(element_type_.get_user_type_id())); + OX (table->set_element_desc(elem_desc)); + } else { + elem_desc.set_data_type(*(element_type_.get_data_type())); + elem_desc.set_field_count(1); + table->set_element_desc(elem_desc); + } + } + for (int64_t i = count; OB_SUCC(ret) && i < max_count; ++i) { + OZ (table->delete_collection_elem(i), K(i), K(max_count), K(count)); + } + } + } + return ret; +} + +int ObCollectionType::convert(ObPLResolveCtx &ctx, ObObj *&src, ObObj *&dst) const +{ + int ret = OB_SUCCESS; + ObPLCollection *src_table = NULL; + ObPLCollection *dst_table = NULL; + int64_t element_init_size = 0; + common::ObIAllocator *collection_allocator = NULL; + char *table_data = NULL; + + CK (OB_NOT_NULL(src)); + CK (OB_NOT_NULL(dst)); + CK (OB_LIKELY(src->is_ext())); + CK (OB_LIKELY(dst->is_ext())); + CK (OB_NOT_NULL(src_table = reinterpret_cast(src->get_ext()))); + CK (OB_NOT_NULL(dst_table = reinterpret_cast(dst->get_ext()))); + OZ (element_type_.get_size(ctx, PL_TYPE_INIT_SIZE, element_init_size)); + + if (OB_SUCC(ret) + && OB_ISNULL(collection_allocator + = static_cast(ctx.allocator_.alloc(sizeof(ObPLCollAllocator))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc collection allocator", K(ret)); + } + OX (new (collection_allocator) ObPLCollAllocator(dst_table)); + if (OB_SUCC(ret) + && OB_ISNULL(table_data + = static_cast( + collection_allocator->alloc(element_init_size * src_table->get_count())))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc table data", K(ret)); + } + if (OB_SUCC(ret)) { + ObObj *src_table_pos = reinterpret_cast(src_table->get_data()); + ObObj *dst_table_pos = reinterpret_cast(table_data); + for (int64_t i = 0; OB_SUCC(ret) && i < src_table->get_count(); i++) { + OZ (element_type_.convert(ctx, src_table_pos, dst_table_pos)); + } + } + if (OB_SUCC(ret)) { + dst_table->set_type(src_table->get_type()); + dst_table->set_allocator(collection_allocator); + dst_table->set_count(src_table->get_count()); + dst_table->set_first(1); + dst_table->set_last(src_table->get_count()); + dst_table->set_data(reinterpret_cast(table_data)); + + ObElemDesc elem_desc; + elem_desc.set_pl_type(element_type_.get_type()); + elem_desc.set_not_null(element_type_.get_not_null()); + if (OB_ISNULL(element_type_.get_data_type())) { + int64_t field_cnt = OB_INVALID_COUNT; + elem_desc.set_obj_type(common::ObExtendType); + elem_desc.set_udt_id(element_type_.get_user_type_id()); + OZ (element_type_.get_field_count(ctx, field_cnt)); + OX (elem_desc.set_field_count(field_cnt)); + } else { + elem_desc.set_data_type(*(element_type_.get_data_type())); + elem_desc.set_field_count(1); + } + OX (dst_table->set_element_desc(elem_desc)); + } + return ret; +} + +//---------- for ObNestedTableType ---------- + +int ObNestedTableType::generate_construct(ObPLCodeGenerator &generator, + const ObPLINS &ns, + jit::ObLLVMValue &value, + const pl::ObPLStmt *stmt) const +{ + int ret = OB_SUCCESS; + OZ (SMART_CALL(ObCollectionType::generate_construct(generator, ns, value, stmt))); + return ret; +} + +int ObNestedTableType::newx(common::ObIAllocator &allocator, const ObPLINS *ns, int64_t &ptr) const +{ + int ret = OB_SUCCESS; + OZ (ObCollectionType::newx(allocator, ns, ptr)); + return ret; +} + +int ObNestedTableType::init_obj(ObSchemaGetterGuard &schema_guard, + ObIAllocator &allocator, + ObObj &obj, + int64_t &init_size) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ObCollectionType::init_obj(schema_guard, allocator, obj, init_size))) { + LOG_WARN("failed to init obj", K(ret)); + } + return ret; +} + +int ObNestedTableType::init_session_var(const ObPLResolveCtx &resolve_ctx, + common::ObIAllocator &obj_allocator, + sql::ObExecContext &exec_ctx, + const sql::ObSqlExpression *default_expr, + bool default_construct, + ObObj &obj) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ObCollectionType::init_session_var(resolve_ctx, + obj_allocator, + exec_ctx, + default_expr, + default_construct, + obj))) { + LOG_WARN("generate copy failed", K(ret)); + } + return ret; +} + +int ObNestedTableType::free_session_var(const ObPLResolveCtx &resolve_ctx, + ObIAllocator &obj_allocator, + ObObj &obj) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ObCollectionType::free_session_var(resolve_ctx, obj_allocator, obj))) { + LOG_WARN("generate copy failed", K(ret)); + } + return ret; +} + +int ObNestedTableType::free_data(const ObPLResolveCtx &resolve_ctx, + ObIAllocator &data_allocator, + void *data) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ObCollectionType::free_data(resolve_ctx, data_allocator, data))) { + LOG_WARN("generate copy failed", K(ret)); + } + return ret; +} + +int ObNestedTableType::serialize(share::schema::ObSchemaGetterGuard &schema_guard, + const ObTimeZoneInfo *tz_info, + MYSQL_PROTOCOL_TYPE type, + char *&src, + char *dst, + const int64_t dst_len, + int64_t &dst_pos) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ObCollectionType::serialize(schema_guard, + tz_info, + type, + src, + dst, + dst_len, + dst_pos))) { + LOG_WARN("failed to serialize ObNestedTableType", K(ret)); + } + return ret; +} + +int ObNestedTableType::deserialize(ObSchemaGetterGuard &schema_guard, + ObIAllocator &allocator, + const ObCharsetType charset, + const ObCollationType cs_type, + const ObCollationType ncs_type, + const common::ObTimeZoneInfo *tz_info, + const char *&src, + char *dst, + const int64_t dst_len, + int64_t &dst_pos) const +{ + int ret = OB_SUCCESS; + OZ (ObCollectionType::deserialize(schema_guard, + allocator, + charset, + cs_type, + ncs_type, + tz_info, + src, dst, dst_len, dst_pos), *this); + return ret; +} + +// --------- for session serialize/deserialize interface --------- +int ObNestedTableType::get_serialize_size( + const ObPLResolveCtx &resolve_ctx, char *&src, int64_t &size) const +{ + return ObCollectionType::get_serialize_size(resolve_ctx, src, size); +} + +int ObNestedTableType::serialize( + const ObPLResolveCtx &resolve_ctx, + char *&src, char* dst, int64_t dst_len, int64_t &dst_pos) const +{ + return ObCollectionType::serialize(resolve_ctx, src, dst, dst_len, dst_pos); +} + +int ObNestedTableType::deserialize( + const ObPLResolveCtx &resolve_ctx, + common::ObIAllocator &allocator, + const char* src, const int64_t src_len, int64_t &src_pos, char *&dst) const +{ + return ObCollectionType::deserialize(resolve_ctx, allocator, src, src_len, src_pos, dst); +} + +int ObNestedTableType::add_package_routine_schema_param(const ObPLResolveCtx &resolve_ctx, + const ObPLBlockNS &block_ns, + const common::ObString &package_name, + const common::ObString ¶m_name, + int64_t mode, int64_t position, + int64_t level, int64_t &sequence, + share::schema::ObRoutineInfo &routine_info) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ObCollectionType::add_package_routine_schema_param(resolve_ctx, + block_ns, + package_name, + param_name, + mode, + position, + level, + sequence, + routine_info))) { + LOG_WARN("generate copy failed", K(ret)); + } + return ret; +} + +int ObNestedTableType::get_all_depended_user_type(const ObPLResolveCtx &resolve_ctx, + const ObPLBlockNS ¤t_ns) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ObCollectionType::get_all_depended_user_type(resolve_ctx, current_ns))) { + LOG_WARN("generate copy failed", K(ret)); + } + return ret; +} + +//---------- for ObVArrayType ---------- + +int ObVArrayType::deep_copy(common::ObIAllocator &alloc, const ObVArrayType &other) +{ + int ret = OB_SUCCESS; + OZ (ObCollectionType::deep_copy(alloc, other)); + OX (capacity_ = other.capacity_); + return ret; +} + +int ObVArrayType::generate_construct(ObPLCodeGenerator &generator, + const ObPLINS &ns, + jit::ObLLVMValue &value, + const pl::ObPLStmt *stmt) const +{ + int ret = OB_SUCCESS; + ObLLVMValue capacity_ptr; + OZ (SMART_CALL(ObCollectionType::generate_construct(generator, ns, value, stmt))); + OZ (generator.extract_capacity_ptr_from_varray(value, capacity_ptr)); + OZ (generator.get_helper().create_istore(capacity_, capacity_ptr)); + return ret; +} + +int ObVArrayType::newx(common::ObIAllocator &allocator, const ObPLINS *ns, int64_t &ptr) const +{ + int ret = OB_SUCCESS; + OZ (ObCollectionType::newx(allocator, ns, ptr)); + OX (reinterpret_cast(ptr)->set_capacity(capacity_)); + return ret; +} + +int ObVArrayType::init_session_var(const ObPLResolveCtx &resolve_ctx, + common::ObIAllocator &obj_allocator, + sql::ObExecContext &exec_ctx, + const sql::ObSqlExpression *default_expr, + bool default_construct, + ObObj &obj) const +{ + int ret = OB_SUCCESS; + ObPLVArray *varray_ptr = NULL; + int64_t data = 0; + OZ (ObCollectionType::init_session_var( + resolve_ctx, obj_allocator, exec_ctx, default_expr, default_construct, obj)); + OZ (obj.get_ext(data)); + OX (varray_ptr = reinterpret_cast(data)); + OX (varray_ptr->set_capacity(capacity_)); + return ret; +} + +int ObVArrayType::convert(ObPLResolveCtx &ctx, ObObj *&src, ObObj *&dst) const +{ + UNUSEDx(ctx, src, dst); + LOG_WARN_RET(OB_NOT_SUPPORTED, "failed to convert to varray type"); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "convert to varray"); + return OB_NOT_SUPPORTED; +} + +//---------- for ObAssocArrayType ---------- + +int ObAssocArrayType::deep_copy(common::ObIAllocator &alloc, const ObAssocArrayType &other) +{ + int ret = OB_SUCCESS; + OZ (ObCollectionType::deep_copy(alloc, other)); + OZ (index_type_.deep_copy(alloc, other.index_type_)); + return ret; +} + +int ObAssocArrayType::generate_construct(ObPLCodeGenerator &generator, + const ObPLINS &ns, + jit::ObLLVMValue &value, + const pl::ObPLStmt *stmt) const +{ + //TODO: @ryan.ly + int ret = OB_SUCCESS; + ObLLVMValue capacity_ptr; + OZ (SMART_CALL(ObCollectionType::generate_construct(generator, ns, value, stmt))); + return ret; +} + +int ObAssocArrayType::newx(common::ObIAllocator &allocator, const ObPLINS *ns, int64_t &ptr) const +{ + int ret = OB_SUCCESS; + OZ (ObCollectionType::newx(allocator, ns, ptr)); + return ret; +} + +int ObAssocArrayType::init_session_var(const ObPLResolveCtx &resolve_ctx, + common::ObIAllocator &obj_allocator, + sql::ObExecContext &exec_ctx, + const sql::ObSqlExpression *default_expr, + bool default_construct, + ObObj &obj) const +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ObCollectionType::init_session_var(resolve_ctx, + obj_allocator, + exec_ctx, + default_expr, + default_construct, + obj))) { + LOG_WARN("generate copy failed", K(ret)); + } else { + ObPLAssocArray *assoc_ptr = NULL; + int64_t data = 0; + if (OB_FAIL(obj.get_ext(data))) { + LOG_WARN("init associate array session var failed.", K(ret)); + } else { + assoc_ptr = reinterpret_cast(data); + assoc_ptr->set_key(NULL); + assoc_ptr->set_sort(NULL); + } + } + return ret; +} + +// --------- for session serialize/deserialize interface --------- +int ObAssocArrayType::get_serialize_size( + const ObPLResolveCtx &resolve_ctx, char *&src, int64_t &size) const +{ + int ret = OB_SUCCESS; + ObPLAssocArray *assoc_table = reinterpret_cast(src); + char *key = NULL; + int64_t *sort = NULL; + int64_t key_sort_cnt = 0; // 紧密数组, key和sort是null + CK (OB_NOT_NULL(assoc_table)); + OZ (ObCollectionType::get_serialize_size(resolve_ctx, src, size)); + OX (key = reinterpret_cast(assoc_table->get_key())); + OX (sort = assoc_table->get_sort()); + OX (key_sort_cnt = OB_NOT_NULL(key) ? assoc_table->get_count() : 0); + OX (size += serialization::encoded_length(key_sort_cnt)); + for (int64_t i = 0; OB_SUCC(ret) && i < key_sort_cnt; ++i) { + OZ (index_type_.get_serialize_size(resolve_ctx, key, size)); + OX (size += serialization::encoded_length(*sort)); + OX (sort++); + } + return ret; +} + +int ObAssocArrayType::serialize( + const ObPLResolveCtx &resolve_ctx, + char *&src, char* dst, int64_t dst_len, int64_t &dst_pos) const +{ + int ret = OB_SUCCESS; + ObPLAssocArray *assoc_table = reinterpret_cast(src); + char *key = NULL; + int64_t *sort = NULL; + int64_t key_sort_cnt = 0; + CK (OB_NOT_NULL(assoc_table)); + OZ (ObCollectionType::serialize(resolve_ctx, src, dst, dst_len, dst_pos)); + OX (key = reinterpret_cast(assoc_table->get_key())); + OX (sort = assoc_table->get_sort()); + OX (key_sort_cnt = OB_NOT_NULL(key) ? assoc_table->get_count() : 0); + OZ (serialization::encode(dst, dst_len, dst_pos, key_sort_cnt)); + for (int64_t i = 0; OB_SUCC(ret) && i < key_sort_cnt; ++i) { + OZ (index_type_.serialize(resolve_ctx, key, dst, dst_len, dst_pos)); + OZ (serialization::encode(dst, dst_len, dst_pos, *sort)); + OX (sort++); + } + return ret; +} + +int ObAssocArrayType::deserialize( + const ObPLResolveCtx &resolve_ctx, + common::ObIAllocator &allocator, + const char* src, const int64_t src_len, int64_t &src_pos, char *&dst) const +{ + int ret = OB_SUCCESS; + ObPLAssocArray *assoc_table = reinterpret_cast(dst); + char *key = NULL; + int64_t *sort = NULL; + int64_t key_sort_cnt = 0; + CK (OB_NOT_NULL(assoc_table)); + OZ (ObCollectionType::deserialize(resolve_ctx, allocator, src, src_len, src_pos, dst)); + if (OB_FAIL(ret)) { + //do nothing + } else if (PL_INVALID_TYPE == (reinterpret_cast(dst))->get_type()) { + // element be delete . do not deserialize continue + } else { + OZ (serialization::decode(src, src_len, src_pos, key_sort_cnt)); + if (OB_FAIL(ret)) { + } else if (0 == key_sort_cnt) { + assoc_table->set_key(NULL); + assoc_table->set_sort(NULL); + } else { + CK (key_sort_cnt == assoc_table->get_count()); + CK (OB_NOT_NULL(key = reinterpret_cast(assoc_table->get_key()))); + CK (OB_NOT_NULL(sort = assoc_table->get_sort())); + CK (OB_NOT_NULL(assoc_table->get_allocator())); + for (int64_t i = 0; OB_SUCC(ret) && i < key_sort_cnt; ++i) { + OZ (index_type_.deserialize( + resolve_ctx, *(assoc_table->get_allocator()), src, src_len, src_pos, key)); + OZ (serialization::decode(src, src_len, src_pos, *sort)); + OX (sort++); + } + } + } + return ret; +} + +int ObAssocArrayType::convert(ObPLResolveCtx &ctx, ObObj *&src, ObObj *&dst) const +{ + UNUSEDx(ctx, src, dst); + LOG_WARN_RET(OB_NOT_SUPPORTED, "failed to convert to assoc array type"); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "convert to associtive array"); + return OB_NOT_SUPPORTED; +} +#endif //---------- for ObPLCollection ---------- @@ -1416,6 +2946,63 @@ int ObPLComposite::deep_copy(ObPLComposite &src, { int ret = OB_SUCCESS; +#ifdef OB_BUILD_ORACLE_PL +#define COPY_COLLECTION(TYPE) \ + do { \ + if (OB_SUCC(ret)) { \ + TYPE *collection = NULL; \ + ObIAllocator *copy_allocator = NULL; \ + ObPLCollAllocator *new_coll_allocator = NULL; \ + if (NULL == dest) { \ + if (OB_ISNULL(dest = reinterpret_cast(allocator.alloc(src.get_init_size())))) { \ + ret = OB_ALLOCATE_MEMORY_FAILED; \ + LOG_WARN("failed to alloc memory for collection", K(ret)); \ + } else { \ + TYPE *collection = static_cast(dest); \ + CK (OB_NOT_NULL(collection)); \ + LOG_INFO("src is: ", KP(&src), K(src), KP(dest), K(src.get_init_size())); \ + OX (new(collection)TYPE(src.get_id())); \ + } \ + } else { \ + if (!need_new_allocator && NULL == static_cast(dest)->get_allocator()) { \ + LOG_ERROR("Why I am here!!!"); \ + } \ + } \ + OX (collection = static_cast(dest)); \ + if (OB_SUCC(ret) \ + && (need_new_allocator || NULL == static_cast(dest)->get_allocator())) { \ + ObIAllocator *coll_allocator = static_cast(dest)->get_allocator(); \ + if (dynamic_cast(coll_allocator) != NULL && &allocator == coll_allocator) { \ + copy_allocator = coll_allocator; /*NOTICE, do not alloc new allocator, use old directly.*/ \ + } else { \ + if (OB_ISNULL(new_coll_allocator = \ + static_cast(allocator.alloc(sizeof(ObPLCollAllocator))))) { \ + ret = OB_ALLOCATE_MEMORY_FAILED; \ + LOG_WARN("failed to allocator memory for new collection allocator", K(ret)); \ + } else { \ + copy_allocator = new(new_coll_allocator)ObPLCollAllocator(collection); \ + } \ + } \ + } else { \ + ret = OB_ERR_UNEXPECTED; \ + LOG_ERROR("Should never be here!!!"); \ + } \ + if (OB_SUCC(ret)) { \ + ObObj destruct_obj; \ + destruct_obj.set_extend(reinterpret_cast(collection), collection->get_type()); \ + OZ (ObUserDefinedType::destruct_obj(destruct_obj, session)); \ + } \ + OX (collection->set_allocator(copy_allocator)); /*约定：在这里设置allocator，而不是通过deep_copy的参数传进去*/ \ + if (OB_FAIL(ret)) { \ + } else if (OB_FAIL(collection->deep_copy(static_cast(&src), NULL, ignore_del_element))) { \ + ObObj destruct_obj; \ + destruct_obj.set_extend(reinterpret_cast(collection), collection->get_type()); \ + int tmp = ObUserDefinedType::destruct_obj(destruct_obj, session); \ + LOG_WARN("fail to deep copy collection, release memory", K(ret), K(tmp)); \ + } \ + } \ + } while(0) +#endif switch (src.get_type()) { case PL_RECORD_TYPE: { @@ -1435,6 +3022,23 @@ int ObPLComposite::deep_copy(ObPLComposite &src, } break; +#ifdef OB_BUILD_ORACLE_PL + case PL_NESTED_TABLE_TYPE: { + COPY_COLLECTION(ObPLNestedTable); + } + break; + case PL_ASSOCIATIVE_ARRAY_TYPE: { + COPY_COLLECTION(ObPLAssocArray); + } + break; + case PL_VARRAY_TYPE: { + COPY_COLLECTION(ObPLVArray); + } + break; + case PL_OPAQUE_TYPE: { //forthrough + + } +#endif default: { ret = OB_ERR_UNEXPECTED; @@ -1471,6 +3075,24 @@ int ObPLComposite::copy_element(const ObObj &src, { int ret = OB_SUCCESS; if (src.is_ext()) { +#ifdef OB_BUILD_ORACLE_PL + if (PL_OPAQUE_TYPE == src.get_meta().get_extend_type()) { + ObPLOpaque *dest_composite = reinterpret_cast(dest.get_ext()); + ObPLOpaque *src_composite = reinterpret_cast(src.get_ext()); + CK (OB_NOT_NULL(src_composite)); + if (OB_SUCC(ret) && src_composite != dest_composite) { + OZ (ObSPIService::spi_copy_opaque(NULL, + &allocator, + *src_composite, + dest_composite, + OB_INVALID_ID)); + } + CK (OB_NOT_NULL(dest_composite)); + OX (dest.set_extend(reinterpret_cast(dest_composite), + src.get_meta().get_extend_type(), + src.get_val_len())); + } else { +#endif ObPLComposite *dest_composite = (dest.get_ext() == src.get_ext()) ? NULL : reinterpret_cast(dest.get_ext()); ObPLComposite *src_composite = reinterpret_cast(src.get_ext()); @@ -1505,6 +3127,9 @@ int ObPLComposite::copy_element(const ObObj &src, OX (dest.set_extend(reinterpret_cast(dest_composite), src.get_meta().get_extend_type(), src.get_val_len())); +#ifdef OB_BUILD_ORACLE_PL + } +#endif } else if (NULL != dest_type && NULL != session && !src.is_null()) { ObExprResType result_type; ObObjParam result; @@ -1532,6 +3157,21 @@ int ObPLComposite::assign(ObPLComposite *src, ObIAllocator *allocator) size = static_cast(this)->assign(static_cast(src), allocator); } break; +#ifdef OB_BUILD_ORACLE_PL + case PL_NESTED_TABLE_TYPE: { + size = static_cast(this)->assign(static_cast(src), + allocator); + } + break; + case PL_ASSOCIATIVE_ARRAY_TYPE: { + size = static_cast(this)->assign(static_cast(src), allocator); + } + break; + case PL_VARRAY_TYPE: { + size = static_cast(this)->assign(static_cast(src), allocator); + } + break; +#endif default: { LOG_WARN_RET(OB_ERR_UNEXPECTED, "unexpected composite to get init size", K(get_type())); } @@ -1551,6 +3191,20 @@ int64_t ObPLComposite::get_init_size() const } break; +#ifdef OB_BUILD_ORACLE_PL + case PL_NESTED_TABLE_TYPE: { + size = static_cast(this)->get_init_size(); + } + break; + case PL_ASSOCIATIVE_ARRAY_TYPE: { + size = static_cast(this)->get_init_size(); + } + break; + case PL_VARRAY_TYPE: { + size = static_cast(this)->get_init_size(); + } + break; +#endif default: { LOG_WARN_RET(OB_ERR_UNEXPECTED, "unexpected composite to get init size", K(get_type())); @@ -1593,6 +3247,14 @@ void ObPLComposite::print() const static_cast(this)->print(); } break; +#ifdef OB_BUILD_ORACLE_PL + case PL_NESTED_TABLE_TYPE: + case PL_ASSOCIATIVE_ARRAY_TYPE: + case PL_VARRAY_TYPE: { + static_cast(this)->print(); + } + break; +#endif default: { LOG_WARN_RET(OB_ERR_UNEXPECTED, "unexpected composite to print", K(get_type())); } @@ -2158,7 +3820,45 @@ int ObPLCollection::deserialize(common::ObIAllocator &allocator, const char *buf, const int64_t len, int64_t &pos) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSEDx(allocator, buf, len, pos); +#else + int64_t count = 0; + int64_t rowsize = 0; + int64_t first = 0; + int64_t last = 0; + + OZ (serialization::decode(buf, len, pos, count)); + OZ (serialization::decode(buf, len, pos, rowsize)); + OZ (serialization::decode(buf, len, pos, first)); + OZ (serialization::decode(buf, len, pos, last)); + CK (rowsize > 0); + + UNUSED(allocator); + CK (OB_NOT_NULL(get_allocator())); + OX (set_inited()); + OX (set_first(first)); + OX (set_last(last)); + if (OB_FAIL(ret)) { + } else if (is_associative_array()) { + ObPLAssocArray *assoc_table = static_cast(this); + OZ (ObSPIService::spi_extend_assoc_array( //TODO:@ryan.ly myst be bug here!!! + OB_INVALID_ID, NULL, *get_allocator(), *assoc_table, count)); + } else { + OZ (ObSPIService::spi_set_collection( + OB_INVALID_ID, NULL, *get_allocator(), *this, count, true)); + } + CK (OB_NOT_NULL(get_data())); + + if (OB_SUCC(ret)) { + char *table_data = reinterpret_cast(get_data()); + for (int64_t i = 0; OB_SUCC(ret) && i < count * rowsize / sizeof(ObObj); ++i) { + ObObj src_obj; + OZ (src_obj.deserialize(buf, len, pos)); + OZ (deep_copy_obj(*get_allocator(), src_obj, reinterpret_cast(table_data)[i])); + } + } +#endif return ret; } @@ -2400,6 +4100,996 @@ int ObPLCollection::exist(int64_t idx, ObObj &result) return ret; } +#ifdef OB_BUILD_ORACLE_PL +//---------- for ObPLAssocArray ---------- + +int ObPLAssocArray::first(ObObj &result) +{ + int ret = OB_SUCCESS; + int64_t first = get_first(); + if (OB_INVALID_INDEX == first) { + result.set_null(); + } else if (NULL == get_key()) { + result.set_int(first); + } else { + CK (OB_NOT_NULL(get_key(first - 1))); + OX (result = *(get_key(first - 1))); + } + return ret; +} + +int ObPLAssocArray::last(ObObj &result) +{ + int ret = OB_SUCCESS; + int64_t last = get_last(); + if (OB_INVALID_INDEX == last) { + result.set_null(); + } else if (NULL == get_key()) { + result.set_int(last); + } else { + CK (OB_NOT_NULL(get_key(last - 1))); + OX (result = *(get_key(last - 1))); + } + return ret; +} + +int ObPLAssocArray::prior(int64_t idx, ObObj &result) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(get_key()) && OB_ISNULL(get_sort())) { + OZ (ObPLCollection::prior(idx, result)); + } else if (OB_NOT_NULL(get_key()) && OB_NOT_NULL(get_sort())) { + bool need_search = false; + if (IndexRangeType::LESS_THAN_FIRST == idx) { + result.set_null(); + } else if (IndexRangeType::LARGE_THAN_LAST == idx) { + OX (idx = get_last()); + OZ (is_elem_deleted(idx - 1, need_search)); + } else if (idx > 0) { + if (idx == get_first()) { + idx = IndexRangeType::LESS_THAN_FIRST; + result.set_null(); + } else { + need_search = true; + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected assoc array", K(ret), K(idx)); + } + if (OB_SUCC(ret) && need_search) { + int64_t search = idx - 1; + do { + for (int64_t i = 0; i < get_count(); ++i) { + if (get_sort()[i] == search) { + search = i; + break; + } + } + if (search == (idx - 1)) { + for (int64_t i = 0; i < get_count(); ++i) { + LOG_WARN("debug assoc sort: ", K(i), K(get_sort()[i])); + } + result.set_null(); + break; + } else { + OZ (is_elem_deleted(search, need_search)); + OX (idx = (search + 1)); + } + } while (OB_SUCC(ret) && need_search); + } + if (OB_SUCC(ret) && !need_search && idx >= 1 && idx <= get_count()) { + CK (OB_NOT_NULL(get_key(idx - 1))); + OX (result = *(get_key(idx - 1))); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected assoc array", K(ret), K(get_key()), K(get_sort())); + } + return ret; +} + +int ObPLAssocArray::next(int64_t idx, ObObj &result) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(get_key()) && OB_ISNULL(get_sort())) { + OZ (ObPLCollection::next(idx, result)); + } else if (OB_NOT_NULL(get_key()) && OB_NOT_NULL(get_sort())) { + bool need_search = false; + if (IndexRangeType::LARGE_THAN_LAST == idx) { + result.set_null(); + } else if (IndexRangeType::LESS_THAN_FIRST == idx) { + if (OB_INVALID_INDEX == get_first()) { + result.set_null(); + } else { + OX (idx = get_first()); + OZ (is_elem_deleted(idx - 1, need_search)); + } + } else if (idx > 0) { + if (idx == get_last()) { + idx = IndexRangeType::LARGE_THAN_LAST; + result.set_null(); + } else { + need_search = true; + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected assoc array", K(ret), K(idx)); + } + CK (idx <= get_count()); + if (OB_SUCC(ret) && need_search) { + do { + int64_t search = get_sort()[idx - 1]; + if (OB_INVALID_INDEX == search) { + result.set_null(); + break; + } + OZ (is_elem_deleted(search, need_search)); + OX (idx = search + 1); + } while (OB_SUCC(ret) && need_search); + } + if (OB_SUCC(ret) && !need_search && idx > 0 && idx <= get_count()) { + CK (OB_NOT_NULL(get_key(idx - 1))); + OX (result = *(get_key(idx - 1))); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected assoc array", K(ret), K(get_key()), K(get_sort())); + } + return ret; +} + +int ObPLAssocArray::exist(int64_t idx, ObObj &result) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(get_key()) && OB_ISNULL(get_sort())) { + OZ (ObPLCollection::exist(idx, result)); + } else if (OB_INVALID_INDEX == idx) { + OX (result.set_tinyint(false)); + } else if (OB_NOT_NULL(get_key()) && OB_NOT_NULL(get_sort())) { + bool is_del = false; + OZ (is_elem_deleted(idx - 1, is_del)); + OX (result.set_tinyint(is_del ? false : true)); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("assoc array is illegal", K(ret)); + } + return ret; +} + +int ObPLAssocArray::deep_copy(ObPLCollection *src, ObIAllocator *allocator, bool ignore_del_element) +{ + int ret = OB_SUCCESS; + ObObj *key = NULL; + int64_t *sort = NULL; + CK (OB_NOT_NULL(src)); + CK (src->is_associative_array()); + OZ (ObPLCollection::deep_copy(src, allocator, ignore_del_element)); + if (OB_SUCC(ret) && src->get_count() > 0) { + ObPLAssocArray *src_aa = static_cast(src); + CK (OB_NOT_NULL(src_aa)); + if (NULL != src_aa->get_key() && NULL != src_aa->get_sort()) { + CK(OB_NOT_NULL(get_allocator())); + CK (OB_NOT_NULL(key + = static_cast(get_allocator()->alloc(src->get_count() * sizeof(ObObj))))); + CK (OB_NOT_NULL(sort + = static_cast(get_allocator()->alloc(src->get_count() * sizeof(int64_t))))); + for (int64_t i = 0; OB_SUCC(ret) && i < src->get_count(); ++i) { + OZ (deep_copy_obj(*get_allocator(), *(src_aa->get_key(i)), key[i])); + OX (sort[i] = src_aa->get_sort(i)); + } + } else if (NULL == src_aa->get_key() && NULL == src_aa->get_sort()) { + //Associative array的优化会出现这种情况，拷贝的时候同样按优化拷 + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected associative array", K(*src_aa), K(ret)); + } + } + OX (set_key(key)); + OX (set_sort(sort)); + return ret; +} + +int ObPLAssocArray::update_first() +{ + int ret = OB_SUCCESS; + if (!is_inited()) { + ret = OB_ERR_COLLECION_NULL; + LOG_WARN("pl collection is not inited", K(ret)); + } else if (0 > get_count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("collection is empty", K(get_count()), K(ret)); + } else { + if (OB_ISNULL(get_key()) && OB_ISNULL(get_sort())) { + update_first_impl(); + } else if (OB_NOT_NULL(get_key()) && OB_NOT_NULL(get_sort())){ + int64_t first = first_ - 1; + if (OB_INVALID_INDEX == first_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid firt property", K(first), K(last_)); + } else { + bool is_deleted = false; + int64_t i = 0; + do { + if (OB_FAIL(is_elem_deleted(first, is_deleted))) { + LOG_WARN("test element deleted failed.", K(ret)); + } else { + if (is_deleted) { + OX (first = get_sort()[first]); + //这是最后一个了，都没有找到，就不存在. + if (0 > first) { + OX (set_first(OB_INVALID_INDEX)); + break; + } + } else { + OX (set_first(first + 1)); + break; + } + ++i; + } + } while (i < count_ && OB_SUCC(ret)); + if (i >= count_) { + OX(set_first(OB_INVALID_INDEX)); + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("illegal status of assocative array.", K(count_)); + } + } + return ret; +} + + +int ObPLAssocArray::update_last() +{ + int ret = OB_SUCCESS; + if (!is_inited()) { + ret = OB_ERR_COLLECION_NULL; + LOG_WARN("pl collection is not inited", K(ret)); + } else if (0 > get_count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("collection is empty", K(get_count()), K(ret)); + } else { + if (OB_ISNULL(get_key()) && OB_ISNULL(get_sort())) { + update_last_impl(); + } else if (OB_NOT_NULL(get_key()) && OB_NOT_NULL(get_sort())){ + int64_t last = last_ - 1; + if (OB_INVALID_INDEX == last_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid last property", K(ret), K(last), K(first_), K(count_)); + } else { + bool is_deleted = false; + int64_t i = 0; + int64_t tmp_last = last; + do { + if (OB_FAIL(is_elem_deleted(tmp_last, is_deleted))) { + LOG_WARN("test element deleted failed.", K(ret)); + } else { + if (is_deleted) { + for (int j = 0; OB_SUCC(ret) && j < count_; ++j) { + if (get_sort()[j] == tmp_last) { + tmp_last = j; + break; + } + } + // 没有前面的元素，可能只有一个 + if (tmp_last == last) { + if (1 == count_) { + OX (set_last(OB_INVALID_INDEX)); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("illegal assoc array status", K(count_), K(first_), K(last_)); + } + break; + } + } else { + OX (set_last(tmp_last + 1)); + break; + } + ++i; + } + } while (i < count_ && OB_SUCC(ret)); + if (i >= count_) { + OX(set_last(OB_INVALID_INDEX)); + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("illegal status of assocative array.", K(count_)); + } + } + return ret; +} + +int64_t ObPLAssocArray::get_first() +{ + return first_; +} + +int64_t ObPLAssocArray::get_last() +{ + return last_; +} + +int ObPLOpaque::deep_copy(ObPLOpaque *dst) +{ + int ret = OB_SUCCESS; + CK (OB_NOT_NULL(dst)); + OX (dst->~ObPLOpaque()); + OX (dst = new(dst)ObPLOpaque()); + return ret; +} + +int64_t ObPLOpaque::get_init_size() const +{ + int64_t init_size = sizeof(ObPLOpaque); + init_size = init_size < sizeof(ObPLAnyData) ? sizeof(ObPLAnyData) : init_size; + init_size = init_size < sizeof(ObPLAnyType) ? sizeof(ObPLAnyType) : init_size; + init_size = init_size < sizeof(ObPLXmlType) ? sizeof(ObPLXmlType) : init_size; + init_size = init_size < sizeof(ObPLJsonBaseType) ? sizeof(ObPLJsonBaseType) : init_size; + return init_size; +} + +bool ObPLAnyType::is_obj_type(ObPLAnyType::TypeCode code) +{ + bool is_obj_type = false; + switch (code) { + case TypeCode::TYPECODE_DATE: + case TypeCode::TYPECODE_NUMBER: + case TypeCode::TYPECODE_RAW: + case TypeCode::TYPECODE_CHAR: + case TypeCode::TYPECODE_VARCHAR2: + case TypeCode::TYPECODE_VARCHAR: + case TypeCode::TYPECODE_MLSLABEL: + case TypeCode::TYPECODE_BLOB: + case TypeCode::TYPECODE_BFILE: + case TypeCode::TYPECODE_CLOB: + case TypeCode::TYPECODE_CFILE: + case TypeCode::TYPECODE_TIMESTAMP: + case TypeCode::TYPECODE_TIMESTAMP_TZ: + case TypeCode::TYPECODE_TIMESTAMP_LTZ: + case TypeCode::TYPECODE_INTERVAL_YM: + case TypeCode::TYPECODE_INTERVAL_DS: + case TypeCode::TYPECODE_NCHAR: + case TypeCode::TYPECODE_NVARCHAR2: + case TypeCode::TYPECODE_NCLOB: + case TypeCode::TYPECODE_BFLOAT: + case TypeCode::TYPECODE_BDOUBLE: + case TypeCode::TYPECODE_UROWID: { + is_obj_type = true; + } break; + default: { + } break; + } + return is_obj_type; +} + +bool ObPLAnyType::is_valid_type(ObPLAnyType::TypeCode code) +{ + bool valid = false; + switch (code) { + case TypeCode::TYPECODE_DATE: + case TypeCode::TYPECODE_NUMBER: + case TypeCode::TYPECODE_RAW: + case TypeCode::TYPECODE_CHAR: + case TypeCode::TYPECODE_VARCHAR2: + case TypeCode::TYPECODE_VARCHAR: + case TypeCode::TYPECODE_MLSLABEL: + case TypeCode::TYPECODE_BLOB: + case TypeCode::TYPECODE_BFILE: + case TypeCode::TYPECODE_CLOB: + case TypeCode::TYPECODE_CFILE: + case TypeCode::TYPECODE_TIMESTAMP: + case TypeCode::TYPECODE_TIMESTAMP_TZ: + case TypeCode::TYPECODE_TIMESTAMP_LTZ: + case TypeCode::TYPECODE_INTERVAL_YM: + case TypeCode::TYPECODE_INTERVAL_DS: + case TypeCode::TYPECODE_REF: + case TypeCode::TYPECODE_OBJECT: + case TypeCode::TYPECODE_VARRAY: + case TypeCode::TYPECODE_TABLE: + case TypeCode::TYPECODE_NAMEDCOLLECTION: + case TypeCode::TYPECODE_OPAQUE: + case TypeCode::TYPECODE_NCHAR: + case TypeCode::TYPECODE_NVARCHAR2: + case TypeCode::TYPECODE_NCLOB: + case TypeCode::TYPECODE_BFLOAT: + case TypeCode::TYPECODE_BDOUBLE: + case TypeCode::TYPECODE_UROWID: { + valid = true; + } break; + default: { + } break; + } + return valid; +} + +int ObPLAnyType::pltype_to_typecode( + const ObPLDataType &pl_type, ObPLAnyType::TypeCode &typecode) +{ + int ret = OB_SUCCESS; + typecode = ObPLAnyType::TypeCode::TYPECODE_INVALID; + if (pl_type.is_obj_type()) { + +#define TO_TYPECODE(obj_type, code) \ + case obj_type: { \ + typecode = code; \ + } break; + + switch (pl_type.get_obj_type()) { + TO_TYPECODE(ObDateType, ObPLAnyType::TypeCode::TYPECODE_DATE); + TO_TYPECODE(ObDateTimeType, ObPLAnyType::TypeCode::TYPECODE_DATE); + + TO_TYPECODE(ObTinyIntType, ObPLAnyType::TypeCode::TYPECODE_NUMBER); + TO_TYPECODE(ObSmallIntType, ObPLAnyType::TypeCode::TYPECODE_NUMBER); + TO_TYPECODE(ObMediumIntType, ObPLAnyType::TypeCode::TYPECODE_NUMBER); + TO_TYPECODE(ObInt32Type, ObPLAnyType::TypeCode::TYPECODE_NUMBER); + TO_TYPECODE(ObIntType, ObPLAnyType::TypeCode::TYPECODE_NUMBER); + TO_TYPECODE(ObUTinyIntType, ObPLAnyType::TypeCode::TYPECODE_NUMBER); + TO_TYPECODE(ObUSmallIntType, ObPLAnyType::TypeCode::TYPECODE_NUMBER); + TO_TYPECODE(ObUMediumIntType, ObPLAnyType::TypeCode::TYPECODE_NUMBER); + TO_TYPECODE(ObUInt32Type, ObPLAnyType::TypeCode::TYPECODE_NUMBER); + TO_TYPECODE(ObUInt64Type, ObPLAnyType::TypeCode::TYPECODE_NUMBER); + TO_TYPECODE(ObNumberType, ObPLAnyType::TypeCode::TYPECODE_NUMBER); + TO_TYPECODE(ObUNumberType, ObPLAnyType::TypeCode::TYPECODE_NUMBER); + TO_TYPECODE(ObNumberFloatType, ObPLAnyType::TypeCode::TYPECODE_NUMBER); + + TO_TYPECODE(ObRawType, ObPLAnyType::TypeCode::TYPECODE_RAW); + TO_TYPECODE(ObCharType, ObPLAnyType::TypeCode::TYPECODE_CHAR); + + TO_TYPECODE(ObVarcharType, ObPLAnyType::TypeCode::TYPECODE_VARCHAR2); + // typecode = ObPLAnyType::TypeCode::TYPECODE_VARCHAR; + // typecode = ObPLAnyType::TypeCode::TYPECODE_MLSLABEL; + + case ObLobType: { + if (IS_CLUSTER_VERSION_BEFORE_4_1_0_0) { + if (pl_type.get_meta_type()->is_blob_locator()) { + typecode = ObPLAnyType::TypeCode::TYPECODE_BLOB; + } else { + typecode = ObPLAnyType::TypeCode::TYPECODE_CLOB; + } + // typecode = ObPLAnyType::TypeCode::TYPECODE_BFILE; + // typecode = ObPLAnyType::TypeCode::TYPECODE_CFILE; + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid lob type for after 4.1", K(ret)); + } + break; + } + + case ObLongTextType: { + if (!IS_CLUSTER_VERSION_BEFORE_4_1_0_0) { + if (pl_type.get_meta_type()->is_blob()) { + typecode = ObPLAnyType::TypeCode::TYPECODE_BLOB; + } else { + typecode = ObPLAnyType::TypeCode::TYPECODE_CLOB; + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid lob type for before 4.1", K(ret)); + } + break; + } + + TO_TYPECODE(ObTimestampType, ObPLAnyType::TypeCode::TYPECODE_TIMESTAMP); + TO_TYPECODE(ObTimestampNanoType, ObPLAnyType::TypeCode::TYPECODE_TIMESTAMP); + TO_TYPECODE(ObTimestampTZType, ObPLAnyType::TypeCode::TYPECODE_TIMESTAMP_TZ); + TO_TYPECODE(ObTimestampLTZType, ObPLAnyType::TypeCode::TYPECODE_TIMESTAMP_LTZ); + TO_TYPECODE(ObIntervalYMType, ObPLAnyType::TypeCode::TYPECODE_INTERVAL_YM); + TO_TYPECODE(ObIntervalDSType, ObPLAnyType::TypeCode::TYPECODE_INTERVAL_DS); + TO_TYPECODE(ObNCharType, ObPLAnyType::TypeCode::TYPECODE_NCHAR); + TO_TYPECODE(ObNVarchar2Type, ObPLAnyType::TypeCode::TYPECODE_NVARCHAR2); + // typecode = ObPLAnyType::TypeCode::TYPECODE_NCLOB; + TO_TYPECODE(ObFloatType, ObPLAnyType::TypeCode::TYPECODE_BFLOAT); + TO_TYPECODE(ObDoubleType, ObPLAnyType::TypeCode::TYPECODE_BDOUBLE); + TO_TYPECODE(ObURowIDType, ObPLAnyType::TypeCode::TYPECODE_UROWID); + default: { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not supported type for type_to_typecode", K(ret), K(pl_type.get_obj_type())); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "type for type_to_type_code"); + } + } +#undef TO_TYPECODE + + } else if (pl_type.is_record_type()) { + typecode = ObPLAnyType::TypeCode::TYPECODE_OBJECT; + } else if (pl_type.is_nested_table_type() || pl_type.is_associative_array_type()) { + if (pl_type.is_udt_type()) { + typecode = ObPLAnyType::TypeCode::TYPECODE_NAMEDCOLLECTION; + } else { + typecode = ObPLAnyType::TypeCode::TYPECODE_TABLE; + } + } else if (pl_type.is_varray_type()) { + typecode = ObPLAnyType::TypeCode::TYPECODE_VARRAY; + } else if (pl_type.is_ref_cursor_type()) { + typecode = ObPLAnyType::TypeCode::TYPECODE_REF; + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not supported type for type_to_typecode", K(ret), K(pl_type.get_obj_type())); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "type for type_to_typecode"); + } + return ret; +} + +int ObPLAnyType::deep_copy(ObPLOpaque *dst) +{ + int ret = OB_SUCCESS; + ObPLAnyType *copy = NULL; + CK (OB_NOT_NULL(dst)); + OZ (ObPLOpaque::deep_copy(dst)); + CK (OB_NOT_NULL(copy = new(dst)ObPLAnyType())); + + if (OB_SUCC(ret) && OB_NOT_NULL(type_)) { + ObPLDataType *dst_type = NULL; + OZ (ObPLDataType::deep_copy_pl_type(copy->get_allocator(), *type_, dst_type)); + OX (copy->set_type_ptr(dst_type)); + } + + OX (copy->set_in_begincreate(in_begincreate_)); + OX (copy->set_typecode(code_)); + OX (copy->set_rowsize(rowsize_)); + + return ret; +} + +bool ObPLAnyType::typecode_compatible(ObPLAnyType::TypeCode &src, ObPLAnyType::TypeCode &dst) +{ + return src == dst + || (ObPLAnyType::TypeCode::TYPECODE_TABLE == dst + && (ObPLAnyType::TypeCode::TYPECODE_VARRAY == src + || ObPLAnyType::TypeCode::TYPECODE_TABLE == src + || ObPLAnyType::TypeCode::TYPECODE_NAMEDCOLLECTION == src)) + || (ObPLAnyType::TYPECODE_VARCHAR == dst && ObPLAnyType::TYPECODE_VARCHAR2 == src) + || (ObPLAnyType::TYPECODE_VARCHAR2 == dst && ObPLAnyType::TYPECODE_VARCHAR == src); +} + +int ObPLAnyData::deep_copy(ObPLOpaque *dst) +{ + int ret = OB_SUCCESS; + + ObPLAnyData *copy = NULL; + OZ (ObPLOpaque::deep_copy(dst)); + CK (OB_NOT_NULL(copy = new(dst)ObPLAnyData())); + OX (copy->set_in_begincreate(in_begincreate_)); + OX (copy->set_piecewise(in_piecewise_)); + OX (copy->set_rowsize(rowsize_)); + OX (copy->set_current_pos(current_pos_)); + OX (copy->set_is_last_elem(is_last_elem_)); + OX (copy->set_is_no_data(is_no_data_)); + OX (copy->set_type_code(type_code_)); + + if (OB_NOT_NULL(data_)) { + OZ (copy->set_data(*data_)); + } + if (OB_NOT_NULL(type_)) { + ObPLDataType *dst_type = NULL; + OZ (ObPLDataType::deep_copy_pl_type( + copy->get_allocator(), *(const_cast(type_)), dst_type)); + CK (OB_NOT_NULL(dst_type)); + OX (copy->set_type(dst_type)); + } + return ret; +} + +int ObPLAnyData::set_data(const ObObj &obj) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(data_)) { + if (OB_ISNULL(data_ = reinterpret_cast(get_allocator().alloc(sizeof(ObObj))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("faild to alloc memory for new object", K(ret)); + } else { + new (data_) ObObj(); + } + } + if (obj.is_ext()) { + OZ (ObUserDefinedType::deep_copy_obj(get_allocator(), obj, *data_, true, true)); + } else { + OZ (deep_copy_obj(get_allocator(), obj, *data_)); + } + return ret; +} + +int ObPLAnyData::get_current_data( + ObObj &obj, ObPLAnyType::TypeCode &dst_typecode) +{ + int ret = OB_SUCCESS; + +#define CHECK_TYPE(src_type) \ + OZ (ObPLAnyType::pltype_to_typecode(src_type, src_typecode)); \ + if (OB_SUCC(ret) && !ObPLAnyType::typecode_compatible(src_typecode, dst_typecode)) { \ + ret = OB_ERR_TYPE_MISMATCH; \ + LOG_WARN("ORA-22626: Type Mismatch while constructing or accessing OCIAnyData", \ + K(ret), KPC(type_), K(dst_typecode), K(src_typecode), K(in_piecewise_)); \ + } + + ObPLAnyType::TypeCode src_typecode = ObPLAnyType::TypeCode::TYPECODE_INVALID; + const ObPLDataType *pl_type = NULL; + CK (OB_NOT_NULL(type_)); + CK (OB_NOT_NULL(data_)); + OZ (get_current_type(pl_type)); + CK (OB_NOT_NULL(pl_type)); + if (OB_FAIL(ret)) { + } else if (is_in_piecewise()) { + if (get_type()->is_record_type()) { + const ObRecordType *c_type = dynamic_cast(get_type()); + ObPLRecord *record = reinterpret_cast(data_->get_ext()); + if (OB_ISNULL(c_type)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("src type is incomplete, can not extract data."); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "extract data with incomplete type"); + } + CK (OB_NOT_NULL(record)); + + CK (OB_NOT_NULL(c_type->get_record_member_type(current_pos_))); + CHECK_TYPE(*(c_type->get_record_member_type(current_pos_))); + OZ (record->get_element(current_pos_, obj)); + if (OB_SUCC(ret) + && current_pos_ == (c_type->get_member_count() - 1)) { + set_is_no_data(true); + } + } else if (get_type()->is_collection_type()) { + const ObCollectionType *c_type = dynamic_cast(get_type()); + ObPLCollection *coll = reinterpret_cast(data_->get_ext()); + ObObj* coll_list = NULL; + + if (OB_ISNULL(c_type)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("src type is incomplete, can not extract data."); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "extract data with incomplete type"); + } + CK (OB_NOT_NULL(coll)); + CK (OB_NOT_NULL(coll_list = reinterpret_cast(coll->get_data()))); + + CHECK_TYPE(c_type->get_element_type()); + + if (OB_FAIL(ret)) { + } else if (current_pos_ >= coll->get_count()) { + ret = OB_ERR_DATA_NOT_WELL_FORMAT; + LOG_WARN("ORA-22625: OCIAnyData is not well-formed", K(ret), K(current_pos_), KPC(coll)); + } else { + obj = coll_list[current_pos_]; + set_is_no_data(current_pos_ == (coll->get_count() - 1)); + } + } + OX (current_pos_ += 1); + } else { + CHECK_TYPE(*pl_type); + OX (obj = *data_); + } +#undef CHECK_TYPE + return ret; +} + +int ObPLAnyData::get_current_type(const ObPLDataType *&pl_type) +{ + int ret = OB_SUCCESS; + CK (OB_NOT_NULL(type_)); + if (is_in_piecewise()) { + if (get_type()->is_record_type()) { + const ObRecordType *r_type = dynamic_cast(type_); + if (OB_ISNULL(r_type)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("src type is incomplete, can not extract data."); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "extract data with incomplete type"); + } else if (current_pos_ >= r_type->get_member_count()) { + ret = OB_ERR_DATA_NOT_WELL_FORMAT; + LOG_WARN("ORA-22625: OCIAnyData is not well-formed", + K(ret), K(current_pos_), KPC(r_type)); + } else { + pl_type = r_type->get_record_member_type(current_pos_); + } + } else if (get_type()->is_collection_type()) { + const ObCollectionType *c_type = dynamic_cast(type_); + if (OB_ISNULL(c_type)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("src type is incomplete, can not extract data."); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "extract data with incomplete type"); + } else { + pl_type = c_type->get_member(0); + } + } + } else { + pl_type = type_; + } + return ret; +} + +int ObPLAnyData::set_record_element(ObObj &obj) +{ + int ret = OB_SUCCESS; + const ObRecordType *r_type = dynamic_cast(type_); + ObObj* record_list = NULL; + ObObj* new_data = NULL; + if (OB_ISNULL(r_type)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("src type is incomplete, can not extract data."); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "extract data with incomplete type"); + } else if (obj.is_ext()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not supported record element also a complex value", K(ret), K(obj)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "record element also a complex value in Set*()"); + } else if (OB_ISNULL(data_)) { + if (OB_ISNULL(new_data = reinterpret_cast(get_allocator().alloc(sizeof(ObObj))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("faild to alloc memory for new object", K(ret)); + } else if (OB_ISNULL(record_list = reinterpret_cast + (get_allocator().alloc( + r_type->get_data_offset(r_type->get_record_member_count()) + sizeof(ObObj) * r_type->get_member_count())))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for new record list", + K(ret), K(r_type->get_member_count())); + } else { + new (new_data) ObObj(ObExtendType); + new_data->set_extend(reinterpret_cast(record_list), PL_RECORD_TYPE); + ObPLRecord *record = new (record_list) ObPLRecord(r_type->get_user_type_id(), r_type->get_record_member_count()); + ObObj *member = NULL; + for (int64_t i = 0; OB_SUCC(ret) && i < r_type->get_record_member_count(); ++i) { + CK (OB_NOT_NULL(r_type->get_member(i))); + OZ (record->get_element(i, member)); + CK (OB_NOT_NULL(member)); + CK (r_type->get_member(i)->is_obj_type()); + OX (new (member) ObObj(ObNullType)); + } + OX (current_pos_ = 0); + OZ (record->get_element(current_pos_, member)); + OZ (deep_copy_obj(get_allocator(), obj, *member)); + OX (data_ = new_data); + } + } else { + ObPLRecord *record = NULL; + ObObj *member = NULL; + CK (data_->is_ext()); + CK (OB_NOT_NULL(record = reinterpret_cast(data_->get_ext()))); + OZ (record->get_element(current_pos_, member)); + CK (OB_NOT_NULL(member)); + OZ (deep_copy_obj(get_allocator(), obj, *member)); + } + OX (current_pos_ += 1); + if (OB_SUCC(ret) && current_pos_ >= r_type->get_member_count()) { + set_is_last_elem(true); + } + return ret; +} + +int ObPLAnyData::set_collection_element(ObObj &obj) +{ + int ret = OB_SUCCESS; + const ObCollectionType *c_type = dynamic_cast(type_); + ObPLNestedTable *new_coll = NULL; + ObObj* new_data = NULL; + int64_t init_size = 0; + if (OB_ISNULL(c_type)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("src type is incomplete, can not extract data."); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "extract data with incomplete type"); + } else if (OB_ISNULL(data_)) { + if (OB_ISNULL(new_data = reinterpret_cast(get_allocator().alloc(sizeof(ObObj))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("faild to alloc memory for new object", K(ret)); + } else if (OB_FAIL(c_type->get_init_size(init_size))) { + LOG_WARN("failed to get init size", K(ret), KPC(c_type)); + } else if (OB_ISNULL(new_coll = + reinterpret_cast(get_allocator().alloc(init_size)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("faild to alloc memory for new collection", K(ret)); + } else if (c_type->is_associative_array_type()) { + new (new_coll) ObPLAssocArray(c_type->get_user_type_id()); + } else if (c_type->is_varray_type()) { + ObPLVArray *varray = NULL; + const ObVArrayType *v_type = NULL; + new (new_coll) ObPLVArray(c_type->get_user_type_id()); + CK (OB_NOT_NULL(varray = static_cast(new_coll))); + CK (OB_NOT_NULL(v_type = static_cast(c_type))); + OX (varray->set_capacity(v_type->get_capacity())); + } else if (c_type->is_nested_table_type()) { + new (new_coll) ObPLNestedTable(c_type->get_user_type_id()); + } + + if (OB_SUCC(ret)) { + ObElemDesc elem_desc; + elem_desc.set_pl_type(c_type->get_element_type().get_type()); + elem_desc.set_not_null(c_type->get_element_type().get_not_null()); + if (OB_ISNULL(c_type->get_element_type().get_data_type())) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not support set composite for anydata", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "set composite for anydata"); + } else { + elem_desc.set_data_type(*(c_type->get_element_type().get_data_type())); + elem_desc.set_field_count(1); + } + OX (new_coll->set_element_desc(elem_desc)); + } + + OX (new_coll->set_inited()); + OZ (ObSPIService::spi_set_collection(OB_INVALID_ID, NULL, get_allocator(), *new_coll, 1, true)); //TODO:@ryan.ly here must by a BUG!!! + OX (new (new_data) ObObj()); + OX (new_data->set_extend(reinterpret_cast(new_coll), c_type->get_type())); + CK (0 == current_pos_); + } else { + CK (data_->is_ext()); + CK (OB_NOT_NULL(new_coll = reinterpret_cast(data_->get_ext()))); + OZ (ObSPIService::spi_set_collection(OB_INVALID_ID, NULL, get_allocator(), *new_coll, 1, true)); + } + if (OB_SUCC(ret)) { + ObObj* elements = reinterpret_cast( + reinterpret_cast(new_coll->get_data()) + + (sizeof(ObObj) * (new_coll->get_count() - 1))); + CK (OB_NOT_NULL(elements)); + if (!obj.is_ext()) { + OZ (deep_copy_obj(*new_coll->get_allocator(), obj, *elements)); + } else { + // TODO: + } + } + if (OB_SUCC(ret) && OB_ISNULL(data_)) { + data_ = new_data; + } + return ret; +} + +int ObPLAnyData::set_current_data( + ObObj &obj, ObPLAnyType::TypeCode &src_typecode, bool is_last_elem) +{ + int ret = OB_SUCCESS; + const ObPLDataType *pl_type = NULL; + ObPLAnyType::TypeCode dst_typecode = ObPLAnyType::TypeCode::TYPECODE_INVALID; + + CK (is_in_begincreate()); + CK (is_in_piecewise()); + CK (OB_NOT_NULL(get_type())); + CK (get_type()->is_record_type() || get_type()->is_collection_type()); + OZ (get_current_type(pl_type)); + CK (OB_NOT_NULL(pl_type)); + OZ (ObPLAnyType::pltype_to_typecode(*pl_type, dst_typecode)); + + if (OB_FAIL(ret)) { + } else if (!ObPLAnyType::typecode_compatible(dst_typecode, src_typecode)) { + ret = OB_ERR_TYPE_MISMATCH; + LOG_WARN("ORA-22626: Type Mismatch while constructing or accessing OCIAnyData", + K(ret), KPC(type_), K(dst_typecode), K(src_typecode), K(in_piecewise_)); + } else if (is_last_elem_) { + ret = OB_ERR_INCORRECT_METHOD_USAGE; + LOG_WARN("ORA-22370: incorrect usage of method Set*()", K(ret), KPC(this)); + LOG_USER_ERROR(OB_ERR_INCORRECT_METHOD_USAGE, "Set*()"); + } else if (get_type()->is_record_type()) { + OZ (set_record_element(obj)); + } else if (get_type()->is_collection_type()) { + OZ (set_collection_element(obj)); + OX (set_is_last_elem(is_last_elem)); + } + return ret; +} + +int ObPLAssocArray::get_serialize_size(int64_t &size) +{ + int ret = OB_SUCCESS; + int64_t key_sort_cnt = 0; // 紧密数组, key和sort是null + OZ (ObPLCollection::get_serialize_size(size)); + OX (key_sort_cnt = OB_NOT_NULL(get_key()) ? get_count() : 0); + OX (size += serialization::encoded_length(key_sort_cnt)); + for (int64_t i = 0; OB_SUCC(ret) && i < key_sort_cnt; ++i) { + OZ (size += get_key(i)->get_serialize_size()); + OX (size += serialization::encoded_length(get_sort(i))); + } + return ret; +} + +int ObPLAssocArray::serialize(char* buf, const int64_t len, int64_t& pos) +{ + int ret = OB_SUCCESS; + int64_t key_sort_cnt = 0; + OZ (ObPLCollection::serialize(buf, len, pos)); + OX (key_sort_cnt = OB_NOT_NULL(get_key()) ? get_count() : 0); + OZ (serialization::encode(buf, len, pos, key_sort_cnt)); + for (int64_t i = 0; OB_SUCC(ret) && i < key_sort_cnt; ++i) { + OZ (get_key(i)->serialize(buf, len, pos)); + OZ (serialization::encode(buf, len, pos, get_sort(i))); + } + return ret; +} + +int ObPLAssocArray::deserialize(common::ObIAllocator &allocator, + const char *buf, const int64_t len, int64_t &pos) +{ + int ret = OB_SUCCESS; + int64_t key_sort_cnt = 0; + OZ (ObPLCollection::deserialize(allocator, buf, len, pos)); + OZ (serialization::decode(buf, len, pos, key_sort_cnt)); + if (OB_FAIL(ret)) { + } else if (0 == key_sort_cnt) { + set_key(NULL); + set_sort(NULL); + } else { + CK (key_sort_cnt == get_count()); + for (int64_t i = 0; OB_SUCC(ret) && i < key_sort_cnt; ++i) { + ObObj src_obj; + OZ (src_obj.deserialize(buf, len, pos)); + OZ (deep_copy_obj(*get_allocator(), src_obj, *get_key(i))); + OZ (serialization::decode(buf, len, pos, get_sort()[i])); + } + } + return ret; +} + +int ObPLXmlType::deep_copy(ObPLOpaque *dst) +{ + int ret = OB_SUCCESS; + + ObPLXmlType *copy = NULL; + OZ (ObPLOpaque::deep_copy(dst)); + CK (OB_NOT_NULL(copy = new(dst)ObPLXmlType())); + + if (OB_NOT_NULL(data_)) { + ObObj *new_data = NULL; + if (OB_ISNULL(new_data = static_cast(copy->get_allocator().alloc(sizeof(ObObj))))) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("alloc memory for start_obj failed", K(ret)); + } else { + copy->set_data(new_data); + } + if (OB_SUCC(ret)) { + OZ(ob_write_obj(copy->get_allocator(), *data_, *(copy->get_data()))); + } + } + + return ret; +} + +int ObPLJsonBaseType::deep_copy(ObPLOpaque *dst) +{ + int ret = OB_SUCCESS; + + ObPLJsonBaseType *copy = NULL; + OZ (ObPLOpaque::deep_copy(dst)); + CK (OB_NOT_NULL(copy = new(dst)ObPLJsonBaseType())); + OX (copy->set_err_behavior(static_cast(behavior_))); + if (OB_NOT_NULL(data_)) { + OX (copy->set_data(data_)); + } + + return ret; +} + +//---------- for ObPLVarray ---------- + +int ObPLVArray::deep_copy(ObPLCollection *src, ObIAllocator *allocator, bool ignore_del_element) +{ + int ret = OB_SUCCESS; + ObPLVArray *src_va = NULL; + CK (OB_NOT_NULL(src)); + CK (src->is_varray()); + OZ (ObPLCollection::deep_copy(src, allocator, ignore_del_element)); + CK (OB_NOT_NULL(src_va = static_cast(src))); + OX (set_capacity(src_va->get_capacity())); + return ret; +} + +int ObPLVArray::get_serialize_size(int64_t &size) +{ + int ret = OB_SUCCESS; + OZ (ObPLCollection::get_serialize_size(size)); + OX (size += serialization::encoded_length(get_capacity())); + return ret; +} + +int ObPLVArray::serialize(char* buf, const int64_t len, int64_t& pos) +{ + int ret = OB_SUCCESS; + OZ (ObPLCollection::serialize(buf, len, pos)); + OZ (serialization::encode(buf, len, pos, get_capacity())); + return ret; +} + +int ObPLVArray::deserialize(common::ObIAllocator &allocator, + const char *buf, const int64_t len, int64_t &pos) +{ + int ret = OB_SUCCESS; + OZ (ObPLCollection::deserialize(allocator, buf, len, pos)); + OZ (serialization::decode(buf, len, pos, capacity_)); + return ret; +} +#endif } // namespace pl } // namespace oceanbase diff --git a/src/pl/ob_pl_user_type.h b/src/pl/ob_pl_user_type.h index 6dbbe8fea..cedfb4e40 100644 --- a/src/pl/ob_pl_user_type.h +++ b/src/pl/ob_pl_user_type.h @@ -153,6 +153,74 @@ protected: common::ObString type_name_; }; +#ifdef OB_BUILD_ORACLE_PL +//---------- for ObUserDefinedSubType ---------- + +class ObUserDefinedSubType : public ObUserDefinedType +{ +public: + ObUserDefinedSubType() + : ObUserDefinedType(PL_SUBTYPE), base_type_() {} + virtual ~ObUserDefinedSubType() {} + + inline void set_base_type(ObPLDataType &base_type) { base_type_ = base_type; } + inline const ObPLDataType* get_base_type() const { return &base_type_; } + + int deep_copy(common::ObIAllocator &alloc, const ObUserDefinedSubType &other); + +public: + virtual int64_t get_member_count() const { return 1; } + virtual const ObPLDataType *get_member(int64_t i) const { return 0 == i ? &base_type_ : NULL; } + virtual int generate_copy(ObPLCodeGenerator &generator, + const ObPLBlockNS &ns, + jit::ObLLVMValue &allocator, + jit::ObLLVMValue &src, + jit::ObLLVMValue &dest, + bool in_notfound, + bool in_warning, + uint64_t package_id = OB_INVALID_ID) const; + virtual int generate_construct(ObPLCodeGenerator &generator, + const ObPLINS &ns, + jit::ObLLVMValue &value, + const pl::ObPLStmt *stmt = NULL) const; + virtual int generate_new(ObPLCodeGenerator &generator, + const ObPLINS &ns, + jit::ObLLVMValue &value, + const pl::ObPLStmt *s = NULL) const; + virtual int newx(common::ObIAllocator &allocator, + const ObPLINS *ns, + int64_t &ptr) const; + + virtual int get_all_depended_user_type(const ObPLResolveCtx &resolve_ctx, + const ObPLBlockNS ¤t_ns) const; + + virtual int get_size(const ObPLINS &ns, ObPLTypeSize type, int64_t &size) const; + virtual int serialize(share::schema::ObSchemaGetterGuard &schema_guard, + const common::ObTimeZoneInfo *tz_info, obmysql::MYSQL_PROTOCOL_TYPE type, + char *&src, char *dst, const int64_t dst_len, int64_t &dst_pos) const; + virtual int deserialize(share::schema::ObSchemaGetterGuard &schema_guard, + common::ObIAllocator &allocator, + const common::ObCharsetType charset, + const common::ObCollationType cs_type, + const common::ObCollationType ncs_type, + const common::ObTimeZoneInfo *tz_info, + const char *&src, + char *dst, + const int64_t dst_len, + int64_t &dst_pos) const; + virtual int convert(ObPLResolveCtx &ctx, ObObj *&src, ObObj *&dst) const; + + TO_STRING_KV(K_(type), + K_(user_type_id), + K_(not_null), + K_(lower), + K_(upper), + K_(base_type)); + +private: + ObPLDataType base_type_; +}; +#endif //---------- for ObRefCursorType ---------- @@ -402,6 +470,42 @@ private: common::ObFixedArray record_members_; }; +#ifdef OB_BUILD_ORACLE_PL +//---------- for ObOpaqueType ---------- + +class ObOpaqueType : public ObUserDefinedType +{ +public: + ObOpaqueType() : ObUserDefinedType(PL_OPAQUE_TYPE) {} + virtual ~ObOpaqueType() {} + +public: + virtual int generate_construct(ObPLCodeGenerator &generator, + const ObPLINS &ns, + jit::ObLLVMValue &value, + const pl::ObPLStmt *stmt = NULL) const; + virtual int64_t get_member_count() const { return 0; } + virtual const ObPLDataType *get_member(int64_t i) const { UNUSED(i); return NULL; } + + virtual int get_size(const ObPLINS &ns, ObPLTypeSize type, int64_t &size) const; + + virtual int get_all_depended_user_type( + const ObPLResolveCtx &resolve_ctx, const ObPLBlockNS ¤t_ns) const + { + UNUSEDx(resolve_ctx, current_ns); return OB_SUCCESS; + } + virtual int newx(common::ObIAllocator &allocator, + const ObPLINS *ns, + int64_t &ptr) const; + virtual int init_session_var(const ObPLResolveCtx &resolve_ctx, + common::ObIAllocator &obj_allocator, + sql::ObExecContext &exec_ctx, + const sql::ObSqlExpression *default_expr, + bool default_construct, + common::ObObj &obj) const; + virtual int free_session_var(const ObPLResolveCtx &resolve_ctx, common::ObIAllocator &obj_allocator, common::ObObj &obj) const; +}; +#endif //---------- for ObCollectionType ---------- class ObPLCollection; @@ -511,6 +615,170 @@ protected: ObPLDataType element_type_; }; +#ifdef OB_BUILD_ORACLE_PL +class ObNestedTableType: public ObCollectionType +{ +public: + ObNestedTableType() + : ObCollectionType(PL_NESTED_TABLE_TYPE) {} + virtual ~ObNestedTableType() {} +public: + virtual int64_t get_member_count() const { return ObCollectionType::get_member_count(); } + virtual const ObPLDataType *get_member(int64_t i) const { return ObCollectionType::get_member(i); } + virtual int generate_construct(ObPLCodeGenerator &generator, + const ObPLINS &ns, + jit::ObLLVMValue &value, + const pl::ObPLStmt *stmt = NULL) const; + virtual int newx(common::ObIAllocator &allocator, + const ObPLINS *ns, + int64_t &ptr) const; + virtual int init_session_var(const ObPLResolveCtx &resolve_ctx, + common::ObIAllocator &obj_allocator, + sql::ObExecContext &exec_ctx, + const sql::ObSqlExpression *default_expr, + bool default_construct, + common::ObObj &obj) const; + virtual int free_session_var(const ObPLResolveCtx &resolve_ctx, common::ObIAllocator &obj_allocator, common::ObObj &obj) const; + virtual int free_data(const ObPLResolveCtx &resolve_ctx, common::ObIAllocator &data_allocator, void *data) const; + + // --------- for session serialize/deserialize interface --------- + virtual int get_serialize_size( + const ObPLResolveCtx &resolve_ctx, char *&src, int64_t &size) const; + virtual int serialize( + const ObPLResolveCtx &resolve_ctx, + char *&src, char* dst, int64_t dst_len, int64_t &dst_pos) const; + virtual int deserialize( + const ObPLResolveCtx &resolve_ctx, + common::ObIAllocator &allocator, + const char* src, const int64_t src_len, int64_t &src_pos, char *&dst) const; + + virtual int add_package_routine_schema_param(const ObPLResolveCtx &resolve_ctx, + const ObPLBlockNS &block_ns, + const common::ObString &package_name, + const common::ObString ¶m_name, + int64_t mode, int64_t position, + int64_t level, int64_t &sequence, + share::schema::ObRoutineInfo &routine_info) const; + virtual int get_all_depended_user_type(const ObPLResolveCtx &resolve_ctx, + const ObPLBlockNS ¤t_ns) const; + virtual int init_obj(share::schema::ObSchemaGetterGuard &schema_guard, + common::ObIAllocator &allocator, + common::ObObj &obj, + int64_t &init_size) const; + virtual int serialize(share::schema::ObSchemaGetterGuard &schema_guard, + const common::ObTimeZoneInfo *tz_info, obmysql::MYSQL_PROTOCOL_TYPE type, + char *&src, char *dst, const int64_t dst_len, int64_t &dst_pos) const; + virtual int deserialize(share::schema::ObSchemaGetterGuard &schema_guard, + common::ObIAllocator &allocator, + const common::ObCharsetType charset, + const common::ObCollationType cs_type, + const common::ObCollationType ncs_type, + const common::ObTimeZoneInfo *tz_info, + const char *&src, + char *dst, + const int64_t dst_len, + int64_t &dst_pos) const; + TO_STRING_KV(K_(type), + K_(user_type_id), + K_(element_type), + K_(not_null)); +private: +}; + +class ObVArrayType: public ObCollectionType +{ +public: + ObVArrayType() + : ObCollectionType(PL_VARRAY_TYPE), + capacity_(OB_INVALID_SIZE) {} + virtual ~ObVArrayType() {} + + int64_t get_capacity() const { return capacity_; } + void set_capacity(int64_t size) { capacity_ = size; } + + int deep_copy(common::ObIAllocator &alloc, const ObVArrayType &other); + +public: + virtual int64_t get_member_count() const { return ObCollectionType::get_member_count(); } + virtual const ObPLDataType *get_member(int64_t i) const { return ObCollectionType::get_member(i); } + virtual int generate_construct(ObPLCodeGenerator &generator, + const ObPLINS &ns, + jit::ObLLVMValue &value, + const pl::ObPLStmt *stmt = NULL) const; + virtual int newx(common::ObIAllocator &allocator, + const ObPLINS *ns, + int64_t &ptr) const; + + virtual int init_session_var( + const ObPLResolveCtx &resolve_ctx, common::ObIAllocator &obj_allocator, + sql::ObExecContext &exec_ctx, const sql::ObSqlExpression *default_expr, + bool default_construct, common::ObObj &obj) const; + + virtual int convert(ObPLResolveCtx &ctx, ObObj *&src, ObObj *&dst) const; + + TO_STRING_KV(K_(type), + K_(user_type_id), + K_(element_type), + K_(not_null), + K_(capacity)); +private: + int64_t capacity_; +}; + +class ObAssocArrayType : public ObCollectionType +{ +public: + ObAssocArrayType() + : ObCollectionType(PL_ASSOCIATIVE_ARRAY_TYPE), + index_type_() {} + virtual ~ObAssocArrayType() {} + + const ObPLDataType &get_index_type() const { return index_type_; } + void set_index_type(const ObPLDataType &index_type) { index_type_ = index_type; } + + int deep_copy(common::ObIAllocator &alloc, const ObAssocArrayType &other); + +public: + virtual int64_t get_member_count() const { return ObCollectionType::get_member_count(); } + virtual const ObPLDataType *get_member(int64_t i) const { return ObCollectionType::get_member(i); } + virtual int generate_construct(ObPLCodeGenerator &generator, + const ObPLINS &ns, + jit::ObLLVMValue &value, + const pl::ObPLStmt *stmt = NULL) const; + virtual int newx(common::ObIAllocator &allocator, + const ObPLINS *ns, + int64_t &ptr) const; + virtual int init_session_var(const ObPLResolveCtx &resolve_ctx, + common::ObIAllocator &obj_allocator, + sql::ObExecContext &exec_ctx, + const sql::ObSqlExpression *default_expr, + bool default_construct, + common::ObObj &obj) const; + + // --------- for session serialize/deserialize interface --------- + virtual int get_serialize_size( + const ObPLResolveCtx &resolve_ctx, char *&src, int64_t &size) const; + + virtual int serialize( + const ObPLResolveCtx &resolve_ctx, + char *&src, char* dst, int64_t dst_len, int64_t &dst_pos) const; + + virtual int deserialize( + const ObPLResolveCtx &resolve_ctx, + common::ObIAllocator &allocator, + const char* src, const int64_t src_len, int64_t &src_pos, char *&dst) const; + + virtual int convert(ObPLResolveCtx &ctx, ObObj *&src, ObObj *&dst) const; + + TO_STRING_KV(K_(type), + K_(user_type_id), + K_(element_type), + K_(not_null), + K_(index_type)); +private: + ObPLDataType index_type_; +}; +#endif class ObPLComposite { @@ -780,12 +1048,409 @@ protected: ObObj *data_; }; +#ifdef OB_BUILD_ORACLE_PL +class ObPLNestedTable : public ObPLCollection +{ +public: + ObPLNestedTable() : ObPLCollection(PL_NESTED_TABLE_TYPE, OB_INVALID_ID) { } + ObPLNestedTable(uint64_t id) : ObPLCollection(PL_NESTED_TABLE_TYPE, id) { } + + TO_STRING_KV(KP_(allocator), K_(type), K_(count), K_(first), K_(last), K_(data)); +private: + +}; +#endif #define IDX_ASSOCARRAY_KEY 9 #define IDX_ASSOCARRAY_SORT 10 +#ifdef OB_BUILD_ORACLE_PL +class ObPLAssocArray : public ObPLCollection +{ +public: + ObPLAssocArray(uint64_t id) : ObPLCollection(PL_ASSOCIATIVE_ARRAY_TYPE, id), key_(NULL), sort_(NULL) {} + + inline const common::ObObj *get_key() const { return key_; } + inline common::ObObj *get_key() { return key_; } + inline void set_key(common::ObObj* key) { key_ = key; } + inline const common::ObObj *get_key(int64_t i) const { return i < 0 || i >= count_ ? NULL : &key_[i]; } + inline common::ObObj *get_key(int64_t i) { return i < 0 || i >= count_ ? NULL : &key_[i]; } + inline int set_key(int64_t i, const common::ObObj &key) + { + int ret = OB_SUCCESS; + if (i < 0 || i >= count_) { + ret = OB_ARRAY_OUT_OF_RANGE; + } else { + key_[i] = key; + } + return ret; + } + inline const int64_t *get_sort() const { return sort_; } + inline int64_t *get_sort() { return sort_; } + inline void set_sort(int64_t* sort) { sort_ = sort; } + inline int64_t get_sort(int64_t i) const { return i < 0 || i >= count_ ? OB_INVALID_INDEX : sort_[i]; } + inline int set_sort(int64_t i, int64_t sort) + { + int ret = OB_SUCCESS; + if (i < 0 || i >= count_) { + ret = OB_ARRAY_OUT_OF_RANGE; + } else { + sort_[i] = sort; + } + return ret; + } + + int deep_copy(ObPLCollection *src, ObIAllocator *allocator, bool ignore_del_element = false); + int64_t get_init_size() const + { + return sizeof(ObPLAssocArray); + } + int update_first(); + int update_last(); + int64_t get_first(); + int64_t get_last(); + + /*serialize functions*/ + int get_serialize_size(int64_t &size); + int serialize(char* buf, const int64_t len, int64_t& pos); + int deserialize(common::ObIAllocator &allocator, + const char *buf, const int64_t len, int64_t &pos); + + static uint32_t key_offset_bits() { return offsetof(ObPLAssocArray, key_) * 8; } + static uint32_t map_offset_bits() { return offsetof(ObPLAssocArray, sort_) * 8; } + + int first(ObObj &result); + int last(ObObj &result); + int next(int64_t idx, ObObj &result); + int prior(int64_t idx, ObObj &result); + int exist(int64_t idx, ObObj &result); + + TO_STRING_KV(KP_(allocator), K_(type), K_(count), K_(first), K_(last), K_(data), K_(key), K_(sort)); +private: + common::ObObj *key_; + int64_t *sort_; //每一个元素的sort属性存储的是排序在自己后面元素的下标，从0开始 +}; +#endif #define IDX_VARRAY_CAPACITY 9 +#ifdef OB_BUILD_ORACLE_PL +class ObPLVArray : public ObPLNestedTable +{ +public: + ObPLVArray(uint64_t id) : ObPLNestedTable(id), capacity_(OB_INVALID_SIZE) { set_type(PL_VARRAY_TYPE); } + + inline int64_t get_capacity() const { return capacity_; } + inline void set_capacity(int64_t size) { capacity_ = size; } + + static uint32_t key_offset_bits() { return offsetof(ObPLVArray, capacity_) * 8; } + + int deep_copy(ObPLCollection *src, ObIAllocator *allocator, bool ignore_del_element = false); + int64_t get_init_size() const + { + return sizeof(ObPLVArray); + } + + /*serialize functions*/ + int get_serialize_size(int64_t &size); + int serialize(char* buf, const int64_t len, int64_t& pos); + int deserialize(common::ObIAllocator &allocator, + const char *buf, const int64_t len, int64_t &pos); + + TO_STRING_KV(KP_(allocator), K_(type), K_(count), K_(first), K_(last), K_(data), K_(capacity)); +private: + int64_t capacity_; +}; + +class ObPLOpaque +{ +public: + ObPLOpaque(ObPLOpaqueType type) + : type_(type), allocator_("PlOpaque", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()) {} + ObPLOpaque() + : type_(ObPLOpaqueType::PL_INVALID), allocator_("PlOpaque", OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()) {} + + virtual ~ObPLOpaque() { allocator_.reset(); } + + inline ObPLOpaqueType get_type() const { return type_; } + + static ObPLOpaqueType get_type(uint64_t id) + { + ObPLOpaqueType type = ObPLOpaqueType::PL_INVALID; + switch (id) + { + case 300001: { + type = ObPLOpaqueType::PL_XML_TYPE; + } break; + case 300004: { + type = ObPLOpaqueType::PL_ANY_TYPE; + } break; + case 300005: { + type = ObPLOpaqueType::PL_ANY_DATA; + } break; + case 300023: + case 300024: { + type = ObPLOpaqueType::PL_JSON_TYPE; + } break; + default :{ + } break; + } + return type; + } + + inline bool is_invalid() const { return ObPLOpaqueType::PL_INVALID == type_; } + inline bool is_anytype() const { return ObPLOpaqueType::PL_ANY_TYPE == type_; } + inline bool is_anydata() const { return ObPLOpaqueType::PL_ANY_DATA == type_; } + inline bool is_xmltype() const { return ObPLOpaqueType::PL_XML_TYPE == type_; } + inline bool is_json_type() const { return ObPLOpaqueType::PL_JSON_TYPE == type_; } + + inline ObIAllocator& get_allocator() { return allocator_; } + + virtual int deep_copy(ObPLOpaque *dst); + + int64_t get_init_size() const; + + TO_STRING_KV(K_(type)); + +private: + ObPLOpaqueType type_; + ObArenaAllocator allocator_; +}; + +class ObPLAnyType : public ObPLOpaque +{ +public: + enum TypeCode { + TYPECODE_INVALID = -1, + TYPECODE_DATE = 12, + TYPECODE_NUMBER = 2, + TYPECODE_RAW = 95, + TYPECODE_CHAR = 96, + TYPECODE_VARCHAR2 = 9, + TYPECODE_VARCHAR = 1, + TYPECODE_MLSLABEL = 105, + TYPECODE_BLOB = 113, + TYPECODE_BFILE = 114, + TYPECODE_CLOB = 112, + TYPECODE_CFILE = 115, + TYPECODE_TIMESTAMP = 187, + TYPECODE_TIMESTAMP_TZ = 188, + TYPECODE_TIMESTAMP_LTZ = 232, + TYPECODE_INTERVAL_YM = 189, + TYPECODE_INTERVAL_DS = 190, + + TYPECODE_REF = 110, + TYPECODE_OBJECT = 108, + TYPECODE_VARRAY = 247, // COLLECTION TYPE + TYPECODE_TABLE = 248, // COLLECTION TYPE + TYPECODE_NAMEDCOLLECTION = 122, + TYPECODE_OPAQUE = 58, // OPAQUE TYPE + + TYPECODE_NCHAR = 286, + TYPECODE_NVARCHAR2 = 287, + TYPECODE_NCLOB = 288, + + TYPECODE_BFLOAT = 100, + TYPECODE_BDOUBLE = 101, + TYPECODE_UROWID = 104, + }; + +public: + ObPLAnyType() + : ObPLOpaque(ObPLOpaqueType::PL_ANY_TYPE), + in_begincreate_(false), + rowsize_(0), + code_(TypeCode::TYPECODE_INVALID), + type_(NULL) {} + + virtual ~ObPLAnyType() {} + +public: + virtual int deep_copy(ObPLOpaque *dst); + static int pltype_to_typecode(const ObPLDataType &pl_type, ObPLAnyType::TypeCode &typecode); + static bool typecode_compatible(ObPLAnyType::TypeCode &src, ObPLAnyType::TypeCode &dst); + +public: + void set_in_begincreate(bool v) { in_begincreate_ = v; } + bool is_in_begincreate() { return in_begincreate_; } + + void set_rowsize(int64_t v) { rowsize_ = v; } + int64_t get_rowsize() { return rowsize_; } + + void set_type_ptr(ObPLDataType *type) { type_ = type; } + ObPLDataType* get_type_ptr() { return type_; } + + void set_typecode(TypeCode code) + { + if (is_valid_type(code)) { + code_ = code; + } + } + TypeCode get_typecode() { return code_; } + + bool is_valid_type() { return is_valid_type(code_); } + + static bool is_obj_type(TypeCode code); + static bool is_valid_type(TypeCode code); + + + TO_STRING_KV(K_(in_begincreate), K_(code), K_(type)); + +private: + bool in_begincreate_; + int64_t rowsize_; + TypeCode code_; + ObPLDataType *type_; +}; + +class ObPLAnyData : public ObPLOpaque +{ +public: + ObPLAnyData() + : ObPLOpaque(ObPLOpaqueType::PL_ANY_DATA), + in_begincreate_(false), + in_piecewise_(false), + is_last_elem_(false), + is_no_data_(false), + rowsize_(0), + current_pos_(0), + type_code_(ObPLAnyType::TypeCode::TYPECODE_INVALID), + type_(NULL), + data_(NULL) {} + + virtual ~ObPLAnyData() + { + in_piecewise_ = false; + in_begincreate_ = false; + is_last_elem_ = false; + is_no_data_ = false; + rowsize_ = 0; + current_pos_ = 0; + type_code_ = ObPLAnyType::TypeCode::TYPECODE_INVALID; + type_ = NULL; + if (NULL != data_) { + (void)ObUserDefinedType::destruct_obj(*data_); + data_ = NULL; + } + } + +public: + virtual int deep_copy(ObPLOpaque *dst); + + int set_data(const ObObj &data); + ObObj* get_data() { return data_; } + + void set_type_code(ObPLAnyType::TypeCode type_code) { type_code_ = type_code; } + ObPLAnyType::TypeCode get_type_code() { return type_code_; } + + void set_type(const ObPLDataType *type) { type_ = type; } + const ObPLDataType* get_type() { return type_; } + + void set_in_begincreate(bool v) { in_begincreate_ = v; } + bool is_in_begincreate() { return in_begincreate_; } + + void set_piecewise(bool v) { in_piecewise_ = v; } + bool is_in_piecewise() { return in_piecewise_; } + + void set_rowsize(int64_t v) { rowsize_ = v; } + int64_t get_rowsize() { return rowsize_; } + + void set_current_pos(int64_t pos) { current_pos_ = pos; } + int64_t get_current_pos() { return current_pos_; } + + void set_is_last_elem(bool v) { is_last_elem_ = v; } + bool is_last_elem() { return is_last_elem_; } + + void set_is_no_data(bool v) { is_no_data_ = v; } + bool is_no_data() { return is_no_data_; } + + int get_current_data(ObObj &obj, ObPLAnyType::TypeCode &dst_typecode); + int set_current_data(ObObj &obj, ObPLAnyType::TypeCode &src_typecode, bool is_last_elem); + + int get_current_type(const ObPLDataType *&pl_type); + int set_record_element(ObObj &obj); + int set_collection_element(ObObj &obj); + + TO_STRING_KV(K_(in_begincreate), + K_(in_piecewise), + K_(current_pos), + KPC(type_), + KPC(data_)); + +private: + bool in_begincreate_; + bool in_piecewise_; + bool is_last_elem_; + bool is_no_data_; + int64_t rowsize_; + int64_t current_pos_; + ObPLAnyType::TypeCode type_code_; + const ObPLDataType *type_; + ObObj *data_; +}; + +class ObPLXmlType : public ObPLOpaque +{ +public: + ObPLXmlType() + : ObPLOpaque(ObPLOpaqueType::PL_XML_TYPE), + data_(NULL) { + } + + virtual ~ObPLXmlType() + { + data_ = NULL; + } + +public: + virtual int deep_copy(ObPLOpaque *dst); + + void set_data(ObObj *data) { data_ = data; } + ObObj* get_data() { return data_; } + + TO_STRING_KV(KPC(data_)); + +private: + ObObj *data_; +}; + +class ObPLJsonBaseType : public ObPLOpaque +{ +public: + enum JSN_ERR_BEHAVIOR { + JSN_PL_NULL_ON_ERR, + JSN_PL_ERR_ON_ERR, + JSN_PL_ERR_ON_EMP, + JSN_PL_ERR_ON_MISMATCH, + JSN_PL_ERR_ON_INVALID = 7 + }; + + ObPLJsonBaseType() + : ObPLOpaque(ObPLOpaqueType::PL_JSON_TYPE), + data_(NULL), + behavior_(0) + {} + + virtual ~ObPLJsonBaseType() + { + data_ = NULL; + behavior_ = 0; + } + +public: + virtual int deep_copy(ObPLOpaque *dst); + void set_data(ObJsonNode *data) { data_ = data; } + void set_err_behavior(int behavior) { behavior_ = behavior; } + int get_err_behavior() { return behavior_ ; } + ObJsonNode* get_data() { return data_; } + + TO_STRING_KV(KPC(data_), K_(behavior)); + +private: + ObJsonNode *data_; + int behavior_; +}; + +#endif } // namespace pl } // namespace oceanbase diff --git a/src/pl/parser/gen_parser.sh b/src/pl/parser/gen_parser.sh index a077e1097..3b70cd759 100755 --- a/src/pl/parser/gen_parser.sh +++ b/src/pl/parser/gen_parser.sh @@ -36,7 +36,12 @@ fi # generate pl_parser bison_parser ../../../src/pl/parser/pl_parser_mysql_mode.y ../../../src/pl/parser/pl_parser_mysql_mode_tab.c - flex -o ../../../src/pl/parser/pl_parser_mysql_mode_lex.c ../../../src/pl/parser/pl_parser_mysql_mode.l ../../../src/pl/parser/pl_parser_mysql_mode_tab.h + +if [ -d "../../../close_modules/oracle_pl/pl/parser/" ]; then + bison_parser ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode.y ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode_tab.c + flex -o ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode_lex.c ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode.l ../../../close_modules/oracle_pl/pl/parser/pl_parser_oracle_mode_tab.h +fi + #./gen_type_name.sh ob_item_type.h >type_name.c diff --git a/src/pl/parser/pl_parser_base.c b/src/pl/parser/pl_parser_base.c index 60039598c..25df7f756 100644 --- a/src/pl/parser/pl_parser_base.c +++ b/src/pl/parser/pl_parser_base.c @@ -10,13 +10,21 @@ * See the Mulan PubL v2 for more details. */ -#include "pl_parser_base.h" +#include "pl/parser/pl_parser_base.h" #define yyconst const typedef void* yyscan_t; #define YY_EXTRA_TYPE void * typedef struct yy_buffer_state *YY_BUFFER_STATE; +#ifdef OB_BUILD_ORACLE_PL +extern YY_BUFFER_STATE obpl_oracle_yy_scan_bytes (yyconst char *bytes,int len ,yyscan_t yyscanner ); +extern void obpl_oracle_yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner ); +extern void obpl_oracle_yy_delete_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner ); +extern int obpl_oracle_yylex_init_extra (YY_EXTRA_TYPE user_defined,yyscan_t* scanner); +extern int obpl_oracle_yyparse(ObParseCtx *parse_ctx); +#define IS_ORACLE_COMPATIBLE (1/*ORACLE_MODE*/ == parse_ctx->comp_mode_) +#endif extern YY_BUFFER_STATE obpl_mysql_yy_scan_bytes (yyconst char *bytes,int len ,yyscan_t yyscanner ); extern void obpl_mysql_yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner ); @@ -30,7 +38,13 @@ int obpl_parser_init(ObParseCtx *parse_ctx) if (NULL_PTR(parse_ctx) || NULL_PTR(parse_ctx->mem_pool_)) { ret = -1; } else { +#ifndef OB_BUILD_ORACLE_PL ret = obpl_mysql_yylex_init_extra(parse_ctx, &(parse_ctx->scanner_ctx_.yyscan_info_)); +#else + ret = IS_ORACLE_COMPATIBLE ? + obpl_oracle_yylex_init_extra(parse_ctx, &(parse_ctx->scanner_ctx_.yyscan_info_)) + : obpl_mysql_yylex_init_extra(parse_ctx, &(parse_ctx->scanner_ctx_.yyscan_info_)); +#endif } return ret; } diff --git a/src/pl/parser/pl_parser_base.h b/src/pl/parser/pl_parser_base.h index eda241f61..1912e2749 100644 --- a/src/pl/parser/pl_parser_base.h +++ b/src/pl/parser/pl_parser_base.h @@ -113,6 +113,9 @@ extern ParseNode *new_terminal_node(void *malloc_pool, ObItemType type); extern ParseNode *new_non_terminal_node(void *malloc_pool, ObItemType node_tag, int num, ...); extern int parse_sql_stmt(ParseResult *parse_result); +#ifdef OB_BUILD_ORACLE_PL +extern const NonReservedKeyword *oracle_pl_non_reserved_keyword_lookup(const char *word); +#endif extern const NonReservedKeyword *mysql_pl_non_reserved_keyword_lookup(const char *word); @@ -193,6 +196,42 @@ extern const NonReservedKeyword *mysql_pl_non_reserved_keyword_lookup(const char } \ } while (0) +#ifdef OB_BUILD_ORACLE_PL +#define get_oracle_non_reserved_node(node, malloc_pool, expr_start, expr_end) \ + do { \ + malloc_terminal_node(node, malloc_pool, T_IDENT); \ + if (OB_UNLIKELY(NULL == node || NULL == parse_ctx || NULL == parse_ctx->stmt_str_)) {\ + YY_FATAL_ERROR("invalid argument, node:%p, result:%p or input_sql is NULL", node, parse_ctx); \ + } else if (OB_UNLIKELY(expr_start <= 0 || expr_end <= 0 || expr_start > expr_end)) { \ + YY_FATAL_ERROR("invalid argument, expr_start:%d, expr_end:%d", expr_start, expr_end); \ + } else { \ + int start = expr_start; \ + char * upper_value = NULL; \ + char * raw_str = NULL; \ + node->str_value_ = NULL; \ + node->str_len_ = 0; \ + node->raw_text_ = NULL; \ + node->text_len_ = 0; \ + while (start <= expr_end && ISSPACE(parse_ctx->stmt_str_[start])) { \ + start++; \ + } \ + if (start >= expr_start \ + && (OB_UNLIKELY((NULL == (upper_value = parse_strndup(parse_ctx->stmt_str_ + start, expr_end - start + 1, parse_ctx->mem_pool_)))))) { \ + YY_FATAL_ERROR("No more space for copying expression string"); \ + } else { \ + if (start >= expr_start \ + && (OB_UNLIKELY((NULL == (raw_str = parse_strndup(parse_ctx->stmt_str_ + start, expr_end - start + 1, parse_ctx->mem_pool_)))))) { \ + YY_FATAL_ERROR("No more space for copying expression string"); \ + } else { \ + node->raw_text_ = raw_str; \ + node->text_len_ = expr_end - start + 1; \ + node->str_value_ = str_toupper(upper_value, expr_end - start + 1); \ + node->str_len_ = expr_end - start + 1; \ + } \ + } \ + } \ + } while(0) +#endif #define make_name_node(node, malloc_pool, name) \ do { \ @@ -209,6 +248,19 @@ extern const NonReservedKeyword *mysql_pl_non_reserved_keyword_lookup(const char // only supports up to 15 decimal place precision: // e.g., (int64_t)3.00000000000000001 == 3.00000000000000001 is true; // oracle supports up to 40 decimal place precision. +#ifdef OB_BUILD_ORACLE_PL +#define CHECK_ASSIGN_ZERO_SUFFIX_INT(decimal_str, result, int_flag) \ + do \ + { \ + double temp_val = strtod(decimal_str, NULL); \ + if ((int64_t)temp_val == temp_val) { \ + int_flag = true; \ + } else { \ + int_flag = false; \ + } \ + result = (int64_t)temp_val; \ + } while (0); +#endif #define ESCAPE_PERCENT_CHARACTER(result, src, dst)\ do {\ diff --git a/src/pl/parser/pl_parser_mysql_mode.l b/src/pl/parser/pl_parser_mysql_mode.l index 3b07d0e53..37160b051 100644 --- a/src/pl/parser/pl_parser_mysql_mode.l +++ b/src/pl/parser/pl_parser_mysql_mode.l @@ -17,7 +17,7 @@ %option prefix="obpl_mysql_yy" %option header-file="../../../src/pl/parser/pl_parser_mysql_mode_lex.h" %{ -#include "pl_parser_base.h" +#include "pl/parser/pl_parser_base.h" #include "pl_parser_mysql_mode_tab.h" extern void obpl_mysql_yyerror(YYLTYPE *yylloc, ObParseCtx *parse_ctx, char *s,...); diff --git a/src/pl/parser/pl_parser_mysql_mode.y b/src/pl/parser/pl_parser_mysql_mode.y index 3a61c8b10..3faafd8ac 100644 --- a/src/pl/parser/pl_parser_mysql_mode.y +++ b/src/pl/parser/pl_parser_mysql_mode.y @@ -66,7 +66,7 @@ typedef struct YYLTYPE %{ #include "pl/parser/pl_parser_mysql_mode_lex.h" -#include "pl_parser_base.h" +#include "pl/parser/pl_parser_base.h" typedef struct _YYLookaheadToken { diff --git a/src/pl/sys_package/ob_dbms_sql.cpp b/src/pl/sys_package/ob_dbms_sql.cpp index afe43d9b2..d43821a0a 100644 --- a/src/pl/sys_package/ob_dbms_sql.cpp +++ b/src/pl/sys_package/ob_dbms_sql.cpp @@ -135,6 +135,29 @@ int ObDbmsInfo::init_params(int64_t param_count) return ret; } +#ifdef OB_BUILD_ORACLE_PL +int ObDbmsInfo::expand_params() +{ + int ret = OB_SUCCESS; + const ObObjParam *param_value = NULL; + if (!exec_params_.empty()) { + OX (exec_params_.reset()); + } + for (int64_t i = 0; OB_SUCC(ret) && i < param_names_.count(); i++) { + OX (param_value = get_bind_param(param_names_.at(i))); + OV (OB_NOT_NULL(param_value), OB_ERR_DBMS_SQL_NOT_ALL_VAR_BIND, param_names_.at(i)); + // bind_params_数组中的param_value_已经做过深拷贝，这里不需要再做。 + OZ (exec_params_.push_back(*param_value), i, param_names_.at(i), *param_value); + } + for (int64_t i = 0; OB_SUCC(ret) && i < into_names_.count(); i++) { + OX (param_value = get_bind_param(into_names_.at(i))); + OV (OB_NOT_NULL(param_value), OB_ERR_DBMS_SQL_NOT_ALL_VAR_BIND, into_names_.at(i)); + } + // oracle can reuse bind_params + // OX (bind_params_.clear()); + return ret; +} +#endif const ObObjParam *ObDbmsInfo::get_bind_param(const ObString ¶m_name) const { @@ -212,6 +235,299 @@ int ObDbmsInfo::set_bind_param(const ObString ¶m_name, const ObObjParam¶ return ret; } +#ifdef OB_BUILD_ORACLE_PL +int64_t ObDbmsCursorInfo::search_array(const ObString &name, ObIArray &array) +{ + int64_t idx = OB_INVALID_INDEX; + ObString left_name; + if (name[0] == ':') { + left_name = ObString(name.length() - 1, name.ptr() + 1); + } else { + left_name = name; + } + for (int64_t i = 0; i < array.count(); i++) { + ObString right_name + = (':' == array.at(i)[0]) + ? (ObString(array.at(i).length() - 1, array.at(i).ptr() + 1)) + : (array.at(i)); + if (0 == left_name.case_compare(right_name)) { + idx = i; + break; + } + } + return idx; +} + +int ObDbmsCursorInfo::variable_value(sql::ObSQLSessionInfo *session, + ObIAllocator *allocator, + const ObString &variable_name, + sql::ObExprResType &result_type, + ObObjParam &result) +{ + int ret = OB_SUCCESS; + if (variable_name.empty()) { + ret = OB_ERR_BIND_VAR_NOT_EXIST; + LOG_WARN("bind variable does not exist", K(ret), K(variable_name)); + } else { + int64_t idx = OB_INVALID_INDEX; + if (get_into_names().count() > 0 || ObStmt::is_select_stmt(get_stmt_type())) { + if (!fetched_ || (fetched_ && 0 == rowcount_)) { + // do nothing ... + } else { + idx = search_array(variable_name, get_into_names()); + if (idx != OB_INVALID_INDEX) { + OZ (ObDbmsInfo::variable_value(session, + allocator, + idx, + get_current_row().get_cell(idx), + result_type, + result)); + } else if ((idx = search_array(variable_name, get_param_names())) != OB_INVALID_INDEX) { + CK (get_exec_params().count() > idx); + OZ (ObDbmsInfo::variable_value(session, + allocator, + idx, + get_exec_params().at(idx), + result_type, + result)); + } else { + ret = OB_ERR_BIND_VAR_NOT_EXIST; + LOG_WARN("bind variable does not exist", K(ret), K(variable_name)); + } + } + } else if ((idx = search_array(variable_name, get_param_names())) != OB_INVALID_INDEX) { + CK (get_exec_params().count() > idx); + OZ (ObDbmsInfo::variable_value(session, + allocator, + idx, + get_exec_params().at(idx), + result_type, + result)); + } else { + ret = OB_ERR_BIND_VAR_NOT_EXIST; + LOG_WARN("bind variable does not exist", K(ret), K(variable_name)); + } + } + return ret; +} + +int ObDbmsInfo::bind_variable(const ObString ¶m_name, const ObObjParam ¶m_value) +{ + int ret = OB_SUCCESS; + OZ (set_bind_param(param_name, param_value)); + return ret; +} + +int ObDbmsInfo::define_column(int64_t col_idx, ObObjType col_type, + ObCollationType col_cs_type, int64_t col_size) +{ + int ret = OB_SUCCESS; + if (col_idx < 0 || col_idx >= fields_.count()) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("define column position is invalid", K(col_idx), K(fields_), K(col_type), K(ret)); + } else if (!cast_supported(fields_.at(col_idx).type_.get_type(), + static_cast(fields_.at(col_idx).charsetnr_), + col_type, col_cs_type)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("define column position is invalid", K(col_idx), K(fields_), K(col_type), K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, + "ORA-06562: type of out argument must match type of column or bind variable"); + } else { + #define ENABLE_RECOVER_EXIST 1 + OZ (define_columns_.set_refactored(col_idx, col_size, ENABLE_RECOVER_EXIST)); + #undef ENABLE_RECOVER_EXIST + } + return ret; +} + +int ObDbmsInfo::define_array(int64_t col_idx, + uint64_t id, + int64_t cnt, + int64_t lower_bnd, + ObDataType &elem_type) +{ + int ret = OB_SUCCESS; + if (col_idx < 0 || col_idx >= fields_.count()) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("define column position is invalid", K(col_idx), K(fields_), K(id), K(ret)); + } else if (cnt <= 0) { + ret = OB_ARRAY_CNT_IS_ILLEGAL; + LOG_WARN("array cnt is illegal in dbms_sql.define_array", K(ret), K(cnt)); + } else if (!cast_supported(fields_.at(col_idx).type_.get_type(), + static_cast(fields_.at(col_idx).charsetnr_), + elem_type.get_obj_type(), elem_type.get_collation_type())) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("define column position is invalid", + K(col_idx), K(fields_), K(id),K(elem_type), K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, + "ORA-06562: type of out argument must match type of column or bind variable"); + } else { + #define ENABLE_RECOVER_EXIST 1 + ArrayDesc desc(id, cnt, lower_bnd, elem_type); + OZ (define_arrays_.set_refactored(col_idx, desc, ENABLE_RECOVER_EXIST)); + #undef ENABLE_RECOVER_EXIST + } + + if (OB_SUCC(ret)) { + /* + * DEFINE_ARRAY replaces DEFINE_COLUMN if DEFINE_ARRAY is after DEFINE_COLUMN. + * Following is a successful example: + * + * declare + * i number :=1; + * names DBMS_SQL.varchar2_Table; + * sals DBMS_SQL.varchar2_Table; + * c NUMBER; + * r NUMBER; + * v number := 0; + * sql_stmt VARCHAR2(32767) := + * 'SELECT last_name, salary FROM employees1'; + * BEGIN + * c := DBMS_SQL.OPEN_CURSOR; + * DBMS_SQL.PARSE(c, sql_stmt, dbms_sql.native); + * DBMS_SQL.DEFINE_COLUMN(c, 2, v); + * DBMS_SQL.DEFINE_ARRAY(c, 1.1, names, 5.3, 1.2); + * DBMS_SQL.DEFINE_ARRAY(c, 2, sals, 4.5, 1.3); + * r := DBMS_SQL.EXECUTE(c); + * r := DBMS_SQL.FETCH_ROWS(c); + * DBMS_SQL.COLUMN_VALUE(c, 1, names); + * DBMS_SQL.COLUMN_VALUE(c, 2, sals); + * DBMS_SQL.CLOSE_CURSOR(c); + * END; + * / + * + * But, DEFINE_COLUMN after DEFINE_ARRAY may cause error in DBMS_SQL.EXECUTE, + * see DBMS_SQL.EXECUTE + * */ + OZ (define_columns_.reuse()); + } + + return ret; +} + +int ObDbmsInfo::column_value(sql::ObSQLSessionInfo *session, + ObIAllocator *allocator, + int64_t col_idx, + const ObObjParam src_obj, + sql::ObExprResType &result_type, + ObObjParam &result) +{ + int ret = OB_SUCCESS; + int64_t column_size = OB_INVALID_SIZE; + if (OB_SUCC(define_columns_.get_refactored(col_idx, column_size))) { + ObObjParam src; + if (OB_FAIL(ob_write_obj(*allocator, src_obj, src))) { + LOG_WARN("write obj failed", K(ret), K(src_obj)); + } else if (OB_INVALID_SIZE != column_size + && !ob_is_accuracy_length_valid_tc(result_type.get_type())) { + ret = OB_ERR_BIND_TYPE_NOT_MATCH_COLUMN; + LOG_WARN("define column type and column value type are not match", + K(col_idx), K(fields_), K(result_type), K(result), K(ret)); + } else if (OB_FAIL((ObSPIService::spi_convert(session, allocator, src, result_type, result)))) { + LOG_WARN("spi convert fail.", K(ret)); + } else if (OB_INVALID_SIZE != column_size && column_size < result.get_val_len()) { + result.set_val_len(column_size); + } else { /*do nothing*/ } + } else if (OB_HASH_NOT_EXIST == ret) { + ret = OB_SUCCESS; + ArrayDesc *desc = const_cast(define_arrays_.get(col_idx)); + if (OB_ISNULL(desc)) { + ret = OB_ERR_VARIABLE_NOT_IN_SELECT_LIST; + LOG_WARN("column is not defined", K(col_idx), K(ret)); + LOG_USER_ERROR(OB_ERR_VARIABLE_NOT_IN_SELECT_LIST); + } else { + if (result.is_pl_extend() && result_type.is_ext() && result_type.get_udt_id() == desc->id_) { + /* + * can not reset Collection here, because if we call DEFINE_ARRAY twice and set ARRAY cnt smaller than first time, we should + * remain the values fetched in first time not be replaces in the gap. + */ + ObPLAssocArray *table = reinterpret_cast(result.get_ext()); + ObObjParam obj; + ObObj key(ObInt32Type); + ObExprResType element_type; + // using reset mode instead of append mode after re-DEFINE_ARRAY + bool reset_mode = desc->lower_bnd_ > 1 ? desc->cur_idx_ == desc->lower_bnd_ - 1 ? true : false + : 0 == desc->cur_idx_ ? true : false;; + // always extend fetch_rows_.count() in append mode, and only extend the gap in reset mode + /* spi_extend_assoc_array 里面的逻辑是这样的 + * if table->get_key != NULL, 在 原来table 基础上扩展 extend_size 的 key. + * key的内存是从新分配的, 长度是 table->get_count + extend_size + * else , 直接分配 extend_size 的 key + */ + int64_t extend_size = reset_mode + ? NULL != table->get_key() && NULL != table->get_sort() + ? fetch_rows_.count() + desc->cur_idx_ - table->get_count() + : fetch_rows_.count() + desc->cur_idx_ + : fetch_rows_.count(); + if (extend_size > 0) { + OZ (ObSPIService::spi_extend_assoc_array(session->get_effective_tenant_id(), + NULL, + *allocator, + *table, + extend_size)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < extend_size - fetch_rows_.count(); ++i) { + OX (new (reinterpret_cast(table->get_data()) + i) ObObj(ObMaxType)); + } + OX (element_type.set_meta(desc->type_.get_meta_type())); + OX (element_type.set_accuracy(desc->type_.get_accuracy())); + for (int64_t i = 0; OB_SUCC(ret) && i < fetch_rows_.count(); ++i) { + // check `table->get_count() > desc->cur_idx_` + // prevent the `table->get_data()[desc->cur_idx_]` is invalid + + /* when lower_bnd_ > 1 + * cur_idx_ will begin from lower_bnd_ - 1, the key is desc->cur_idx_ + 1 + * else + * cur_id_ will begin from 0, he key is desc->lower_bnd_ + desc->cur_idx_ + */ + if (table->get_count() > desc->cur_idx_) { + ObNewRow &row = fetch_rows_.at(i); + ObObjParam src = row.get_cell(col_idx); + OZ (ObSPIService::spi_convert(session, table->get_allocator(), src, element_type, obj)); + OZ (deep_copy_obj(*table->get_allocator(), + obj, + reinterpret_cast(table->get_data())[desc->cur_idx_])); + OX (key.set_int32(desc->lower_bnd_ > 1 ? desc->cur_idx_ + 1 : desc->lower_bnd_ + desc->cur_idx_)); + CK (OB_NOT_NULL(table->get_key())); + CK (OB_NOT_NULL(table->get_sort())); + OX (table->set_key(desc->cur_idx_, key)); + OX (table->set_sort(desc->cur_idx_, desc->cur_idx_ + 1)); + LOG_DEBUG("column add key ", K(key.get_int32()), K(desc->cur_idx_), K(table->get_key(desc->cur_idx_)->get_int32()), K(table->get_sort(desc->cur_idx_))); + OX (++desc->cur_idx_); + } + } + + OX (table->set_first(reset_mode ? desc->lower_bnd_ > 1 ? desc->lower_bnd_ : 1 + : table->get_first())); + OX (table->set_last(table->get_first() - 1 + table->get_actual_count())); + LOG_DEBUG("column value set last and first", K(reset_mode), K(desc->lower_bnd_), K(table->get_first()), K(table->get_actual_count()), K(table->get_last())); + } else { + ret = OB_ERR_INVALID_TYPE_FOR_ARGUMENT; + LOG_WARN("type of out argument must match type of column or bind variable", + K(result_type), K(result)); + } + } + } else { + //do nothing, just report error + } + return ret; +} + + int ObDbmsInfo::variable_value(sql::ObSQLSessionInfo *session, + ObIAllocator *allocator, + int64_t col_idx, + const ObObjParam src_obj, + sql::ObExprResType &result_type, + ObObjParam &result) +{ + int ret = OB_SUCCESS; + ObObjParam src; + UNUSED(col_idx); + OZ (ob_write_obj(*allocator, src_obj, src)); + OZ (ObSPIService::spi_convert(session, allocator, src, result_type, result)); + return ret; +} +#endif int ObDbmsInfo::add_param_name(ObString &clone_name) { @@ -361,7 +677,1120 @@ int64_t ObDbmsCursorInfo::convert_to_dbms_cursor_id(int64_t id) return new_id; } +#ifdef OB_BUILD_ORACLE_PL +int ObDbmsCursorInfo::column_value(sql::ObSQLSessionInfo *session, + ObIAllocator *allocator, + int64_t col_idx, + sql::ObExprResType &result_type, + ObObjParam &result) +{ + int ret = OB_SUCCESS; + // check whether col_idx is defined first + if (nullptr == get_define_columns().get(col_idx) && + nullptr == get_define_arrays().get(col_idx)) { + ret = OB_ERR_VARIABLE_NOT_IN_SELECT_LIST; + LOG_WARN("column is not defined", K(col_idx), K(ret)); + LOG_USER_ERROR(OB_ERR_VARIABLE_NOT_IN_SELECT_LIST); + } else if (!fetched_ || (fetched_ && 0 == rowcount_)) { + // do nothing; + } else if (col_idx >= get_current_row().get_count()) { + ret = OB_ERROR_OUT_OF_RANGE; + LOG_WARN("column column idx out of range", K(ret), K(col_idx), + K(get_current_row().get_count())); + } else { + ret = ObDbmsInfo::column_value(session, allocator, col_idx, + get_current_row().get_cell(col_idx), + result_type, result); + } + return ret; +} +#endif + +#ifdef OB_BUILD_ORACLE_PL +int ObPLDbmsSql::open_cursor(ObExecContext &exec_ctx, ParamStore ¶ms, ObObj &result) +{ + int ret = OB_SUCCESS; + ObDbmsCursorInfo *cursor = NULL; + ObSQLSessionInfo *session = exec_ctx.get_my_session(); + int64_t param_count = params.count(); + number::ObNumber num; + OV (param_count == 0 || param_count == 1, OB_INVALID_ARGUMENT, params); + OV (OB_NOT_NULL(session)); + OZ (session->make_dbms_cursor(cursor)); + OV (OB_NOT_NULL(cursor)); + OZ (num.from(cursor->get_dbms_id(), exec_ctx.get_allocator())); + OX (result.set_number(num)); + return ret; +} + +int ObPLDbmsSql::parse_6p(ObExecContext &exec_ctx, ParamStore ¶ms, ObObj &result) +{ + UNUSED(result); + int ret = OB_SUCCESS; + int64_t param_count = params.count(); + OV (param_count == 6, OB_INVALID_ARGUMENT, params); + OV (params.at(1).is_ext(), OB_INVALID_ARGUMENT, params); + OV (params.at(2).is_number(), OB_INVALID_ARGUMENT, params); + OV (params.at(3).is_number(), OB_INVALID_ARGUMENT, params); + OV (params.at(4).is_tinyint(), OB_INVALID_ARGUMENT, params); + OV (params.at(5).is_number(), OB_INVALID_ARGUMENT, params); + + ObPLCollection *sql_arr = NULL; + OX (sql_arr = reinterpret_cast(params.at(1).get_ext())); + CK (OB_NOT_NULL(sql_arr)); + CK (sql_arr->is_associative_array()); + int64_t low_bound = -1, upper_bound = -1; + bool linefeed = false; + int8_t flag = 0; + OV (params.at(2).get_number().is_valid_int64(low_bound), OB_INVALID_ARGUMENT, params.at(2)); + OV (params.at(3).get_number().is_valid_int64(upper_bound), OB_INVALID_ARGUMENT, params.at(3)); + OZ (params.at(4).get_tinyint(flag), params.at(4)); + OX (linefeed = static_cast(flag)); + CK (sql_arr->get_count() >= upper_bound - low_bound); + CK (low_bound <= upper_bound); + ObSqlString sql_txt; + if (OB_SUCC(ret)) { + ObString elem_txt; + ObPLAssocArray *assoc_arr = static_cast(sql_arr); + int64_t *sort_arr = assoc_arr->get_sort(); + int64_t idx = assoc_arr->get_first() - 1; + ObObj *keys = assoc_arr->get_key(); + CK (OB_NOT_NULL(keys)); + int32_t key = -1; + int64_t low_idx = OB_INVALID_INDEX, upper_idx = OB_INVALID_INDEX; + #define GET_NEXT() { \ + if (OB_SUCC(ret) && 0 <= idx && idx < assoc_arr->get_count()) { \ + if (OB_NOT_NULL(sort_arr)) { \ + idx = sort_arr[idx]; \ + } else { \ + idx++; \ + } \ + } else { \ + idx = OB_INVALID_INDEX; \ + } \ + } + // 首先拿到low_bound和upper_bound对应的index; + do { + if (OB_SUCC(ret) && 0 <= idx && idx < assoc_arr->get_count()) { + OZ (keys[idx].get_int32(key), keys[idx]); + if (OB_SUCC(ret)) { + if (static_cast(key) == low_bound) { + low_idx = idx; + } + if (static_cast(key) == upper_bound) { + upper_idx = idx; + } + } + } + if (OB_INVALID_INDEX != low_idx && OB_INVALID_INDEX != upper_idx) { + break; + } + GET_NEXT(); + } while (OB_SUCC(ret) && OB_INVALID_INDEX != idx); + + CK (0 <= low_idx); + CK (sql_arr->get_count() > low_idx); + CK (sql_arr->get_count() > upper_idx); + CK (low_idx <= upper_idx); + + if (OB_SUCC(ret)) { + // reset idx + OX (idx = assoc_arr->get_first() - 1); + // 定位low bound + while(0 < low_idx) { + GET_NEXT(); + low_idx--; + upper_idx--; + } + // 开始组装 + while (OB_SUCC(ret) && 0 <= upper_idx) { + bool is_del = false; + OZ (assoc_arr->is_elem_deleted(idx, is_del)); + if (OB_SUCC(ret)) { + if (is_del) { + ret = OB_READ_NOTHING; + } else { + ObObj &elem = reinterpret_cast(assoc_arr->get_data())[idx]; + if (elem.is_null()) { + ret = OB_READ_NOTHING; + } else { + OZ (elem.get_varchar(elem_txt), elem); + OZ (sql_txt.append(elem_txt)); + if (linefeed) { + OZ (sql_txt.append("\n")); + } + GET_NEXT(); + } + } + } + upper_idx--; + } + } + } + ObDbmsCursorInfo *cursor = NULL; + ObString sql_stmt; + OZ (get_cursor(exec_ctx, params, cursor));// 这儿只用了params的第一个参数，cursor id + OZ (ob_write_string(exec_ctx.get_allocator(), sql_txt.string(), sql_stmt)); + LOG_DEBUG("parse 6p, concated sql stmt", K(sql_stmt), K(sql_txt.string())); + OZ (do_parse(exec_ctx, cursor, sql_stmt)); + return ret; +} + +int ObPLDbmsSql::parse(ObExecContext &exec_ctx, ParamStore ¶ms, ObObj &result) +{ + UNUSED(result); + int ret = OB_SUCCESS; + ObDbmsCursorInfo *cursor = NULL; + int64_t param_count = params.count(); + + // parse all param. + ObString sql_stmt; + OV (3 == param_count || 6 == param_count, OB_INVALID_ARGUMENT, params); + if (OB_SUCC(ret)) { + if (3 == param_count) { + OV (params.at(1).is_varchar(), OB_INVALID_ARGUMENT, params); + /* + * TODO: support all string type for param 1. + */ + OZ (params.at(1).get_varchar(sql_stmt), params.at(1)); + + OZ (get_cursor(exec_ctx, params, cursor)); + OZ (do_parse(exec_ctx, cursor, sql_stmt)); + } else if (6 == param_count) { + OZ (parse_6p(exec_ctx, params, result)); + } else { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("unexpected argument count", K(ret), K(param_count)); + } + } + return ret; +} + +int ObPLDbmsSql::do_parse(ObExecContext &exec_ctx, ObDbmsCursorInfo *cursor, ObString &sql_stmt) +{ + int ret = OB_SUCCESS; + CK (OB_NOT_NULL(cursor)); + // do parse. + ObSQLSessionInfo *session = exec_ctx.get_my_session(); + OZ (cursor->parse(sql_stmt, *session), sql_stmt); + if (OB_SUCC(ret)) { + ObString ps_sql; + stmt::StmtType stmt_type = stmt::StmtType::T_NONE; + bool for_update = false; + bool hidden_rowid = false; + int64_t into_cnt = 0; + ParamStore dummy_params; + ObSqlString sql_str; + ObPLExecCtx pl_ctx(cursor->get_allocator(), &exec_ctx, &dummy_params, + NULL/*result*/, &ret, NULL/*func*/, true); + CK (OB_NOT_NULL(exec_ctx.get_my_session())); + OZ (sql_str.append(sql_stmt)); + OX (cursor->get_field_columns().set_allocator(&cursor->get_dbms_entity()->get_arena_allocator())); + OZ (ObSPIService::prepare_dynamic(&pl_ctx, + cursor->get_dbms_entity()->get_arena_allocator(), + false/*is_returning*/, + true, + cursor->get_param_name_count(), + sql_str, + ps_sql, + stmt_type, + for_update, + hidden_rowid, + into_cnt, + &cursor->get_field_columns())); + if (OB_SUCC(ret)) { + cursor->set_ps_sql(ps_sql); + cursor->set_stmt_type(stmt_type); + if (for_update) { + cursor->set_for_update(); + } + if (hidden_rowid) { + cursor->set_hidden_rowid(); + } + } + OZ (cursor->set_into_names(into_cnt)); + } + //if (ret == OB_ERR_PARSE_SQL) { + // ret = OB_ERR_DBMS_SQL_INVALID_STMT; + //} + // execute if stmt is not dml nor select. + if (OB_SUCC(ret) && check_stmt_need_to_be_executed_when_parsing(*cursor)) { + OZ (do_execute(exec_ctx, *cursor)); + OX (cursor->get_ps_sql().reset()); + OX (cursor->get_sql_stmt().reset()); + } + return ret; +} + +bool ObPLDbmsSql::check_stmt_need_to_be_executed_when_parsing(ObDbmsCursorInfo &cursor) +{ + bool flag = ObStmt::is_ddl_stmt(cursor.get_stmt_type(), true); + return flag; +} + +int ObPLDbmsSql::bind_variable(ObExecContext &exec_ctx, ParamStore ¶ms, ObObj &result) +{ + UNUSED(result); + int ret = OB_SUCCESS; + ObDbmsCursorInfo *cursor = NULL; + int64_t param_count = params.count(); + int64_t out_value_size = 0; + + // parse all param. + ObString param_name; + ObObjParam param_value; + OV (3 == param_count || 4 == param_count, OB_INVALID_ARGUMENT, params); + OV (params.at(1).is_varchar(), OB_INVALID_ARGUMENT, params); + /* + * TODO: support all string type for param 1. + */ + OZ (params.at(1).get_varchar(param_name), params.at(1)); + OX (param_value = params.at(2)); + if (OB_SUCC(ret) && 4 == param_count) { + OV (params.at(3).is_number(), OB_INVALID_ARGUMENT, params); + OZ (params.at(3).get_number().extract_valid_int64_with_trunc(out_value_size)); + OX (param_value.set_length(out_value_size)); + } + OZ (get_cursor(exec_ctx, params, cursor)); + CK (OB_NOT_NULL(cursor)); + if (OB_SUCC(ret) && cursor->get_sql_stmt().empty()) { + ret = OB_NO_STMT_PARSE; + LOG_WARN("no statement parsed or DDL is executed when parsed", K(ret)); + } + + // do bind. + OZ (cursor->bind_variable(param_name, param_value), param_name, param_value); + return ret; +} + +int ObPLDbmsSql::define_column_number(ObExecContext &exec_ctx, ParamStore ¶ms, ObObj &result) +{ + UNUSED(exec_ctx); + UNUSED(params); + UNUSED(result); + int ret = OB_SUCCESS; + return ret; +} + +int ObPLDbmsSql::define_column_varchar(ObExecContext &exec_ctx, ParamStore ¶ms, ObObj &result) +{ + UNUSED(exec_ctx); + UNUSED(params); + UNUSED(result); + int ret = OB_SUCCESS; + return ret; +} + +int ObPLDbmsSql::define_column(ObExecContext &exec_ctx, ParamStore ¶ms, ObObj &result) +{ + UNUSED(result); + int ret = OB_SUCCESS; + ObDbmsCursorInfo *cursor = NULL; + int64_t param_count = params.count(); + + // parse all param. + int64_t column_pos = -1; + ObObjType column_type = ObNullType; + ObCollationType column_cs_type = CS_TYPE_INVALID; + int64_t column_size = OB_INVALID_SIZE; + OV (3 == param_count || 4 == param_count, OB_INVALID_ARGUMENT, params); + OV (params.at(1).is_number(), OB_INVALID_ARGUMENT, params); + OV (params.at(1).get_number().is_valid_int64(column_pos), OB_INVALID_ARGUMENT, params.at(1)); + OX (column_type = + params.at(2).is_null() ? params.at(2).get_null_meta().get_type() : params.at(2).get_type()); + OX (column_cs_type = params.at(2).is_null() + ? params.at(2).get_null_meta().get_collation_type() + : params.at(2).get_collation_type()); + if (OB_SUCC(ret)) { + if (ob_is_accuracy_length_valid_tc(column_type)) { + if (4 == param_count) { + OV (params.at(3).is_number(), OB_INVALID_ARGUMENT, params); + OV (params.at(3).get_number().is_valid_int64(column_size), OB_INVALID_ARGUMENT, params.at(3)); + } + } else if (3 == param_count) { + //do nothing + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("column size cannot be used for this type", K(column_type), K(column_size), K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, + "ORA-06562: type of out argument must match type of column or bind variable"); + } + } + + OZ (get_cursor(exec_ctx, params, cursor)); + CK (OB_NOT_NULL(cursor)); + if (OB_SUCC(ret) && cursor->get_sql_stmt().empty()) { + ret = OB_NO_STMT_PARSE; + LOG_WARN("no statement parsed or DDL is executed when parsed", K(ret)); + } + + // do define. + OZ (cursor->define_column(column_pos - 1, column_type, column_cs_type, column_size), + column_pos, column_type); + + return ret; +} + +int ObPLDbmsSql::define_array(ObExecContext &exec_ctx, ParamStore ¶ms, ObObj &result) +{ + UNUSED(result); + int ret = OB_SUCCESS; + ObDbmsCursorInfo *cursor = NULL; + ObSQLSessionInfo *session = exec_ctx.get_my_session(); + int64_t param_count = params.count(); + + // parse all param. + int64_t column_pos = OB_INVALID_INDEX; + int64_t cnt = OB_INVALID_COUNT; + int64_t lower_bnd = OB_INVALID_INDEX; + uint64_t type_id = OB_INVALID_ID; + ObDataType elem_type; + OV (5 == param_count, OB_INVALID_ARGUMENT, params); + /* + * DBMS_SQL.DEFINE_ARRAY ( + * c IN INTEGER, + * position IN INTEGER, IN , + * cnt IN INTEGER, + * lower_bnd IN INTEGER); + * */ + OV (params.at(1).is_number(), OB_INVALID_ARGUMENT, params); + OV (params.at(3).is_number(), OB_INVALID_ARGUMENT, params); + OV (params.at(4).is_number(), OB_INVALID_ARGUMENT, params); + OZ (params.at(1).get_number().extract_valid_int64_with_round(column_pos), + OB_INVALID_ARGUMENT, params.at(1)); + OZ (params.at(3).get_number().extract_valid_int64_with_round(cnt), + OB_INVALID_ARGUMENT, params.at(3)); + OV ((cnt < (1L << 31)), OB_NUMERIC_OVERFLOW, cnt); + OZ (params.at(4).get_number().extract_valid_int64_with_round(lower_bnd), + OB_INVALID_ARGUMENT, params.at(4)); + OV (params.at(2).is_pl_extend(), OB_INVALID_ARGUMENT, params.at(2).get_meta()); + OX (type_id = params.at(2).get_udt_id()); + if (OB_SUCC(ret)) { + const ObCollectionType *coll_type = NULL; + ObPLPackageGuard package_guard(exec_ctx.get_my_session()->get_effective_tenant_id()); + const ObUserDefinedType *user_type = NULL; + CK (OB_NOT_NULL(exec_ctx.get_sql_ctx()->schema_guard_)); + OZ (ObResolverUtils::get_user_type(&exec_ctx.get_allocator(), + exec_ctx.get_my_session(), + exec_ctx.get_sql_proxy(), + exec_ctx.get_sql_ctx()->schema_guard_, + package_guard, + type_id, + user_type)); + CK (OB_NOT_NULL( + coll_type = static_cast(user_type))); + CK (OB_NOT_NULL(coll_type->get_element_type().get_data_type())); + OX (elem_type = *coll_type->get_element_type().get_data_type()); + } + + OZ (get_cursor(exec_ctx, params, cursor)); + CK (OB_NOT_NULL(cursor)); + if (OB_SUCC(ret) && cursor->get_sql_stmt().empty()) { + ret = OB_NO_STMT_PARSE; + LOG_WARN("no statement parsed or DDL is executed when parsed", K(ret)); + } + + // do define. + OZ (cursor->define_array(column_pos - 1, type_id, cnt, lower_bnd, elem_type), + column_pos, type_id, cnt, lower_bnd); + + return ret; +} + +int ObPLDbmsSql::execute(ObExecContext &exec_ctx, ParamStore ¶ms, ObObj &result) +{ + int ret = OB_SUCCESS; + ObDbmsCursorInfo *cursor = NULL; + + OZ (get_cursor(exec_ctx, params, cursor)); + CK (OB_NOT_NULL(cursor)); + if (OB_SUCC(ret)) { + if (0 != cursor->get_define_columns().size() && 0 != cursor->get_define_arrays().size()) { + /* + * DEFINE_COLUMN after DEFINE_ARRAY cause error in DBMS_SQL.EXECUTE. + * Following is a failed example: + * + * declare + * names DBMS_SQL.varchar2_Table; + * sals DBMS_SQL.varchar2_Table; + * c NUMBER; + * r NUMBER; + * v number := 0; + * sql_stmt VARCHAR2(32767) := + * 'SELECT last_name, salary FROM employees1'; + * BEGIN + * c := DBMS_SQL.OPEN_CURSOR; + * DBMS_SQL.PARSE(c, sql_stmt, dbms_sql.native); + * DBMS_SQL.DEFINE_ARRAY(c, 1.1, names, 5.3, 1.2); + * DBMS_SQL.DEFINE_ARRAY(c, 2, sals, 4.5, 1.3); --whatever this statement exists + * DBMS_SQL.DEFINE_COLUMN(c, 2, v); + * r := DBMS_SQL.EXECUTE(c); + * DBMS_SQL.CLOSE_CURSOR(c); + * END; + * / + * */ + ret = OB_ERR_CURSOR_CONTAIN_BOTH_REGULAR_AND_ARRAY; + LOG_WARN("Cursor contains both regular and array defines which is illegal", K(ret)); + } else { + // do execute. + if (!check_stmt_need_to_be_executed_when_parsing(*cursor)) { + OZ (do_execute(exec_ctx, *cursor, params, result)); + //every ececute should reset current index of Array + for (ObDbmsCursorInfo::DefineArrays::iterator iter = cursor->get_define_arrays().begin(); + OB_SUCC(ret) && iter != cursor->get_define_arrays().end(); + ++iter) { + ObDbmsCursorInfo::ArrayDesc &array_info = iter->second; + array_info.cur_idx_ = array_info.lower_bnd_ > 1 ? array_info.lower_bnd_ - 1 : 0; + } + } else { + number::ObNumber num; + int64_t res_num = 0; + OZ (num.from(res_num, exec_ctx.get_allocator())); + result.set_number(num); + } + } + } + + return ret; +} + +int ObPLDbmsSql::do_execute(ObExecContext &exec_ctx, + ObDbmsCursorInfo &dbms_cursor) +{ + int ret = OB_SUCCESS; + ObPLExecCtx pl_ctx(dbms_cursor.get_allocator(), &exec_ctx, NULL/*params*/, + NULL/*result*/, &ret, NULL/*func*/, true); + OZ (ObSPIService::dbms_dynamic_open(&pl_ctx, dbms_cursor)); + return ret; +} + +int ObPLDbmsSql::do_execute(ObExecContext &exec_ctx, + ObDbmsCursorInfo &cursor, + ParamStore ¶ms, + ObObj &result) +{ + int ret = OB_SUCCESS; + number::ObNumber num; + ObPLExecCtx pl_ctx(cursor.get_allocator(), &exec_ctx, ¶ms, + NULL/*result*/, &ret, NULL/*func*/, true); + OZ (cursor.expand_params()); + OZ (ObSPIService::dbms_dynamic_open(&pl_ctx, cursor)); + if (OB_SUCC(ret) && cursor.get_into_names().count() > 0) { // DML Returning + OZ (do_fetch(exec_ctx, params, result, cursor)); + } else { + OZ (num.from(cursor.get_affected_rows() < 0 ? 0 : cursor.get_affected_rows(), exec_ctx.get_allocator())); + OX (result.set_number(num)); + } + return ret; +} + +int ObPLDbmsSql::fetch_rows(ObExecContext &exec_ctx, ParamStore ¶ms, ObObj &result) +{ + int ret = OB_SUCCESS; + ObDbmsCursorInfo *cursor = NULL; + int64_t param_count = params.count(); + // parse all param. + OV (1 == param_count, OB_INVALID_ARGUMENT, params); + + OZ (get_cursor(exec_ctx, params, cursor)); + CK (OB_NOT_NULL(cursor)); + if (OB_SUCC(ret) && cursor->get_sql_stmt().empty()) { + ret = OB_NO_STMT_PARSE; + LOG_WARN("no statement parsed or DDL is executed when parsed", K(ret)); + } + OZ (do_fetch(exec_ctx, params, result, *cursor)); + return ret; +} + +int ObPLDbmsSql::do_fetch(ObExecContext &exec_ctx, + ParamStore ¶ms, + ObObj &result, + ObDbmsCursorInfo &cursor) { + int ret = OB_SUCCESS; + bool is_first_fetch = !cursor.get_fetched(); + bool is_last_fetch_with_row = cursor.get_fetched_with_row(); + + if (OB_SUCC(ret)) { + ObPLExecCtx pl_ctx(cursor.get_allocator(), &exec_ctx, ¶ms, + NULL/*result*/, &ret, NULL/*func*/, true); + int64_t fetch_cnt = 0; + ObNumber row_count; + if (0 == cursor.get_define_arrays().size()) { + OZ (ObSPIService::dbms_cursor_fetch(&pl_ctx, cursor)); + if (OB_SUCC(ret)) { + bool found = false; + bool isnull = false; + OZ (cursor.get_found(found, isnull)); + OX (fetch_cnt = found ? 1 : 0); + } + } else { + // if DEFINE_ARRAYs using different cnt, choose the small one + int64_t define_cnt = OB_INVALID_COUNT; + ObNewRow row; + for (ObDbmsCursorInfo::DefineArrays::const_iterator iter = cursor.get_define_arrays().begin(); + OB_SUCC(ret) && iter != cursor.get_define_arrays().end(); + ++iter) { + const ObDbmsCursorInfo::ArrayDesc &array_info = iter->second; + if (OB_INVALID_COUNT == define_cnt || define_cnt > array_info.cnt_) { + define_cnt = array_info.cnt_; + } + } + CK (define_cnt > 0); + OX (cursor.get_fetch_rows().reuse()); + for (int64_t i = 0; OB_SUCC(ret) && i < define_cnt; ++i) { + OX (row.reset()); + OZ (ObSPIService::dbms_cursor_fetch(&pl_ctx, cursor)); + OZ (ob_write_row(cursor.get_dbms_entity()->get_arena_allocator(), + cursor.get_current_row(), + row)); + OZ (cursor.get_fetch_rows().push_back(row)); + OX (++fetch_cnt); + } + if (OB_SUCC(ret)) { + if (fetch_cnt != define_cnt) { + ret = OB_ERR_FETCH_OUT_SEQUENCE; + LOG_WARN("rows fetched unexpected", K(ret), K(fetch_cnt), K(define_cnt), K(cursor)); + } + } + } + + if (OB_READ_NOTHING == ret) { + // 第一次读，重置success，因此这个时候用户不知道能否fetch到数据，返回success，并且置0. + // 如果上次fetch有返回数据，那么下次fetch就算没有数据也不报错，逻辑同上，这个时候用户不知道是否这次能否fetch + // 到数据，所以不抛异常。如果上次fetch结果是0，再调用fetch，那么该报错报错. + if (is_first_fetch || (cursor.get_fetched() && is_last_fetch_with_row)) { + ret = OB_SUCCESS; + LOG_DEBUG("first fetch, reset ret to OB_SUCCESS"); + } else { + ret = OB_ERR_FETCH_OUT_SEQUENCE; + LOG_WARN("rows fetched unexpected", K(ret), K(fetch_cnt), K(is_first_fetch), K(cursor)); + } + } + OZ (row_count.from(static_cast(fetch_cnt), exec_ctx.get_allocator())); + OX (result.set_number(row_count)); + } + return ret; +} + +int ObPLDbmsSql::column_value(ObExecContext &exec_ctx, ParamStore ¶ms, ObObj &result) +{ + UNUSED(result); + int ret = OB_SUCCESS; + ObDbmsCursorInfo *cursor = NULL; + int64_t column_pos = -1; + int64_t param_count = params.count(); + + // parse all param. + ObExprResType result_type; + if (5 == param_count) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("five parameters for column_value not supported", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "five parameters for column_value series procedure"); + } + OV (3 == param_count, OB_INVALID_ARGUMENT, params); + OV (params.at(1).is_number(), OB_INVALID_ARGUMENT, params); + OV (params.at(1).get_number().is_valid_int64(column_pos), OB_INVALID_ARGUMENT, params.at(1)); + OX (result_type.set_meta(params.at(2).is_null() + ? params.at(2).get_null_meta() : params.at(2).get_meta())); + OX (result_type.set_accuracy(params.at(2).get_accuracy())); + + OZ (get_cursor(exec_ctx, params, cursor)); + CK(OB_NOT_NULL(cursor)); + + OZ(cursor->column_value(exec_ctx.get_my_session(), + &exec_ctx.get_allocator(), + column_pos - 1, + result_type, + params.at(2))); + + return ret; +} + +int ObPLDbmsSql::variable_value(ObExecContext &exec_ctx, ParamStore ¶ms, ObObj &result) +{ + UNUSED(result); + int ret = OB_SUCCESS; + ObDbmsCursorInfo *cursor = NULL; + ObString name; + int64_t param_count = params.count(); + // parse all param. + ObExprResType result_type; + OV (3 == param_count, OB_INVALID_ARGUMENT, params); + OV (params.at(1).is_varchar(), OB_INVALID_ARGUMENT, params); + OZ (params.at(1).get_string(name)); + OX (result_type.set_meta(params.at(2).is_null() + ? params.at(2).get_null_meta() : params.at(2).get_meta())); + OX (result_type.set_accuracy(params.at(2).get_accuracy())); + OZ (get_cursor(exec_ctx, params, cursor)); + CK (OB_NOT_NULL(cursor)); + OZ (cursor->variable_value(exec_ctx.get_my_session(), + &exec_ctx.get_allocator(), + name, + result_type, + params.at(2))); + return ret; +} + +int ObPLDbmsSql::close_cursor(ObExecContext &exec_ctx, ParamStore ¶ms, ObObj &result) +{ + UNUSED(result); + int ret = OB_SUCCESS; + ObDbmsCursorInfo *cursor = NULL; + OZ (get_cursor(exec_ctx, params, cursor)); + CK (OB_NOT_NULL(cursor)); + // do close. + int64_t cursor_id = OB_INVALID_ID; + OX (cursor_id = cursor->get_id()); + CK (OB_NOT_NULL(exec_ctx.get_my_session())); + OZ (cursor->close(*exec_ctx.get_my_session()), cursor_id); + OZ (exec_ctx.get_my_session()->close_dbms_cursor(cursor_id), cursor_id); + + OX (params.at(0).set_null()); + + return ret; +} + +int ObPLDbmsSql::describe_columns(ObExecContext &exec_ctx, ParamStore ¶ms, ObObj &result) +{ + UNUSED(result); + return do_describe(exec_ctx, params, DESCRIBE); +} + +int ObPLDbmsSql::describe_columns2(ObExecContext &exec_ctx, ParamStore ¶ms, ObObj &result) +{ + UNUSED(result); + return do_describe(exec_ctx, params, DESCRIBE2); +} + +int ObPLDbmsSql::describe_columns3(ObExecContext &exec_ctx, ParamStore ¶ms, ObObj &result) +{ + UNUSED(result); + return do_describe(exec_ctx, params, DESCRIBE3); +} + +int ObPLDbmsSql::do_describe(ObExecContext &exec_ctx, ParamStore ¶ms, DescribeType type) +{ + int ret = OB_SUCCESS; + ObDbmsCursorInfo *cursor = NULL; + OZ (get_cursor(exec_ctx, params, cursor)); + CK (OB_NOT_NULL(cursor)); + if (OB_SUCC(ret) && cursor->get_sql_stmt().empty()) { + ret = OB_NO_STMT_PARSE; + LOG_WARN("no statement parsed or DDL is executed when parsed", K(ret)); + } + + if (OB_SUCC(ret) && !ObStmt::is_select_stmt(cursor->get_stmt_type())) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("Only select statement can be described", K(cursor->get_stmt_type()), K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, + "ORA-00900: invalid SQL statement, only select statement can be described"); + } + + OV (3 == params.count(), OB_INVALID_ARGUMENT, params); + OV (params.at(1).is_number() || params.at(1).is_null(), OB_INVALID_ARGUMENT, params); + OV (params.at(2).is_pl_extend(), OB_INVALID_ARGUMENT, params); + + if (OB_SUCC(ret)) { + ObNumber num; + OZ (num.from(cursor->get_field_columns().count(), exec_ctx.get_allocator())); + OX (params.at(1).set_number(num)); + } + + if (OB_SUCC(ret)) { + ObPLAssocArray *table = reinterpret_cast(params.at(2).get_ext()); + CK (OB_NOT_NULL(exec_ctx.get_my_session())); + CK (OB_NOT_NULL(table)); + OZ (ObSPIService::spi_extend_assoc_array(exec_ctx.get_my_session()->get_effective_tenant_id(), + NULL, + exec_ctx.get_allocator(), + *table, + cursor->get_field_columns().count())); + + /* + * TYPE desc_rec IS RECORD ( col_type BINARY_INTEGER := 0, + * col_max_len BINARY_INTEGER := 0, + * col_name VARCHAR2(32) := '', + * col_name_len BINARY_INTEGER := 0, + * col_schema_name VARCHAR2(32) := '', + * col_schema_name_len BINARY_INTEGER := 0, + * col_precision BINARY_INTEGER := 0, + * col_scale BINARY_INTEGER := 0, + * col_charsetid BINARY_INTEGER := 0, + * col_charsetform BINARY_INTEGER := 0, + * col_null_ok BOOLEAN := TRUE); + * + * TYPE desc_tab IS TABLE OF desc_rec INDEX BY BINARY_INTEGER; + * */ + ObSEArray row; + ObObj obj; + for (int64_t i = 0; OB_SUCC(ret) && i < cursor->get_field_columns().count(); ++i) { + ObField &field = cursor->get_field_columns().at(i); + row.reuse(); + + // col_type BINARY_INTEGER := 0, + OX (obj.set_int32(field.type_.get_type())); + OZ (row.push_back(obj)); + + //col_max_len BINARY_INTEGER := 0, + OX (obj.set_int32(field.length_)); + OZ (row.push_back(obj)); + + // col_name VARCHAR2(32) := '', + if (OB_SUCC(ret)) { + if (DESCRIBE == type) { + if (field.cname_.length() > 32) { + ret = OB_ERR_NUMERIC_OR_VALUE_ERROR; + LOG_WARN("character string buffer too small", K(ret), K(field.cname_)); + } else { + OX (obj.set_varchar(field.cname_)); + } + } else { //varchar2(32767) + OX (obj.set_varchar(field.cname_)); + } + OZ (row.push_back(obj)); + } + + // col_name_len BINARY_INTEGER := 0, + OX (obj.set_int32(field.cname_.length())); + OZ (row.push_back(obj)); + + // col_schema_name VARCHAR2(32) := '', + OX (obj.set_varchar(field.dname_)); + OZ (row.push_back(obj)); + + // col_schema_name_len BINARY_INTEGER := 0, + OX (obj.set_int32(field.dname_.length())); + OZ (row.push_back(obj)); + + // col_precision BINARY_INTEGER := 0, + OX (obj.set_int32(field.accuracy_.get_precision())); + OZ (row.push_back(obj)); + + // col_scale BINARY_INTEGER := 0, + OX (obj.set_int32(field.accuracy_.get_scale())); + OZ (row.push_back(obj)); + + // col_charsetid BINARY_INTEGER := 0, + OX (OX (obj.set_int32(field.charsetnr_))); + OZ (row.push_back(obj)); + + // col_charsetform BINARY_INTEGER := 0, + OX (obj.set_int32(field.charsetnr_)); + OZ (row.push_back(obj)); + + // col_null_ok BOOLEAN := TRUE + OX (obj.set_bool(0 == (field.flags_ & NOT_NULL_FLAG))); + OZ (row.push_back(obj)); + + if (OB_SUCC(ret) && DESCRIBE3 == type) { + // col_type_name varchar2(32) := '', + OX (obj.set_null()); + OZ (row.push_back(obj)); + + // col_type_name_len binary_integer := 0, + OX (obj.set_int32(0)); + OZ (row.push_back(obj)); + } + OZ (table->set_row(row, i)); + } + + OX (table->set_first(1)); + OX (table->set_last(cursor->get_field_columns().count())); + OX (table->set_key(NULL)); + OX (table->set_sort(NULL)); + } + + return ret; +} + +int ObPLDbmsSql::get_cursor(ObExecContext &exec_ctx, ParamStore ¶ms, ObDbmsCursorInfo *&cursor) +{ + int ret = OB_SUCCESS; + cursor = NULL; + ObSQLSessionInfo *session = exec_ctx.get_my_session(); + int64_t security_level = 0; + + // parse and check all param. + int64_t cursor_id = -1; + int64_t id = -1; + if (params.at(0).is_float()) { + ret = OB_ERR_DBMS_SQL_CURSOR_NOT_EXIST; + LOG_WARN("cursor id is invalid int64 ", K(cursor_id), K(id), K(ret)); + } else if (params.at(0).is_number()) { + params.at(0).get_number().is_valid_int64(id); + } + + if (OB_SUCC(ret) && 0 == id) { + ret = OB_OBEN_CURSOR_NUMBER_IS_ZERO; + LOG_WARN("dbms_sql open cursor fail, cursor id is 0", K(ret), K(id), K(security_level)); + } + // get cursor. + OV (OB_NOT_NULL(session), OB_INVALID_ARGUMENT, id); + OX (cursor_id = ObDbmsCursorInfo::convert_to_dbms_cursor_id(id)) + OV (OB_NOT_NULL(cursor = session->get_dbms_cursor(cursor_id)), + OB_ERR_DBMS_SQL_CURSOR_NOT_EXIST, cursor_id); + + return ret; +} + +int ObPLDbmsSql::is_open(ObExecContext &exec_ctx, ParamStore ¶ms, ObObj &result) +{ + int ret = OB_SUCCESS; + bool is_open = false; + ObDbmsCursorInfo *cursor = NULL; + if (params.at(0).is_null() + || params.at(0).get_number().compare(static_cast(0)) <= 0) { + // cursor id is null, return false. + } else if (OB_SUCC(get_cursor(exec_ctx, params, cursor))) { + is_open = true; + } + result.set_bool(const_cast(is_open)); + return ret; +} + +int ObPLDbmsSql::execute_and_fetch(ObExecContext &exec_ctx, ParamStore ¶ms, ObObj &result) +{ + int ret = OB_SUCCESS; + ObDbmsCursorInfo *cursor = NULL; + int64_t row_count = 0; + uint64_t param_count = params.count(); + ObNumber fetch_cnt; + bool has_open = true; + OZ (get_cursor(exec_ctx, params, cursor)); + CK (OB_NOT_NULL(cursor)); + if (OB_SUCC(ret) && cursor->get_sql_stmt().empty()) { + ret = OB_NO_STMT_PARSE; + LOG_WARN("no statement parsed or DDL is executed when parsed", K(ret)); + } + if (OB_SUCC(ret)) { + //every ececute should reset current index of Array + for (ObDbmsCursorInfo::DefineArrays::iterator iter = cursor->get_define_arrays().begin(); + OB_SUCC(ret) && iter != cursor->get_define_arrays().end(); + ++iter) { + ObDbmsCursorInfo::ArrayDesc &array_info = iter->second; + array_info.cur_idx_ = array_info.lower_bnd_ > 1 ? array_info.lower_bnd_ - 1 : 0; + } + } + if (OB_SUCC(ret) && !cursor->isopen() + && !check_stmt_need_to_be_executed_when_parsing(*cursor)) { + // do execute. + has_open = false; + OZ (do_execute(exec_ctx, *cursor, params, result)); + } + if (OB_SUCC(ret)) { + if (2 == param_count) { + if (params.at(1).is_null() && ObNullType == params.at(1).get_type()) { + // not exact_fetch + } else if (ObTinyIntType != params.at(1).get_type()) { + ret = OB_ERR_WRONG_FUNC_ARGUMENTS_TYPE; + LOG_USER_ERROR(OB_ERR_WRONG_FUNC_ARGUMENTS_TYPE, 17, "EXECUTE_AND_FETCH"); + LOG_WARN("wrong types of arguments", K(params.at(1).get_type())); + } else if (params.at(1).get_bool()) { + // exact fetch + if (OB_NOT_NULL(cursor->get_spi_cursor())) { + row_count = cursor->get_spi_cursor()->row_store_.get_row_cnt(); + } + if (row_count < 1) { + ret = OB_READ_NOTHING; + LOG_WARN("OCI_EXACT_FETCH has not enoughrows.", K(ret)); + } else if (row_count > 1) { + ret = OB_ERR_TOO_MANY_ROWS; + LOG_WARN("OCI_EXACT_FETCH has too many rows.", K(ret)); + } + } else { /* do nothing */ } + } + } + if (!has_open || (OB_NOT_NULL(cursor) && !cursor->get_fetched())) { + OZ (do_fetch(exec_ctx, params, result, *cursor)); + } else { + OZ (fetch_cnt.from(static_cast(1), exec_ctx.get_allocator())); + OX (result.set_number(fetch_cnt)); + } + return ret; +} + +/* now dbms cursor use [unstream cursor] + * so we should fetch orig cursor info into a new unstream cursor + * there is three things we should do + * 1. fill cursor field + * 2. fill spi_cursor. (which result row_store include) + * 3. close old cursor + * + * orig cursor maybe two part + * 1. stream cursor + * use fill_cursor to fill dbms cursor + * 2. unstream cursor + * fetch row store to fill dbms cursor + * + */ +int ObPLDbmsSql::fill_dbms_cursor(ObSQLSessionInfo *session, + ObPLCursorInfo *cursor, + ObDbmsCursorInfo *new_cursor) +{ + int ret = OB_SUCCESS; + uint64_t size = 0; + ObSPICursor *spi_cursor = NULL; + OV (OB_NOT_NULL(session) && OB_NOT_NULL(new_cursor), OB_ERR_UNEXPECTED); + OV (OB_NOT_NULL(cursor), OB_ERR_INVALID_CURSOR); + + // 1. fill cursor field + OV (cursor->is_streaming() ? OB_NOT_NULL(cursor->get_cursor_handler()) + : OB_NOT_NULL(cursor->get_spi_cursor())); + OZ (ObDbmsInfo::deep_copy_field_columns( + new_cursor->get_dbms_entity()->get_arena_allocator(), + cursor->is_streaming() + ? cursor->get_cursor_handler()->get_result_set()->get_field_columns() + : &(cursor->get_spi_cursor()->fields_), + new_cursor->get_field_columns())); + + // 2. fill spi_cursor + OZ (session->get_tmp_table_size(size)); + OZ (new_cursor->prepare_spi_cursor(spi_cursor, + session->get_effective_tenant_id(), + size)); + OV (OB_NOT_NULL(spi_cursor)); + + if OB_FAIL(ret) { + // do nothing + } else { + // 2.* fill row store + if (cursor->is_streaming()) { + // we can't reopen the cursor, so if fill cursor has error. we will report to client. + OZ (ObSPIService::fill_cursor(*(cursor->get_cursor_handler()->get_result_set()), spi_cursor)); + } else { + ObSPICursor *orig_spi_cursor = cursor->get_spi_cursor(); + for (int64_t i = 0; OB_SUCC(ret) && i < orig_spi_cursor->fields_.count(); ++i) { + ObDataType type; + type.set_meta_type(orig_spi_cursor->fields_.at(i).type_.get_meta()); + type.set_accuracy(orig_spi_cursor->fields_.at(i).accuracy_); + if (OB_FAIL(spi_cursor->row_desc_.push_back(type))) { + LOG_WARN("push back error", K(i), K(orig_spi_cursor->fields_.at(i).type_), + K(orig_spi_cursor->fields_.at(i).accuracy_), K(ret)); + } + } + for (int64_t i = orig_spi_cursor->cur_; OB_SUCC(ret) && i < orig_spi_cursor->row_store_.get_row_cnt(); i++) { + // pay attention to ra rowstore overload function of get_row + const ObNewRow *cur_row = NULL; + OZ (orig_spi_cursor->row_store_.get_row(i, cur_row)); + OZ (spi_cursor->row_store_.add_row(*cur_row)); + } + } + } + OX (spi_cursor->row_store_.finish_add_row()); + OX (new_cursor->open(spi_cursor)); + if (OB_FAIL(ret) && NULL != spi_cursor) { + spi_cursor->~ObSPICursor(); + LOG_WARN("fill cursor failed.", K(ret), K(new_cursor->get_id()), K(session->get_sessid())); + } + + // 3. set old cursor invalid + OX (cursor->set_invalid_cursor()); + //OZ (OB_INVALID_ID == cursor->get_id() ? cursor->close(*session) : session->close_cursor(cursor->get_id())); + return ret; +} + +int ObPLDbmsSql::to_cursor_number(ObExecContext &exec_ctx, ParamStore ¶ms, ObObj &result) +{ + int ret = OB_SUCCESS; + /* TODO : to_cursor_number param type check + * oracle : + * 1. NULL type report OB_ERR_EXP_NOT_ASSIGNABLE + * 2. wrong type report OB_ERR_CALL_WRONG_ARG + * 3. cursor type but cursor not init or be set to null report OB_ERR_INVALID_CURSOR + * OB : + * 1. in out param not support const value, to_cursor_number(NULL) report -5592 + * 2. wrong type will report -5555 when pick_routine + * 3. oracle report OB_ERR_INVALID_CURSOR when cursor type be set to null, other ext type will report OB_ERR_CALL_WRONG_ARG . + * IN OB, when a ext type be set to null, get_extend_type() will be set to a invalid type. we can't distinguish them by get_extend_type(), + * we will all report OB_ERR_CALL_WRONG_ARG now. + */ + if (params.at(0).is_ext() && (params.at(0).get_meta().get_extend_type() == PL_CURSOR_TYPE + || params.at(0).get_meta().get_extend_type() == PL_REF_CURSOR_TYPE)) { + ObPLCursorInfo *cursor = reinterpret_cast(params.at(0).get_ext()); + if (NULL != cursor) { + ObDbmsCursorInfo *new_cursor = NULL; + ObSQLSessionInfo *session = exec_ctx.get_my_session(); + ObNumber cursor_id; + OV (!cursor->is_invalid_cursor(), OB_ERR_INVALID_CURSOR); + OV (cursor->isopen(), OB_ERR_INVALID_CURSOR); + OV (OB_NOT_NULL(session)); + OZ (session->make_dbms_cursor(new_cursor)); + OV (OB_NOT_NULL(new_cursor)); + OZ (new_cursor->prepare_entity(*session)); + OZ (fill_dbms_cursor(session, cursor, new_cursor)); + OX (new_cursor->set_ref_by_refcursor()); + OX (new_cursor->set_stmt_type(stmt::T_SELECT)); + OX (new_cursor->get_sql_stmt() = "SELECT"); // placeholder for SQL stmt + if (OB_FAIL(ret) && NULL != new_cursor) { + int close_ret = session->close_cursor(new_cursor->get_id()); + if (OB_SUCCESS != close_ret) { + LOG_WARN("close new cursor fail.", K(ret)); + } + } + + // set out param + //OX (new_cursor->set_is_session_cursor()); + // OX (params.at(0).set_extend(reinterpret_cast(new_cursor), PL_REF_CURSOR_TYPE)); + // OX (params.at(0).set_param_meta()); + + // set return value + OZ (cursor_id.from(new_cursor->get_dbms_id(), exec_ctx.get_allocator())); + OX (result.set_number(cursor_id)); + } else { + ret = OB_ERR_INVALID_CURSOR; + LOG_WARN("cursor not be inited yet. ", K(ret), K(params.at(0).get_type()), K(params.at(0).get_meta().get_extend_type())); + } + } else if (params.at(0).is_null()) { + ret = OB_ERR_EXP_NOT_ASSIGNABLE; + LOG_WARN("NULL type can't to cursor number. ", K(ret)); + } else { + ret = OB_ERR_CALL_WRONG_ARG; + LOG_WARN("use wrong type. ", K(ret), K(params.at(0).get_type()), K(params.at(0).get_meta().get_extend_type())); + } + return ret; +} + +int ObPLDbmsSql::last_error_position(ObExecContext &exec_ctx, ParamStore ¶ms, ObObj &result) { + int ret = OB_SUCCESS; + + ObSQLSessionInfo *session = exec_ctx.get_my_session(); + CK(OB_NOT_NULL(session)); + CK(params.count() == 0); + + ObNumber res; + int64_t pos = -1; + OX (pos = session->get_warnings_buffer().get_error_column()); + OZ (res.from(pos, exec_ctx.get_eval_res_allocator())); + OX (result.set_number(res)); + + return ret; +} + +// start of dbms_sql +// TODO: not support long type. define_column_long, +// The implementation of define_column_long and column_value_long not support yet + +int ObPLDbmsSql::define_column_long(ObExecContext &exec_ctx, ParamStore ¶ms, ObObj &result) +{ + UNUSED(exec_ctx); + UNUSED(params); + UNUSED(result); + int ret = OB_ERR_BIND_TYPE_NOT_MATCH_COLUMN; + LOG_WARN("not support long type yet.", K(ret)); + return ret; +} + +int ObPLDbmsSql::column_value_long(ObExecContext &exec_ctx, ParamStore ¶ms, ObObj &result) +{ + UNUSED(exec_ctx); + UNUSED(params); + UNUSED(result); + int ret = OB_ERR_BIND_TYPE_NOT_MATCH_COLUMN; + LOG_WARN("not support long type yet.", K(ret)); + return ret; +} +#endif } } diff --git a/src/pl/sys_package/ob_dbms_sql.h b/src/pl/sys_package/ob_dbms_sql.h index 836331271..7c1a06e60 100644 --- a/src/pl/sys_package/ob_dbms_sql.h +++ b/src/pl/sys_package/ob_dbms_sql.h @@ -82,6 +82,34 @@ public: int add_param_name(ObString &clone_name); int set_into_names(int64_t into_cnt); ObIArray &get_into_names() { return into_names_; } +#ifdef OB_BUILD_ORACLE_PL + int bind_variable(const common::ObString ¶m_name, const common::ObObjParam ¶m_value); + /* + * 考虑到BIND_ARRAY接口，后面需要做如下修改： + * 1. 接口名改为expand_next_params(); + * 2. 接口逻辑改为尝试构造下一组执行params，如果某个param的数组已经迭代完，返回4008; + */ + int expand_params(); + int define_column(int64_t col_idx, ObObjType col_type, + ObCollationType col_cs_type, int64_t col_size); + int define_array(int64_t col_idx, + uint64_t id, + int64_t cnt, + int64_t lower_bnd, + ObDataType &elem_type); + int column_value(sql::ObSQLSessionInfo *session, + ObIAllocator *allocator, + int64_t col_idx, + const ObObjParam src_obj, + sql::ObExprResType &result_type, + ObObjParam &result); + int variable_value(sql::ObSQLSessionInfo *session, + ObIAllocator *allocator, + int64_t col_idx, + const ObObjParam src_obj, + sql::ObExprResType &result_type, + ObObjParam &result); +#endif protected: class BindParam @@ -190,6 +218,18 @@ public: virtual int close(sql::ObSQLSessionInfo &session, bool is_cursor_reuse = false, bool is_dbms_reuse = false); +#ifdef OB_BUILD_ORACLE_PL + int column_value(sql::ObSQLSessionInfo *session, + ObIAllocator *allocator, + int64_t col_idx, + sql::ObExprResType &result_type, + ObObjParam &result); + int variable_value(sql::ObSQLSessionInfo *session, + ObIAllocator *allocator, + const ObString &variable_name, + sql::ObExprResType &result_type, + ObObjParam &result); +#endif public: int init(); @@ -208,6 +248,115 @@ private: int64_t affected_rows_; }; +#ifdef OB_BUILD_ORACLE_PL +class ObPLDbmsSql +{ +public: + static int open_cursor(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + common::ObObj &result); + static int parse(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + common::ObObj &result); + static int bind_variable(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + common::ObObj &result); + static int define_column_number(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + common::ObObj &result); + static int define_column_varchar(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + common::ObObj &result); + static int define_column(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + ObObj &result); + static int define_array(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + ObObj &result); + static int execute(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + common::ObObj &result); + static int fetch_rows(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + common::ObObj &result); +//static int column_value_number(sql::ObExecContext &exec_ctx, +// ParamStore ¶ms, +// common::ObObj &result); +//static int column_value_varchar(sql::ObExecContext &exec_ctx, +// ParamStore ¶ms, +// common::ObObj &result); + static int column_value(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + common::ObObj &result); + static int variable_value(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + common::ObObj &result); + static int close_cursor(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + common::ObObj &result); + + static int describe_columns(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + ObObj &result); + static int describe_columns2(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + ObObj &result); + static int describe_columns3(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + ObObj &result); + static int is_open(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + ObObj &result); + static int execute_and_fetch(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + ObObj &result); + static int to_cursor_number(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + ObObj &result); + static int define_column_long(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + ObObj &result); + static int column_value_long(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + ObObj &result); + static int last_error_position(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + ObObj &result); + +private: + static int do_execute(sql::ObExecContext &exec_ctx, + ObDbmsCursorInfo &cursor); + static int do_execute(sql::ObExecContext &exec_ctx, + ObDbmsCursorInfo &cursor, + ParamStore ¶ms, + common::ObObj &result); + static int do_fetch(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + common::ObObj &result, + ObDbmsCursorInfo &cursor); + static int get_cursor(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + ObDbmsCursorInfo *&cursor); + static bool check_stmt_need_to_be_executed_when_parsing(ObDbmsCursorInfo &cursor); + + enum DescribeType { + DESCRIBE = 0, + DESCRIBE2, + DESCRIBE3, + }; + + static int do_describe(sql::ObExecContext &exec_ctx, ParamStore ¶ms, DescribeType type); + static int do_parse(sql::ObExecContext &exec_ctx, + ObDbmsCursorInfo *cursor, + common::ObString &sql_stmt); + static int parse_6p(sql::ObExecContext &exec_ctx, + ParamStore ¶ms, + common::ObObj &result); + static int fill_dbms_cursor(sql::ObSQLSessionInfo *session, + ObPLCursorInfo *cursor, + ObDbmsCursorInfo *new_cursor); +}; +#endif } } diff --git a/src/pl/sys_package/ob_pl_dbms_resource_manager.cpp b/src/pl/sys_package/ob_pl_dbms_resource_manager.cpp deleted file mode 100644 index cac818fdf..000000000 --- a/src/pl/sys_package/ob_pl_dbms_resource_manager.cpp +++ /dev/null @@ -1,379 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#define USING_LOG_PREFIX PL -#include "ob_pl_dbms_resource_manager.h" -#include "share/resource_manager/ob_resource_manager_proxy.h" -#include "sql/privilege_check/ob_ora_priv_check.h" - -using namespace oceanbase::common; -using namespace oceanbase::share; -using namespace oceanbase::pl; - -int ObPlDBMSResourceManager::create_plan( - sql::ObExecContext &ctx, - sql::ParamStore ¶ms, - common::ObObj &result) -{ - enum { - PLAN = 0, - COMMENT = 1, - MAX_PARAM - }; - - int ret = OB_SUCCESS; - uint64_t tenant_id; - UNUSED(result); - ObResourceManagerProxy proxy; - ObString plan; - sql::ObSQLSessionInfo *sess = GET_MY_SESSION(ctx); - if (OB_ISNULL(sess) || params.count() < MAX_PARAM) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("err unexpected", K(params.count()), K(MAX_PARAM), K(ret)); - } else { - tenant_id = sess->get_effective_tenant_id(); - } - for (int64_t i = 0; OB_SUCC(ret) && i < params.count() && i < COMMENT; ++i) { - ObObj &obj = params.at(i); - if (PLAN == i) { - ret = obj.get_string(plan); - } - } - if (OB_SUCC(ret)) { - if (OB_UNLIKELY(0 == plan.length())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("name of plan cannot be null or empty", K(ret)); - // plan 肯定存在，所以可以 get_string 读取， - // COMMENT 是可选的可能为 null，所以传入 ObObj - } else if (OB_FAIL(proxy.create_plan(tenant_id, plan, params.at(COMMENT)))) { - LOG_WARN("fail create plan", K(tenant_id), K(plan), K(ret)); - } - } - return ret; -} - - -int ObPlDBMSResourceManager::delete_plan( - sql::ObExecContext &ctx, - sql::ParamStore ¶ms, - common::ObObj &result) -{ - enum { - PLAN = 0, - MAX_PARAM - }; - - int ret = OB_SUCCESS; - uint64_t tenant_id; - UNUSED(result); - ObResourceManagerProxy proxy; - ObString plan; - sql::ObSQLSessionInfo *sess = GET_MY_SESSION(ctx); - if (OB_ISNULL(sess) || params.count() < MAX_PARAM) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("err unexpected", K(params.count()), K(MAX_PARAM), K(ret)); - } else { - tenant_id = sess->get_effective_tenant_id(); - } - for (int64_t i = 0; OB_SUCC(ret) && i < params.count() && i < MAX_PARAM; ++i) { - ObObj &obj = params.at(i); - if (PLAN == i) { - ret = obj.get_string(plan); - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(proxy.delete_plan(tenant_id, plan))) { - LOG_WARN("fail create plan", K(tenant_id), K(plan), K(ret)); - } - } - return ret; -} - -int ObPlDBMSResourceManager::create_consumer_group( - sql::ObExecContext &ctx, - sql::ParamStore ¶ms, - common::ObObj &result) -{ - enum { - CONSUMER_GROUP = 0, - COMMENT = 1, - MAX_PARAM - }; - - int ret = OB_SUCCESS; - uint64_t tenant_id; - UNUSED(result); - ObResourceManagerProxy proxy; - ObString consumer_group; - sql::ObSQLSessionInfo *sess = GET_MY_SESSION(ctx); - if (OB_ISNULL(sess) || params.count() < MAX_PARAM) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("err unexpected", K(params.count()), K(MAX_PARAM), K(ret)); - } else { - tenant_id = sess->get_effective_tenant_id(); - } - for (int64_t i = 0; OB_SUCC(ret) && i < params.count() && i < COMMENT; ++i) { - ObObj &obj = params.at(i); - if (CONSUMER_GROUP == i) { - ret = obj.get_string(consumer_group); - } - } - if (OB_SUCC(ret)) { - if (OB_UNLIKELY(0 == consumer_group.length())) { - ret = OB_INVALID_ARGUMENT; - LOG_WARN("name of consumer group cannot be null or empty", K(ret)); - // consumer_group 肯定存在，所以可以 get_string 读取， - // COMMENT 是可选的可能为 null，所以传入 ObObj - } else if (OB_FAIL(proxy.create_consumer_group(tenant_id, consumer_group, params.at(COMMENT)))) { - LOG_WARN("fail create consumer_group", K(tenant_id), K(consumer_group), K(ret)); - } - } - return ret; -} - - -int ObPlDBMSResourceManager::delete_consumer_group( - sql::ObExecContext &ctx, - sql::ParamStore ¶ms, - common::ObObj &result) -{ - enum { - CONSUMER_GROUP = 0, - MAX_PARAM - }; - - int ret = OB_SUCCESS; - uint64_t tenant_id; - UNUSED(result); - ObResourceManagerProxy proxy; - ObString consumer_group; - sql::ObSQLSessionInfo *sess = GET_MY_SESSION(ctx); - if (OB_ISNULL(sess) || params.count() < MAX_PARAM) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("err unexpected", K(params.count()), K(MAX_PARAM), K(ret)); - } else { - tenant_id = sess->get_effective_tenant_id(); - } - for (int64_t i = 0; OB_SUCC(ret) && i < params.count() && i < MAX_PARAM; ++i) { - ObObj &obj = params.at(i); - if (CONSUMER_GROUP == i) { - ret = obj.get_string(consumer_group); - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(proxy.delete_consumer_group(tenant_id, consumer_group))) { - LOG_WARN("fail create consumer_group", K(tenant_id), K(consumer_group), K(ret)); - } - } - return ret; -} - -int ObPlDBMSResourceManager::create_plan_directive( - sql::ObExecContext &ctx, - sql::ParamStore ¶ms, - common::ObObj &result) -{ - enum { - PLAN = 0, - GROUP = 1, - COMMENT = 2, - MGMT_P1 = 3, - UTILIZATION_LIMIT = 4, - MIN_IOPS = 5, - MAX_IOPS = 6, - WEIGHT_IOPS = 7, - MAX_PARAM - }; - - int ret = OB_SUCCESS; - uint64_t tenant_id; - UNUSED(result); - ObResourceManagerProxy proxy; - ObString plan; - ObString group; - sql::ObSQLSessionInfo *sess = GET_MY_SESSION(ctx); - if (OB_ISNULL(sess) || params.count() < MAX_PARAM) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("err unexpected", K(params.count()), K(MAX_PARAM), K(ret)); - } else { - tenant_id = sess->get_effective_tenant_id(); - } - for (int64_t i = 0; OB_SUCC(ret) && i < params.count() && i < COMMENT; ++i) { - LOG_INFO("pl params", K(params.at(i))); - ObObj &obj = params.at(i); - if (PLAN == i) { - ret = obj.get_string(plan); - } else if (GROUP == i) { - ret = obj.get_string(group); - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(proxy.create_plan_directive(tenant_id, - plan, - group, - params.at(COMMENT), - params.at(MGMT_P1), - params.at(UTILIZATION_LIMIT), - params.at(MIN_IOPS), - params.at(MAX_IOPS), - params.at(WEIGHT_IOPS)))) { - LOG_WARN("fail create plan directive", K(tenant_id), K(plan), K(group), K(ret)); - } - } - return ret; -} - - -int ObPlDBMSResourceManager::delete_plan_directive( - sql::ObExecContext &ctx, - sql::ParamStore ¶ms, - common::ObObj &result) -{ - enum { - PLAN = 0, - GROUP = 1, - MAX_PARAM - }; - - int ret = OB_SUCCESS; - uint64_t tenant_id; - UNUSED(result); - ObResourceManagerProxy proxy; - ObString plan; - ObString group; - sql::ObSQLSessionInfo *sess = GET_MY_SESSION(ctx); - if (OB_ISNULL(sess) || params.count() < MAX_PARAM) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("err unexpected", K(params.count()), K(MAX_PARAM), K(ret)); - } else { - tenant_id = sess->get_effective_tenant_id(); - } - for (int64_t i = 0; OB_SUCC(ret) && i < params.count() && i < MAX_PARAM; ++i) { - ObObj &obj = params.at(i); - if (PLAN == i) { - ret = obj.get_string(plan); - } else if (GROUP == i) { - ret = obj.get_string(group); - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(proxy.delete_plan_directive(tenant_id, plan, group))) { - LOG_WARN("fail create plan", K(tenant_id), K(plan), K(ret)); - } - } - return ret; -} - -int ObPlDBMSResourceManager::update_plan_directive( - sql::ObExecContext &ctx, - sql::ParamStore ¶ms, - common::ObObj &result) -{ - enum { - PLAN = 0, - GROUP = 1, - COMMENT = 2, - MGMT_P1 = 3, - UTILIZATION_LIMIT = 4, - MIN_IOPS = 5, - MAX_IOPS = 6, - WEIGHT_IOPS = 7, - MAX_PARAM - }; - int ret = OB_SUCCESS; - uint64_t tenant_id; - UNUSED(result); - ObResourceManagerProxy proxy; - ObString plan; - ObString group; - sql::ObSQLSessionInfo *sess = GET_MY_SESSION(ctx); - if (OB_ISNULL(sess) || params.count() < MAX_PARAM) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("err unexpected", K(params.count()), K(MAX_PARAM), K(ret)); - } else { - tenant_id = sess->get_effective_tenant_id(); - } - for (int64_t i = 0; OB_SUCC(ret) && i < params.count() && i < 2; ++i) { - ObObj &obj = params.at(i); - if (PLAN == i) { - ret = obj.get_string(plan); - } else if (GROUP == i) { - ret = obj.get_string(group); - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(proxy.update_plan_directive(tenant_id, - plan, - group, - params.at(COMMENT), - params.at(MGMT_P1), - params.at(UTILIZATION_LIMIT), - params.at(MIN_IOPS), - params.at(MAX_IOPS), - params.at(WEIGHT_IOPS)))) { - LOG_WARN("fail update plan directive", K(tenant_id), K(plan), K(group), K(ret)); - } - } - return ret; -} - -int ObPlDBMSResourceManager::set_consumer_group_mapping( - sql::ObExecContext &ctx, - sql::ParamStore ¶ms, - common::ObObj &result) -{ - enum { - ATTR = 0, - VALUE = 1, - GROUP = 2, - MAX_PARAM - }; - - int ret = OB_SUCCESS; - uint64_t tenant_id = OB_INVALID_ID; - UNUSED(result); - ObResourceManagerProxy proxy; - ObString attr; - ObString value; - ObString group; - sql::ObSQLSessionInfo *sess = GET_MY_SESSION(ctx); - if (OB_ISNULL(sess) || params.count() < MAX_PARAM) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("err unexpected", K(params.count()), K(MAX_PARAM), K(ret)); - } else { - tenant_id = sess->get_effective_tenant_id(); - } - for (int64_t i = 0; OB_SUCC(ret) && i < params.count(); ++i) { - ObObj &obj = params.at(i); - if (ATTR == i) { - ret = obj.get_string(attr); - } else if (VALUE == i) { - ret = obj.get_string(value); - } else if (GROUP == i) { - // note: group 不指定 (obj is null) - // 时表示取消 value 对应的 group 绑定 - if (!obj.is_null()) { - ret = obj.get_string(group); - } - } - } - if (OB_SUCC(ret)) { - if (OB_FAIL(proxy.replace_mapping_rule(tenant_id, - attr, - value, - group, - *sess))) { - LOG_WARN("fail update plan directive", K(tenant_id), K(attr), K(value), K(group), K(ret)); - } - } - return ret; -} diff --git a/src/pl/sys_package/ob_pl_dbms_resource_manager.h b/src/pl/sys_package/ob_pl_dbms_resource_manager.h deleted file mode 100644 index 675e33faf..000000000 --- a/src/pl/sys_package/ob_pl_dbms_resource_manager.h +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#ifndef _OCEANBASE_SRC_PL_SYS_PACKAGE_DBMS_RESOURCE_MANANGER_PL_H_ -#define _OCEANBASE_SRC_PL_SYS_PACKAGE_DBMS_RESOURCE_MANANGER_PL_H_ - -#include "sql/engine/ob_exec_context.h" - -namespace oceanbase -{ -namespace pl -{ - -class ObPlDBMSResourceManager -{ -public: - ObPlDBMSResourceManager() {} - virtual ~ObPlDBMSResourceManager() {} -public: - static int create_plan( - sql::ObExecContext &ctx, - sql::ParamStore ¶ms, - common::ObObj &result); - static int delete_plan( - sql::ObExecContext &ctx, - sql::ParamStore ¶ms, - common::ObObj &result); - static int create_consumer_group( - sql::ObExecContext &ctx, - sql::ParamStore ¶ms, - common::ObObj &result); - static int delete_consumer_group( - sql::ObExecContext &ctx, - sql::ParamStore ¶ms, - common::ObObj &result); - static int create_plan_directive( - sql::ObExecContext &ctx, - sql::ParamStore ¶ms, - common::ObObj &result); - static int delete_plan_directive( - sql::ObExecContext &ctx, - sql::ParamStore ¶ms, - common::ObObj &result); - static int update_plan_directive( - sql::ObExecContext &ctx, - sql::ParamStore ¶ms, - common::ObObj &result); - static int set_consumer_group_mapping( - sql::ObExecContext &ctx, - sql::ParamStore ¶ms, - common::ObObj &result); -private: - /* functions */ - /* variables */ - DISALLOW_COPY_AND_ASSIGN(ObPlDBMSResourceManager); -}; - -} -} -#endif /* _OCEANBASE_SRC_PL_SYS_PACKAGE_DBMS_RESOURCE_MANANGER_PL_H_ */ -//// end of header file diff --git a/src/rootserver/backup/ob_archive_scheduler_service.cpp b/src/rootserver/backup/ob_archive_scheduler_service.cpp index 7378eefd8..5a5d2c8e4 100644 --- a/src/rootserver/backup/ob_archive_scheduler_service.cpp +++ b/src/rootserver/backup/ob_archive_scheduler_service.cpp @@ -393,7 +393,8 @@ void ObArchiveSchedulerService::set_checkpoint_interval_(const int64_t interval_ int ObArchiveSchedulerService::open_archive_mode(const uint64_t tenant_id, const common::ObIArray &archive_tenant_ids) { - // TODO: print error trace to user + + // TODO(wangxiaohui.wxh):4.3, print error trace to user int ret = OB_SUCCESS; ObArray bak_archive_tenant_ids; if (IS_NOT_INIT) { @@ -430,7 +431,8 @@ int ObArchiveSchedulerService::open_archive_mode(const uint64_t tenant_id, const int ObArchiveSchedulerService::open_tenant_archive_mode_( const common::ObIArray &tenant_ids_array) { - // TODO: return failed if any tenant failed + + // TODO(wangxiaohui.wxh):4.3, return failed if any tenant failed int ret = OB_SUCCESS; for (int64_t i = 0; i < tenant_ids_array.count(); i++) { int tmp_ret = OB_SUCCESS; diff --git a/src/rootserver/backup/ob_backup_clean_scheduler.cpp b/src/rootserver/backup/ob_backup_clean_scheduler.cpp index 9aaa8332c..f5605298f 100644 --- a/src/rootserver/backup/ob_backup_clean_scheduler.cpp +++ b/src/rootserver/backup/ob_backup_clean_scheduler.cpp @@ -1255,7 +1255,7 @@ int ObUserTenantBackupDeleteMgr::get_delete_backup_set_infos_(ObArray &piece_list) { int ret = OB_SUCCESS; - // TODO 4.1 support + // TODO(wenjinyu.wjy) 4.3 support return ret; } diff --git a/src/rootserver/backup/ob_backup_clean_task_mgr.cpp b/src/rootserver/backup/ob_backup_clean_task_mgr.cpp index cf3ea5066..ed4dfbc07 100644 --- a/src/rootserver/backup/ob_backup_clean_task_mgr.cpp +++ b/src/rootserver/backup/ob_backup_clean_task_mgr.cpp @@ -101,7 +101,7 @@ int ObBackupCleanTaskMgr::init( backup_service_ = &backup_service; is_inited_ = true; } - // TODO: integrate sql_proxy and lease_service 4.1 + // TODO(wenjinyu.wjy): integrate sql_proxy and lease_service 4.3 return ret; } @@ -604,6 +604,7 @@ int ObBackupCleanTaskMgr::get_ls_ids_from_traverse_(const ObBackupPath &path, Ob LOG_WARN("failed to set log stream prefix", K(ret)); } else if (OB_FAIL(prefix_op.init(logstream_prefix, strlen(logstream_prefix)))) { LOG_WARN("failed to init dir prefix", K(ret), K(logstream_prefix)); + // TODO(wenjinyu.wjy) iterate dir sequentially 4.3 } else if (OB_FAIL(util.list_directories(path.get_obstr(), backup_dest_.get_storage_info(), prefix_op))) { LOG_WARN("failed to list files", K(ret)); } else { @@ -761,6 +762,7 @@ int ObBackupCleanTaskMgr::delete_data_info_turn_files_(const ObBackupPath &infos ObDirPrefixEntryNameFilter prefix_op(d_entrys); if (OB_FAIL(prefix_op.init(info_turn_prefix, strlen(info_turn_prefix)))) { LOG_WARN("failed to init dir prefix", K(ret), K(info_turn_prefix)); + // TODO(wenjinyu.wjy) iterate dir sequentially 4.3 } else if (OB_FAIL(util.list_directories(infos_path.get_obstr(), backup_dest_.get_storage_info(), prefix_op))) { LOG_WARN("failed to list directories", K(ret)); } else { diff --git a/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp b/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp index b063b2fe1..78036c45a 100644 --- a/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp +++ b/src/rootserver/backup/ob_backup_data_set_task_mgr.cpp @@ -1218,7 +1218,9 @@ int ObBackupSetTaskMgr::update_inner_task_( if (nullptr == ls_attr) { ret = OB_ERR_UNEXPECTED; LOG_WARN("[DATA_BACKUP]null ls ptr", K(ret)); - // TODO use another error code to determine change turn in 4.1 + // else if (OB_LS_NOT_EXIST == ls_attr->result_) { + // LOG_WARN("[DATA_BACKUP]ls has been delete, need not to redo backup", K(ret), "ls_id", ls_attr->ls_id_); + // } // TODO(yangyi.yyy): use another error code to determine change turn in 4.1 } else if (OB_FAIL(calc_task_turn_(ls_attr->task_type_, turn_id))) { LOG_WARN("failed to calc task turn id", K(ret)); } else if (OB_FAIL(ObBackupDataLSTaskMgr::redo_ls_task( diff --git a/src/rootserver/backup/ob_backup_service.cpp b/src/rootserver/backup/ob_backup_service.cpp index 3b281bd40..6eb6c2c9b 100644 --- a/src/rootserver/backup/ob_backup_service.cpp +++ b/src/rootserver/backup/ob_backup_service.cpp @@ -321,7 +321,7 @@ int ObBackupCleanService::handle_backup_delete(const obrpc::ObBackupCleanArg &ar }; case ObNewBackupCleanType::DELETE_BACKUP_SET: case ObNewBackupCleanType::DELETE_BACKUP_PIECE: { - // TODO 4.1 support delete backup set/piece + // TODO(wenjinyu.wjy) 4.3 support delete backup set/piece ret = OB_NOT_SUPPORTED; break; }; @@ -333,7 +333,7 @@ int ObBackupCleanService::handle_backup_delete(const obrpc::ObBackupCleanArg &ar break; }; case ObNewBackupCleanType::DELETE_BACKUP_ALL: { - // TODO 4.1 support delete backup all function + // TODO(wenjinyu.wjy) 4.3 support delete backup all function ret = OB_NOT_SUPPORTED; break; }; diff --git a/src/rootserver/backup/ob_backup_task_scheduler.cpp b/src/rootserver/backup/ob_backup_task_scheduler.cpp index 680be7d8f..4efd2fc19 100644 --- a/src/rootserver/backup/ob_backup_task_scheduler.cpp +++ b/src/rootserver/backup/ob_backup_task_scheduler.cpp @@ -277,7 +277,8 @@ int ObBackupTaskSchedulerQueue::pop_task(ObBackupScheduleTask *&output_task, com DLIST_FOREACH(t, wait_list_) { if (!backup_zone.empty() || !backup_region.empty()) { - // TODO: when backup zone and backup region scheme is ready, adjust this code. + // TODO(chongrong.th): when backup zone and backup region scheme is ready, adjust this code in 4.3 + // only backup ls task need the defensive operation ObArray empty_block_server; if (!t->can_execute_on_any_server() && BackupJobType::BACKUP_BACKUP_DATA_JOB == t->get_type()) { ObBackupDataLSTask *tmp_task = static_cast(t); diff --git a/src/rootserver/ob_bootstrap.cpp b/src/rootserver/ob_bootstrap.cpp index a6912e2d7..1bcabd35f 100644 --- a/src/rootserver/ob_bootstrap.cpp +++ b/src/rootserver/ob_bootstrap.cpp @@ -270,6 +270,23 @@ int ObPreBootstrap::prepare_bootstrap(ObAddr &master_rs) int ObPreBootstrap::notify_sys_tenant_root_key() { int ret = OB_SUCCESS; +#ifdef OB_BUILD_TDE_SECURITY + ObArray addrs; + obrpc::ObRootKeyArg arg; + if (OB_FAIL(addrs.reserve(rs_list_.count()))) { + LOG_WARN("fail to reserve array", KR(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < rs_list_.count(); i++) { + if (OB_FAIL(addrs.push_back(rs_list_[i].server_))) { + LOG_WARN("fail to push back server", KR(ret)); + } + } // end for + if (FAILEDx(ObDDLService::create_root_key( + rpc_proxy_, OB_SYS_TENANT_ID, addrs))) { + LOG_WARN("fail to create sys tenant root key", KR(ret), K(addrs)); + } + BOOTSTRAP_CHECK_SUCCESS(); +#endif return ret; } diff --git a/src/rootserver/ob_ddl_operator.cpp b/src/rootserver/ob_ddl_operator.cpp index 0eaf690a0..8a929780f 100644 --- a/src/rootserver/ob_ddl_operator.cpp +++ b/src/rootserver/ob_ddl_operator.cpp @@ -5606,6 +5606,18 @@ int ObDDLOperator::init_tenant_users(const ObTenantSchema &tenant_schema, "system administrator", trans))) { RS_LOG(WARN, "failed to init sys user", K(ret), K(tenant_id)); } +#ifdef OB_BUILD_TDE_SECURITY + if (OB_SUCC(ret)) { + if (OB_FAIL(share::ObKeyGenerator::generate_encrypt_key(ora_auditor_password, + ENCRYPT_KEY_LENGTH))) { + RS_LOG(WARN, "failed to generate auditor's password", K(ret), K(tenant_id)); + } else if (OB_FAIL(init_tenant_user(tenant_id, ora_auditor_user_name, + ObString(ENCRYPT_KEY_LENGTH, ora_auditor_password), + OB_ORA_AUDITOR_USER_ID, "system administrator", trans, true))) { + RS_LOG(WARN, "failed to init mysql audit user", K(ret), K(tenant_id)); + } + } +#endif } //TODO in standby cluster, temp logical, will be deleted after inner sql ready diff --git a/src/rootserver/ob_ddl_service.cpp b/src/rootserver/ob_ddl_service.cpp index b17ab4cf3..5bac6bb92 100755 --- a/src/rootserver/ob_ddl_service.cpp +++ b/src/rootserver/ob_ddl_service.cpp @@ -104,6 +104,14 @@ #include "logservice/data_dictionary/ob_data_dict_storager.h" // ObDataDictStorage #include "share/scn.h" #include "share/backup/ob_backup_config.h" // ObBackupConfigParserMgr +#ifdef OB_BUILD_ARBITRATION +#include "share/arbitration_service/ob_arbitration_service_table_operator.h" +#include "share/arbitration_service/ob_arbitration_service_utils.h" // ObArbitrationServiceUtils +#endif +#ifdef OB_BUILD_ORACLE_PL +#include "pl/sys_package/ob_dbms_audit_mgmt.h" // ObDbmsAuditMgmt +#include "share/backup/ob_log_restore_config.h"//ObLogRestoreSourceServiceConfigParser +#endif #include "storage/tx_storage/ob_ls_map.h" #include "storage/tx_storage/ob_ls_service.h" #include "storage/tablelock/ob_lock_inner_connection_util.h" @@ -21188,6 +21196,35 @@ int ObDDLService::create_tenant_schema( } else if (OB_FAIL(trans.start(sql_proxy_, OB_SYS_TENANT_ID, refreshed_schema_version))) { LOG_WARN("start transaction failed, ", KR(ret), K(refreshed_schema_version)); } +#ifdef OB_BUILD_ARBITRATION + // check arbitration service if needed + ObArbitrationServiceTableOperator arbitration_service_table_operator; + const ObString arbitration_service_key("default"); + const bool lock_line = true; + ObArbitrationServiceInfo arbitration_service_info; + if (OB_FAIL(ret)) { + } else if (meta_tenant_schema.get_arbitration_service_status() + != user_tenant_schema.get_arbitration_service_status()) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("tenant has different arbitration service status with its meta tenant", KR(ret), + "meta_tenant_arb_status", meta_tenant_schema.get_arbitration_service_status(), + "user_tenant_arb_status", user_tenant_schema.get_arbitration_service_status()); + } else if (meta_tenant_schema.get_arbitration_service_status().is_enabling() + || meta_tenant_schema.get_arbitration_service_status().is_enabled()) { + if (OB_FAIL(arbitration_service_table_operator.get( + trans, + arbitration_service_key, + lock_line, + arbitration_service_info))) { + if (OB_ARBITRATION_SERVICE_NOT_EXIST == ret) { + ret = OB_OP_NOT_ALLOW; + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "arbitration service not exist, create tenant"); + } + LOG_WARN("fail to get arbitration service", KR(ret), K(arbitration_service_key), + K(lock_line), K(arbitration_service_info)); + } + } +#endif // 1. create tenant schema if (OB_SUCC(ret)) { LOG_INFO("[CREATE_TENANT] STEP 1.1. start create tenant schema", K(arg)); @@ -21257,6 +21294,29 @@ int ObDDLService::create_tenant_schema( "cost", ObTimeUtility::fast_current_time() - tmp_start_time); } +#ifdef OB_BUILD_TDE_SECURITY + if (OB_SUCC(ret)) { + LOG_INFO("[CREATE_TENANT] STEP 1.5. start create root key", K(user_tenant_id)); + const int64_t tmp_start_time = ObTimeUtility::fast_current_time(); + ObArray addrs; + bool need_create = false; + if (OB_FAIL(check_need_create_root_key(arg, need_create))) { + LOG_WARN("fail to check need create root key", K(ret)); + } else if (!need_create) { + // do nothing + } else if (OB_FAIL(unit_mgr_->get_servers_by_pools(pools, addrs))) { + LOG_WARN("fail to get tenant's servers", KR(ret), K(user_tenant_id)); + } else if (arg.is_creating_standby_) { + if (OB_FAIL(standby_create_root_key(user_tenant_id, arg, addrs))) { + LOG_WARN("failed to create root key", KR(ret), K(user_tenant_id), K(arg)); + } + } else if (OB_FAIL(create_root_key(*rpc_proxy_, user_tenant_id, addrs))) { + LOG_WARN("fail to create root key", KR(ret), K(addrs)); + } + LOG_INFO("[CREATE_TENANT] STEP 1.5. finish create root key", + KR(ret), K(user_tenant_id), "cost", ObTimeUtility::fast_current_time() - tmp_start_time); + } +#endif if (OB_SUCC(ret)) { ObArray addrs; @@ -21363,6 +21423,305 @@ int ObDDLService::notify_init_tenant_config( return ret; } +#ifdef OB_BUILD_TDE_SECURITY +int ObDDLService::check_need_create_root_key(const ObCreateTenantArg &arg, bool &need_create) +{ + int ret = OB_SUCCESS; + need_create = false; + if (GET_MIN_CLUSTER_VERSION() < CLUSTER_VERSION_4_2_0_0) { + need_create = false; + } else if (arg.is_restore_) { + need_create = false; + } else { + need_create = true; + } + return ret; +} + +int ObDDLService::standby_create_root_key( + const uint64_t tenant_id, + const obrpc::ObCreateTenantArg &arg, + const common::ObIArray &addrs) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(check_inner_stat())) { + LOG_WARN("variable is not init", KR(ret)); + } else if (OB_UNLIKELY(!is_user_tenant(tenant_id) || !arg.is_creating_standby_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(arg)); + } else { + obrpc::RootKeyType key_type = obrpc::RootKeyType::INVALID; + RootKeyValue root_key; + if (OB_FAIL(get_root_key_from_primary(arg, tenant_id, key_type, root_key))) { + LOG_WARN("failed to get root key", KR(ret), K(arg), K(tenant_id)); + } else { + obrpc::ObRootKeyArg root_key_arg; + obrpc::ObRootKeyResult dummy_result; + ObString key_value_str(root_key.ptr()); + if (OB_FAIL(root_key_arg.init(tenant_id, key_type, key_value_str))) { + LOG_WARN("failed to init root key arg", KR(ret), K(tenant_id), K(key_type), K(root_key)); + } else if (OB_FAIL(notify_root_key(*rpc_proxy_, root_key_arg, addrs, dummy_result))) { + LOG_WARN("fail to notify root key", K(ret), K(root_key_arg)); + } + } + } + return ret; +} + +int ObDDLService::get_root_key_from_primary(const obrpc::ObCreateTenantArg &arg, + const uint64_t tenant_id, obrpc::RootKeyType &key_type, + RootKeyValue &key_value) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(check_inner_stat())) { + LOG_WARN("variable is not init", KR(ret)); + } else if (OB_UNLIKELY(!is_user_tenant(tenant_id) || !arg.is_creating_standby_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(arg)); + } else { + uint64_t primary_tenant_id = OB_INVALID_TENANT_ID; + uint64_t cluster_id = OB_INVALID_ID; + ObArray addr_list; + ObLogRestoreSourceServiceConfigParser log_restore_source(ObBackupConfigType::LOG_RESTORE_SOURCE, tenant_id); + common::ObSqlString value; + obrpc::ObRootKeyArg root_key_arg; + if (OB_FAIL(value.assign(arg.log_restore_source_))) { + LOG_WARN("fail to assign value", KR(ret), K(log_restore_source)); + } else if (OB_FAIL(log_restore_source.get_primary_server_addr( + value, primary_tenant_id, cluster_id, addr_list))) { + LOG_WARN("failed to get primary server addr", KR(ret), K(value)); + } else if (OB_FAIL(root_key_arg.init_for_get(primary_tenant_id))) { + LOG_WARN("failed to init for get", KR(ret), K(primary_tenant_id)); + } + if (FAILEDx(get_root_key_from_obs(cluster_id, *rpc_proxy_, root_key_arg, + addr_list, key_type, key_value))) { + LOG_WARN("failed to get root key from obs", KR(ret), K(cluster_id), + K(root_key_arg), K(addr_list)); + } + if (OB_INVALID_ROOT_KEY == ret) { + LOG_USER_ERROR(OB_INVALID_ROOT_KEY, "Can not get root key from primary tenant"); + } + LOG_INFO("get root key from primary tenant", K(primary_tenant_id), K(tenant_id), K(value), + K(addr_list), K(key_type), K(key_value), K(cluster_id)); + } + return ret; +} + +int ObDDLService::create_root_key( + obrpc::ObSrvRpcProxy &rpc_proxy, + const uint64_t tenant_id, + const common::ObIArray &addrs) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id || addrs.count() <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant_id is invalid", KR(ret), K(tenant_id), K(addrs)); + } else { + char root_key[OB_ROOT_KEY_LEN] = {0}; + obrpc::ObRootKeyArg arg; + obrpc::ObRootKeyResult dummy_result; + arg.tenant_id_ = tenant_id; + arg.is_set_ = true; + arg.key_type_ = obrpc::RootKeyType::NORMAL; + if (OB_FAIL(ObKeyGenerator::generate_encrypt_key(root_key, OB_ROOT_KEY_LEN))) { + LOG_WARN("fail to generate root key", K(ret)); + } else if (FALSE_IT(arg.root_key_.assign_ptr(root_key, OB_ROOT_KEY_LEN))) { + } else if (OB_FAIL(notify_root_key(rpc_proxy, arg, addrs, dummy_result))) { + LOG_WARN("fail to notify root key", K(ret)); + } + } + return ret; +} + +int ObDDLService::get_root_key_from_obs( + const uint64_t &cluster_id, + obrpc::ObSrvRpcProxy &rpc_proxy, + const obrpc::ObRootKeyArg &arg, + const common::ObIArray &addrs, + obrpc::RootKeyType &key_type, + RootKeyValue &key_value) +{ + int ret = OB_SUCCESS; + key_type = obrpc::RootKeyType::INVALID; + key_value.reset(); + if (OB_UNLIKELY(OB_INVALID_CLUSTER_ID == cluster_id + || !arg.is_valid() || arg.is_set_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(arg)); + } else { + ObTimeoutCtx ctx; + bool has_failed = false; + const int64_t DEFAULT_TIMEOUT = 10 * 1000 * 1000L; // 10s + if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, DEFAULT_TIMEOUT))) { + LOG_WARN("fail to set default timeout", KR(ret)); + } else { + // 1. send rpc + rootserver::ObSetRootKeyProxy proxy( + rpc_proxy, &obrpc::ObSrvRpcProxy::set_root_key); + int tmp_ret = OB_SUCCESS; + for (int64_t i = 0; OB_SUCC(ret) && i < addrs.count(); i++) { + int64_t timeout = ctx.get_timeout(); + const ObAddr &addr = addrs.at(i); + if (OB_TMP_FAIL(proxy.call(addr, timeout, cluster_id, OB_SYS_TENANT_ID, arg))) { + has_failed = true; + LOG_WARN("send rpc failed", KR(ret), KR(tmp_ret), K(addr), K(timeout), K(arg), K(cluster_id)); + } + } // end for + // 2. check result + ObArray return_ret_array; + if (OB_SUCCESS != (tmp_ret = proxy.wait_all(return_ret_array))) { + LOG_WARN("wait batch result failed", KR(tmp_ret), KR(ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } + for (int64_t i = 0; OB_SUCC(ret) && i < return_ret_array.count() && !has_failed; ++i) { + tmp_ret = return_ret_array.at(i); + if (OB_TMP_FAIL(tmp_ret)) { + has_failed = true; + LOG_WARN("failed to get root key from observer", KR(tmp_ret), K(i)); + } + } + for (int64_t i = 0; OB_SUCC(ret) && i < proxy.get_results().count(); i++) { + const ObRootKeyResult *rpc_result = proxy.get_results().at(i); + if (OB_ISNULL(rpc_result)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get empty result", KR(ret), K(i), K(addrs)); + } else if (obrpc::RootKeyType::INVALID == rpc_result->key_type_) { + //There may be no root_key information on some observers + } else if (rpc_result->key_type_ != key_type) { + if (OB_UNLIKELY(obrpc::RootKeyType::INVALID != key_type)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("root key type is conflict", KR(ret), K(key_type), KPC(rpc_result)); + } else if (OB_FAIL(key_value.assign(rpc_result->root_key_))) { + LOG_WARN("failed to assign result", KR(ret), KPC(rpc_result)); + } else { + key_type = rpc_result->key_type_; + } + } else if (OB_UNLIKELY(0 != key_value.str().compare(rpc_result->root_key_))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("root key is conflict", KR(ret), K(key_value), KPC(rpc_result)); + } + } // end for + if (OB_SUCC(ret) && obrpc::RootKeyType::INVALID == key_type) { + if (has_failed) { + ret = OB_INVALID_ROOT_KEY; + LOG_WARN("failed to get root key from obs", KR(ret), K(cluster_id), + K(addrs), K(key_type), K(key_value)); + } else { + //If the root_key cannot be obtained from all current observers, + //set default. This tenant may be an upgraded tenant. + //The scope of this observer is obtained from the __all_virtual_log_stat, + //it may not be all observers, ignore this situation + key_type = obrpc::RootKeyType::DEFAULT; + LOG_INFO("can not get root key from all observer, set default", K(cluster_id), + K(addrs), K(key_type), K(key_value)); + } + } + } + } + return ret; +} + +int ObDDLService::notify_root_key( + obrpc::ObSrvRpcProxy &rpc_proxy, + const obrpc::ObRootKeyArg &arg, + const common::ObIArray &addrs, + obrpc::ObRootKeyResult &result) +{ + int ret = OB_SUCCESS; + ObTimeoutCtx ctx; + const int64_t DEFAULT_TIMEOUT = 10 * 1000 * 1000L; // 10s + if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, DEFAULT_TIMEOUT))) { + LOG_WARN("fail to set default timeout", KR(ret)); + } else { + int64_t server_cnt = addrs.count(); + // 1. send rpc + rootserver::ObSetRootKeyProxy proxy( + rpc_proxy, &obrpc::ObSrvRpcProxy::set_root_key); + bool call_rs = false; + ObAddr rs_addr = GCONF.self_addr_; + int64_t timeout = ctx.get_timeout(); + for (int64_t i = 0; OB_SUCC(ret) && i < addrs.count(); i++) { + const ObAddr &addr = addrs.at(i); + if (OB_FAIL(proxy.call(addr, timeout, arg))) { + LOG_WARN("send rpc failed", KR(ret), K(addr), K(timeout), K(arg)); + } else if (rs_addr == addr) { + call_rs = true; + } + } // end for + if (OB_FAIL(ret) || call_rs) { + } else if (OB_FAIL(proxy.call(rs_addr, timeout, arg))) { + LOG_WARN("fail to call rs", KR(ret), K(rs_addr), K(timeout), K(arg)); + } else { + server_cnt++; + } + // 2. check result + ObArray return_ret_array; + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = proxy.wait_all(return_ret_array))) { // ignore ret + LOG_WARN("wait batch result failed", KR(tmp_ret), KR(ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } else if (return_ret_array.count() != server_cnt || + proxy.get_results().count() != server_cnt) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("result cnt not match", KR(ret), K(server_cnt), "ret_cnt", return_ret_array.count(), + "result_cnt", proxy.get_results().count()); + } + for (int64_t i = 0; OB_SUCC(ret) && i < return_ret_array.count(); i++) { + int return_ret = return_ret_array.at(i); + const ObAddr &addr = proxy.get_dests().at(i); + const ObRootKeyResult *result = proxy.get_results().at(i); + if (OB_SUCCESS != return_ret) { + ret = return_ret; + LOG_WARN("rpc return error", KR(ret), K(addr), K(timeout)); + } else if (OB_ISNULL(result)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get empty result", KR(ret), K(addr), K(timeout)); + } + } // end for + if (OB_SUCC(ret) && !arg.is_set_) { + obrpc::RootKeyType key_type = obrpc::RootKeyType::INVALID; + ObString root_key; + for (int64_t i = 0; OB_SUCC(ret) && i < proxy.get_results().count(); ++i) { + const ObRootKeyResult *rpc_result = proxy.get_results().at(i); + if (OB_ISNULL(rpc_result)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get empty result", KR(ret), "addr", proxy.get_dests().at(i), K(timeout)); + } else if (obrpc::RootKeyType::INVALID == rpc_result->key_type_) { + // do nothing + } else if (rpc_result->key_type_ != key_type) { + if (OB_UNLIKELY(obrpc::RootKeyType::INVALID != key_type)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("root key type is conflict", K(key_type), K(rpc_result->key_type_), K(ret)); + } else { + key_type = rpc_result->key_type_; + root_key = rpc_result->root_key_; + } + } else if (OB_UNLIKELY(0 != root_key.compare(rpc_result->root_key_))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("root key is conflict", K(root_key), K(rpc_result->root_key_), K(ret)); + } + } + if (OB_SUCC(ret)) { + if (obrpc::RootKeyType::INVALID == key_type) { + key_type = obrpc::RootKeyType::DEFAULT; + } + if (OB_FAIL(ObMasterKeyGetter::instance().set_root_key(arg.tenant_id_, + key_type, root_key))) { + LOG_WARN("failed to set root key", K(ret)); + } else if (OB_FAIL(ObMasterKeyGetter::instance().get_root_key(arg.tenant_id_, + result.key_type_, result.root_key_))) { + LOG_WARN("failed to get root key", K(ret)); + } else if (OB_UNLIKELY(key_type != result.key_type_ || + 0 != root_key.compare(result.root_key_))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpect root key", K(ret)); + } + } + } + } + return ret; +} +#endif // 1. create tenant's sys ls // 2. broadcast sys table schemas @@ -23342,9 +23701,26 @@ int ObDDLService::modify_tenant_inner_phase(const ObModifyTenantArg &arg, const if (OB_FAIL(ret)) { } else if (arg.alter_option_bitset_.has_member(obrpc::ObModifyTenantArg::ENABLE_ARBITRATION_SERVICE)) { +#ifndef OB_BUILD_ARBITRATION ret = OB_OP_NOT_ALLOW; LOG_WARN("modify tenant arbitration service status in CE version not supprted", KR(ret)); LOG_USER_ERROR(OB_OP_NOT_ALLOW, "modify tenant arbitration service status in CE version"); +#else + const ObArbitrationServiceStatus old_status = orig_tenant_schema->get_arbitration_service_status(); + const ObArbitrationServiceStatus new_status = arg.tenant_schema_.get_arbitration_service_status(); + if ((new_status.is_enabling() && old_status.is_enable_like()) + || ((new_status.is_disabling() && old_status.is_disable_like()))) { + // do nothing + } else if (OB_FAIL(check_tenant_arbitration_service_status_( + trans, + tenant_id, + old_status, + new_status))) { + LOG_WARN("fail to check tenant arbitration service", KR(ret), K(tenant_id), K(old_status), K(new_status)); + } else { + new_tenant_schema.set_arbitration_service_status(new_status); + } +#endif } if (OB_FAIL(ret)) { @@ -24406,6 +24782,10 @@ int ObDDLService::modify_system_variable(const ObModifySysVarArg &arg) LOG_WARN("alter tenant info failed", K(ret)); } else if (OB_FAIL(sys_variable_schema->get_oracle_mode(is_oracle_mode))) { LOG_WARN("failed to get oracle mode", K(ret)); +#ifdef OB_BUILD_ORACLE_PL + } else if (!is_oracle_mode && OB_FAIL(pl::ObDbmsAuditMgmt::handle_audit_param_mysql(new_sys_variable_schema, trans))) { + LOG_WARN("failed to refresh audit log trail jobs", K(ret), K(new_sys_variable_schema)); +#endif } if (trans.is_started()) { int temp_ret = OB_SUCCESS; @@ -30081,6 +30461,41 @@ int ObDDLService::get_schema_primary_regions( return ret; } +#ifdef OB_BUILD_ARBITRATION +int ObDDLService::check_tenant_arbitration_service_status_( + ObMySQLTransaction &trans, + const uint64_t tenant_id, + const share::ObArbitrationServiceStatus &old_status, + const share::ObArbitrationServiceStatus &new_status) +{ + int ret = OB_SUCCESS; + bool is_compatible = false; + bool can_promote = false; + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id + || !old_status.is_valid() + || !new_status.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(old_status), K(new_status)); + } else if (OB_FAIL(ObShareUtil::check_compat_version_for_arbitration_service(tenant_id, is_compatible))) { + LOG_WARN("fail to check data version", KR(ret), K(tenant_id)); + } else if (!is_compatible) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("can not change arbitration service status with data version below 4.1", KR(ret), K(tenant_id)); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "data version is below 4.1, change tenant arbitration service status"); + } else if ((new_status.is_enabled() && !old_status.is_enabling()) + || (new_status.is_disabled() && !old_status.is_disabling())) { + ret = OB_STATE_NOT_MATCH; + LOG_WARN("unexpected status", KR(ret), K(new_status), K(old_status)); + } else if (OB_FAIL(ObArbitrationServiceUtils::check_can_promote_arbitration_service_status(trans, tenant_id, old_status, new_status, can_promote))) { + LOG_WARN("fail to check whether can enable arb service", KR(ret), K(tenant_id), K(old_status), K(new_status)); + } else if (!can_promote) { + // LOG_USER_ERROR will raise inside check_can_promote_arbitration_service_status + ret = OB_STATE_NOT_MATCH; + LOG_WARN("promote conditions not satisfied", KR(ret), K(tenant_id), K(old_status), K(new_status), K(can_promote)); + } + return ret; +} +#endif int ObDDLService::check_tenant_primary_zone_( share::schema::ObSchemaGetterGuard &schema_guard, diff --git a/src/rootserver/ob_ddl_service.h b/src/rootserver/ob_ddl_service.h index bfb01df29..3c9e2db30 100644 --- a/src/rootserver/ob_ddl_service.h +++ b/src/rootserver/ob_ddl_service.h @@ -1852,6 +1852,32 @@ public: obrpc::ObSrvRpcProxy &rpc_proxy, const common::ObIArray &init_configs, const common::ObIArray &addrs); +#ifdef OB_BUILD_TDE_SECURITY + int check_need_create_root_key(const obrpc::ObCreateTenantArg &arg, bool &need_create); + int get_root_key_from_primary(const obrpc::ObCreateTenantArg &arg, + const uint64_t tenant_id, obrpc::RootKeyType &key_type, + RootKeyValue &key_value); + static int get_root_key_from_obs( + const uint64_t &cluster_id, + obrpc::ObSrvRpcProxy &rpc_proxy, + const obrpc::ObRootKeyArg &arg, + const common::ObIArray &addrs, + obrpc::RootKeyType &key_type, + RootKeyValue &key_value); + int standby_create_root_key( + const uint64_t tenant_id, + const obrpc::ObCreateTenantArg &arg, + const common::ObIArray &addrs); + static int create_root_key( + obrpc::ObSrvRpcProxy &rpc_proxy, + const uint64_t tenant_id, + const common::ObIArray &addrs); + static int notify_root_key( + obrpc::ObSrvRpcProxy &rpc_proxy, + const obrpc::ObRootKeyArg &arg, + const common::ObIArray &addrs, + obrpc::ObRootKeyResult &result); +#endif private: int handle_security_audit_for_stmt(const obrpc::ObSecurityAuditArg &arg, share::schema::ObSAuditSchema &audit_schema); @@ -2282,6 +2308,13 @@ private: ObTableSchema &meta_schema, ObTableSchema &data_schema); int check_has_multi_autoinc(share::schema::ObTableSchema &table_schema); private: +#ifdef OB_BUILD_ARBITRATION + int check_tenant_arbitration_service_status_( + ObMySQLTransaction &trans, + const uint64_t tenant_id, + const share::ObArbitrationServiceStatus &old_status, + const share::ObArbitrationServiceStatus &new_status); +#endif int check_tenant_primary_zone_( share::schema::ObSchemaGetterGuard &schema_guard, const share::schema::ObTenantSchema &new_tenant_schema); diff --git a/src/rootserver/ob_root_service.cpp b/src/rootserver/ob_root_service.cpp index 1237f8242..0aa650467 100755 --- a/src/rootserver/ob_root_service.cpp +++ b/src/rootserver/ob_root_service.cpp @@ -56,6 +56,12 @@ #include "observer/ob_server_event_history_table_operator.h" #include "share/ob_upgrade_utils.h" #include "share/deadlock/ob_deadlock_inner_table_service.h" +#ifdef OB_BUILD_TDE_SECURITY +#include "share/ob_master_key_getter.h" +#endif +#ifdef OB_BUILD_ARBITRATION +#include "share/arbitration_service/ob_arbitration_service_utils.h" // ObArbitrationServiceUtils +#endif #include "share/ob_max_id_fetcher.h" // ObMaxIdFetcher #include "share/backup/ob_backup_config.h" #include "share/backup/ob_backup_helper.h" @@ -655,6 +661,9 @@ ObRootService::ObRootService() rs_status_(), fail_count_(0), schema_history_recycler_(), +#ifdef OB_BUILD_TDE_SECURITY + master_key_mgr_(), +#endif disaster_recovery_task_executor_(), disaster_recovery_task_mgr_(), global_ctx_task_(*this) @@ -834,6 +843,9 @@ int ObRootService::init(ObServerConfig &config, lst_operator, unit_manager_, sql_proxy_ +#ifdef OB_BUILD_TDE_SECURITY + , &master_key_mgr_ +#endif ))) { FLOG_WARN("init server zone op service failed", KR(ret)); } else if (OB_FAIL(hb_checker_.init(server_manager_))) { @@ -895,6 +907,10 @@ int ObRootService::init(ObServerConfig &config, &sql_proxy_, schema_service_))) { FLOG_WARN("init ObDBMSSchedJobMaster failed", KR(ret)); +#ifdef OB_BUILD_TDE_SECURITY + } else if (OB_FAIL(master_key_mgr_.init(&zone_manager_, schema_service_))) { + FLOG_WARN("init master key mgr failed", KR(ret)); +#endif } else if (OB_FAIL(disaster_recovery_task_executor_.init(lst_operator, rpc_proxy_))) { FLOG_WARN("init disaster recovery task executor failed", KR(ret)); @@ -986,6 +1002,14 @@ void ObRootService::destroy() ddl_scheduler_.destroy(); FLOG_INFO("ddl task scheduler destroy"); +#ifdef OB_BUILD_TDE_SECURITY + if (OB_FAIL(master_key_mgr_.destroy())) { + FLOG_WARN("master key mgr destroy failed", KR(ret)); + fail_ret = OB_SUCCESS == fail_ret ? ret : fail_ret; + } else { + FLOG_INFO("master key mgr destroy"); + } +#endif if (OB_FAIL(disaster_recovery_task_mgr_.destroy())) { FLOG_WARN("disaster recovery task mgr destroy failed", KR(ret)); @@ -1203,6 +1227,10 @@ int ObRootService::stop() FLOG_INFO("ddl task scheduler stop"); dbms_job::ObDBMSJobMaster::get_instance().stop(); FLOG_INFO("dbms job master stop"); +#ifdef OB_BUILD_TDE_SECURITY + master_key_mgr_.stop(); + FLOG_INFO("master key mgr stop"); +#endif disaster_recovery_task_mgr_.stop(); FLOG_INFO("disaster_recovery_task_mgr stop"); dbms_scheduler::ObDBMSSchedJobMaster::get_instance().stop(); @@ -1243,6 +1271,10 @@ void ObRootService::wait() FLOG_INFO("inspect queue exit success"); ddl_scheduler_.wait(); FLOG_INFO("ddl task scheduler exit success"); +#ifdef OB_BUILD_TDE_SECURITY + master_key_mgr_.wait(); + FLOG_INFO("master key mgr exit success"); +#endif disaster_recovery_task_mgr_.wait(); FLOG_INFO("rebalance task mgr exit success"); TG_WAIT(lib::TGDefIDs::GlobalCtxTimer); @@ -1982,6 +2014,33 @@ int ObRootService::execute_bootstrap(const obrpc::ObBootstrapArg &arg) return ret; } +#ifdef OB_BUILD_TDE_SECURITY +int ObRootService::check_sys_tenant_initial_master_key_valid() +{ + int ret = OB_SUCCESS; + const int64_t start = ObTimeUtility::current_time(); + const int64_t MAX_WAIT_US = 120L * 1000L * 1000L; //120s + const int64_t end = start + MAX_WAIT_US; + const int64_t IDLING_US = 100L * 1000L; // 100ms + while (OB_SUCC(ret)) { + if (ObTimeUtility::current_time() >= end) { + ret = OB_TIMEOUT; + LOG_WARN("wait sys tenant initial master key valid timeout", KR(ret)); + } else { + bool has_available_master_key = false; + if (OB_FAIL(master_key_mgr_.check_if_tenant_has_available_master_keys( + OB_SYS_TENANT_ID, has_available_master_key))) { + LOG_WARN("fail to check if tenant has available master key", KR(ret)); + } else if (!has_available_master_key) { + ob_usleep(std::min(IDLING_US, end - ObTimeUtility::current_time())); + } else { + break; + } + } + } + return ret; +} +#endif int ObRootService::check_config_result(const char *name, const char* value) { @@ -2172,6 +2231,12 @@ int ObRootService::renew_lease(const ObLeaseRequest &lease_request, ObLeaseRespo lease_response.rs_server_status_ = is_stopped ? RSS_IS_STOPPED : RSS_IS_WORKING; } } +#ifdef OB_BUILD_TDE_SECURITY + if (OB_SUCCESS != (temp_ret = master_key_mgr_.input_server_master_key( + lease_request.server_, lease_request.tenant_max_flushed_key_version_))) { + LOG_WARN("fail to input server master key", KR(temp_ret), K(lease_request)); + } +#endif } if (OB_SUCC(ret)) { lease_response.version_ = ObLeaseResponse::LEASE_VERSION; @@ -2192,6 +2257,14 @@ int ObRootService::renew_lease(const ObLeaseRequest &lease_request, ObLeaseRespo LOG_WARN("fail to get refresh_schema_info", K(temp_ret)); } +#ifdef OB_BUILD_TDE_SECURITY + if (OB_SUCCESS != (temp_ret = master_key_mgr_.get_all_tenant_master_key( + lease_request.zone_, + lease_response.tenant_max_key_version_))) { + LOG_WARN("fail to get all tenant master key", KR(temp_ret), + "server", lease_request.server_, "zone", lease_request.zone_); + } +#endif LOG_TRACE("lease_request", K(lease_request), K(lease_response)); } } @@ -4983,6 +5056,13 @@ int ObRootService::do_restart() } +#ifdef OB_BUILD_TDE_SECURITY + if (FAILEDx(master_key_mgr_.start())) { + FLOG_WARN("fail to start master key manager", KR(ret)); + } else { + FLOG_INFO("success to start master key manager"); + } +#endif if (FAILEDx(disaster_recovery_task_mgr_.start())) { FLOG_WARN("disaster recovery task manager start failed", KR(ret)); @@ -6402,6 +6482,104 @@ int ObRootService::drop_synonym(const obrpc::ObDropSynonymArg &arg) } //-----End of functions for managing synonyms----- +#ifdef OB_BUILD_SPM +//-----Functions for managing plan_baselines----- +int ObRootService::accept_plan_baseline(const ObModifyPlanBaselineArg &arg) +{ + int ret = OB_SUCCESS; + ObZone null_zone; + ObSEArray server_list; + if (!inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_FAIL(SVR_TRACER.get_alive_servers(null_zone, server_list))) { + LOG_WARN("fail to get alive server", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < server_list.count(); i++) { + if (OB_FAIL(rpc_proxy_.to(server_list.at(i)) + .by(arg.tenant_id_) + .as(arg.tenant_id_) + .svr_accept_plan_baseline(arg))) { + LOG_WARN("fail to accept plan baseline", K(ret), K(server_list.at(i))); + ret = OB_SUCCESS; + } else { /*do nothing*/} + } + } + return ret; +} + +int ObRootService::cancel_evolve_task(const ObModifyPlanBaselineArg &arg) +{ + int ret = OB_SUCCESS; + ObZone null_zone; + ObSEArray server_list; + if (!inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_FAIL(SVR_TRACER.get_alive_servers(null_zone, server_list))) { + LOG_WARN("fail to get alive server", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < server_list.count(); i++) { + if (OB_FAIL(rpc_proxy_.to(server_list.at(i)) + .by(arg.tenant_id_) + .as(arg.tenant_id_) + .svr_cancel_evolve_task(arg))) { + LOG_WARN("fail to accept plan baseline", K(ret), K(server_list.at(i))); + ret = OB_SUCCESS; + } else { /*do nothing*/} + } + } + return ret; +} + +int ObRootService::admin_load_baseline(const obrpc::ObLoadPlanBaselineArg &arg) +{ + int ret = OB_SUCCESS; + if (!inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else { + ObSystemAdminCtx ctx; + if (OB_FAIL(init_sys_admin_ctx(ctx))) { + LOG_WARN("init_sys_admin_ctx failed", K(ret)); + } else { + ObAdminLoadBaseline admin_util(ctx); + if (OB_FAIL(admin_util.execute(arg))) { + LOG_WARN("dispatch flush cache failed", K(arg), K(ret)); + } + ROOTSERVICE_EVENT_ADD("root_service", "admin_load_baseline", K(ret), K(arg)); + } + } + return ret; +} + +int ObRootService::admin_load_baseline_v2(const obrpc::ObLoadPlanBaselineArg &arg, + obrpc::ObLoadBaselineRes &res) +{ + int ret = OB_SUCCESS; + if (!inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else { + ObSystemAdminCtx ctx; + uint64_t load_count = 0; + if (OB_FAIL(init_sys_admin_ctx(ctx))) { + LOG_WARN("init_sys_admin_ctx failed", K(ret)); + } else { + ObAdminLoadBaselineV2 admin_util(ctx); + if (OB_FAIL(admin_util.execute(arg, load_count))) { + LOG_WARN("dispatch flush cache failed", K(arg), K(ret)); + } else { + res.load_count_ = load_count; + } + ROOTSERVICE_EVENT_ADD("root_service", "admin_load_baseline", K(ret), K(arg)); + } + } + return ret; +} + +//-----End of functions for managing plan_baselines----- +#endif int ObRootService::admin_sync_rewrite_rules(const obrpc::ObSyncRewriteRuleArg &arg) { @@ -6760,9 +6938,36 @@ int ObRootService::do_keystore_ddl(const obrpc::ObKeystoreDDLArg &arg) if (!inited_) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); +#ifndef OB_BUILD_TDE_SECURITY } else if (OB_FAIL(ddl_service_.do_keystore_ddl(arg))) { LOG_WARN("do sequence ddl failed", K(arg), K(ret)); } +#else + } else { + // exclude add server + common::ObArray tenant_id_array; + common::ObArray > max_key_version; + SpinRLockGuard sync_guard(master_key_mgr_.sync()); + if (OB_FAIL(ddl_service_.do_keystore_ddl(arg))) { + LOG_WARN("do sequence ddl failed", K(arg), K(ret)); + } else if (arg.type_ == ObKeystoreDDLArg::ALTER_KEYSTORE_SET_KEY) { + if (OB_FAIL(tenant_id_array.push_back(arg.exec_tenant_id_))) { + LOG_WARN("fail to push back", KR(ret)); + } else if (OB_FAIL(ObMasterKeyGetter::instance().get_latest_key_versions( + tenant_id_array, max_key_version))) { + LOG_WARN("fail to get latest key versions", KR(ret)); + } else if (max_key_version.count() != 1 || max_key_version.at(0).second <= 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("max key version unexpected", KR(ret), K(arg), K(max_key_version)); + } else if (OB_FAIL(master_key_mgr_.forward_tenant_max_key_version( + arg.exec_tenant_id_, max_key_version.at(0).second))) { + LOG_WARN("fail to forward tenant max key version", KR(ret), K(arg), K(max_key_version)); + } + } else { + // no new master key generated, ignore + } + } +#endif return ret; } @@ -6883,6 +7088,10 @@ int ObRootService::old_add_server(const obrpc::ObAdminServerArg &arg) int ret = OB_SUCCESS; uint64_t sys_data_version = 0; // argument +#ifdef OB_BUILD_TDE_SECURITY + ObWaitMasterKeyInSyncArg wms_in_sync_arg; + // master key mgr sync +#endif if (!inited_) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); @@ -6891,6 +7100,10 @@ int ObRootService::old_add_server(const obrpc::ObAdminServerArg &arg) LOG_WARN("invalid arg", K(arg), K(ret)); } else if (OB_FAIL(GET_MIN_DATA_VERSION(OB_SYS_TENANT_ID, sys_data_version))) { LOG_WARN("fail to get sys data version", KR(ret)); +#ifdef OB_BUILD_TDE_SECURITY + } else if (OB_FAIL(server_zone_op_service_.construct_rs_list_arg(wms_in_sync_arg.rs_list_arg_))) { + LOG_WARN("fail to construct rs list arg", KR(ret)); +#endif } else { LOG_INFO("add_server", K(arg), "timeout_ts", THIS_WORKER.get_timeout_ts()); ObCheckServerEmptyArg new_arg(ObCheckServerEmptyArg::ADD_SERVER, @@ -6898,6 +7111,9 @@ int ObRootService::old_add_server(const obrpc::ObAdminServerArg &arg) ObCheckDeploymentModeArg dp_arg; dp_arg.single_zone_deployment_on_ = OB_FILE_SYSTEM_ROUTER.is_single_zone_deployment_on(); +#ifdef OB_BUILD_TDE_SECURITY + SpinRLockGuard sync_guard(master_key_mgr_.sync()); +#endif for (int64_t i = 0; OB_SUCC(ret) && i < arg.servers_.count(); ++i) { const ObAddr &addr = arg.servers_[i]; Bool is_empty(false); @@ -6918,6 +7134,13 @@ int ObRootService::old_add_server(const obrpc::ObAdminServerArg &arg) ret = OB_OP_NOT_ALLOW; LOG_WARN("add server deployment mode mot match not allowed", K(ret)); LOG_USER_ERROR(OB_OP_NOT_ALLOW, "add server deployment mode not match"); +#ifdef OB_BUILD_TDE_SECURITY + } else if (OB_FAIL(server_zone_op_service_.master_key_checking_for_adding_server( + addr, + arg.zone_, + wms_in_sync_arg))) { + LOG_WARN("master key checking for adding server is failed", KR(ret), K(addr), K(arg.zone_)); +#endif } if (OB_SUCC(ret)) { if (OB_FAIL(server_manager_.add_server(addr, arg.zone_))) { @@ -7181,6 +7404,31 @@ int ObRootService::stop_server(const obrpc::ObAdminServerArg &arg) return ret; } +#ifdef OB_BUILD_TDE_SECURITY +int ObRootService::try_check_encryption_zone_cond( + const obrpc::ObAdminZoneArg &arg) +{ + int ret = OB_SUCCESS; + bool has_available_master_key = false; + if (OB_UNLIKELY(!inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", K(ret)); + } else if (OB_UNLIKELY(!arg.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(arg)); + } else if (arg.zone_type_ != ZONE_TYPE_ENCRYPTION) { + // good, no need to check + } else if (OB_FAIL(master_key_mgr_.check_if_tenant_has_available_master_keys( + OB_SYS_TENANT_ID, has_available_master_key))) { + LOG_WARN("fail to check if tenant has available master key", KR(ret)); + } else if (!has_available_master_key) { + ret = OB_OP_NOT_ALLOW; + LOG_WARN("add encryption zone without available master key in sys tenant is not allowed", KR(ret)); + LOG_USER_ERROR(OB_OP_NOT_ALLOW, "add encryption zone without available master key in sys tenant"); + } + return ret; +} +#endif int ObRootService::add_zone(const obrpc::ObAdminZoneArg &arg) { @@ -7191,6 +7439,10 @@ int ObRootService::add_zone(const obrpc::ObAdminZoneArg &arg) } else if (!arg.is_valid()) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arg", K(arg), K(ret)); +#ifdef OB_BUILD_TDE_SECURITY + } else if (OB_FAIL(try_check_encryption_zone_cond(arg))) { + LOG_WARN("fail to check encryption zone", KR(ret), K(arg)); +#endif } else if (OB_FAIL(zone_manager_.add_zone(arg.zone_, arg.region_, arg.idc_, arg.zone_type_))) { LOG_WARN("failed to add zone", K(ret), K(arg)); } else { @@ -7934,6 +8186,70 @@ int ObRootService::admin_clear_merge_error(const obrpc::ObAdminMergeArg &arg) return ret; } +#ifdef OB_BUILD_ARBITRATION +int ObRootService::admin_add_arbitration_service(const obrpc::ObAdminAddArbitrationServiceArg &arg) +{ + int ret = OB_SUCCESS; + if (!inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (!arg.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", KR(ret), K(arg)); + } else if (OB_FAIL(share::ObArbitrationServiceUtils::add_arbitration_service(sql_proxy_, arg))) { + LOG_WARN("fail to add arbitration service", KR(ret), K(arg)); + } + ROOTSERVICE_EVENT_ADD("arb_service", "admin_add_arbitration_service", K(ret), K(arg)); + return ret; +} + +int ObRootService::admin_remove_arbitration_service(const obrpc::ObAdminRemoveArbitrationServiceArg &arg) +{ + int ret = OB_SUCCESS; + if (!inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (!arg.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", KR(ret), K(arg)); + } else if (OB_FAIL(share::ObArbitrationServiceUtils::remove_arbitration_service(sql_proxy_, arg))) { + LOG_WARN("fail to remove arbitration service", KR(ret), K(arg)); + } + ROOTSERVICE_EVENT_ADD("arb_service", "admin_remove_arbitration_service", K(ret), K(arg)); + return ret; +} + +int ObRootService::admin_replace_arbitration_service(const obrpc::ObAdminReplaceArbitrationServiceArg &arg) +{ + int ret = OB_SUCCESS; + if (!inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (!arg.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", KR(ret), K(arg)); + } else if (OB_FAIL(share::ObArbitrationServiceUtils::replace_arbitration_service(sql_proxy_, arg))) { + LOG_WARN("fail to replace arbitration service", KR(ret), K(arg)); + } + ROOTSERVICE_EVENT_ADD("arb_service", "admin_replace_arbitration_service", K(ret), K(arg)); + return ret; +} + +int ObRootService::remove_cluster_info_from_arb_server(const obrpc::ObRemoveClusterInfoFromArbServerArg &arg) +{ + int ret = OB_SUCCESS; + if (!inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (!arg.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", KR(ret), K(arg)); + } else if (OB_FAIL(ObArbitrationServiceUtils::remove_cluster_info_from_arb_server(arg.get_arbitration_service()))) { + LOG_WARN("fail to remove cluster info from arb server", KR(ret), K(arg)); + } + return ret; +} +#endif int ObRootService::admin_migrate_unit(const obrpc::ObAdminMigrateUnitArg &arg) { @@ -10064,6 +10380,42 @@ int ObRootService::try_add_dep_infos_for_synonym_batch(const obrpc::ObTryAddDepI return ret; } +#ifdef OB_BUILD_TDE_SECURITY +int ObRootService::handle_get_root_key(const obrpc::ObRootKeyArg &arg, + obrpc::ObRootKeyResult &result) +{ + int ret = OB_SUCCESS; + ObRootKey root_key; + if (OB_UNLIKELY(arg.is_set_ || !arg.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg", K(arg), K(ret)); + } else if (OB_FAIL(ObMasterKeyGetter::instance().get_root_key(arg.tenant_id_, root_key))) { + LOG_WARN("failed to get root key", K(ret)); + } else if (obrpc::RootKeyType::INVALID != root_key.key_type_) { + result.key_type_ = root_key.key_type_; + result.root_key_ = root_key.key_; + } else if (OB_FAIL(get_root_key_from_obs(arg, result))) { + LOG_WARN("failed to get root key from obs", K(ret)); + } + return ret; +} + +int ObRootService::get_root_key_from_obs(const obrpc::ObRootKeyArg &arg, + obrpc::ObRootKeyResult &result) +{ + int ret = OB_SUCCESS; + ObZone empty_zone; + ObArray active_server_list; + ObArray inactive_server_list; + if (OB_FAIL(SVR_TRACER.get_servers_by_status(empty_zone, active_server_list, + inactive_server_list))) { + LOG_WARN("get alive servers failed", K(ret)); + } else if (OB_FAIL(ObDDLService::notify_root_key(rpc_proxy_, arg, active_server_list, result))) { + LOG_WARN("failed to notify root key"); + } + return ret; +} +#endif } // end namespace rootserver } // end namespace oceanbase diff --git a/src/rootserver/ob_root_service.h b/src/rootserver/ob_root_service.h index ad31d428b..c8a1ee7db 100644 --- a/src/rootserver/ob_root_service.h +++ b/src/rootserver/ob_root_service.h @@ -55,6 +55,9 @@ #include "rootserver/ob_empty_server_checker.h" #include "rootserver/ob_lost_replica_checker.h" #include "rootserver/ob_server_zone_op_service.h" +#ifdef OB_BUILD_TDE_SECURITY +#include "rootserver/ob_rs_master_key_manager.h" +#endif namespace oceanbase { @@ -424,6 +427,9 @@ public: int not_implement(); int execute_bootstrap(const obrpc::ObBootstrapArg &arg); +#ifdef OB_BUILD_TDE_SECURITY + int check_sys_tenant_initial_master_key_valid(); +#endif int check_config_result(const char *name, const char *value); int check_ddl_allowed(); @@ -570,6 +576,15 @@ public: int drop_synonym(const obrpc::ObDropSynonymArg &arg); //----End of functions for managing synonyms---- +#ifdef OB_BUILD_SPM + //----Functions for managing plan_baselines---- + int accept_plan_baseline(const obrpc::ObModifyPlanBaselineArg &arg); + int cancel_evolve_task(const obrpc::ObModifyPlanBaselineArg &arg); + int admin_load_baseline(const obrpc::ObLoadPlanBaselineArg &arg); + int admin_load_baseline_v2(const obrpc::ObLoadPlanBaselineArg &arg, obrpc::ObLoadBaselineRes &res); + // int drop_plan_baseline(const obrpc::ObDropPlanBaselineArg &arg); + //----End of functions for managing plan_baselines---- +#endif //----Functions for sync rewrite rules---- int admin_sync_rewrite_rules(const obrpc::ObSyncRewriteRuleArg &arg); @@ -681,6 +696,12 @@ public: int admin_reload_server(); int admin_reload_zone(); int admin_clear_merge_error(const obrpc::ObAdminMergeArg &arg); +#ifdef OB_BUILD_ARBITRATION + int admin_add_arbitration_service(const obrpc::ObAdminAddArbitrationServiceArg &arg); + int admin_remove_arbitration_service(const obrpc::ObAdminRemoveArbitrationServiceArg &arg); + int admin_replace_arbitration_service(const obrpc::ObAdminReplaceArbitrationServiceArg &arg); + int remove_cluster_info_from_arb_server(const obrpc::ObRemoveClusterInfoFromArbServerArg &arg); +#endif int admin_migrate_unit(const obrpc::ObAdminMigrateUnitArg &arg); int admin_upgrade_virtual_schema(); int run_job(const obrpc::ObRunJobArg &arg); @@ -777,7 +798,15 @@ public: int update_rslist(); int recompile_all_views_batch(const obrpc::ObRecompileAllViewsBatchArg &arg); int try_add_dep_infos_for_synonym_batch(const obrpc::ObTryAddDepInofsForSynonymBatchArg &arg); +#ifdef OB_BUILD_TDE_SECURITY + int handle_get_root_key(const obrpc::ObRootKeyArg &arg, obrpc::ObRootKeyResult &result); + int get_root_key_from_obs(const obrpc::ObRootKeyArg &arg, obrpc::ObRootKeyResult &result); +#endif private: +#ifdef OB_BUILD_TDE_SECURITY + int try_check_encryption_zone_cond( + const obrpc::ObAdminZoneArg &arg); +#endif int check_parallel_ddl_conflict( share::schema::ObSchemaGetterGuard &schema_guard, const obrpc::ObDDLArg &arg); @@ -954,6 +983,10 @@ private: int64_t fail_count_; ObSchemaHistoryRecycler schema_history_recycler_; +#ifdef OB_BUILD_TDE_SECURITY + // master key manager + ObRsMasterKeyManager master_key_mgr_; +#endif // Disaster Recovery related ObDRTaskExecutor disaster_recovery_task_executor_; ObDRTaskMgr disaster_recovery_task_mgr_; diff --git a/src/rootserver/ob_rs_async_rpc_proxy.h b/src/rootserver/ob_rs_async_rpc_proxy.h index ec5cb26be..19cffa609 100644 --- a/src/rootserver/ob_rs_async_rpc_proxy.h +++ b/src/rootserver/ob_rs_async_rpc_proxy.h @@ -78,6 +78,10 @@ RPC_F(obrpc::OB_NOTIFY_SWITCH_LEADER, obrpc::ObNotifySwitchLeaderArg, obrpc::ObSrvRpcProxy::ObRpc::Response, ObNotifySwitchLeaderProxy); RPC_F(obrpc::OB_UPDATE_TENANT_INFO_CACHE, obrpc::ObUpdateTenantInfoCacheArg, obrpc::ObUpdateTenantInfoCacheRes, ObUpdateTenantInfoCacheProxy); RPC_F(obrpc::OB_BROADCAST_CONSENSUS_VERSION, obrpc::ObBroadcastConsensusVersionArg, obrpc::ObBroadcastConsensusVersionRes, ObBroadcstConsensusVersionProxy); +#ifdef OB_BUILD_TDE_SECURITY +RPC_F(obrpc::OB_RESTORE_KEY, obrpc::ObRestoreKeyArg, obrpc::ObRestoreKeyResult, ObRestoreKeyProxy); +RPC_F(obrpc::OB_SET_ROOT_KEY, obrpc::ObRootKeyArg, obrpc::ObRootKeyResult, ObSetRootKeyProxy); +#endif }//end namespace rootserver }//end namespace oceanbase diff --git a/src/rootserver/ob_rs_rpc_processor.h b/src/rootserver/ob_rs_rpc_processor.h index 6b0b1eb69..097b699c4 100644 --- a/src/rootserver/ob_rs_rpc_processor.h +++ b/src/rootserver/ob_rs_rpc_processor.h @@ -455,6 +455,12 @@ DEFINE_RS_RPC_PROCESSOR(obrpc::OB_ADMIN_RELOAD_SERVER, ObRpcAdminReloadServerP, DEFINE_RS_RPC_PROCESSOR(obrpc::OB_ADMIN_RELOAD_ZONE, ObRpcAdminReloadZoneP, admin_reload_zone()); DEFINE_RS_RPC_PROCESSOR(obrpc::OB_ADMIN_CLEAR_MERGE_ERROR, ObRpcAdminClearMergeErrorP, admin_clear_merge_error(arg_)); DEFINE_RS_RPC_PROCESSOR(obrpc::OB_ADMIN_MIGRATE_UNIT, ObRpcAdminMigrateUnitP, admin_migrate_unit(arg_)); +#ifdef OB_BUILD_ARBITRATION +DEFINE_RS_RPC_PROCESSOR(obrpc::OB_ADMIN_ADD_ARBITRATION_SERVICE, ObRpcAdminAddArbitrationServiceP, admin_add_arbitration_service(arg_)); +DEFINE_RS_RPC_PROCESSOR(obrpc::OB_ADMIN_REMOVE_ARBITRATION_SERVICE, ObRpcAdminRemoveArbitrationServiceP, admin_remove_arbitration_service(arg_)); +DEFINE_RS_RPC_PROCESSOR(obrpc::OB_ADMIN_REPLACE_ARBITRATION_SERVICE, ObRpcAdminReplaceArbitrationServiceP, admin_replace_arbitration_service(arg_)); +DEFINE_RS_RPC_PROCESSOR(obrpc::OB_REMOVE_CLUSTER_INFO_FROM_ARB_SERVER, ObRpcRemoveClusterInfoFromArbServerP, remove_cluster_info_from_arb_server(arg_)); +#endif DEFINE_RS_RPC_PROCESSOR(obrpc::OB_ADMIN_UPGRADE_VIRTUAL_SCHEMA, ObRpcAdminUpgradeVirtualSchemaP, admin_upgrade_virtual_schema()); DEFINE_RS_RPC_PROCESSOR(obrpc::OB_RUN_JOB, ObRpcRunJobP, run_job(arg_)); DEFINE_RS_RPC_PROCESSOR(obrpc::OB_RUN_UPGRADE_JOB, ObRpcRunUpgradeJobP, run_upgrade_job(arg_)); @@ -517,12 +523,23 @@ DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_DROP_DIRECTORY, ObRpcDropDirectoryP, drop_ DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_DO_CONTEXT_DDL, ObRpcDoContextDDLP, do_context_ddl(arg_)); DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_RECOMPILE_ALL_VIEWS_BATCH, ObRpcRecompileAllViewsBatchP, recompile_all_views_batch(arg_)); DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_TRY_ADD_DEP_INFOS_FOR_SYNONYM_BATCH, ObRpcTryAddDepInfosForSynonymBatchP,try_add_dep_infos_for_synonym_batch(arg_)); +#ifdef OB_BUILD_SPM +// sql plan baseline +DEFINE_RS_RPC_PROCESSOR(obrpc::OB_RS_ACCEPT_PLAN_BASELINE, ObRpcAcceptPlanBaselineP, accept_plan_baseline(arg_)); +DEFINE_RS_RPC_PROCESSOR(obrpc::OB_RS_CANCEL_EVOLVE_TASK, ObRpcCancelEvolveTaskP, cancel_evolve_task(arg_)); +DEFINE_RS_RPC_PROCESSOR(obrpc::OB_ADMIN_LOAD_BASELINE, ObRpcAdminLoadBaselineP, admin_load_baseline(arg_)); +DEFINE_RS_RPC_PROCESSOR(obrpc::OB_ADMIN_LOAD_BASELINE_V2, ObRpcAdminLoadBaselineV2P, admin_load_baseline_v2(arg_, result_)); + +#endif DEFINE_RS_RPC_PROCESSOR(obrpc::OB_ADMIN_SYNC_REWRITE_RULES, ObRpcAdminSyncRewriteRulesP, admin_sync_rewrite_rules(arg_)); // row level security ddl DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_HANDLE_RLS_POLICY_DDL, ObRpcHandleRlsPolicyDDLP, handle_rls_policy_ddl(arg_)); DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_HANDLE_RLS_GROUP_DDL, ObRpcHandleRlsGroupDDLP, handle_rls_group_ddl(arg_)); DEFINE_DDL_RS_RPC_PROCESSOR(obrpc::OB_HANDLE_RLS_CONTEXT_DDL, ObRpcHandleRlsContextDDLP, handle_rls_context_ddl(arg_)); +#ifdef OB_BUILD_TDE_SECURITY +DEFINE_RS_RPC_PROCESSOR(obrpc::OB_GET_ROOT_KEY, ObGetRootKeyP, handle_get_root_key(arg_, result_)); +#endif #undef DEFINE_RS_RPC_PROCESSOR_ #undef DEFINE_RS_RPC_PROCESSOR diff --git a/src/rootserver/ob_server_zone_op_service.cpp b/src/rootserver/ob_server_zone_op_service.cpp index 4a179c435..65d6eece6 100644 --- a/src/rootserver/ob_server_zone_op_service.cpp +++ b/src/rootserver/ob_server_zone_op_service.cpp @@ -37,6 +37,9 @@ ObServerZoneOpService::ObServerZoneOpService() sql_proxy_(NULL), lst_operator_(NULL), unit_manager_(NULL) +#ifdef OB_BUILD_TDE_SECURITY + , master_key_mgr_() +#endif { } ObServerZoneOpService::~ObServerZoneOpService() @@ -48,12 +51,20 @@ int ObServerZoneOpService::init( ObLSTableOperator &lst_operator, ObUnitManager &unit_manager, ObMySQLProxy &sql_proxy +#ifdef OB_BUILD_TDE_SECURITY + , ObRsMasterKeyManager *master_key_mgr +#endif ) { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_inited_)) { ret = OB_INIT_TWICE; LOG_WARN("server zone operation service has been inited already", KR(ret), K(is_inited_)); +#ifdef OB_BUILD_TDE_SECURITY + } else if (OB_ISNULL(master_key_mgr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("master key mgr is null", KR(ret), KP(master_key_mgr)); +#endif } else if (OB_FAIL(st_operator_.init(&sql_proxy))) { LOG_WARN("fail to init server table operator", KR(ret)); } else { @@ -62,6 +73,9 @@ int ObServerZoneOpService::init( sql_proxy_ = &sql_proxy; lst_operator_ = &lst_operator; unit_manager_ = &unit_manager; +#ifdef OB_BUILD_TDE_SECURITY + master_key_mgr_ = master_key_mgr; +#endif is_inited_ = true; } return ret; @@ -74,6 +88,10 @@ int ObServerZoneOpService::add_servers(const ObIArray &servers, const Ob ObCheckServerForAddingServerResult rpc_result; ObZone picked_zone; ObTimeoutCtx ctx; +#ifdef OB_BUILD_TDE_SECURITY + ObWaitMasterKeyInSyncArg wms_in_sync_arg; + // master key mgr sync +#endif if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("not init", KR(ret), K(is_inited_)); @@ -82,9 +100,19 @@ int ObServerZoneOpService::add_servers(const ObIArray &servers, const Ob } else if (OB_ISNULL(rpc_proxy_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("rpc_proxy_ is null", KR(ret), KP(rpc_proxy_)); +#ifdef OB_BUILD_TDE_SECURITY + } else if (OB_ISNULL(master_key_mgr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("master_key_mgr_ is null", KR(ret), KP(master_key_mgr_)); + } else if (OB_FAIL(construct_rs_list_arg(wms_in_sync_arg.rs_list_arg_))) { + LOG_WARN("fail to construct rs list arg", KR(ret)); +#endif } else if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) { LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx)); } else { +#ifdef OB_BUILD_TDE_SECURITY + SpinRLockGuard sync_guard(master_key_mgr_->sync()); +#endif for (int64_t i = 0; OB_SUCC(ret) && i < servers.count(); ++i) { const ObAddr &addr = servers.at(i); int64_t timeout = ctx.get_timeout(); @@ -124,6 +152,10 @@ int ObServerZoneOpService::add_servers(const ObIArray &servers, const Ob LOG_USER_ERROR(OB_OP_NOT_ALLOW, non_empty_server_err_msg); } else if (OB_FAIL(zone_checking_for_adding_server_(zone, rpc_result.get_zone(), picked_zone))) { LOG_WARN("zone checking for adding server is failed", KR(ret), K(zone), K(rpc_result.get_zone())); +#ifdef OB_BUILD_TDE_SECURITY + } else if (!is_bootstrap && OB_FAIL(master_key_checking_for_adding_server(addr, picked_zone, wms_in_sync_arg))) { + LOG_WARN("master key checking for adding server is failed", KR(ret), K(addr), K(picked_zone)); +#endif } else if (OB_FAIL(add_server_( addr, server_id, @@ -273,6 +305,61 @@ int ObServerZoneOpService::start_servers( } return ret; } +#ifdef OB_BUILD_TDE_SECURITY +int ObServerZoneOpService::master_key_checking_for_adding_server( + const common::ObAddr &server, + const ObZone &zone, + obrpc::ObWaitMasterKeyInSyncArg &wms_in_sync_arg) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret), K(is_inited_)); + } else if (OB_ISNULL(master_key_mgr_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("master_key_mgr_ is null", KR(ret), KP(master_key_mgr_)); + } else { + bool master_key_empty = true; + share::ObLeaseResponse tmp_lease_response; + bool encryption = false; + ObTimeoutCtx ctx; + if (OB_FAIL(master_key_mgr_->check_master_key_empty(master_key_empty))) { + LOG_WARN("fail to check whether master key is empty", KR(ret)); + } else if (master_key_empty) { + LOG_INFO("empty master key, no need to sync master key info"); + } else if (!master_key_empty && zone.is_empty()) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "not support to add a server " + "without a specified zone when the master key is valid"); + } else if (OB_FAIL(ObZoneTableOperation::check_encryption_zone(*sql_proxy_, zone, encryption))) { + LOG_WARN("fail to check zone encryption", KR(ret), "zone", zone); + } else if (encryption) { + LOG_INFO("server in encrypted zone, no need to sync master key info", "zone", zone); + } else if (OB_FAIL(master_key_mgr_->get_all_tenant_master_key( + zone, wms_in_sync_arg.tenant_max_key_version_))) { + LOG_WARN("fail to get all tenant master key", KR(ret)); + } else if (OB_FAIL(OTC_MGR.get_lease_response(tmp_lease_response))) { + LOG_WARN("fail to get lease response", KR(ret)); + } else if (OB_FAIL(wms_in_sync_arg.tenant_config_version_.assign( + tmp_lease_response.tenant_config_version_))) { + LOG_WARN("fail to assign tenant config version", KR(ret)); + } else if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) { + LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx)); + } else { + int64_t timeout = ctx.get_timeout(); + if (OB_UNLIKELY(timeout <= 0)) { + ret = OB_TIMEOUT; + LOG_WARN("ctx time out", KR(ret), K(timeout)); + } else if (OB_FAIL(rpc_proxy_->to(server) + .timeout(timeout) + .wait_master_key_in_sync(wms_in_sync_arg))) { + LOG_WARN("fail to wait master key in sync", KR(ret), K(server)); + } else {} + } + } + return ret; +} +#endif int ObServerZoneOpService::stop_server_precheck( const ObIArray &servers, const obrpc::ObAdminServerArg::AdminServerOp &op) diff --git a/src/rootserver/ob_server_zone_op_service.h b/src/rootserver/ob_server_zone_op_service.h index 2ae88ec08..4945d50b0 100644 --- a/src/rootserver/ob_server_zone_op_service.h +++ b/src/rootserver/ob_server_zone_op_service.h @@ -15,6 +15,9 @@ #include "share/ob_server_table_operator.h" #include "share/ob_rpc_struct.h" +#ifdef OB_BUILD_TDE_SECURITY +#include "rootserver/ob_rs_master_key_manager.h" +#endif namespace oceanbase { @@ -22,6 +25,9 @@ namespace obrpc { class ObSrvRpcProxy; struct ObRsListArg; +#ifdef OB_BUILD_TDE_SECURITY +struct ObWaitMasterKeyInSyncArg; +#endif // struct ObAdminServerArg; } namespace share @@ -44,6 +50,9 @@ public: share::ObLSTableOperator &lst_operator, ObUnitManager &unit_manager, ObMySQLProxy &sql_proxy +#ifdef OB_BUILD_TDE_SECURITY + , ObRsMasterKeyManager *master_key_mgr +#endif ); // Add new servers to a specified（optional) zone in the cluster. // The servers should be empty and the zone should be active. @@ -156,6 +165,12 @@ public: int start_servers( const ObIArray &servers, const ObZone &zone); +#ifdef OB_BUILD_TDE_SECURITY + int master_key_checking_for_adding_server( + const common::ObAddr &server, + const common::ObZone &zone, + obrpc::ObWaitMasterKeyInSyncArg &wms_in_sync_arg); +#endif int stop_server_precheck( const ObIArray &servers, const obrpc::ObAdminServerArg::AdminServerOp &op); @@ -207,6 +222,9 @@ private: share::ObLSTableOperator *lst_operator_; share::ObServerTableOperator st_operator_; ObUnitManager *unit_manager_; +#ifdef OB_BUILD_TDE_SECURITY + ObRsMasterKeyManager *master_key_mgr_; +#endif private: DISALLOW_COPY_AND_ASSIGN(ObServerZoneOpService); diff --git a/src/rootserver/ob_system_admin_util.cpp b/src/rootserver/ob_system_admin_util.cpp index 5a44c76b9..8340db9a1 100644 --- a/src/rootserver/ob_system_admin_util.cpp +++ b/src/rootserver/ob_system_admin_util.cpp @@ -1885,6 +1885,92 @@ int ObAdminFlushCache::execute(const obrpc::ObAdminFlushCacheArg &arg) return ret; } +#ifdef OB_BUILD_SPM +int ObAdminLoadBaseline::execute(const obrpc::ObLoadPlanBaselineArg &arg) +{ + int ret = OB_SUCCESS; + ObSEArray server_list; + if (OB_FAIL(get_tenant_servers(arg.tenant_id_, server_list))) { + LOG_WARN("fail to get tenant servers", "tenant_id", arg.tenant_id_, KR(ret)); + } else { + //call tenant servers; + for (int64_t j = 0; OB_SUCC(ret) && j < server_list.count(); ++j) { + if (OB_FAIL(call_server(server_list.at(j), arg))) { + LOG_WARN("fail to call tenant server", + "tenant_id", arg.tenant_id_, + "server addr", server_list.at(j), + KR(ret)); + } + } + } + server_list.reset(); + return ret; +} + +int ObAdminLoadBaseline::call_server(const common::ObAddr &server, + const obrpc::ObLoadPlanBaselineArg &arg) +{ + int ret = OB_SUCCESS; + if (!ctx_.is_inited()) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (!server.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid server", K(server), KR(ret)); + } else if (OB_FAIL(ctx_.rpc_proxy_->to(server) + .by(arg.tenant_id_) + .as(arg.tenant_id_) + .load_baseline(arg))) { + LOG_WARN("request server load baseline failed", KR(ret), K(server)); + } + + return ret; +} + +int ObAdminLoadBaselineV2::execute(const obrpc::ObLoadPlanBaselineArg &arg, uint64_t &total_load_count) +{ + int ret = OB_SUCCESS; + ObSEArray server_list; + if (OB_FAIL(get_tenant_servers(arg.tenant_id_, server_list))) { + LOG_WARN("fail to get tenant servers", "tenant_id", arg.tenant_id_, KR(ret)); + } else { + //call tenant servers; + for (int64_t j = 0; OB_SUCC(ret) && j < server_list.count(); ++j) { + obrpc::ObLoadBaselineRes res; + if (OB_FAIL(call_server(server_list.at(j), arg, res))) { + LOG_WARN("fail to call tenant server", + "tenant_id", arg.tenant_id_, + "server addr", server_list.at(j), + KR(ret)); + } else { + total_load_count += res.load_count_; + } + } + } + server_list.reset(); + return ret; +} + +int ObAdminLoadBaselineV2::call_server(const common::ObAddr &server, + const obrpc::ObLoadPlanBaselineArg &arg, + obrpc::ObLoadBaselineRes &res) +{ + int ret = OB_SUCCESS; + if (!ctx_.is_inited()) { + ret = OB_NOT_INIT; + LOG_WARN("not init", KR(ret)); + } else if (!server.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid server", K(server), KR(ret)); + } else if (OB_FAIL(ctx_.rpc_proxy_->to(server) + .by(arg.tenant_id_) + .as(arg.tenant_id_) + .load_baseline_v2(arg, res))) { + LOG_WARN("request server load baseline failed", KR(ret), K(server)); + } + return ret; +} +#endif int ObTenantServerAdminUtil::get_tenant_servers(const uint64_t tenant_id, common::ObIArray &servers) { diff --git a/src/rootserver/ob_system_admin_util.h b/src/rootserver/ob_system_admin_util.h index d94ae13e8..275935505 100644 --- a/src/rootserver/ob_system_admin_util.h +++ b/src/rootserver/ob_system_admin_util.h @@ -527,6 +527,42 @@ private: DISALLOW_COPY_AND_ASSIGN(ObAdminFlushCache); }; +#ifdef OB_BUILD_SPM +class ObAdminLoadBaseline : public ObTenantServerAdminUtil +{ +public: + explicit ObAdminLoadBaseline(const ObSystemAdminCtx &ctx) + : ObTenantServerAdminUtil(ctx) + {} + virtual ~ObAdminLoadBaseline() {} + + int call_server(const common::ObAddr &server, + const obrpc::ObLoadPlanBaselineArg &arg); + + int execute(const obrpc::ObLoadPlanBaselineArg &arg); + +private: + DISALLOW_COPY_AND_ASSIGN(ObAdminLoadBaseline); +}; + +class ObAdminLoadBaselineV2 : public ObTenantServerAdminUtil +{ +public: + explicit ObAdminLoadBaselineV2(const ObSystemAdminCtx &ctx) + : ObTenantServerAdminUtil(ctx) + {} + virtual ~ObAdminLoadBaselineV2() {} + + int call_server(const common::ObAddr &server, + const obrpc::ObLoadPlanBaselineArg &arg, + obrpc::ObLoadBaselineRes &res); + + int execute(const obrpc::ObLoadPlanBaselineArg &arg, uint64_t &total_load_count); + +private: + DISALLOW_COPY_AND_ASSIGN(ObAdminLoadBaselineV2); +}; +#endif class ObAdminSetTP : public ObAdminCallServer { diff --git a/src/rootserver/ob_upgrade_executor.cpp b/src/rootserver/ob_upgrade_executor.cpp index 67649682e..f70492dd6 100644 --- a/src/rootserver/ob_upgrade_executor.cpp +++ b/src/rootserver/ob_upgrade_executor.cpp @@ -806,6 +806,12 @@ int ObUpgradeExecutor::run_upgrade_system_package_job_() LOG_WARN("fail to check schema sync", KR(ret), K(tenant_id)); } else if (OB_FAIL(upgrade_mysql_system_package_job_())) { LOG_WARN("fail to upgrade mysql system package", KR(ret)); +#ifdef OB_BUILD_ORACLE_PL + } else if (OB_FAIL(check_schema_sync_(tenant_id))) { + LOG_WARN("fail to check schema sync", KR(ret), K(tenant_id)); + } else if (OB_FAIL(upgrade_oracle_system_package_job_())) { + LOG_WARN("fail to upgrade mysql system package", KR(ret)); +#endif } return ret; } @@ -867,6 +873,68 @@ int ObUpgradeExecutor::upgrade_mysql_system_package_job_() return ret; } +#ifdef OB_BUILD_ORACLE_PL +int ObUpgradeExecutor::upgrade_oracle_system_package_job_() +{ + int ret = OB_SUCCESS; + int64_t start_ts = ObTimeUtility::current_time(); + FLOG_INFO("[UPGRADE] start to run upgrade oracle system package job"); + ObCompatibilityMode mode = ObCompatibilityMode::ORACLE_MODE; + int64_t timeout = GCONF._ob_ddl_timeout; + const char *create_package_sql = + "CREATE OR REPLACE PACKAGE \"__DBMS_UPGRADE\" IS \ + PROCEDURE UPGRADE(package_name VARCHAR2); \ + PROCEDURE UPGRADE_ALL; \ + END;"; + const char *create_package_body_sql = + "CREATE OR REPLACE PACKAGE BODY \"__DBMS_UPGRADE\" IS \ + PROCEDURE UPGRADE(package_name VARCHAR2); \ + PRAGMA INTERFACE(c, UPGRADE_SINGLE); \ + PROCEDURE UPGRADE_ALL; \ + PRAGMA INTERFACE(c, UPGRADE_ALL); \ + END;"; + const char *upgrade_sql = "CALL \"__DBMS_UPGRADE\".UPGRADE_ALL();"; + ObTimeoutCtx ctx; + int64_t affected_rows = 0; + if (OB_FAIL(check_inner_stat_())) { + LOG_WARN("fail to check inner stat", KR(ret)); + } else if (OB_FAIL(ctx.set_timeout(timeout))) { + LOG_WARN("fail to set timeout", KR(ret)); + } else if (OB_FAIL(sql_proxy_->write( + OB_SYS_TENANT_ID, create_package_sql, + affected_rows, static_cast(mode)))) { + LOG_WARN("fail to execute sql", KR(ret), "sql", create_package_sql); + } else if (0 != affected_rows) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("affected_rows expected to be zero", KR(ret), K(affected_rows)); + } else if (OB_FAIL(check_stop())) { + LOG_WARN("executor is stop", KR(ret)); + } else if (OB_FAIL(ctx.set_timeout(timeout))) { + LOG_WARN("fail to set timeout", KR(ret)); + } else if (OB_FAIL(sql_proxy_->write( + OB_SYS_TENANT_ID, create_package_body_sql, + affected_rows, static_cast(mode)))) { + LOG_WARN("fail to execute sql", KR(ret), "sql", create_package_body_sql); + } else if (0 != affected_rows) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("affected_rows expected to be zero", KR(ret), K(affected_rows)); + } else if (OB_FAIL(check_stop())) { + LOG_WARN("executor is stop", KR(ret)); + } else if (OB_FAIL(ctx.set_timeout(timeout))) { + LOG_WARN("fail to set timeout", KR(ret)); + } else if (OB_FAIL(sql_proxy_->write( + OB_SYS_TENANT_ID, upgrade_sql, + affected_rows, static_cast(mode)))) { + LOG_WARN("fail to execute sql", KR(ret), "sql", upgrade_sql); + } else if (0 != affected_rows) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("affected_rows expected to be zero", KR(ret), K(affected_rows)); + } + FLOG_INFO("[UPGRADE] finish run upgrade oracle system package job", + KR(ret), "cost", ObTimeUtility::current_time() - start_ts); + return ret; +} +#endif int ObUpgradeExecutor::run_upgrade_all_post_action_( const common::ObIArray &tenant_ids) diff --git a/src/rootserver/ob_upgrade_executor.h b/src/rootserver/ob_upgrade_executor.h index a4557901c..a9f78e58f 100644 --- a/src/rootserver/ob_upgrade_executor.h +++ b/src/rootserver/ob_upgrade_executor.h @@ -89,6 +89,9 @@ private: int check_table_schema_(const uint64_t tenant_id, const share::schema::ObTableSchema &hard_code_table); int upgrade_mysql_system_package_job_(); +#ifdef OB_BUILD_ORACLE_PL + int upgrade_oracle_system_package_job_(); +#endif int run_upgrade_all_post_action_(const uint64_t tenant_id); int run_upgrade_end_action_(const uint64_t tenant_id); diff --git a/src/rootserver/restore/ob_restore_scheduler.cpp b/src/rootserver/restore/ob_restore_scheduler.cpp index 60d2d6bca..ded984324 100644 --- a/src/rootserver/restore/ob_restore_scheduler.cpp +++ b/src/rootserver/restore/ob_restore_scheduler.cpp @@ -37,6 +37,9 @@ #include "share/ob_primary_standby_service.h" #include "logservice/palf/log_define.h"//scn #include "share/scn.h" +#ifdef OB_BUILD_TDE_SECURITY +#include "share/ob_master_key_getter.h" +#endif namespace oceanbase { @@ -521,18 +524,153 @@ int ObRestoreService::convert_parameters( const ObPhysicalRestoreJob &job_info) { int ret = OB_SUCCESS; +#ifdef OB_BUILD_TDE_SECURITY + uint64_t tenant_id = tenant_id_; + if (!inited_) { + ret = OB_NOT_INIT; + LOG_WARN("not inited", K(ret)); + } else if (OB_INVALID_TENANT_ID == tenant_id + || OB_SYS_TENANT_ID == tenant_id) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid tenant id", K(ret), K(tenant_id)); + } else if (OB_FAIL(check_stop())) { + LOG_WARN("restore scheduler stopped", K(ret)); + } else if (job_info.get_kms_dest().empty()) { + // do nothing + } else { + ObArenaAllocator allocator; + int64_t affected_row = 0; + ObString tde_method; + ObString kms_info; + ObSqlString sql; + // set tde_method + if (OB_FAIL(ObMasterKeyUtil::restore_encrypt_params(allocator, job_info.get_kms_dest(), + job_info.get_kms_encrypt_key(), tde_method, kms_info))) { + LOG_WARN("failed to restore encrypt params", K(ret)); + } else if (OB_UNLIKELY(tde_method.empty())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tde_method is empty", K(ret)); + } else if (!job_info.get_kms_info().empty()) { + kms_info = job_info.get_kms_info(); + } + if (OB_FAIL(ret)) { + } else if (!ObTdeMethodUtil::is_valid(tde_method)) { + // do nothing + } else if (OB_FAIL(sql.assign_fmt("ALTER SYSTEM SET tde_method = '%s'", tde_method.ptr()))) { + LOG_WARN("failed to assign fmt", K(ret), K(sql)); + } else if (OB_FAIL(sql_proxy_->write(tenant_id, sql.ptr(), affected_row))) { + LOG_WARN("failed to execute", K(ret), K(affected_row), K(sql)); + } else if (ObTdeMethodUtil::is_internal(tde_method)) { + // do nothing + } else if (FALSE_IT(sql.reset())) { + } else if (OB_FAIL(sql.assign_fmt("ALTER SYSTEM SET external_kms_info= '%s'", kms_info.ptr()))) { + LOG_WARN("failed to assign fmt", K(ret), K(sql)); + } else if (OB_FAIL(sql_proxy_->write(tenant_id, sql.ptr(), affected_row))) { + LOG_WARN("failed to execute", K(ret), K(affected_row), K(sql)); + } + } +#endif return ret; } int ObRestoreService::restore_root_key(const share::ObPhysicalRestoreJob &job_info) { int ret = OB_SUCCESS; +#ifdef OB_BUILD_TDE_SECURITY + int64_t idx = job_info.get_multi_restore_path_list().get_backup_set_path_list().count() - 1; + if (idx < 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid job info", K(ret), K(idx), K(job_info)); + } else if (OB_ISNULL(srv_rpc_proxy_) || OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null svr rpc proxy or sql proxy", K(ret)); + } else { + storage::ObBackupDataStore store; + const share::ObBackupSetPath &backup_set_path = job_info.get_multi_restore_path_list().get_backup_set_path_list().at(idx); + ObRootKey root_key; + if (OB_FAIL(store.init(backup_set_path.ptr()))) { + LOG_WARN("fail to init backup data store", K(ret)); + } else if (OB_FAIL(store.read_root_key_info(tenant_id_))) { + LOG_WARN("fail to read root key info", K(ret)); + } else if (OB_FAIL(ObMasterKeyGetter::instance().get_root_key(tenant_id_, root_key))) { + LOG_WARN("fail to get root key", K(ret)); + } else if (obrpc::RootKeyType::INVALID == root_key.key_type_) { + // do nothing + } else { + obrpc::ObRootKeyArg arg; + obrpc::ObRootKeyResult result; + ObUnitTableOperator unit_operator; + ObArray units; + ObArray addrs; + arg.tenant_id_ = tenant_id_; + arg.is_set_ = true; + arg.key_type_ = root_key.key_type_; + arg.root_key_ = root_key.key_; + if (OB_FAIL(unit_operator.init(*sql_proxy_))) { + LOG_WARN("failed to init unit operator", KR(ret)); + } else if (OB_FAIL(unit_operator.get_units_by_tenant(tenant_id_, units))) { + LOG_WARN("failed to get tenant unit", KR(ret), K_(tenant_id)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < units.count(); i++) { + const ObUnit &unit = units.at(i); + if (OB_FAIL(addrs.push_back(unit.server_))) { + LOG_WARN("failed to push back addr", KR(ret)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(ObDDLService::notify_root_key(*srv_rpc_proxy_, arg, addrs, result))) { + LOG_WARN("failed to notify root key", K(ret)); + } + } + } +#endif return ret; } int ObRestoreService::restore_keystore(const share::ObPhysicalRestoreJob &job_info) { int ret = OB_SUCCESS; +#ifdef OB_BUILD_TDE_SECURITY + const int64_t DEFAULT_TIMEOUT = 10 * 1000 * 1000L; + ObUnitTableOperator unit_operator; + common::ObArray units; + ObArray return_code_array; + obrpc::ObRestoreKeyArg arg; + if (job_info.get_kms_dest().empty()) { + // do nothing + } else if (OB_ISNULL(srv_rpc_proxy_) || OB_ISNULL(sql_proxy_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null svr rpc proxy or sql proxy", K(ret)); + } else if (OB_FAIL(unit_operator.init(*sql_proxy_))) { + LOG_WARN("failed to init unit operator", KR(ret)); + } else if (OB_FAIL(unit_operator.get_units_by_tenant(tenant_id_, units))) { + LOG_WARN("failed to get tenant unit", KR(ret), K_(tenant_id)); + } else { + ObRestoreKeyProxy proxy(*srv_rpc_proxy_, &obrpc::ObSrvRpcProxy::restore_key); + arg.tenant_id_ = job_info.get_tenant_id(); + arg.backup_dest_ = job_info.get_kms_dest(); + arg.encrypt_key_ = job_info.get_kms_encrypt_key(); + for (int64_t i = 0; OB_SUCC(ret) && i < units.count(); i++) { + const ObUnit &unit = units.at(i); + if (OB_FAIL(proxy.call(unit.server_, DEFAULT_TIMEOUT, arg))) { + LOG_WARN("failed to send rpc", KR(ret)); + } + } + + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = proxy.wait_all(return_code_array))) { + LOG_WARN("wait batch result failed", KR(tmp_ret), KR(ret)); + ret = OB_SUCC(ret) ? tmp_ret : ret; + } + for (int64_t i = 0; OB_SUCC(ret) && i < return_code_array.count(); i++) { + ret = return_code_array.at(i); + const ObAddr &addr = proxy.get_dests().at(i); + if (OB_FAIL(ret)) { + LOG_WARN("failed to restore key", KR(ret), K(addr)); + } + } + } +#endif return ret; } diff --git a/src/rootserver/restore/ob_restore_util.cpp b/src/rootserver/restore/ob_restore_util.cpp index c16cd2f9a..30e141a92 100644 --- a/src/rootserver/restore/ob_restore_util.cpp +++ b/src/rootserver/restore/ob_restore_util.cpp @@ -239,9 +239,7 @@ int ObRestoreUtil::fill_multi_backup_path( share::ObPhysicalRestoreJob &job) { int ret = OB_SUCCESS; - // TODO: use restore preview url - return ret; } @@ -423,6 +421,34 @@ int ObRestoreUtil::fill_encrypt_info_( share::ObPhysicalRestoreJob &job) { int ret = OB_SUCCESS; +#ifdef OB_BUILD_TDE_SECURITY + ObArenaAllocator allocator; + ObArray kms_path_array; + ObString kms_dest_str; + ObBackupDest dest; + ObBackupIoAdapter util; + bool is_exist = false; + if (OB_FAIL(job.set_encrypt_key(arg.encrypt_key_))) { + LOG_WARN("failed to fill encrypt key", KR(ret), K(arg)); + } else if (arg.kms_uri_.empty()) { + // do nothing + } else if (OB_FAIL(ObPhysicalRestoreUriParser::parse(arg.kms_uri_, allocator, kms_path_array))) { + LOG_WARN("fail to parse uri", K(ret), K(arg)); + } else if (OB_FAIL(get_encrypt_backup_dest_format_str(kms_path_array, allocator, kms_dest_str))) { + LOG_WARN("failed to convert uri", K(ret), K(arg), K(kms_path_array)); + } else if (OB_FAIL(dest.set(kms_dest_str))) { + LOG_WARN("failed to set dest", K(ret)); + } else if (OB_FAIL(util.is_exist(dest.get_root_path(), dest.get_storage_info(), is_exist))) { + LOG_WARN("failed to check file is exists", K(ret)); + } else if (OB_UNLIKELY(!is_exist)) { + ret = OB_BACKUP_FILE_NOT_EXIST; + LOG_WARN("kms backup file is not exist", K(ret)); + } else if (OB_FAIL(job.set_kms_dest(kms_dest_str))) { + LOG_WARN("failed to copy kms dest", K(ret), K(arg)); + } else if (OB_FAIL(job.set_kms_encrypt_key(arg.kms_encrypt_key_))) { + LOG_WARN("failed to fill kms encrypt key", KR(ret), K(arg)); + } +#endif return ret; } diff --git a/src/share/CMakeLists.txt b/src/share/CMakeLists.txt index d2fae2503..1ded37bb0 100755 --- a/src/share/CMakeLists.txt +++ b/src/share/CMakeLists.txt @@ -20,7 +20,6 @@ ob_set_subtarget(ob_share allocator allocator/ob_tenant_mutil_allocator_mgr.cpp ) - ob_set_subtarget(ob_share backup backup/ob_archive_piece.cpp backup/ob_archive_struct.cpp @@ -92,7 +91,7 @@ ob_set_subtarget(ob_share common ob_disk_usage_table_operator.cpp ob_dml_sql_splicer.cpp ob_encryption_struct.cpp - ob_encryption_util.cpp + ob_encryption_util_os.cpp ob_errno.cpp ob_event_history_table_operator.cpp ob_tenant_id_schema_version.cpp diff --git a/src/share/backup/ob_backup_clean_struct.h b/src/share/backup/ob_backup_clean_struct.h index c4ff09cd8..9f8d1e865 100644 --- a/src/share/backup/ob_backup_clean_struct.h +++ b/src/share/backup/ob_backup_clean_struct.h @@ -146,6 +146,7 @@ public: int64_t task_count_; int64_t success_task_count_; }; + // TODO(wenjinyu.wjy) 4.3 Split the structure of set and piece struct ObBackupCleanTaskAttr final { diff --git a/src/share/backup/ob_backup_config.cpp b/src/share/backup/ob_backup_config.cpp index d5aa0a762..88155af9f 100644 --- a/src/share/backup/ob_backup_config.cpp +++ b/src/share/backup/ob_backup_config.cpp @@ -487,7 +487,8 @@ int ObDataBackupDestConfigParser::update_inner_config_table(common::ObISQLClient ObBackupDestMgr dest_mgr; share::ObBackupPathString backup_dest; ObBackupDestType::TYPE dest_type = ObBackupDestType::TYPE::DEST_TYPE_BACKUP_DATA; - // TODO: handle trans failed after write format file. + + // TODO(wangxiaohui.wxh):4.3, handle trans failed after write format file in 4.1. if (!type_.is_valid() || 1 != config_items_.count() ) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid parser", K(ret), KPC(this)); @@ -592,7 +593,8 @@ int ObLogArchiveDestConfigParser::update_archive_dest_config_(common::ObISQLClie LOG_WARN("fail to gen archive config items", K(ret)); } - // TODO: handle trans failed after write format file. + + // TODO(wangxiaohui.wxh):4.3, handle trans failed after write format file in 4.1. ARRAY_FOREACH_X(config_items_, i, cnt, OB_SUCC(ret)) { const BackupConfigItemPair &config_item = config_items_.at(i); if (OB_FAIL(helper.set_kv_item(trans, dest_no_, config_item.key_, config_item.value_))) { @@ -783,7 +785,7 @@ int ObLogArchiveDestConfigParser::do_parse_compression_(const common::ObString & LOG_WARN("fail to push back pair", K(ret), K(pair)); } } else { - // TODO: when log archive support compression, remove this. + // TODO(chongrong.th): when log archive support compression, remove this in 4.3 ret = OB_NOT_SUPPORTED; LOG_WARN("compression not support value", K(ret), K(value)); } diff --git a/src/share/backup/ob_backup_connectivity.cpp b/src/share/backup/ob_backup_connectivity.cpp index 002bd92ae..3c5bc1c16 100644 --- a/src/share/backup/ob_backup_connectivity.cpp +++ b/src/share/backup/ob_backup_connectivity.cpp @@ -62,7 +62,7 @@ int ObBackupConnectivityCheckManager::schedule_connectivity_check_( const share::ObBackupPath &path) { int ret = OB_SUCCESS; - // TODO in 4.1, this code logic needs to be adjusted. + // TODO(wenjinyu.wjy) in 4.3, this code logic needs to be rewritten. Since server_mgr needs to be removed, first comment the code // obrpc::ObCheckBackupConnectivityArg args; // args.tenant_id_ = tenant_id_; // common::ObArray server_list; @@ -202,6 +202,9 @@ int ObBackupConnectivityCheckManager::check_backup_dest_connectivity( LOG_WARN("failed to check oss/cos io permission", K(ret), K_(tenant_id), K(backup_dest)); } else if (OB_FAIL(set_connectivity_check_path_(backup_dest, path))) { LOG_WARN("failed to get check file", K(ret), K_(tenant_id), K(backup_dest)); + // TODO(wenjinyu.wjy) in 4.3, support check connectivity + //} else if (OB_FAIL(schedule_connectivity_check_(backup_dest, path))) { + // LOG_WARN("failed to schedule connectivity check", K(ret), K_(tenant_id)); } else if (OB_FAIL(set_last_check_time_(backup_dest))) { LOG_WARN("failed to set last check time", K(ret), K_(tenant_id), K(backup_dest)); } else { @@ -359,7 +362,6 @@ int ObBackupCheckFile::generate_format_desc_(const share::ObBackupDest &dest, sh int ret = OB_SUCCESS; schema::ObSchemaGetterGuard schema_guard; const schema::ObTenantSchema *tenant_schema = nullptr; - if (OB_ISNULL(GCTX.schema_service_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid argument", K(ret), K(GCTX.schema_service_)); diff --git a/src/share/backup/ob_backup_io_adapter.cpp b/src/share/backup/ob_backup_io_adapter.cpp index 845b52d24..e403ddc84 100644 --- a/src/share/backup/ob_backup_io_adapter.cpp +++ b/src/share/backup/ob_backup_io_adapter.cpp @@ -568,7 +568,7 @@ int ObCheckDirEmptOp::func(const dirent *entry) file_cnt_++; return OB_ERR_EXIST_OBJECT; } - // TODO need to be refactored 4.1 + // TODO(wenjinyu.wjy) need to be refactored 4.3 int ObBackupIoAdapter::is_empty_directory(const common::ObString &uri, const share::ObBackupStorageInfo *storage_info, bool &is_empty_directory) diff --git a/src/share/backup/ob_backup_store.cpp b/src/share/backup/ob_backup_store.cpp index 83e0de67d..89bc6ba2d 100644 --- a/src/share/backup/ob_backup_store.cpp +++ b/src/share/backup/ob_backup_store.cpp @@ -487,7 +487,6 @@ int ObBackupDestMgr::generate_format_desc_( schema::ObSchemaGetterGuard schema_guard; share::ObBackupPathString root_path; const schema::ObTenantSchema *tenant_schema = nullptr; - if (OB_ISNULL(GCTX.schema_service_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("invalid argument", K(ret), K(GCTX.schema_service_)); diff --git a/src/share/backup/ob_backup_struct.cpp b/src/share/backup/ob_backup_struct.cpp index 6ae2d90eb..935b3207f 100755 --- a/src/share/backup/ob_backup_struct.cpp +++ b/src/share/backup/ob_backup_struct.cpp @@ -1177,6 +1177,15 @@ int ObBackupStorageInfo::set_access_key_(const char *buf, const bool need_decryp } else if (0 != strlen(access_key_)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("access_key has been set value", K(ret), K_(access_key)); +#ifdef OB_BUILD_TDE_SECURITY + } else if (need_decrypt) { + char encrypted_key[OB_MAX_BACKUP_ENCRYPTKEY_LENGTH] = { 0 }; + if (OB_FAIL(set_storage_info_field_(buf, encrypted_key, sizeof(encrypted_key)))) { + LOG_WARN("failed to set access_key", K(ret), K(buf)); + } else if (OB_FAIL(decrypt_access_key_(encrypted_key))) { + LOG_WARN("failed to set access key", K(ret), K(buf)); + } +#endif } else if (OB_FAIL(set_storage_info_field_(buf, access_key_, sizeof(access_key_)))) { LOG_WARN("failed to set endpoint",K(ret), K(buf)); } @@ -1241,6 +1250,12 @@ int ObBackupStorageInfo::set( if (OB_FAIL(set_storage_info_field_(token, access_id_, sizeof(access_id_)))) { LOG_WARN("failed to set access id", K(ret), K(token)); } +#ifdef OB_BUILD_TDE_SECURITY + } else if (0 == strncmp(ENCRYPT_KEY, token, strlen(ENCRYPT_KEY))) { + if (OB_FAIL(set_access_key_(token, true/*need decrypt*/))) { + LOG_WARN("failed to set oss extension", K(ret), K(token)); + } +#endif } else if (0 == strncmp(ACCESS_KEY, token, strlen(ACCESS_KEY))) { if (OB_FAIL(set_access_key_(token, false/*need decrypt*/))) { LOG_WARN("failed to set oss extension", K(ret), K(token)); @@ -1311,6 +1326,15 @@ int ObBackupStorageInfo::parse_authorization_(const char *authorization) if (OB_FAIL(set_storage_info_field_(token, access_id_, sizeof(access_id_)))) { LOG_WARN("failed to set access id", K(ret), K(token)); } +#ifdef OB_BUILD_TDE_SECURITY + } else if (0 == strncmp(ENCRYPT_KEY, token, strlen(ENCRYPT_KEY))) { + char serialize_key[OB_MAX_BACKUP_SERIALIZEKEY_LENGTH] = { 0 }; + if (OB_FAIL(set_storage_info_field_(token, serialize_key, sizeof(serialize_key)))) { + LOG_WARN("failed to set encrypt_key", K(ret), K(token)); + } else if (OB_FAIL(decrypt_access_key_(serialize_key))) { + LOG_WARN("failed to decrypt access key", K(ret), K(token)); + } +#endif } else if (0 == strncmp(ACCESS_KEY, token, strlen(ACCESS_KEY))) { if (OB_FAIL(set_storage_info_field_(token, access_key_, sizeof(access_key_)))) { LOG_WARN("failed to set access key", K(ret), K(token)); @@ -1337,8 +1361,15 @@ int ObBackupStorageInfo::get_authorization_info(char *authorization, int64_t len LOG_WARN("storage info is invalid", K(ret)); } else if (OB_STORAGE_FILE == device_type_) { // do nothing +#ifndef OB_BUILD_TDE_SECURITY } else if (OB_FAIL(databuff_printf(authorization, length, "%s&%s", access_id_, access_key_))) { LOG_WARN("failed to set authorization", K(ret), K_(access_id), K(access_key_)); +#else + } else if (OB_FAIL(encrypt_access_key_(encrypt_key, sizeof(encrypt_key)))) { + LOG_WARN("failed to encrept access key", K(ret)); + } else if (OB_FAIL(databuff_printf(authorization, length, "%s&%s", access_id_, encrypt_key))) { + LOG_WARN("failed to set authorization", K(ret), K_(access_id), K(encrypt_key)); +#endif } return ret; } @@ -1356,6 +1387,14 @@ int ObBackupStorageInfo::get_storage_info_str(char *storage_info, int64_t info_l LOG_WARN("storage info is invalid", K(ret)); } else if (OB_STORAGE_FILE == device_type_) { // do nothing +#ifdef OB_BUILD_TDE_SECURITY + } else if (need_encrypt) { + if (OB_FAIL(encrypt_access_key_(encrypt_key, sizeof(encrypt_key)))) { + LOG_WARN("failed to encrypt access key", K(ret)); + } else { + key = encrypt_key; + } +#endif } else { key = access_key_; } @@ -1428,6 +1467,66 @@ int ObBackupStorageInfo::set_storage_info_field_(const char *info, char *field, return ret; } +#ifdef OB_BUILD_TDE_SECURITY +int ObBackupStorageInfo::encrypt_access_key_(char *encrypt_key, int64_t length) const +{ + int ret = OB_SUCCESS; + char encrypted_key[OB_MAX_BACKUP_ENCRYPTKEY_LENGTH] = { 0 }; + char serialize_buf[OB_MAX_BACKUP_SERIALIZEKEY_LENGTH] = { 0 }; + int64_t serialize_pos = 0; + int64_t key_len = 0; + if (0 == strlen(access_key_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("access_key is empty, shouldn't encrypt", K(ret)); + } else if (0 != strncmp(ACCESS_KEY, access_key_, strlen(ACCESS_KEY))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("parameter is not access_key", K(ret)); + } else if (OB_FAIL(ObEncryptionUtil::encrypt_sys_data(OB_SYS_TENANT_ID, + access_key_ + strlen(ACCESS_KEY), strlen(access_key_) - strlen(ACCESS_KEY), + encrypted_key, OB_MAX_BACKUP_ENCRYPTKEY_LENGTH, key_len))) { + LOG_WARN("failed to encrypt authorization key", K(ret)); + } else if (OB_FAIL(hex_print(encrypted_key, key_len, serialize_buf, sizeof(serialize_buf), serialize_pos))) { + LOG_WARN("failed to serialize encrypted key", K(ret), K(encrypted_key)); + } else if (serialize_pos >= sizeof(serialize_buf) || serialize_pos >= length) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("encode error", K(ret), K(serialize_pos), K(sizeof(serialize_buf)), K(length)); + } else if (FALSE_IT(serialize_buf[serialize_pos] = '\0')) { + } else if (OB_FAIL(databuff_printf(encrypt_key, length, "%s%s", ENCRYPT_KEY, serialize_buf))) { + LOG_WARN("failed to get encrypted key", K(ret), K(serialize_buf)); + } + + return ret; +} + +int ObBackupStorageInfo::decrypt_access_key_(const char *buf) +{ + int ret = OB_SUCCESS; + int64_t key_len = 0; + char deserialize_buf[OB_MAX_BACKUP_SERIALIZEKEY_LENGTH] = { 0 }; + char decrypt_key[OB_MAX_BACKUP_ACCESSKEY_LENGTH] = { 0 }; + int64_t buf_len = strlen(buf); + int64_t deserialize_size = 0; + if (0 == buf_len) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("encrypted_key is empty", K(ret)); + } else if (OB_FAIL(hex_to_cstr(buf + strlen(ENCRYPT_KEY), buf_len - strlen(ENCRYPT_KEY), + deserialize_buf, sizeof(deserialize_buf), deserialize_size))) { + LOG_WARN("failed to get cstr from hex", KR(ret), K(buf_len), K(sizeof(deserialize_buf))); + } else if (OB_FAIL(ObEncryptionUtil::decrypt_sys_data(OB_SYS_TENANT_ID, + deserialize_buf, deserialize_size, + decrypt_key, sizeof(decrypt_key), key_len))) { + LOG_WARN("failed to decrypt authorization key", K(ret), K(deserialize_buf), K(deserialize_size)); + } else if (key_len >= sizeof(decrypt_key) || (key_len + strlen(ACCESS_KEY)) >= sizeof(access_key_)) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("decrypt key size overflow", K(ret), K(key_len), K(sizeof(decrypt_key))); + } else if (FALSE_IT(decrypt_key[key_len] = '\0')) { + } else if (OB_FAIL(databuff_printf(access_key_, sizeof(access_key_), "%s%s", ACCESS_KEY, decrypt_key))) { + LOG_WARN("failed to set access key", K(ret)); + } + + return ret; +} +#endif ObBackupDest::ObBackupDest() : root_path_(NULL), diff --git a/src/share/backup/ob_backup_struct.h b/src/share/backup/ob_backup_struct.h index 68e855b20..cd6f8381f 100755 --- a/src/share/backup/ob_backup_struct.h +++ b/src/share/backup/ob_backup_struct.h @@ -883,6 +883,10 @@ public: TO_STRING_KV(K_(endpoint), K_(access_id), K_(extension), "type", get_type_str()); private: int set_storage_info_field_(const char *info, char *field, const int64_t length); +#ifdef OB_BUILD_TDE_SECURITY + int encrypt_access_key_(char *encrypt_key, int64_t length) const; + int decrypt_access_key_(const char *buf); +#endif int set_access_key_(const char *buf, const bool need_decrypt); int parse_authorization_(const char *authorization); int check_delete_mode_(const char *delete_mode); diff --git a/src/share/backup/ob_log_restore_struct.cpp b/src/share/backup/ob_log_restore_struct.cpp index 3fb11a787..eccc67162 100644 --- a/src/share/backup/ob_log_restore_struct.cpp +++ b/src/share/backup/ob_log_restore_struct.cpp @@ -258,9 +258,15 @@ int ObRestoreSourceServiceAttr::set_service_passwd_to_encrypt(const char *passwd if (OB_ISNULL(passwd) || OB_UNLIKELY(0 == STRLEN(passwd) || STRLEN(passwd) > OB_MAX_PASSWORD_LENGTH)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument"); +#ifdef OB_BUILD_TDE_SECURITY + } else if (OB_FAIL(set_encrypt_password_key_(passwd))) { + LOG_WARN("encrypt password failed"); + } +#else } else if (OB_FAIL(databuff_printf(encrypt_passwd_, sizeof(encrypt_passwd_), "%s", passwd))) { LOG_WARN("fail to print encrypt password"); } +#endif LOG_INFO("set service password success"); return ret; } @@ -632,21 +638,88 @@ int ObRestoreSourceServiceAttr::get_is_encrypted_str_(char *buf, const int64_t b if (OB_ISNULL(buf) || OB_UNLIKELY(buf_size <= 0)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid encrypted str argument", KP(buf), K(buf_size)); +#ifdef OB_BUILD_TDE_SECURITY + } else if (OB_FAIL(databuff_printf(buf, buf_size, "%s", "true"))) { + LOG_WARN("fail to print is_encrypted str"); + } +#else } else if (OB_FAIL(databuff_printf(buf, buf_size, "%s", "false"))) { LOG_WARN("fail to print encrypted str"); } +#endif return ret; } int ObRestoreSourceServiceAttr::get_is_encrypted_str_(ObSqlString &encrypted_str) const { int ret = OB_SUCCESS; +#ifdef OB_BUILD_TDE_SECURITY + if (OB_FAIL(encrypted_str.assign("true"))) { + LOG_WARN("fail to print is_encrypted str"); + } +#else if (OB_FAIL(encrypted_str.assign("false"))) { LOG_WARN("fail to print str"); } +#endif + return ret; +} + +#ifdef OB_BUILD_TDE_SECURITY +int ObRestoreSourceServiceAttr::set_encrypt_password_key_(const char *passwd) +{ + int ret = OB_SUCCESS; + char encrypted_key[OB_MAX_BACKUP_ENCRYPTKEY_LENGTH] = { 0 }; + char serialize_buf[OB_MAX_BACKUP_SERIALIZEKEY_LENGTH] = { 0 }; + int64_t serialize_pos = 0; + int64_t key_len = 0; + + if (OB_ISNULL(passwd) || OB_UNLIKELY(0 == STRLEN(passwd))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("password is empty, shouldn't encrypt"); + } else if (OB_FAIL(ObEncryptionUtil::encrypt_sys_data(OB_SYS_TENANT_ID, passwd, strlen(passwd), encrypted_key, + OB_MAX_BACKUP_ENCRYPTKEY_LENGTH, key_len))) { + LOG_WARN("failed to encrypt authorization key"); + } else if (OB_FAIL(hex_print(encrypted_key, key_len, serialize_buf, sizeof(serialize_buf), serialize_pos))) { + LOG_WARN("failed to serialize encrypted key", K(encrypted_key)); + } else if (serialize_pos >= sizeof(serialize_buf) || serialize_pos >= sizeof(encrypt_passwd_)) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("encode error", K(serialize_pos), K(sizeof(serialize_buf)), K(sizeof(encrypt_passwd_))); + } else if (FALSE_IT(serialize_buf[serialize_pos] = '\0')) { + } else if (OB_FAIL(databuff_printf(encrypt_passwd_, sizeof(encrypt_passwd_), "%s", serialize_buf))) { + LOG_WARN("failed to get encrypted key", K(serialize_buf)); + } return ret; } +int ObRestoreSourceServiceAttr::get_decrypt_password_key_(char *unencrypt_key, const int64_t buf_size) const +{ + int ret = OB_SUCCESS; + int64_t key_len = 0; + char deserialize_buf[OB_MAX_BACKUP_SERIALIZEKEY_LENGTH] = { 0 }; + char decrypt_key[OB_MAX_BACKUP_ACCESSKEY_LENGTH] = { 0 }; + int64_t buf_len = strlen(encrypt_passwd_); + int64_t deserialize_size = 0; + + if (OB_ISNULL(unencrypt_key) || OB_UNLIKELY((0 == buf_len) || (buf_size <= 0) || (buf_size < OB_MAX_PASSWORD_LENGTH))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get decrypt password, parameter is invalid", K(ret)); + } else if (OB_FAIL(hex_to_cstr(encrypt_passwd_, buf_len, + deserialize_buf, sizeof(deserialize_buf), deserialize_size))) { + LOG_WARN("failed to get cstr from hex", K(buf_len), K(sizeof(deserialize_buf))); + } else if (OB_FAIL(ObEncryptionUtil::decrypt_sys_data(OB_SYS_TENANT_ID, deserialize_buf, deserialize_size, + decrypt_key, sizeof(decrypt_key), key_len))) { + LOG_WARN("failed to decrypt authorization key", K(ret), K(deserialize_buf), K(deserialize_size)); + } else if (key_len >= sizeof(decrypt_key) || (key_len) >= buf_size) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("decrypt key size overflow", K(ret), K(key_len), K(sizeof(decrypt_key)), K(sizeof(unencrypt_key))); + } else if (FALSE_IT(decrypt_key[key_len] = '\0')) { + } else if (OB_FAIL(databuff_printf(unencrypt_key, buf_size, "%s", decrypt_key))) { + LOG_WARN("failed to set unencrypt key", K(ret)); + } + return ret; +} +#endif int ObRestoreSourceServiceAttr::get_password(char *passwd, const int64_t buf_size) const { @@ -655,9 +728,17 @@ int ObRestoreSourceServiceAttr::get_password(char *passwd, const int64_t buf_siz if (OB_ISNULL(passwd) || OB_UNLIKELY((buf_size <= 0) || (buf_size < OB_MAX_PASSWORD_LENGTH))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid parameter when get password", K(passwd), K(STRLEN(passwd))); +#ifdef OB_BUILD_TDE_SECURITY + } else if (OB_FAIL(get_decrypt_password_key_(tmp_passwd, sizeof(tmp_passwd)))) { + LOG_WARN("failed to get decrypt password key"); + } else if (OB_FAIL(databuff_printf(passwd, buf_size, "%s", tmp_passwd))) { + LOG_WARN("failed to print encrypt_passwd_ key", K(encrypt_passwd_)); + } +#else } else if (OB_FAIL(databuff_printf(passwd, buf_size, "%s", encrypt_passwd_))) { LOG_WARN("failed to print encrypt_passwd_ key", K(encrypt_passwd_)); } +#endif return ret; } diff --git a/src/share/ls/ob_ls_creator.cpp b/src/share/ls/ob_ls_creator.cpp index 3c8a511f9..9daf99d13 100755 --- a/src/share/ls/ob_ls_creator.cpp +++ b/src/share/ls/ob_ls_creator.cpp @@ -28,6 +28,10 @@ #include "share/scn.h" #include "share/ls/ob_ls_life_manager.h" #include "rootserver/ob_root_utils.h"//notify_switch_leader +#ifdef OB_BUILD_ARBITRATION +#include "share/arbitration_service/ob_arbitration_service_info.h" // for ObArbitrationServiceInfo +#include "share/arbitration_service/ob_arbitration_service_table_operator.h" // for ObArbitrationServiceTableOperator +#endif using namespace oceanbase::common; using namespace oceanbase::share; @@ -426,11 +430,130 @@ int ObLSCreator::create_ls_(const ObILSAddr &addrs, LOG_WARN("failed to check ls result", KR(ret), K(rpc_count), K(paxos_replica_num), K(return_code_array), K(learner_list)); } +#ifdef OB_BUILD_ARBITRATION + // try to create A-replica if needed + // (1) ignore any erros because majority of F-replica is successfully created, + // arb replica task will generated by back groud process later to add A-replica + bool need_create_arb_replica = false; + ObAddr arbitration_service_addr; + if (OB_FAIL(ret)) { + } else if (OB_SUCCESS != (tmp_ret = check_need_create_arb_replica_( + need_create_arb_replica, + arbitration_service_addr))) { + LOG_WARN("fail to check need create arb replica", KR(tmp_ret)); + } else if (!need_create_arb_replica) { + // do nothing + LOG_INFO("no need to create A-replica for this log stream", K_(tenant_id), K_(id)); + } else if (OB_SUCCESS != (tmp_ret = try_create_arbitration_service_replica_( + tenant_info.get_tenant_role(), + arbitration_service_addr))) { + LOG_WARN("fail to create arbitration service replica", KR(tmp_ret), K(tenant_info)); + } else { + int64_t timestamp = 1; + arbitration_service = ObMember(arbitration_service_addr, timestamp); + } +#endif } } return ret; } +#ifdef OB_BUILD_ARBITRATION +int ObLSCreator::check_need_create_arb_replica_( + bool &need_create_arb_replica, + ObAddr &arbitration_service) +{ + int ret = OB_SUCCESS; + need_create_arb_replica = false; + ObSqlString sql; + const uint64_t sql_tenant_id = OB_SYS_TENANT_ID; + const ObString arbitration_service_key = "default"; + const bool lock_line = false; + ObArbitrationServiceInfo arbitration_service_info; + ObArbitrationServiceTableOperator arbitration_service_table_operator; + if (OB_UNLIKELY(!is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K_(tenant_id), K_(id)); + } else if (OB_ISNULL(proxy_) + || OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), KP(proxy_), KP(GCTX.sql_proxy_)); + } else if (OB_FAIL(sql.assign_fmt("SELECT arbitration_service_status IN ('ENABLING', 'ENABLED') AS is_enabling " + "FROM %s WHERE tenant_id = %ld", + OB_ALL_TENANT_TNAME, tenant_id_))) { + LOG_WARN("fail to assign sql", KR(ret), K_(tenant_id)); + } else { + int64_t is_enabling = 0; + SMART_VAR(ObISQLClient::ReadResult, result) { + if (OB_FAIL(GCTX.sql_proxy_->read(result, sql_tenant_id, sql.ptr()))) { + LOG_WARN("execute sql failed", KR(ret), K(sql_tenant_id), "sql", sql.ptr()); + } else if (OB_ISNULL(result.get_result())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get sql result failed", KR(ret), "sql", sql.ptr()); + } else if (OB_FAIL(result.get_result()->next())) { + if (OB_ITER_END == ret) { + ret = OB_TENANT_NOT_EXIST; + } + LOG_WARN("get result next failed", KR(ret), "sql", sql.ptr()); + } else if (OB_FAIL(result.get_result()->get_int(0L, is_enabling))) { + if (OB_ERR_COLUMN_NOT_FOUND != ret) { + LOG_WARN("get arbitration status failed", KR(ret), "sql", sql.ptr()); + } else { + // ignore column not found + ret = OB_SUCCESS; + } + } else { + need_create_arb_replica = (1 == is_enabling); + } + } + } + + // no need to lock __all_arbitration_service line because: + // (1) ls status is setted in creating status BEFORE get arbitration status from __all_tenant + // (2) there is no ls in creating status BEFORE arbitration service is removed + // after get a enabling status in __all_tenant means + // any remove operations later could see ls in creating status in __all_ls_status, thus disallowing remove it + if (OB_FAIL(ret) || !need_create_arb_replica) { + } else if (OB_FAIL(arbitration_service_table_operator.get( + *proxy_, + arbitration_service_key, + lock_line, + arbitration_service_info))) { + LOG_WARN("fail to get arbitration service info", KR(ret), K(arbitration_service_key), K(lock_line)); + } else if (OB_FAIL(arbitration_service_info.get_arbitration_service_addr(arbitration_service))) { + LOG_WARN("fail to get arbitration service addr", KR(ret), K(arbitration_service_info)); + } + return ret; +} + +int ObLSCreator::try_create_arbitration_service_replica_( + const ObTenantRole &tenant_role, + const ObAddr &arbitration_service) +{ + int ret = OB_SUCCESS; + ObCreateArbArg create_arb_arg; + ObCreateArbResult result; + ObTimeoutCtx ctx; + if (OB_UNLIKELY(!is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret)); + } else if (!tenant_role.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("tenant role is invalid", KR(ret), K_(tenant_id), K_(id), K(tenant_role)); + } else if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, GCONF.rpc_timeout))) { + LOG_WARN("fail to set timeout ctx", KR(ret)); + } else if (OB_ISNULL(GCTX.srv_rpc_proxy_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret)); + } else if (OB_FAIL(create_arb_arg.init(tenant_id_, id_, tenant_role))) { + LOG_WARN("fail to init ObCreateArbArg", KR(ret), K_(tenant_id), K_(id), K(tenant_role)); + } else if (OB_FAIL(GCTX.srv_rpc_proxy_->to(arbitration_service).timeout(ctx.get_timeout()).create_arb(create_arb_arg, result))) { + // do nothing, let arb service add replica + LOG_WARN("fail to create arbitration service replica", KR(ret), K(arbitration_service), "timeout", ctx.get_timeout(), K(create_arb_arg)); + } + return ret; +} +#endif int ObLSCreator::check_create_ls_result_(const int64_t rpc_count, const int64_t paxos_replica_num, diff --git a/src/share/ls/ob_ls_creator.h b/src/share/ls/ob_ls_creator.h index 8159d0895..4aaa37802 100755 --- a/src/share/ls/ob_ls_creator.h +++ b/src/share/ls/ob_ls_creator.h @@ -142,6 +142,12 @@ private: const common::ObMember &arbitration_service, const int64_t paxos_replica_num, const common::GlobalLearnerList &learner_list); +#ifdef OB_BUILD_ARBITRATION + int set_arbitration_service_list_( + const common::ObMember &arbitration_service, + const ObTimeoutCtx &ctx, + const obrpc::ObSetMemberListArgV2 &arg); +#endif int persist_ls_member_list_(const common::ObMemberList &member_list, const ObMember &arb_member, const common::GlobalLearnerList &learner_list); @@ -160,6 +166,15 @@ private: const share::ObZoneReplicaAttrSet &zone_locality, const common::ObIArray &unit_info_array, ObLSReplicaAddr &ls_replica_addr); +#ifdef OB_BUILD_ARBITRATION + int check_need_create_arb_replica_( + bool &need_create_arb_replica, + ObAddr &arbitration_service); + + int try_create_arbitration_service_replica_( + const ObTenantRole &tenant_role, + const ObAddr &arbitration_service); +#endif int check_create_ls_result_(const int64_t rpc_count, const int64_t paxos_replica_num, const ObIArray &return_code_array, diff --git a/src/share/ob_common_rpc_proxy.h b/src/share/ob_common_rpc_proxy.h index 8ad271623..2465940e9 100644 --- a/src/share/ob_common_rpc_proxy.h +++ b/src/share/ob_common_rpc_proxy.h @@ -293,6 +293,12 @@ public: RPC_S(PR5 admin_sync_rewrite_rules, obrpc::OB_ADMIN_SYNC_REWRITE_RULES, (ObSyncRewriteRuleArg)); //----End of Definitions for sync rewrite rules---- +#ifdef OB_BUILD_ARBITRATION + RPC_S(PR5 admin_add_arbitration_service, obrpc::OB_ADMIN_ADD_ARBITRATION_SERVICE, (ObAdminAddArbitrationServiceArg)); + RPC_S(PR5 admin_remove_arbitration_service, obrpc::OB_ADMIN_REMOVE_ARBITRATION_SERVICE, (ObAdminRemoveArbitrationServiceArg)); + RPC_S(PR5 admin_replace_arbitration_service, obrpc::OB_ADMIN_REPLACE_ARBITRATION_SERVICE, (ObAdminReplaceArbitrationServiceArg)); + RPC_S(PR5 remove_cluster_info_from_arb_server, obrpc::OB_REMOVE_CLUSTER_INFO_FROM_ARB_SERVER, (ObRemoveClusterInfoFromArbServerArg)); +#endif //----Definitions for managing row level security---- RPC_S(PRD handle_rls_policy_ddl, obrpc::OB_HANDLE_RLS_POLICY_DDL, (ObRlsPolicyDDLArg)); RPC_S(PRD handle_rls_group_ddl, obrpc::OB_HANDLE_RLS_GROUP_DDL, (ObRlsGroupDDLArg)); @@ -301,6 +307,9 @@ public: RPC_S(PRD recompile_all_views_batch, obrpc::OB_RECOMPILE_ALL_VIEWS_BATCH, (ObRecompileAllViewsBatchArg)); RPC_S(PRD try_add_dep_infos_for_synonym_batch, obrpc::OB_TRY_ADD_DEP_INFOS_FOR_SYNONYM_BATCH, (ObTryAddDepInofsForSynonymBatchArg)); +#ifdef OB_BUILD_TDE_SECURITY + RPC_S(PR5 get_root_key, obrpc::OB_GET_ROOT_KEY, (obrpc::ObRootKeyArg), obrpc::ObRootKeyResult); +#endif public: void set_rs_mgr(share::ObRsMgr &rs_mgr) { diff --git a/src/share/ob_encryption_util.h b/src/share/ob_encryption_util.h index d8785bff6..f7334b5e1 100644 --- a/src/share/ob_encryption_util.h +++ b/src/share/ob_encryption_util.h @@ -111,21 +111,68 @@ const int64_t OB_CLOG_ENCRYPT_TABLE_KEY_LEN = 32; class ObEncryptionUtil { public: - static const ObAesOpMode DEFAULT_TABLE_KEY_ENCRYPT_ALGORITHM = share::ObAesOpMode::ob_aes_128_ecb; - static const ObAesOpMode MASTER_KEY_ENCRYPT_ALGORITHM = share::ObAesOpMode::ob_aes_128_cbc; static int init_ssl_malloc(); +#ifdef OB_BUILD_TDE_SECURITY + static bool need_encrypt(int64_t encrypt_id); + static int get_tde_method(int64_t tenant_id, common::ObString &tde_method); + static int get_tde_kms_info(int64_t tenant_id, common::ObString &kms_info); +#endif static int parse_encryption_algorithm(const common::ObString &str, ObAesOpMode &encryption_algorithm); static int parse_encryption_algorithm(const char *str, ObAesOpMode &encryption_algorithm); static int parse_encryption_id(const char *str, int64_t &encrypt_id); static int parse_encryption_id(const common::ObString &str, int64_t &encrypt_id); +#ifdef OB_BUILD_TDE_SECURITY + static int64_t sys_encrypted_length(int64_t data_len); + + static int encrypt_data(const char *key, const int64_t key_len, enum ObAesOpMode mode, + const char *data, const int64_t data_len, + char *buf, const int64_t buf_len, int64_t &out_len); + static int decrypt_data(const char *key, const int64_t key_len, enum ObAesOpMode mode, + const char *data, const int64_t data_len, + char *buf, const int64_t buf_len, int64_t &out_len); + static int encrypt_data(const share::ObEncryptMeta &meta, + const char *from_buf, const int64_t from_len, + char *to_buf, int64_t to_buf_size, int64_t &to_len); + static int decrypt_data(const share::ObEncryptMeta &meta, + const char *from_buf, const int64_t from_len, + char *to_buf, int64_t to_buf_size, int64_t &to_len); + static int encrypt_table_key(const share::ObEncryptMeta &meta, + char *out_buf, const int64_t out_buf_len, int64_t &out_len); + static int decrypt_table_key(share::ObEncryptMeta &meta, + const char *in_buf, const int64_t in_buf_len); + static int encrypt_master_key(const uint64_t tenant_id, const char *data, const int64_t data_len, + char *buf, const int64_t buf_len, int64_t &out_len); + static int decrypt_master_key(const uint64_t tenant_id, const char *data, const int64_t data_len, + char *buf, const int64_t buf_len, int64_t &out_len); + static int encrypt_sys_data(const uint64_t tenant_id, const char *data, const int64_t data_len, + char *buf, const int64_t buf_len, int64_t &out_len); + static int encrypt_sys_data_default(const char *data, const int64_t data_len, + char *buf, const int64_t buf_len, int64_t &out_len); + + static int decrypt_sys_data(const uint64_t tenant_id, const char *data, const int64_t data_len, + char *buf, const int64_t buf_len, int64_t &out_len); + static int decrypt_sys_data_default(const char *data, const int64_t data_len, + char *buf, const int64_t buf_len, int64_t &out_len); + static int encrypt_zone_data(share::ObZoneEncryptMeta &meta, + const char *data, const int64_t data_len, + char *buf, const int64_t buf_len, int64_t &out_len); + static int decrypt_zone_data(const share::ObZoneEncryptMeta &meta, + const char *data, const int64_t data_len, + char *buf, const int64_t buf_len, int64_t &out_len); +#endif // return the max length after encryption static int64_t encrypted_length(const int64_t data_len); // return an unencrypted data length whose length after encryption is always less than data_len static int64_t decrypted_length(const int64_t data_len); // return the max length after decryption static int64_t safe_buffer_length(const int64_t data_len); +#ifdef OB_BUILD_TDE_SECURITY +private: + static const char* system_encrypt_key_; + static const char* system_encrypt_iv_; +#endif }; struct ObBackupEncryptionMode final @@ -146,6 +193,17 @@ struct ObBackupEncryptionMode final static EncryptionMode parse_str(const common::ObString &str); }; +#ifdef OB_BUILD_TDE_SECURITY +class ObTdeMethodUtil +{ +public: + static bool is_valid(const common::ObString &tde_method); + static bool is_internal(const common::ObString &tde_method); + static bool is_kms(const common::ObString &tde_method); + static bool is_sm_algorithm(const common::ObString &tde_method); + static bool use_external_key_id(const common::ObString &tde_method); +}; +#endif enum ObHashAlgorithm { OB_HASH_INVALID = 0, diff --git a/src/share/ob_encryption_util.cpp b/src/share/ob_encryption_util_os.cpp similarity index 96% rename from src/share/ob_encryption_util.cpp rename to src/share/ob_encryption_util_os.cpp index 52b539035..56870626b 100644 --- a/src/share/ob_encryption_util.cpp +++ b/src/share/ob_encryption_util_os.cpp @@ -10,12 +10,14 @@ * See the Mulan PubL v2 for more details. */ +#ifndef OB_BUILD_TDE_SECURITY + #define USING_LOG_PREFIX SHARE -#include "ob_encryption_util.h" +#include "share/ob_encryption_util.h" #include #include #include -#include "ob_encryption_struct.h" +#include "share/ob_encryption_struct.h" #include "lib/alloc/alloc_assist.h" #include "share/ob_errno.h" #include "lib/atomic/atomic128.h" @@ -31,7 +33,6 @@ using namespace common; namespace share { - int ObKeyGenerator::generate_encrypt_key(char *buf, int64_t len) { int ret = OB_SUCCESS; @@ -105,6 +106,10 @@ static const EVP_CIPHER *aes_evp_type(const ObAesOpMode mode) case ob_aes_256_cfb8: return EVP_aes_256_cfb8(); case ob_aes_256_cfb128: return EVP_aes_256_cfb128(); case ob_aes_256_ofb: return EVP_aes_256_ofb(); +#ifdef OB_USE_BABASSL + case ob_sm4_mode: return EVP_sm4_ctr(); + case ob_sm4_cbc_mode: return EVP_sm4_cbc(); +#endif default: return NULL; } } @@ -265,6 +270,7 @@ int ObAesEncryption::aes_decrypt(const char *key, const int64_t &key_len, const return ret; } +#ifndef OB_USE_BABASSL static void* ob_malloc_openssl(size_t nbytes) { ObMemAttr attr; @@ -283,15 +289,37 @@ static void ob_free_openssl(void *ptr) { ob_free(ptr); } +#else +static void* ob_malloc_openssl(size_t nbyte, const char *, int) +{ + ObMemAttr attr; + attr.label_ = ObModIds::OB_BUFFER; + return ob_malloc(nbyte, attr); +} + +static void* ob_realloc_openssl(void *ptr, size_t nbyte, const char *, int) +{ + ObMemAttr attr; + attr.label_ = ObModIds::OB_BUFFER; + return ob_realloc(ptr, nbyte, attr); +} + +static void ob_free_openssl(void *ptr, const char *, int) +{ + ob_free(ptr); +} +#endif int ObEncryptionUtil::init_ssl_malloc() { int ret = OB_SUCCESS; +#ifdef OB_USE_BABASSL int tmp_ret = CRYPTO_set_mem_functions(ob_malloc_openssl, ob_realloc_openssl, ob_free_openssl); if (OB_UNLIKELY(tmp_ret != 1)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to set crypto mem functions", K(tmp_ret), K(ret)); } +#endif return ret; } @@ -373,7 +401,6 @@ ObBackupEncryptionMode::EncryptionMode ObBackupEncryptionMode::parse_str(const c return mode; } - int ObHashUtil::hash(const enum ObHashAlgorithm algo, const ObString data, ObIAllocator &allocator, ObString &output) { @@ -626,3 +653,5 @@ int ObTdeEncryptEngineLoader::reload_config() }//end share } //end oceanbase + +#endif diff --git a/src/share/ob_label_security_os.cpp b/src/share/ob_label_security_os.cpp index 90091042c..e61773575 100644 --- a/src/share/ob_label_security_os.cpp +++ b/src/share/ob_label_security_os.cpp @@ -10,6 +10,7 @@ * See the Mulan PubL v2 for more details. */ +#ifndef OB_BUILD_LABEL_SECURITY #define USING_LOG_PREFIX SHARE #include "share/ob_label_security.h" @@ -26,6 +27,14 @@ int ObLabelSeResolver::resolve_label_text(const ObString &label_text, ObLabelSeD return ret; } +int ObLabelSeResolver::construct_label_text(const ObLabelSeDecomposedLabel &label_comps, + ObIAllocator *allocator, + ObString &label_text) +{ + int ret = OB_NOT_IMPLEMENT; + return ret; +} + int ObLabelSeResolver::resolve_policy_name(uint64_t tenant_id, const ObString &policy_name, ObSchemaGetterGuard &schema_guard, @@ -50,6 +59,18 @@ int ObLabelSeResolver::deserialize_session_labels(const ObString &labels_str, return ret; } +int ObLabelSeUtil::validate_user_auth( + uint64_t tenant_id, + uint64_t policy_id, + uint64_t user_id, + ObSchemaGetterGuard &schema_guard, + const ObLabelSeLabelCompNums &label_comp_nums, + bool check_lower_bound) +{ + int ret = OB_NOT_IMPLEMENT; + return ret; +} + int ObLabelSeUtil::convert_label_comps_name_to_num( uint64_t tenant_id, uint64_t policy_id, @@ -61,6 +82,28 @@ int ObLabelSeUtil::convert_label_comps_name_to_num( return ret; } +int ObLabelSeUtil::load_default_session_label( + uint64_t tenant_id, + uint64_t policy_id, + uint64_t user_id, + ObSchemaGetterGuard &schema_guard, + ObLabelSeSessionLabel &session_label) +{ + int ret = OB_NOT_IMPLEMENT; + return ret; +} + +int ObLabelSeUtil::check_policy_column(uint64_t tenant_id, + const ObString &schema_name, + const ObString &table_name, + const ObString &column_name, + ObSchemaGetterGuard &schema_guard, + bool &is_policy_column_exist, + bool &is_policy_already_applied_to_column) +{ + int ret = OB_NOT_IMPLEMENT; + return ret; +} OB_SERIALIZE_MEMBER(ObLabelSeSessionLabel, policy_id_, @@ -72,5 +115,4 @@ OB_SERIALIZE_MEMBER(ObLabelSeLabelTag, } } - - +#endif \ No newline at end of file diff --git a/src/share/ob_rpc_struct.cpp b/src/share/ob_rpc_struct.cpp index 6b9686d13..201e2d470 100755 --- a/src/share/ob_rpc_struct.cpp +++ b/src/share/ob_rpc_struct.cpp @@ -3529,6 +3529,97 @@ int ObDRTaskExistArg::init( return ret; } +#ifdef OB_BUILD_ARBITRATION +OB_SERIALIZE_MEMBER(ObAddArbArg, + tenant_id_, + ls_id_, + arb_member_, + timeout_us_); +int ObAddArbArg::assign( + const ObAddArbArg &that) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(arb_member_.assign(that.arb_member_))) { + SHARE_LOG(WARN, "fail to assign arb member", K(ret)); + } else { + tenant_id_ = that.tenant_id_; + ls_id_ = that.ls_id_; + timeout_us_ = that.timeout_us_; + } + return ret; +} +int ObAddArbArg::init( + const uint64_t tenant_id, + const share::ObLSID &ls_id, + const common::ObMember &arb_member, + const int64_t timeout_us) +{ + int ret = OB_SUCCESS; + if (tenant_id == OB_INVALID_ID + || !ls_id.is_valid() + || !ls_id.is_valid_with_tenant(tenant_id) + || !arb_member.is_valid() + || timeout_us == OB_INVALID_TIMESTAMP) { + ret = OB_INVALID_ARGUMENT; + SHARE_LOG(WARN, "failed to init ObAddArbArg", K(ret), K(tenant_id), + K(ls_id), K(arb_member), K(timeout_us)); + } else if (OB_FAIL(arb_member_.assign(arb_member))) { + SHARE_LOG(WARN, "fail to assign arb member", K(ret)); + } else { + tenant_id_ = tenant_id; + ls_id_ = ls_id; + timeout_us_ = timeout_us; + } + return ret; +} + +OB_SERIALIZE_MEMBER(ObRemoveArbArg, + tenant_id_, + ls_id_, + arb_member_, + timeout_us_); + +int ObRemoveArbArg::assign( + const ObRemoveArbArg &that) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(arb_member_.assign(that.arb_member_))) { + SHARE_LOG(WARN, "fail to assign arb member", K(ret)); + } else { + tenant_id_ = that.tenant_id_; + ls_id_ = that.ls_id_; + timeout_us_ = that.timeout_us_; + } + return ret; +} + +int ObRemoveArbArg::init( + const uint64_t tenant_id, + const share::ObLSID &ls_id, + const common::ObMember &arb_member, + const int64_t timeout_us) +{ + int ret = OB_SUCCESS; + if (tenant_id == OB_INVALID_ID + || !ls_id.is_valid() + || !ls_id.is_valid_with_tenant(tenant_id) + || !arb_member.is_valid() + || timeout_us == OB_INVALID_TIMESTAMP) { + ret = OB_INVALID_ARGUMENT; + SHARE_LOG(WARN, "failed to init ObRemoveArbArg", K(ret), K(tenant_id), + K(ls_id), K(arb_member), K(timeout_us)); + } else if (OB_FAIL(arb_member_.assign(arb_member))) { + SHARE_LOG(WARN, "fail to assign arb member", K(ret)); + } else { + tenant_id_ = tenant_id; + ls_id_ = ls_id; + timeout_us_ = timeout_us; + } + return ret; +} + +OB_SERIALIZE_MEMBER(ObForceClearArbClusterInfoArg, cluster_id_); +#endif //----End structs for partition online/offline---- @@ -4573,6 +4664,87 @@ OB_SERIALIZE_MEMBER(ObAutoincSyncArg, OB_SERIALIZE_MEMBER(ObAdminChangeReplicaArg, force_cmd_); +#ifdef OB_BUILD_ARBITRATION +int ObAdminAddArbitrationServiceArg::init(const ObString &arbitration_service) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(arbitration_service.length() > OB_MAX_ARBITRATION_SERVICE_LENGTH)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(arbitration_service)); + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "arbitration service, length oversize"); + } else if (OB_FAIL(arbitration_service_.assign(arbitration_service))) { + LOG_WARN("fali to assign arbitration service", KR(ret), K(arbitration_service)); + } + return ret; +} + +int ObAdminRemoveArbitrationServiceArg::init(const ObString &arbitration_service) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(arbitration_service.length() > OB_MAX_ARBITRATION_SERVICE_LENGTH)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(arbitration_service)); + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "arbitration service, length oversize"); + } else if (OB_FAIL(arbitration_service_.assign(arbitration_service))) { + LOG_WARN("fali to assign arbitration service", KR(ret), K(arbitration_service)); + } + return ret; +} + +int ObAdminReplaceArbitrationServiceArg::init( + const ObString &arbitration_service, + const ObString &previous_arbitration_service) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(arbitration_service.length() > OB_MAX_ARBITRATION_SERVICE_LENGTH)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(arbitration_service)); + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "current arbitration service, length oversize"); + } else if (OB_UNLIKELY(previous_arbitration_service.length() > OB_MAX_ARBITRATION_SERVICE_LENGTH)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(previous_arbitration_service)); + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "new arbitration service, length oversize"); + } else if (OB_FAIL(arbitration_service_.assign(arbitration_service))) { + LOG_WARN("fail to assign arbitration_service", KR(ret), K(arbitration_service)); + } else if (OB_FAIL(previous_arbitration_service_.assign(previous_arbitration_service))) { + LOG_WARN("fail to assign previous arbitration service", KR(ret), K(previous_arbitration_service)); + } + return ret; +} + +int ObAdminReplaceArbitrationServiceArg::assign(const ObAdminReplaceArbitrationServiceArg &other) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(arbitration_service_.assign(other.arbitration_service_))) { + LOG_WARN("fail to assign arbitration service", KR(ret), K(other)); + } else if (OB_FAIL(previous_arbitration_service_.assign(other.previous_arbitration_service_))) { + LOG_WARN("fail to assign previous arbitration service", KR(ret), K(other)); + } + return ret; +} + +int ObRemoveClusterInfoFromArbServerArg::init( + const ObString &arbitration_service) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(arbitration_service.length() > OB_MAX_ARBITRATION_SERVICE_LENGTH)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(arbitration_service)); + } else if (OB_FAIL(arbitration_service_.assign(arbitration_service))) { + LOG_WARN("fail to assign arbitration_service", KR(ret), K(arbitration_service)); + } + return ret; +} + +int ObRemoveClusterInfoFromArbServerArg::assign(const ObRemoveClusterInfoFromArbServerArg &other) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(arbitration_service_.assign(other.arbitration_service_))) { + LOG_WARN("fail to assign arbitration service", KR(ret), K(other)); + } + return ret; +} +#endif bool ObUpdateIndexStatusArg::is_allow_when_disable_ddl() const { @@ -4610,6 +4782,15 @@ OB_SERIALIZE_MEMBER(ObDumpTxDataMemtableArg, tenant_id_, ls_id_); OB_SERIALIZE_MEMBER(ObDumpSingleTxDataArg, tenant_id_, ls_id_, tx_id_); +#ifdef OB_BUILD_ARBITRATION +OB_SERIALIZE_MEMBER(ObAdminAddArbitrationServiceArg, arbitration_service_); + +OB_SERIALIZE_MEMBER(ObAdminRemoveArbitrationServiceArg, arbitration_service_); + +OB_SERIALIZE_MEMBER(ObAdminReplaceArbitrationServiceArg, arbitration_service_, previous_arbitration_service_); + +OB_SERIALIZE_MEMBER(ObRemoveClusterInfoFromArbServerArg, arbitration_service_); +#endif int ObRootMajorFreezeArg::assign(const ObRootMajorFreezeArg &other) { @@ -6061,6 +6242,26 @@ int ObCheckServerForAddingServerResult::assign(const ObCheckServerForAddingServe return ret; } OB_SERIALIZE_MEMBER(ObCheckDeploymentModeArg, single_zone_deployment_on_); +#ifdef OB_BUILD_TDE_SECURITY +OB_SERIALIZE_MEMBER(ObWaitMasterKeyInSyncArg, + tenant_max_key_version_, + tenant_config_version_, + rs_list_arg_); + +int ObWaitMasterKeyInSyncArg::assign( + const ObWaitMasterKeyInSyncArg &that) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(tenant_max_key_version_.assign(that.tenant_max_key_version_))) { + LOG_WARN("fail to assign this member", KR(ret)); + } else if (OB_FAIL(tenant_config_version_.assign(that.tenant_config_version_))) { + LOG_WARN("fail to assign this member", KR(ret)); + } else if (OB_FAIL(rs_list_arg_.assign(that.rs_list_arg_))) { + LOG_WARN("fail to assign this member", KR(ret)); + } + return ret; +} +#endif OB_SERIALIZE_MEMBER(ObArchiveLogArg, enable_, tenant_id_, archive_tenant_ids_); int ObArchiveLogArg::assign(const ObArchiveLogArg &other) @@ -6599,6 +6800,110 @@ int ObFetchLocationArg::assign(const ObFetchLocationArg &other) } return ret; } +#ifdef OB_BUILD_TDE_SECURITY +OB_SERIALIZE_MEMBER(ObGetMasterKeyResultArg, str_); + +OB_SERIALIZE_MEMBER(ObRestoreKeyArg, tenant_id_, backup_dest_, encrypt_key_); +int ObRestoreKeyArg::assign(const ObRestoreKeyArg &other) +{ + int ret = OB_SUCCESS; + tenant_id_ = other.tenant_id_; + backup_dest_ = other.backup_dest_; + encrypt_key_ = other.encrypt_key_; + return ret; +} + +OB_SERIALIZE_MEMBER(ObRestoreKeyResult, ret_); +int ObRestoreKeyResult::assign(const ObRestoreKeyResult &other) +{ + int ret = OB_SUCCESS; + ret_ = other.ret_; + return ret; +} + +OB_SERIALIZE_MEMBER(ObRootKeyArg, tenant_id_, is_set_, key_type_, root_key_); + +int ObRootKeyArg::init_for_get(const uint64_t tenant_id) +{ + int ret = OB_SUCCESS; + if (!is_user_tenant(tenant_id)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id)); + } else { + tenant_id_ = tenant_id; + is_set_ = false; + } + return ret; +} + +int ObRootKeyArg::init(const uint64_t tenant_id, RootKeyType key_type, + ObString &root_key) +{ + int ret = OB_SUCCESS; + if (!is_user_tenant(tenant_id) + || RootKeyType::INVALID == key_type) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(key_type), K(root_key)); + } else { + tenant_id_ = tenant_id; + is_set_ = true; + key_type_ = key_type; + root_key_ = root_key; + } + return ret; +} + +int ObRootKeyArg::assign(const ObRootKeyArg &other) +{ + int ret = OB_SUCCESS; + tenant_id_ = other.tenant_id_; + is_set_ = other.is_set_; + key_type_ = other.key_type_; + root_key_ = other.root_key_; + return ret; +} + +OB_DEF_SERIALIZE(ObRootKeyResult) +{ + int ret = OB_SUCCESS; + LST_DO_CODE(OB_UNIS_ENCODE, + key_type_, + root_key_); + return ret; +} + +OB_DEF_DESERIALIZE(ObRootKeyResult) +{ + int ret = OB_SUCCESS; + ObString tmp_str; + LST_DO_CODE(OB_UNIS_DECODE, + key_type_, + tmp_str); + if (OB_FAIL(ob_write_string(allocator_, tmp_str, root_key_))) { + LOG_WARN("failed to copy string", K(ret)); + } + return ret; +} + +OB_DEF_SERIALIZE_SIZE(ObRootKeyResult) +{ + int len = 0; + LST_DO_CODE(OB_UNIS_ADD_LEN, + key_type_, + root_key_); + return len; +} + +int ObRootKeyResult::assign(const ObRootKeyResult &other) +{ + int ret = OB_SUCCESS; + key_type_ = other.key_type_; + if (OB_FAIL(ob_write_string(allocator_, other.root_key_, root_key_))) { + LOG_WARN("failed to write string", K(ret)); + } + return ret; +} +#endif OB_SERIALIZE_MEMBER(ObTrxToolArg, trans_id_, status_, trans_version_, end_log_ts_, cmd_); OB_SERIALIZE_MEMBER(ObTrxToolRes, trans_info_); @@ -6755,6 +7060,22 @@ int ObDropDirectoryArg::assign(const ObDropDirectoryArg &other) OB_SERIALIZE_MEMBER((ObDropDirectoryArg, ObDDLArg), tenant_id_, directory_name_); +#ifdef OB_BUILD_TDE_SECURITY +int ObDumpCacheMasterKeyResultArg::ObMasterKeyVersionPair::assign( + const ObDumpCacheMasterKeyResultArg::ObMasterKeyVersionPair &other) +{ + int ret = OB_SUCCESS; + tenant_id_ = other.tenant_id_; + master_key_version_ = other.master_key_version_; + master_key_.assign_buffer(buf_, share::OB_CLOG_ENCRYPT_MASTER_KEY_LEN); + master_key_.write(other.master_key_.ptr(), other.master_key_.length()); + return ret; +} + +OB_SERIALIZE_MEMBER(ObDumpCacheMasterKeyResultArg, master_key_version_array_); +OB_SERIALIZE_MEMBER(ObDumpCacheMasterKeyResultArg::ObMasterKeyVersionPair, + tenant_id_, master_key_version_, master_key_); +#endif int ObCreateDupLSArg::assign(const ObCreateDupLSArg &arg) { @@ -6873,6 +7194,113 @@ DEF_TO_STRING(ObCreateLSArg) OB_SERIALIZE_MEMBER(ObCreateLSArg, tenant_id_, id_, replica_type_, replica_property_, tenant_info_, create_scn_, compat_mode_, create_ls_type_, palf_base_info_); +#ifdef OB_BUILD_ARBITRATION +bool ObCreateArbArg::is_valid() const +{ + return OB_INVALID_TENANT_ID != tenant_id_ + && ls_id_.is_valid() + && tenant_role_.is_valid() + && ls_id_.is_valid_with_tenant(tenant_id_); +} + +void ObCreateArbArg::reset() +{ + tenant_id_ = OB_INVALID_TENANT_ID; + ls_id_.reset(); + tenant_role_.reset(); +} + +int ObCreateArbArg::assign(const ObCreateArbArg &arg) +{ + int ret = OB_SUCCESS; + if (this == &arg) { + } else { + tenant_id_ = arg.tenant_id_; + ls_id_ = arg.ls_id_; + tenant_role_ = arg.tenant_role_; + } + return ret; +} + +int ObCreateArbArg::init( + const int64_t tenant_id, + const share::ObLSID &ls_id, + const share::ObTenantRole &tenant_role) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id + || !ls_id.is_valid() + || !ls_id.is_valid_with_tenant(tenant_id) + || !tenant_role.is_valid())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id), K(tenant_role)); + } else { + tenant_id_ = tenant_id; + ls_id_ = ls_id; + tenant_role_ = tenant_role; + } + return ret; +} + +DEF_TO_STRING(ObCreateArbArg) +{ + int64_t pos = 0; + J_KV(K_(tenant_id), K_(ls_id), K_(tenant_role)); + return pos; +} + +OB_SERIALIZE_MEMBER(ObCreateArbArg, tenant_id_, ls_id_, tenant_role_); + +bool ObDeleteArbArg::is_valid() const +{ + return OB_INVALID_TENANT_ID != tenant_id_ + && ls_id_.is_valid() + && ls_id_.is_valid_with_tenant(tenant_id_); +} + +void ObDeleteArbArg::reset() +{ + tenant_id_ = OB_INVALID_TENANT_ID; + ls_id_.reset(); +} + +int ObDeleteArbArg::assign(const ObDeleteArbArg &arg) +{ + int ret = OB_SUCCESS; + if (this == &arg) { + } else { + tenant_id_ = arg.tenant_id_; + ls_id_ = arg.ls_id_; + } + return ret; +} + +int ObDeleteArbArg::init( + const int64_t tenant_id, + const share::ObLSID &ls_id) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id + || !ls_id.is_valid() + || !ls_id.is_valid_with_tenant(tenant_id))) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id)); + } else { + tenant_id_ = tenant_id; + ls_id_ = ls_id; + } + return ret; +} + +DEF_TO_STRING(ObDeleteArbArg) +{ + int64_t pos = 0; + J_KV(K_(tenant_id), K_(ls_id)); + return pos; +} + +OB_SERIALIZE_MEMBER(ObDeleteArbArg, tenant_id_, ls_id_); +#endif bool ObSetMemberListArgV2::is_valid() const { @@ -7440,6 +7868,71 @@ int ObCreateLSResult::assign(const ObCreateLSResult &other) return ret; } +#ifdef OB_BUILD_ARBITRATION +OB_SERIALIZE_MEMBER(ObCreateArbResult, ret_); +bool ObCreateArbResult::is_valid() const +{ + return true; +} + +int ObCreateArbResult::assign(const ObCreateArbResult &other) +{ + int ret = OB_SUCCESS; + if (this == &other) { + } else { + ret_ = other.ret_; + } + return ret; +} + +OB_SERIALIZE_MEMBER(ObDeleteArbResult, ret_); +bool ObDeleteArbResult::is_valid() const +{ + return true; +} + +int ObDeleteArbResult::assign(const ObDeleteArbResult &other) +{ + int ret = OB_SUCCESS; + if (this == &other) { + } else { + ret_ = other.ret_; + } + return ret; +} + +OB_SERIALIZE_MEMBER(ObAddArbResult, ret_); +bool ObAddArbResult::is_valid() const +{ + return true; +} + +int ObAddArbResult::assign(const ObAddArbResult &other) +{ + int ret = OB_SUCCESS; + if (this == &other) { + } else { + ret_ = other.ret_; + } + return ret; +} + +OB_SERIALIZE_MEMBER(ObRemoveArbResult, ret_); +bool ObRemoveArbResult::is_valid() const +{ + return true; +} + +int ObRemoveArbResult::assign(const ObRemoveArbResult &other) +{ + int ret = OB_SUCCESS; + if (this == &other) { + } else { + ret_ = other.ret_; + } + return ret; +} +#endif OB_SERIALIZE_MEMBER(ObSetMemberListResult, ret_); bool ObSetMemberListResult::is_valid() const { @@ -8311,6 +8804,98 @@ int ObFetchLocationResult::set_servers( return servers_.assign(servers); } +#ifdef OB_BUILD_ARBITRATION +OB_SERIALIZE_MEMBER(ObArbGCNotifyArg, epoch_, ls_ids_); + +int ObArbGCNotifyArg::assign(const ObArbGCNotifyArg &other) +{ + int ret = OB_SUCCESS; + if (OB_FAIL(ls_ids_.assign(other.ls_ids_))) { + LOG_WARN("assign ls_ids_ failed", K(ret), KPC(this), K(other)); + } else { + epoch_ = other.epoch_; + } + return ret; +} + +int ObArbGCNotifyArg::init(const arbserver::GCMsgEpoch &epoch, + const arbserver::TenantLSIDSArray &ls_ids) +{ + int ret = OB_SUCCESS; + if (!epoch.is_valid() || !ls_ids.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(ret), KPC(this), K(epoch), K(ls_ids)); + } else if (OB_FAIL(ls_ids_.assign(ls_ids))) { + LOG_WARN("failed to assign ls_ids to ls_ids_", K(ret), KPC(this), K(ls_ids)); + } else { + epoch_ = epoch; + LOG_INFO("init ObArbGCNotifyArg success", KPC(this)); + } + return ret; +} + +bool ObArbGCNotifyArg::is_valid() const +{ + return epoch_.is_valid() && ls_ids_.is_valid(); +} + +const arbserver::GCMsgEpoch &ObArbGCNotifyArg::get_epoch() const +{ + return epoch_; +} + +arbserver::TenantLSIDSArray &ObArbGCNotifyArg::get_ls_ids() +{ + return ls_ids_; +} + +OB_SERIALIZE_MEMBER(ObArbGCNotifyResult, ret_); + +void ObArbClusterOpArg::reset() +{ + type_ = MSG_TYPE::INVALID_MSG; + cluster_id_ = OB_INVALID_CLUSTER_ID; + cluster_name_.reset(); + epoch_.reset(); +} + +int ObArbClusterOpArg::assign(const ObArbClusterOpArg &other) +{ + type_ = other.type_; + cluster_id_ = other.cluster_id_; + cluster_name_ = other.cluster_name_; + epoch_ = other.epoch_; + return OB_SUCCESS; +} + +int ObArbClusterOpArg::init(const int64_t cluster_id, + const ObString &cluster_name, + const arbserver::GCMsgEpoch &epoch, + const bool is_add_arb) +{ + int ret = OB_SUCCESS; + if (OB_INVALID_CLUSTER_ID == cluster_id || !epoch.is_valid()) { + ret = OB_INVALID_ARGUMENT; + } else { + type_ = (is_add_arb)? MSG_TYPE::CLUSTER_ADD_ARB: MSG_TYPE::CLUSTER_REMOVE_ARB; + cluster_id_ = cluster_id; + cluster_name_ = cluster_name; + epoch_ = epoch; + } + return ret; +} + +bool ObArbClusterOpArg::is_valid() const +{ + return type_ != MSG_TYPE::INVALID_MSG && + cluster_id_ != OB_INVALID_CLUSTER_ID && + epoch_.is_valid(); +} + +OB_SERIALIZE_MEMBER(ObArbClusterOpArg, type_, cluster_id_, cluster_name_, epoch_); + +OB_SERIALIZE_MEMBER(ObArbClusterOpResult, ret_); +#endif OB_SERIALIZE_MEMBER(ObSyncRewriteRuleArg, tenant_id_); diff --git a/src/share/ob_rpc_struct.h b/src/share/ob_rpc_struct.h index 4bf889437..f293d343e 100755 --- a/src/share/ob_rpc_struct.h +++ b/src/share/ob_rpc_struct.h @@ -63,6 +63,9 @@ #include "logservice/palf/palf_options.h"//access mode #include "logservice/palf/palf_base_info.h"//PalfBaseInfo #include "logservice/palf/log_define.h"//INVALID_PROPOSAL_ID +#ifdef OB_BUILD_ARBITRATION +#include "logservice/arbserver/ob_arb_gc_utils.h" // TenantLSIDS +#endif #include "share/location_cache/ob_vtable_location_service.h" // share::ObVtableLocationType #include "share/ob_balance_define.h" // share::ObTransferTaskID #include "share/config/ob_config.h" // ObConfigArray @@ -2914,6 +2917,183 @@ private: common::ObReplicaType replica_type_; }; +#ifdef OB_BUILD_ARBITRATION +// send to arbitration service to create A-replica +struct ObCreateArbArg +{ + OB_UNIS_VERSION(1); +public: + ObCreateArbArg() : tenant_id_(OB_INVALID_TENANT_ID), + ls_id_(), + tenant_role_() {} + ~ObCreateArbArg() {} + bool is_valid() const; + void reset(); + int assign(const ObCreateArbArg &arg); + int init(const int64_t tenant_id, + const share::ObLSID &id, + const share::ObTenantRole &tenant_role); + int64_t get_tenant_id() const + { + return tenant_id_; + } + const share::ObTenantRole &get_tenant_role() const { return tenant_role_; } + share::ObLSID get_ls_id() const + { + return ls_id_; + } + DECLARE_TO_STRING; +private: + uint64_t tenant_id_; + share::ObLSID ls_id_; + share::ObTenantRole tenant_role_; +private: + DISALLOW_COPY_AND_ASSIGN(ObCreateArbArg); +}; + +struct ObCreateArbResult +{ + OB_UNIS_VERSION(1); +public: + ObCreateArbResult(): ret_(common::OB_SUCCESS) {} + ~ObCreateArbResult() {} + bool is_valid() const; + int assign(const ObCreateArbResult &other); + TO_STRING_KV(K_(ret)); + void set_result(const int ret) + { + ret_ = ret; + } + int get_result() const + { + return ret_; + } +private: + DISALLOW_COPY_AND_ASSIGN(ObCreateArbResult); +private: + int ret_; +}; + +// send to arbitration service to delete A-replica +struct ObDeleteArbArg +{ + OB_UNIS_VERSION(1); +public: + ObDeleteArbArg() : tenant_id_(OB_INVALID_TENANT_ID), + ls_id_() {} + ~ObDeleteArbArg() {} + bool is_valid() const; + void reset(); + int assign(const ObDeleteArbArg &arg); + int init(const int64_t tenant_id, + const share::ObLSID &id); + int64_t get_tenant_id() const + { + return tenant_id_; + } + share::ObLSID get_ls_id() const + { + return ls_id_; + } + DECLARE_TO_STRING; +private: + uint64_t tenant_id_; + share::ObLSID ls_id_; +private: + DISALLOW_COPY_AND_ASSIGN(ObDeleteArbArg); +}; + +struct ObDeleteArbResult +{ + OB_UNIS_VERSION(1); +public: + ObDeleteArbResult(): ret_(common::OB_SUCCESS) {} + ~ObDeleteArbResult() {} + bool is_valid() const; + int assign(const ObDeleteArbResult &other); + TO_STRING_KV(K_(ret)); + void set_result(const int ret) + { + ret_ = ret; + } + int get_result() const + { + return ret_; + } +private: + DISALLOW_COPY_AND_ASSIGN(ObDeleteArbResult); +private: + int ret_; +}; + +struct ObAddArbResult +{ + OB_UNIS_VERSION(1); +public: + ObAddArbResult(): ret_(common::OB_SUCCESS) {} + ~ObAddArbResult() {} + bool is_valid() const; + int assign(const ObAddArbResult &other); + TO_STRING_KV(K_(ret)); + void set_result(const int ret) + { + ret_ = ret; + } + int get_result() const + { + return ret_; + } +private: + DISALLOW_COPY_AND_ASSIGN(ObAddArbResult); +private: + int ret_; +}; + +struct ObRemoveArbResult +{ + OB_UNIS_VERSION(1); +public: + ObRemoveArbResult(): ret_(common::OB_SUCCESS) {} + ~ObRemoveArbResult() {} + bool is_valid() const; + int assign(const ObRemoveArbResult &other); + TO_STRING_KV(K_(ret)); + void set_result(const int ret) + { + ret_ = ret; + } + int get_result() const + { + return ret_; + } +private: + DISALLOW_COPY_AND_ASSIGN(ObRemoveArbResult); +private: + int ret_; +}; + +struct ObForceClearArbClusterInfoArg +{ + OB_UNIS_VERSION(1); +public: + ObForceClearArbClusterInfoArg() + : cluster_id_(OB_INVALID_ID) { } + ObForceClearArbClusterInfoArg(const int64_t cluster_id) + : cluster_id_(cluster_id) { } + bool is_valid() const { return is_valid_cluster_id(cluster_id_); } + int assign(const ObForceClearArbClusterInfoArg &other) + { + cluster_id_ = other.cluster_id_; + return OB_SUCCESS; + } + int64_t get_cluster_id() const { return cluster_id_; } + TO_STRING_KV(K_(cluster_id)); +private: + DISALLOW_COPY_AND_ASSIGN(ObForceClearArbClusterInfoArg); +private: + int64_t cluster_id_; +}; +#endif struct ObSetMemberListArgV2 { @@ -3797,6 +3977,79 @@ public: share::ObLSID ls_id_; }; +#ifdef OB_BUILD_ARBITRATION +// send to leader to add A-replica for log stream +struct ObAddArbArg +{ +public: + OB_UNIS_VERSION(1); +public: + ObAddArbArg() + : tenant_id_(OB_INVALID_ID), + ls_id_(), + arb_member_(), + timeout_us_(OB_INVALID_TIMESTAMP) {} +public: + int assign(const ObAddArbArg &that); + int init( + const uint64_t tenant_id, + const share::ObLSID &ls_id, + const common::ObMember &arb_member, + const int64_t timeout_us); + TO_STRING_KV(K_(tenant_id), + K_(ls_id), + K_(arb_member), + K_(timeout_us)); + bool is_valid() const { + return common::OB_INVALID_ID != tenant_id_ + && ls_id_.is_valid() + && ls_id_.is_valid_with_tenant(tenant_id_) + && arb_member_.is_valid() + && timeout_us_ != OB_INVALID_TIMESTAMP; + } +public: + uint64_t tenant_id_; + share::ObLSID ls_id_; + common::ObMember arb_member_; + int64_t timeout_us_; +}; + +// send to leader to remove A-replica for log stream +struct ObRemoveArbArg +{ +public: + OB_UNIS_VERSION(1); +public: + ObRemoveArbArg() + : tenant_id_(OB_INVALID_ID), + ls_id_(), + arb_member_(), + timeout_us_(OB_INVALID_TIMESTAMP) {} +public: + int assign(const ObRemoveArbArg &that); + int init( + const uint64_t tenant_id, + const share::ObLSID &ls_id, + const common::ObMember &arb_member, + const int64_t timeout_us); + TO_STRING_KV(K_(tenant_id), + K_(ls_id), + K_(arb_member), + K_(timeout_us)); + bool is_valid() const { + return common::OB_INVALID_ID != tenant_id_ + && ls_id_.is_valid() + && ls_id_.is_valid_with_tenant(tenant_id_) + && arb_member_.is_valid() + && timeout_us_ != OB_INVALID_TIMESTAMP; + } +public: + uint64_t tenant_id_; + share::ObLSID ls_id_; + common::ObMember arb_member_; + int64_t timeout_us_; +}; +#endif struct ObBackupTaskRes { @@ -5189,6 +5442,78 @@ public: common::ObAddr destination_; }; +#ifdef OB_BUILD_ARBITRATION +struct ObAdminAddArbitrationServiceArg +{ + OB_UNIS_VERSION(1); +public: + ObAdminAddArbitrationServiceArg(): + arbitration_service_() + {} + ~ObAdminAddArbitrationServiceArg() {} + int init(const ObString &arbitration_service); + int assign(const ObAdminAddArbitrationServiceArg &other) { return arbitration_service_.assign(other.arbitration_service_); } + bool is_valid() const { return !arbitration_service_.is_empty(); } + const ObString get_arbitration_service() const { return arbitration_service_.str(); } + TO_STRING_KV(K(arbitration_service_)); +private: + common::ObFixedLengthString arbitration_service_; +}; + +struct ObAdminRemoveArbitrationServiceArg +{ + OB_UNIS_VERSION(1); +public: + ObAdminRemoveArbitrationServiceArg(): + arbitration_service_() + {} + ~ObAdminRemoveArbitrationServiceArg() {} + int init(const ObString &arbitration_service); + int assign(const ObAdminRemoveArbitrationServiceArg &other) { return arbitration_service_.assign(other.arbitration_service_); } + const ObString get_arbitration_service() const { return arbitration_service_.str(); } + bool is_valid() const { return !arbitration_service_.is_empty(); } + TO_STRING_KV(K(arbitration_service_)); +private: + common::ObFixedLengthString arbitration_service_; +}; + +struct ObAdminReplaceArbitrationServiceArg +{ + OB_UNIS_VERSION(1); +public: + ObAdminReplaceArbitrationServiceArg(): + arbitration_service_(), + previous_arbitration_service_() + {} + ~ObAdminReplaceArbitrationServiceArg() {} + int init(const ObString &arbitration_service, const ObString &previous_arbitration_service); + int assign(const ObAdminReplaceArbitrationServiceArg &other); + bool is_valid() const { return !arbitration_service_.is_empty() && !previous_arbitration_service_.is_empty(); } + const ObString get_arbitration_service() const { return arbitration_service_.str(); } + const ObString get_previous_arbitration_service() const { return previous_arbitration_service_.str(); } + TO_STRING_KV(K(arbitration_service_), K(previous_arbitration_service_)); +private: + common::ObFixedLengthString arbitration_service_; + common::ObFixedLengthString previous_arbitration_service_; +}; + +struct ObRemoveClusterInfoFromArbServerArg +{ + OB_UNIS_VERSION(1); +public: + ObRemoveClusterInfoFromArbServerArg(): + arbitration_service_() + {} + ~ObRemoveClusterInfoFromArbServerArg() {} + int init(const ObString &arbitration_service); + int assign(const ObRemoveClusterInfoFromArbServerArg &other); + bool is_valid() const { return !arbitration_service_.is_empty(); } + const ObString get_arbitration_service() const { return arbitration_service_.str(); } + TO_STRING_KV(K(arbitration_service_)); +private: + common::ObFixedLengthString arbitration_service_; +}; +#endif struct ObCheckpointSlogArg { @@ -7933,6 +8258,20 @@ public: ObAddr master_rs_; }; +#ifdef OB_BUILD_TDE_SECURITY +struct ObWaitMasterKeyInSyncArg +{ + OB_UNIS_VERSION(1); +public: + ObWaitMasterKeyInSyncArg() : tenant_max_key_version_(), tenant_config_version_(), rs_list_arg_() {} + TO_STRING_KV(K_(rs_list_arg)); + void reset() { tenant_max_key_version_.reset(); tenant_config_version_.reset(); rs_list_arg_.reset(); } + int assign(const ObWaitMasterKeyInSyncArg &that); + common::ObSEArray, 10> tenant_max_key_version_; + common::ObSEArray, 10> tenant_config_version_; + ObRsListArg rs_list_arg_; +}; +#endif struct ObCheckDeploymentModeArg { @@ -7985,6 +8324,99 @@ private: share::ObVtableLocationType vtable_type_; }; +#ifdef OB_BUILD_TDE_SECURITY +struct ObGetMasterKeyResultArg +{ + OB_UNIS_VERSION(1); +public: + ObGetMasterKeyResultArg() : str_(share::OB_CLOG_ENCRYPT_MASTER_KEY_LEN, 0, buf_) {} + ~ObGetMasterKeyResultArg() {} + bool is_valid() const { return true; } + TO_STRING_KV(K_(str)); + char buf_[share::OB_CLOG_ENCRYPT_MASTER_KEY_LEN]; + common::ObString str_; +}; + +struct ObRestoreKeyArg +{ + OB_UNIS_VERSION(1); +public: + ObRestoreKeyArg() + : tenant_id_(OB_INVALID_ID), + backup_dest_(), + encrypt_key_() + {} + ~ObRestoreKeyArg() {} + bool is_valid() const { return OB_INVALID_ID != tenant_id_ && !backup_dest_.empty(); } + int assign(const ObRestoreKeyArg &other); + TO_STRING_KV(K_(tenant_id), K_(backup_dest), K_(encrypt_key)); + uint64_t tenant_id_; + common::ObString backup_dest_; + common::ObString encrypt_key_; +}; + +struct ObRestoreKeyResult +{ + OB_UNIS_VERSION(1); +public: + ObRestoreKeyResult() : ret_(common::OB_ERROR) {} + ~ObRestoreKeyResult() {} + int assign(const ObRestoreKeyResult &other); + void set_ret(int ret) { ret_ = ret; } + int64_t get_ret() const { return ret_; } + TO_STRING_KV(K_(ret)); +private: + int ret_; +}; + +enum RootKeyType +{ + INVALID = 0, + DEFAULT, + NORMAL +}; + +struct ObRootKeyArg +{ + OB_UNIS_VERSION(1); +public: + ObRootKeyArg() + : tenant_id_(OB_INVALID_ID), + is_set_(false), + key_type_(RootKeyType::INVALID), + root_key_() + {} + ~ObRootKeyArg() {} + bool is_valid() const { return OB_INVALID_ID != tenant_id_; } + int assign(const ObRootKeyArg &other); + int init_for_get(const uint64_t tenant_id); + int init(const uint64_t tenant_id, RootKeyType key_type, + ObString &root_key); + TO_STRING_KV(K_(tenant_id), K_(is_set), K_(key_type), K_(root_key)); + uint64_t tenant_id_; + bool is_set_; + enum RootKeyType key_type_; + common::ObString root_key_; +}; + +struct ObRootKeyResult +{ + OB_UNIS_VERSION(1); +public: + ObRootKeyResult() + : key_type_(RootKeyType::INVALID), + root_key_(), + allocator_() + {} + ~ObRootKeyResult() {} + int assign(const ObRootKeyResult &other); + TO_STRING_KV(K_(key_type), K_(root_key)); + enum RootKeyType key_type_; + common::ObString root_key_; +private: + common::ObArenaAllocator allocator_; +}; +#endif enum TransToolCmd { @@ -8331,6 +8763,32 @@ public: int progress_; }; +#ifdef OB_BUILD_TDE_SECURITY +struct ObDumpCacheMasterKeyResultArg +{ + OB_UNIS_VERSION(1); +public: + struct ObMasterKeyVersionPair + { + OB_UNIS_VERSION(1); + public: + ObMasterKeyVersionPair() : tenant_id_(-1), master_key_version_(-1), + master_key_(share::OB_CLOG_ENCRYPT_MASTER_KEY_LEN, 0, buf_) {} + ~ObMasterKeyVersionPair() {} + int assign(const ObMasterKeyVersionPair &other); + TO_STRING_KV(K_(master_key_version), K_(master_key)); + int64_t tenant_id_; + int64_t master_key_version_; + char buf_[share::OB_CLOG_ENCRYPT_MASTER_KEY_LEN]; + common::ObString master_key_; + }; + ObDumpCacheMasterKeyResultArg() {} + ~ObDumpCacheMasterKeyResultArg() {} + bool is_valid() const { return true; } + TO_STRING_KV(K_(master_key_version_array)); + common::ObSEArray master_key_version_array_; +}; +#endif struct ObDetectMasterRsArg { @@ -9037,6 +9495,126 @@ private: DISALLOW_COPY_AND_ASSIGN(ObFinishTransferTaskArg); }; +#ifdef OB_BUILD_ARBITRATION +struct ObArbGCNotifyArg +{ +public: + OB_UNIS_VERSION(1); +public: + static const int64_t DEFAULT_NUM = 16; +public: + ObArbGCNotifyArg() {} + ~ObArbGCNotifyArg() {} + int assign(const ObArbGCNotifyArg &other); + int init(const arbserver::GCMsgEpoch &epoch, + const arbserver::TenantLSIDSArray &ls_ids); + bool is_valid() const; + const arbserver::GCMsgEpoch &get_epoch() const; + arbserver::TenantLSIDSArray &get_ls_ids(); + TO_STRING_KV(K_(epoch), K_(ls_ids)); +private: + DISALLOW_COPY_AND_ASSIGN(ObArbGCNotifyArg); + arbserver::GCMsgEpoch epoch_; + arbserver::TenantLSIDSArray ls_ids_; +}; + +struct ObArbGCNotifyResult +{ +public: + OB_UNIS_VERSION(1); +public: + ObArbGCNotifyResult(): ret_(common::OB_SUCCESS) {} + ~ObArbGCNotifyResult() {} + bool is_valid() const + { + return true; + } + int assign(const ObArbGCNotifyResult &other) + { + ret_ = other.ret_; + return common::OB_SUCCESS; + } + void set_result(const int ret) + { + ret_ = ret; + } + int get_result() const + { + return ret_; + } + TO_STRING_KV(K_(ret)); +private: + DISALLOW_COPY_AND_ASSIGN(ObArbGCNotifyResult); +private: + int ret_; +}; + +struct ObArbClusterOpArg +{ +public: + OB_UNIS_VERSION(1); +public: + ObArbClusterOpArg() + : type_(MSG_TYPE::INVALID_MSG), + cluster_id_(OB_INVALID_CLUSTER_ID), + cluster_name_(), + epoch_() {} + ~ObArbClusterOpArg() { reset(); } + void reset(); + int assign(const ObArbClusterOpArg &other); + int init(const int64_t cluster_id, + const ObString &cluster_name, + const arbserver::GCMsgEpoch &epoch, + const bool is_add_arb); + bool is_valid() const; + int64_t get_cluster_id() const { return cluster_id_; } + const ObString &get_cluster_name() const { return cluster_name_; } + const arbserver::GCMsgEpoch &get_epoch() const { return epoch_; } + bool is_cluster_add_arb_req() const { return type_ == MSG_TYPE::CLUSTER_ADD_ARB; } + TO_STRING_KV(K_(type), K_(cluster_id), K_(cluster_name), K_(epoch)); +private: + enum MSG_TYPE + { + INVALID_MSG = 0, + CLUSTER_ADD_ARB, + CLUSTER_REMOVE_ARB, + }; + DISALLOW_COPY_AND_ASSIGN(ObArbClusterOpArg); +private: + MSG_TYPE type_; + int64_t cluster_id_; + common::ObString cluster_name_; + arbserver::GCMsgEpoch epoch_; +}; + +struct ObArbClusterOpResult +{ +public: + OB_UNIS_VERSION(1); +public: + ObArbClusterOpResult(): ret_(common::OB_SUCCESS) {} + ~ObArbClusterOpResult() {} + bool is_valid() const { return true; } + int assign(const ObArbClusterOpResult &other) + { + ret_ = other.ret_; + return common::OB_SUCCESS; + } + void set_result(const int ret) + { + ret_ = ret; + } + int get_result() const + { + return ret_; + } + TO_STRING_KV(K_(ret)); +private: + DISALLOW_COPY_AND_ASSIGN(ObArbClusterOpResult); +private: + int ret_; +}; +#endif struct ObSyncRewriteRuleArg { diff --git a/src/share/ob_share_util.cpp b/src/share/ob_share_util.cpp index dac964fe0..7e4622b52 100644 --- a/src/share/ob_share_util.cpp +++ b/src/share/ob_share_util.cpp @@ -17,6 +17,9 @@ #include "lib/time/ob_time_utility.h" #include "lib/oblog/ob_log_module.h" #include "share/ob_cluster_version.h" // for GET_MIN_DATA_VERSION +#ifdef OB_BUILD_ARBITRATION +#include "share/arbitration_service/ob_arbitration_service_utils.h" // ObArbitrationServiceUtils +#endif #include "lib/mysqlclient/ob_isql_client.h" #include "observer/omt/ob_tenant_config_mgr.h" // ObTenantConfigGuard @@ -124,6 +127,13 @@ int ObShareUtil::generate_arb_replica_num( || !ls_id.is_valid_with_tenant(tenant_id))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id)); +#ifdef OB_BUILD_ARBITRATION + } else if (OB_FAIL(ObArbitrationServiceUtils::generate_arb_replica_num( + tenant_id, + ls_id, + arb_replica_num))) { + LOG_WARN("fail to generate arb replica number", KR(ret), K(tenant_id), K(ls_id)); +#endif } return ret; } diff --git a/src/share/ob_srv_rpc_proxy.h b/src/share/ob_srv_rpc_proxy.h index 5da9efd7c..f0fe791be 100755 --- a/src/share/ob_srv_rpc_proxy.h +++ b/src/share/ob_srv_rpc_proxy.h @@ -80,6 +80,10 @@ public: RPC_S(PR5 ls_remove_nonpaxos_replica, OB_LS_REMOVE_NONPAXOS_REPLICA, (ObLSDropNonPaxosReplicaArg)); RPC_S(PR5 ls_modify_paxos_replica_number, OB_LS_MODIFY_PAXOS_REPLICA_NUMBER, (ObLSModifyPaxosReplicaNumberArg)); RPC_S(PR5 ls_check_dr_task_exist, OB_LS_CHECK_DR_TASK_EXIST, (ObDRTaskExistArg), obrpc::Bool); +#ifdef OB_BUILD_ARBITRATION + RPC_S(PR5 add_arb, OB_ADD_ARB, (ObAddArbArg), obrpc::ObAddArbResult); + RPC_S(PR5 remove_arb, OB_REMOVE_ARB, (ObRemoveArbArg), obrpc::ObRemoveArbResult); +#endif RPC_S(PR5 checkpoint_slog, OB_CHECKPOINT_SLOG, (ObCheckpointSlogArg)); RPC_AP(PR5 minor_freeze, OB_MINOR_FREEZE, (ObMinorFreezeArg), obrpc::Int64); @@ -99,6 +103,9 @@ public: RPC_S(PR5 is_empty_server, OB_IS_EMPTY_SERVER, (ObCheckServerEmptyArg), Bool); RPC_S(PR5 check_server_for_adding_server, OB_CHECK_SERVER_FOR_ADDING_SERVER, (ObCheckServerForAddingServerArg), ObCheckServerForAddingServerResult); RPC_S(PR5 check_deployment_mode_match, OB_CHECK_DEPLOYMENT_MODE, (ObCheckDeploymentModeArg), Bool); +#ifdef OB_BUILD_TDE_SECURITY + RPC_S(PR5 wait_master_key_in_sync, OB_WAIT_MASTER_KEY_IN_SYNC, (ObWaitMasterKeyInSyncArg)); +#endif RPC_S(PR5 notify_create_tenant_user_ls, OB_NOTIFY_CREATE_TENANT_USER_LS, (obrpc::UInt64)); RPC_S(PR5 report_replica, OB_REPORT_REPLICA); RPC_S(PR5 recycle_replica, OB_RECYCLE_REPLICA); @@ -162,10 +169,25 @@ public: RPC_S(PR5 update_tenant_memory, OB_UPDATE_TENANT_MEMORY, (obrpc::ObTenantMemoryArg)); RPC_S(PR5 renew_in_zone_hb, OB_RENEW_IN_ZONE_HB, (share::ObInZoneHbRequest), share::ObInZoneHbResponse); RPC_S(PR5 pre_process_server_status, OB_PRE_PROCESS_SERVER, (obrpc::ObPreProcessServerArg)); +#ifdef OB_BUILD_TDE_SECURITY + RPC_S(PR5 get_master_key, OB_GET_MASTER_KEY, (obrpc::Int64), ObGetMasterKeyResultArg); + RPC_AP(PR5 restore_key, OB_RESTORE_KEY, (obrpc::ObRestoreKeyArg), obrpc::ObRestoreKeyResult); + RPC_AP(PR5 set_root_key, OB_SET_ROOT_KEY, (obrpc::ObRootKeyArg), obrpc::ObRootKeyResult); +#endif RPC_S(PR5 handle_part_trans_ctx, OB_HANDLE_PART_TRANS_CTX, (obrpc::ObTrxToolArg), ObTrxToolRes); RPC_S(PR5 flush_local_opt_stat_monitoring_info, obrpc::OB_SERVER_FLUSH_OPT_STAT_MONITORING_INFO, (obrpc::ObFlushOptStatArg)); +#ifdef OB_BUILD_TDE_SECURITY + RPC_S(PR5 dump_tenant_cache_master_key, OB_DUMP_TENANT_CACHE_MASTER_KEY, (obrpc::UInt64), ObDumpCacheMasterKeyResultArg); +#endif RPC_AP(PR5 set_member_list, OB_SET_MEMBER_LIST, (obrpc::ObSetMemberListArgV2), obrpc::ObSetMemberListResult); RPC_AP(PR5 create_ls, OB_CREATE_LS, (obrpc::ObCreateLSArg), obrpc::ObCreateLSResult); +#ifdef OB_BUILD_ARBITRATION + RPC_S(PR5 create_arb, OB_CREATE_ARB, (obrpc::ObCreateArbArg), obrpc::ObCreateArbResult); + RPC_S(PR5 delete_arb, OB_DELETE_ARB, (obrpc::ObDeleteArbArg), obrpc::ObDeleteArbResult); + RPC_S(PR5 arb_gc_notify, OB_ARB_GC_NOTIFY, (obrpc::ObArbGCNotifyArg), obrpc::ObArbGCNotifyResult); + RPC_S(PR5 force_clear_arb_cluster_info, OB_LOG_FORCE_CLEAR_ARB_CLUSTER_INFO, (obrpc::ObForceClearArbClusterInfoArg)); + RPC_S(PR5 arb_cluster_op, OB_ARB_CLUSTER_OP, (obrpc::ObArbClusterOpArg), obrpc::ObArbClusterOpResult); +#endif RPC_AP(PR5 create_tablet, OB_CREATE_TABLET, (obrpc::ObBatchCreateTabletArg), obrpc::ObCreateTabletBatchRes); RPC_AP(PR5 batch_broadcast_schema, OB_BATCH_BROADCAST_SCHEMA, (obrpc::ObBatchBroadcastSchemaArg), obrpc::ObBatchBroadcastSchemaResult); RPC_AP(PR5 drop_tablet, OB_DROP_TABLET, (obrpc::ObBatchRemoveTabletArg), obrpc::ObRemoveTabletRes); @@ -189,6 +211,12 @@ public: RPC_S(PR5 check_backup_dest_connectivity, OB_CHECK_BACKUP_DEST_CONNECTIVITY, (ObCheckBackupConnectivityArg)); RPC_AP(PR1 get_ls_access_mode, OB_GET_LS_ACCESS_MODE, (obrpc::ObGetLSAccessModeInfoArg), obrpc::ObLSAccessModeInfo); RPC_AP(PR1 change_ls_access_mode, OB_CHANGE_LS_ACCESS_MODE, (obrpc::ObLSAccessModeInfo), obrpc::ObChangeLSAccessModeRes); +#ifdef OB_BUILD_ARBITRATION + RPC_S(PR5 svr_accept_plan_baseline, obrpc::OB_SERVER_ACCEPT_PLAN_BASELINE, (obrpc::ObModifyPlanBaselineArg)); + RPC_S(PR5 svr_cancel_evolve_task, obrpc::OB_SERVER_CANCEL_EVOLVE_TASK, (obrpc::ObModifyPlanBaselineArg)); + RPC_S(PR5 load_baseline, OB_LOAD_BASELINE, (ObLoadPlanBaselineArg)); + RPC_S(PR5 load_baseline_v2, OB_LOAD_BASELINE_V2, (ObLoadPlanBaselineArg), obrpc::ObLoadBaselineRes); +#endif RPC_S(PR5 estimate_tablet_block_count, OB_ESTIMATE_TABLET_BLOCK_COUNT, (ObEstBlockArg), ObEstBlockRes); RPC_S(PR5 gen_unique_id, OB_GEN_UNIQUE_ID, (obrpc::UInt64), share::ObCommonID); RPC_S(PR5 start_transfer_task, OB_START_TRANSFER_TASK, (ObStartTransferTaskArg)); diff --git a/src/share/ob_thread_define.h b/src/share/ob_thread_define.h index 0eeace0ee..28ee385e5 100755 --- a/src/share/ob_thread_define.h +++ b/src/share/ob_thread_define.h @@ -126,7 +126,14 @@ TG_DEF(DDLScanTask, DDLScanTask, TIMER) TG_DEF(TenantLSMetaChecker, LSMetaCh, TIMER) TG_DEF(TenantTabletMetaChecker, TbMetaCh, TIMER) TG_DEF(ServerMetaChecker, SvrMetaCh, TIMER) +#ifdef OB_BUILD_ARBITRATION +TG_DEF(ArbNormalRpcQueueTh, ArbNormalRpcQueueTh, THREAD_POOL, ThreadCountPair(arbserver::ObArbSrvDeliver::get_normal_rpc_thread_num(), arbserver::ObArbSrvDeliver::MINI_MODE_RPC_QUEUE_CNT)) +TG_DEF(ArbServerRpcQueueTh, ArbSrvRpcQueueTh, THREAD_POOL, ThreadCountPair(arbserver::ObArbSrvDeliver::get_server_rpc_thread_num(), arbserver::ObArbSrvDeliver::MINI_MODE_RPC_QUEUE_CNT)) +#endif TG_DEF(ArbGCSTh, ArbGCTimerP, TIMER) +#ifdef OB_BUILD_ARBITRATION +TG_DEF(ArbServerTimer, ArbServerTimer, TIMER) +#endif TG_DEF(DataDictTimer, DataDictTimer, TIMER) TG_DEF(CDCService, CDCSrv, THREAD_POOL, 1) TG_DEF(LogUpdater, LogUpdater, TIMER) diff --git a/src/share/ob_thread_mgr.cpp b/src/share/ob_thread_mgr.cpp index e6699262d..1976da320 100644 --- a/src/share/ob_thread_mgr.cpp +++ b/src/share/ob_thread_mgr.cpp @@ -17,6 +17,9 @@ #include "storage/tx_storage/ob_ls_freeze_thread.h" #include "rootserver/ob_index_builder.h" #include "observer/ob_srv_deliver.h" +#ifdef OB_BUILD_ARBITRATION +#include "logservice/arbserver/ob_arb_srv_deliver.h" +#endif #include "logservice/palf/log_io_task_cb_thread_pool.h" #include "logservice/palf/log_io_worker.h" #include "logservice/palf/log_define.h" diff --git a/src/share/object/ob_obj_cast.cpp b/src/share/object/ob_obj_cast.cpp index 4be225988..dd2a89349 100644 --- a/src/share/object/ob_obj_cast.cpp +++ b/src/share/object/ob_obj_cast.cpp @@ -32,6 +32,10 @@ #include "sql/engine/expr/ob_expr_lob_utils.h" #include "lib/charset/ob_charset.h" #include "lib/geo/ob_geometry_cast.h" +#ifdef OB_BUILD_ORACLE_XML +#include "lib/xml/ob_xml_util.h" +#include "lib/xml/ob_xml_parser.h" +#endif // from sql_parser_base.h #define DEFAULT_STR_LENGTH -1 @@ -106,7 +110,28 @@ static int cast_extend_types_not_support(const ObObjType expect_type, UNUSED(out); UNUSED(cast_mode); int ret = OB_SUCCESS; +#ifdef OB_BUILD_ORACLE_XML + if (in.is_pl_extend()) { + if (pl::PL_OPAQUE_TYPE == in.get_meta().get_extend_type()) { + pl::ObPLOpaque *pl_src = reinterpret_cast(in.get_ext()); + if (OB_ISNULL(pl_src)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("failed to get pl data type info", K(ret), K(in)); + } else if (pl_src->get_type() == pl::ObPLOpaqueType::PL_XML_TYPE) { + LOG_WARN("inconsistent datatypes", K(ret), K(expect_type), K(in.get_type())); + ret = OB_ERR_INVALID_TYPE_FOR_OP; + } else { + LOG_WARN("not expected obj type convert", K(ret), K(expect_type), K(in), K(out), K(cast_mode)); + ret = OB_ERR_UNEXPECTED; + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected type to convert format", K(ret), K(expect_type), K(in)); + } +#else ret = OB_ERR_UNEXPECTED; +#endif return ret; } @@ -9133,7 +9158,44 @@ static int sql_udt_pl_extend(const ObObjType expect_type, ObObjCastParams ¶m const ObObj &in, ObObj &out, const ObCastMode cast_mode) { int ret = OB_SUCCESS; +#ifdef OB_BUILD_ORACLE_XML + if (in.is_xml_sql_type()) { + // no need to read blob full data + pl::ObPLXmlType *xmltype = NULL; + void *ptr = NULL; + ObObj* data = NULL; // obobj for blob; + if (OB_ISNULL(ptr = params.alloc(sizeof(pl::ObPLXmlType)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("Failed to allocate memory for pl xml data type", K(ret), K(sizeof(pl::ObPLXmlType))); + } else if (FALSE_IT(xmltype = new (ptr)pl::ObPLXmlType())) { + } else if (OB_ISNULL(data = static_cast(params.alloc(sizeof(ObObj))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory for pl object", K(ret)); + } else { + ObString xml_data = in.get_string(); + int64_t xml_data_size = xml_data.length(); + char *xml_data_buff = static_cast(params.alloc(xml_data_size)); + if (OB_ISNULL(xml_data_buff)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("failed to allocate memory for xmldata", K(ret), K(xml_data_size)); + } else { + MEMCPY(xml_data_buff, xml_data.ptr(), xml_data_size); + + data->set_string(ObLongTextType, xml_data_buff, static_cast(xml_data_size)); + data->set_has_lob_header(); // must has lob header + data->set_collation_type(CS_TYPE_UTF8MB4_BIN); + xmltype->set_data(data); + out.set_extend(reinterpret_cast(xmltype), pl::PL_OPAQUE_TYPE); + // need deep copy? + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected type to convert format", K(ret), K(expect_type), K(in)); + } +#else ret = OB_NOT_SUPPORTED; +#endif return ret; } @@ -9141,7 +9203,68 @@ static int string_sql_udt(const ObObjType expect_type, ObObjCastParams ¶ms, const ObObj &in, ObObj &out, const ObCastMode cast_mode) { int ret = OB_SUCCESS; +#ifdef OB_BUILD_ORACLE_XML + if (in.is_string_type()) { + ObMulModeMemCtx* mem_ctx = nullptr; + ObIAllocator &temp_allocator = *params.allocator_v2_; + bool is_null_result = false; + ObString in_str; + ObObj tmp_out; + ObXmlDocument* doc = nullptr; + ObString binary_str; + ObString blob_locator; + if (ObCharset::charset_type_by_coll(in.get_collation_type()) != CHARSET_UTF8MB4) { + if (OB_FAIL(string_string(ObObjType::ObVarcharType, params, in, tmp_out, cast_mode))) { + LOG_WARN("fail to cast string to longtext", K(ret), K(in)); + } else if (tmp_out.is_null()) { + is_null_result = true; + } else { + in_str = tmp_out.get_string(); + } + } else if (OB_FAIL(in.get_string(in_str))) { + LOG_WARN("Failed to get string from in obj", K(ret), K(tmp_out)); + } + + if (OB_FAIL(ret)) { + } else if (is_null_result) { + out.set_sql_udt("", 0, ObXMLSqlType); + } else if (OB_FAIL(ObXmlUtil::create_mulmode_tree_context(&temp_allocator, mem_ctx))) { + LOG_WARN("fail to create tree memory context", K(ret)); + } else { + ObXmlParser parser(mem_ctx); + if (OB_FAIL(parser.parse_document(in_str))) { + ret = OB_ERR_XML_PARSE; + LOG_USER_ERROR(OB_ERR_XML_PARSE); + LOG_WARN("parse xml plain text as document failed.", K(in_str)); + } else if (OB_ISNULL(doc = parser.document())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get doc null", K(ret)); + } else if (!doc->get_encoding().empty() || doc->get_encoding_flag()) { + doc->set_encoding(ObXmlUtil::get_charset_name(in.get_collation_type())); + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(doc->get_raw_binary(binary_str, mem_ctx->allocator_))) { + LOG_WARN("get raw binary failed", K(ret)); + } else { + ObTextStringResult blob_res(ObLongTextType, true, mem_ctx->allocator_); + int64_t total_length = binary_str.length(); + if (OB_FAIL(blob_res.init(total_length))) { + LOG_WARN("failed to init blob res", K(ret), K(blob_res), K(total_length)); + } else if (OB_FAIL(blob_res.append(binary_str))) { + LOG_WARN("failed to append xml binary data", K(ret), K(blob_res)); + } else { + blob_res.get_result_buffer(blob_locator); + out.set_sql_udt(blob_locator.ptr(), blob_locator.length(), ObXMLSqlType); + } + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected type to convert format", K(ret), K(expect_type), K(in)); + } +#else ret = OB_NOT_SUPPORTED; +#endif return ret; } @@ -9149,7 +9272,47 @@ static int pl_extend_sql_udt(const ObObjType expect_type, ObObjCastParams ¶m const ObObj &in, ObObj &out, const ObCastMode cast_mode) { int ret = OB_SUCCESS; +#ifdef OB_BUILD_ORACLE_XML + if (in.is_pl_extend()) { + if (pl::PL_OPAQUE_TYPE == in.get_meta().get_extend_type()) { + pl::ObPLOpaque *pl_src = reinterpret_cast(in.get_ext()); + if (OB_ISNULL(pl_src)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("failed to get pl data type info", K(ret), K(in)); + } else if (pl_src->get_type() == pl::ObPLOpaqueType::PL_XML_TYPE) { + pl::ObPLXmlType * xmltype = static_cast(pl_src); + ObObj *blob_obj = xmltype->get_data(); + if (OB_ISNULL(blob_obj) || blob_obj->is_null()) { + out.set_sql_udt("", 0, ObXMLSqlType); + } else { + // deep copy here or by the caller ? + ObString xml_data = blob_obj->get_string(); + int64_t xml_data_size = xml_data.length(); + char *xml_data_buff = static_cast(params.alloc(xml_data_size)); + if (OB_ISNULL(xml_data_buff)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory for xmldata", + K(ret), K(expect_type), K(xml_data_size)); + } else { + MEMCPY(xml_data_buff, xml_data.ptr(), xml_data_size); + out.set_sql_udt(xml_data_buff, static_cast(xml_data_size), ObXMLSqlType); + } + } + } else if (pl_src->get_type() == pl::ObPLOpaqueType::PL_INVALID) { + // possibly an un-initiated pl opaque variable + out.set_null(); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("not expected obj type convert", K(ret), K(expect_type), K(in), K(out), K(cast_mode)); + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected type to convert format", K(ret), K(expect_type), K(in)); + } +#else ret = OB_NOT_SUPPORTED; +#endif return ret; } @@ -9157,7 +9320,60 @@ static int pl_extend_string(const ObObjType expect_type, ObObjCastParams ¶ms const ObObj &in, ObObj &out, const ObCastMode cast_mode) { int ret = OB_SUCCESS; +#ifdef OB_BUILD_ORACLE_XML + if (in.is_pl_extend()) { + if (pl::PL_OPAQUE_TYPE == in.get_meta().get_extend_type()) { + pl::ObPLOpaque *pl_src = reinterpret_cast(in.get_ext()); + if (OB_ISNULL(pl_src)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("failed to get pl data type info", K(ret), K(in)); + } else if (pl_src->get_type() == pl::ObPLOpaqueType::PL_XML_TYPE) { + pl::ObPLXmlType * xmltype = static_cast(pl_src); + ObObj *blob_obj = xmltype->get_data(); + if (OB_ISNULL(blob_obj)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("Unexpected xml data", K(ret), K(*xmltype)); + } else { + ObString blob_data = blob_obj->get_string(); + ObStringBuffer xml_plain_text(params.allocator_v2_); + common::ObArenaAllocator temp_allocator(ObModIds::OB_LOB_ACCESS_BUFFER, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); + ObCollationType session_cs_type = params.dest_collation_; + if (OB_FAIL(sql::ObTextStringHelper::read_real_string_data(params.allocator_v2_, + ObLongTextType, + CS_TYPE_BINARY, + true, + blob_data))) { + LOG_WARN("fail to get xml data.", K(ret), K(blob_data)); + } else if (OB_FAIL(ObXmlUtil::cast_to_string(blob_data, temp_allocator, xml_plain_text, session_cs_type))) { + LOG_WARN("failed to convert xml to string", K(ret), KP(blob_data.ptr()), K(blob_data.length())); + } else { + ObObj tmp_in = in; + tmp_in.set_string(ObVarcharType, xml_plain_text.string()); + tmp_in.set_collation_type(CS_TYPE_UTF8MB4_BIN); + if (OB_FAIL(string_string(expect_type, params, tmp_in, out, cast_mode))) { + LOG_WARN("cast string to string in udt string failed", K(ret)); + } + } + } + } else if (pl_src->get_type() == pl::ObPLOpaqueType::PL_INVALID) { + // un-initiated pl variable, for example, xml_data and xml_data2 are declared without assignment + // then call this directly: select replace(xml_data,xml_data2 ,'1') into stringval from dual; + out.set_null(); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("not expected obj type convert", K(ret), K(expect_type), K(in), K(out), K(cast_mode)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("not expected obj type convert", K(ret), K(expect_type), K(in), K(out), K(cast_mode)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected type to convert format", K(ret), K(expect_type), K(in)); + } +#else ret = OB_NOT_SUPPORTED; +#endif return ret; } @@ -9185,7 +9401,37 @@ static int udt_string(const ObObjType expect_type, ObObjCastParams ¶ms, const ObObj &in, ObObj &out, const ObCastMode cast_mode) { int ret = OB_SUCCESS; +#ifdef OB_BUILD_ORACLE_XML + if (in.is_xml_sql_type()) { + ObString blob_data = in.get_string(); + ObStringBuffer xml_plain_text(params.allocator_v2_); + common::ObArenaAllocator temp_allocator(ObModIds::OB_LOB_ACCESS_BUFFER, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); + ObCollationType session_cs_type = params.dest_collation_; + if (OB_FAIL(sql::ObTextStringHelper::read_real_string_data(params.allocator_v2_, + ObLongTextType, + CS_TYPE_BINARY, + true, + blob_data))) { + LOG_WARN("fail to get xml data.", K(ret), K(blob_data)); + } else if (OB_FAIL(ObXmlUtil::cast_to_string(blob_data, temp_allocator, xml_plain_text, session_cs_type))) { + LOG_WARN("failed to convert xml to string", K(ret), KP(blob_data.ptr()), K(blob_data.length())); + } else { + ObObj tmp_in = in; + tmp_in.set_string(ObVarcharType, xml_plain_text.string()); + tmp_in.set_collation_type(CS_TYPE_UTF8MB4_BIN); + if (OB_FAIL(string_string(expect_type, params, tmp_in, out, cast_mode))) { + LOG_WARN("cast string to string in udt string failed", K(ret)); + } + } + } else { + ret = OB_ERR_INVALID_TYPE_FOR_OP; + // get udt type name from ctx + LOG_WARN_RET(OB_ERR_INVALID_TYPE_FOR_OP, "inconsistent datatypes", + "expected", expect_type, "got", in.get_type(), K(in.get_udt_subschema_id())); + } +#else ret = OB_NOT_SUPPORTED; +#endif return ret; } diff --git a/src/share/rc/ob_tenant_base.h b/src/share/rc/ob_tenant_base.h index 94770c6ee..5c3ac18f6 100755 --- a/src/share/rc/ob_tenant_base.h +++ b/src/share/rc/ob_tenant_base.h @@ -22,6 +22,9 @@ #include "lib/thread/thread_mgr.h" #include "lib/allocator/ob_malloc.h" #include "share/ob_tenant_role.h"//ObTenantRole +#ifdef OB_BUILD_DBLINK +#include "lib/oracleclient/ob_oci_environment.h" +#endif #include "lib/mysqlclient/ob_tenant_oci_envs.h" namespace oceanbase { @@ -188,7 +191,11 @@ namespace detector class ObDeadLockDetectorMgr; } +#ifndef OB_BUILD_ARBITRATION #define ArbMTLMember +#else +#define ArbMTLMember rootserver::ObArbitrationService*, +#endif // 在这里列举需要添加的租户局部变量的类型，租户会为每种类型创建一个实例。 // 实例的初始化和销毁逻辑由MTL_BIND接口指定。 diff --git a/src/share/schema/ob_schema_service_sql_impl.cpp b/src/share/schema/ob_schema_service_sql_impl.cpp index af0dc7a14..b8cbbba9d 100644 --- a/src/share/schema/ob_schema_service_sql_impl.cpp +++ b/src/share/schema/ob_schema_service_sql_impl.cpp @@ -8408,6 +8408,93 @@ int ObSchemaServiceSQLImpl::sort_table_partition_info_v2( return ret; } +#ifdef OB_BUILD_TDE_SECURITY +int ObSchemaServiceSQLImpl::fetch_master_key( + ObISQLClient &sql_client, + const uint64_t tenant_id, + const uint64_t master_key_id, + ObMasterKey *key, + ObString &encrypt_out) +{ + int ret = OB_SUCCESS; + ObSqlString sql; + ObMySQLResult *result = NULL; + DEFINE_SQL_CLIENT_RETRY_WEAK(sql_client); + + if (OB_INVALID_TENANT_ID == tenant_id || + OB_INVALID_ID == master_key_id || + OB_ISNULL(key) || + OB_ISNULL(key->key_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("master key id is not invalid", K(ret)); + } else { + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + if (OB_FAIL(sql.append_fmt("SELECT * FROM %s WHERE MASTER_KEY_ID = %lu LIMIT 1", + OB_ALL_TENANT_KEYSTORE_HISTORY_TNAME, + ObSchemaUtils::get_extract_schema_id(tenant_id, master_key_id)))) { + LOG_WARN("append sql failed", K(ret)); + } else if (OB_FAIL(sql_client_retry_weak.read(res, tenant_id, sql.ptr()))) { + LOG_WARN("execute sql failed", K(sql), K(ret)); + } else if (OB_UNLIKELY(NULL == (result = res.get_result()))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to get result. ", K(ret)); + } else if (OB_FAIL(result->next())) { + if (common::OB_ITER_END == ret) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("select master_key return no row", K(ret), K(master_key_id)); + } else { + LOG_WARN("failed to get value about master_key", K(ret), K(master_key_id)); + } + } else { + ObString encrypted_key; + ObString empty_str(""); + EXTRACT_VARCHAR_FIELD_MYSQL_WITH_DEFAULT_VALUE( + *result, "encrypted_key", encrypted_key, true, + ObSchemaService::g_ignore_column_retrieve_error_, empty_str); + if (OB_FAIL(ret)) { + LOG_WARN("failed to get encrypted_key value", K(ret), K(master_key_id)); + } else if (!encrypted_key.empty()) { + char real_master_key[OB_MAX_ENCRYPTED_KEY_LENGTH] = {0}; + int64_t master_key_len = 0; + if (encrypt_out.size() > 0) { + if (OB_UNLIKELY(0 == encrypt_out.write(encrypted_key.ptr(), encrypted_key.length()))) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("encrypted key length too long", K( encrypted_key.length()), K(ret)); + } + } else if (OB_FAIL(ObEncryptionUtil::decrypt_master_key(tenant_id, + encrypted_key.ptr(), + encrypted_key.length(), + real_master_key, + OB_MAX_ENCRYPTED_KEY_LENGTH, + master_key_len))) { + LOG_WARN("failed to decrypt_master_key", K(encrypted_key), K(ret)); + } else if (OB_UNLIKELY(master_key_len > OB_MAX_MASTER_KEY_LENGTH)) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("master key length too long", K(master_key_len), K(ret)); + } else { + MEMCPY(key->key_, real_master_key, master_key_len); + key->len_ = master_key_len; + } + } else { + ObString master_key; + EXTRACT_VARCHAR_FIELD_MYSQL(*result, "master_key", master_key); + if (OB_FAIL(ret)) { + LOG_WARN("failed to get master_key value", K(ret), K(master_key_id)); + } else if (OB_UNLIKELY(master_key.length() > OB_MAX_MASTER_KEY_LENGTH)) { + ret = OB_SIZE_OVERFLOW; + LOG_WARN("master key length too long", K(master_key), K(ret)); + } else { + MEMCPY(key->key_, master_key.ptr(), master_key.length()); + key->len_ = master_key.length(); + } + } + } + } + } + LOG_INFO("fetch the mater_key", K(master_key_id), K(ret)); + return ret; +} +#endif // link table. int ObSchemaServiceSQLImpl::get_link_table_schema(const ObDbLinkSchema *dblink_schema, @@ -8620,6 +8707,12 @@ int ObSchemaServiceSQLImpl::fetch_link_table_info(dblink_param_ctx ¶m_ctx, OB_SUCCESS != (tmp_ret = result->close())) { LOG_WARN("failed to close result", K(tmp_ret)); } +#ifdef OB_BUILD_DBLINK + if (DBLINK_DRV_OCI == param_ctx.link_type_ && + OB_SUCCESS != (tmp_ret = static_cast(dblink_conn)->free_oci_stmt())) { + LOG_WARN("failed to close oci result", K(tmp_ret)); + } +#endif if (OB_SUCCESS != (tmp_ret = dblink_proxy_->release_dblink(param_ctx.link_type_, dblink_conn))) { LOG_WARN("failed to relese connection", K(tmp_ret)); } diff --git a/src/share/schema/ob_schema_service_sql_impl.h b/src/share/schema/ob_schema_service_sql_impl.h index 2a903a95f..fc51407a2 100644 --- a/src/share/schema/ob_schema_service_sql_impl.h +++ b/src/share/schema/ob_schema_service_sql_impl.h @@ -42,6 +42,9 @@ #include "share/schema/ob_directory_sql_service.h" #include "share/schema/ob_context_sql_service.h" #include "share/schema/ob_rls_sql_service.h" +#ifdef OB_BUILD_TDE_SECURITY +#include "share/ob_master_key_getter.h" +#endif #include "sql/dblink/ob_dblink_utils.h" #include "lib/string/ob_string.h" @@ -618,6 +621,13 @@ public: const uint64_t tenant_id, common::ObCompatibilityMode &mode); +#ifdef OB_BUILD_TDE_SECURITY + static int fetch_master_key(common::ObISQLClient &sql_client, + const uint64_t tenant_id, + const uint64_t master_key_id, + share::ObMasterKey *key, + common::ObString &encrypt_out); +#endif virtual int get_drop_tenant_infos( common::ObISQLClient &sql_client, diff --git a/src/share/schema/ob_schema_struct.cpp b/src/share/schema/ob_schema_struct.cpp index 38b97a31c..62162d44c 100644 --- a/src/share/schema/ob_schema_struct.cpp +++ b/src/share/schema/ob_schema_struct.cpp @@ -10239,13 +10239,78 @@ int ObDbLinkBaseInfo::do_encrypt_reverse_password() int ObDbLinkBaseInfo::dblink_encrypt(common::ObString &src, common::ObString &dst) { int ret = OB_SUCCESS; +#ifdef OB_BUILD_DBLINK + if (src.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("src is empty", K(ret)); + } else { + char encrypted_string[common::OB_MAX_ENCRYPTED_PASSWORD_LENGTH] = {0}; + + char hex_buff[common::OB_MAX_ENCRYPTED_PASSWORD_LENGTH + 1] = {0}; // +1 to reserve space for \0 + int64_t encrypt_len = -1; + if (OB_FAIL(oceanbase::share::ObEncryptionUtil::encrypt_sys_data(tenant_id_, + src.ptr(), + src.length(), + encrypted_string, + common::OB_MAX_ENCRYPTED_PASSWORD_LENGTH, + encrypt_len))) { + + LOG_WARN("fail to encrypt_sys_data", KR(ret), K(src)); + } else if (0 >= encrypt_len || common::OB_MAX_ENCRYPTED_PASSWORD_LENGTH < encrypt_len * 2) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("encrypt_len is invalid", K(ret), K(encrypt_len), K(common::OB_MAX_ENCRYPTED_PASSWORD_LENGTH)); + } else if (OB_FAIL(to_hex_cstr(encrypted_string, encrypt_len, hex_buff, common::OB_MAX_ENCRYPTED_PASSWORD_LENGTH + 1))) { + LOG_WARN("fail to print to hex str", K(ret)); + } else if (OB_FAIL(deep_copy_str(ObString(hex_buff), dst))) { + LOG_WARN("failed to deep copy encrypted_string", K(ret)); + } else { + LOG_TRACE("succ to encrypt src", K(src), K(dst)); + } + } +#else ret = OB_NOT_SUPPORTED; +#endif return ret; } int ObDbLinkBaseInfo::dblink_decrypt(common::ObString &src, common::ObString &dst) { int ret = OB_SUCCESS; +#ifdef OB_BUILD_DBLINK + if (src.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("src is empty", K(ret)); + } else if (0 != src.length() % 2) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid src", K(src.length()), K(ret)); + } else { + char encrypted_password_not_hex[common::OB_MAX_ENCRYPTED_PASSWORD_LENGTH] = {0}; + char plain_string[common::OB_MAX_PASSWORD_LENGTH + 1] = { 0 }; // need +1 to reserve space for \0 + int64_t plain_string_len = -1; + if (OB_FAIL(hex_to_cstr(src.ptr(), + src.length(), + encrypted_password_not_hex, + common::OB_MAX_ENCRYPTED_PASSWORD_LENGTH))) { + LOG_WARN("failed to hex to cstr", K(src.length()), K(ret)); + } else if (OB_FAIL(ObEncryptionUtil::decrypt_sys_data(tenant_id_, + encrypted_password_not_hex, + + src.length() / 2, + plain_string, + common::OB_MAX_PASSWORD_LENGTH + 1, + plain_string_len))) { + LOG_WARN("failed to decrypt_sys_data", K(ret), K(src.length())); + } else if (0 >= plain_string_len) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("decrypt dblink password failed", K(ret), K(plain_string_len)); + } else if (OB_FAIL(deep_copy_str(ObString(plain_string_len, plain_string), dst))) { + LOG_WARN("failed to deep copy plain_string", K(ret)); + } else { + LOG_TRACE("succ to decrypt src", K(plain_string_len), K(src), K(dst)); + } + } +#else ret = OB_NOT_SUPPORTED; +#endif return ret; } diff --git a/src/share/schema/ob_tenant_sql_service.cpp b/src/share/schema/ob_tenant_sql_service.cpp index 9c6f3d2e7..babfe223e 100644 --- a/src/share/schema/ob_tenant_sql_service.cpp +++ b/src/share/schema/ob_tenant_sql_service.cpp @@ -256,11 +256,36 @@ int ObTenantSqlService::replace_tenant( || OB_FAIL(dml.add_column("in_recyclebin", tenant_schema.is_in_recyclebin())))) { LOG_WARN("add column failed", K(ret)); } +#ifndef OB_BUILD_ARBITRATION if (OB_SUCC(ret) && !tenant_schema.get_arbitration_service_status().is_disabled()) { ret = OB_OP_NOT_ALLOW; LOG_WARN("arbitration service is not supported in CE version", KR(ret), K(tenant_schema)); LOG_USER_ERROR(OB_OP_NOT_ALLOW, "create tenant with arbitration service in CE version"); } +#else + // If this ddl is a create tenant stmt + // (1) Only need to make sure sys tenant data version is above 4.1 to compate with arbitration service. + // (2) Do not check data version of the creating tenant and its meta tenant, because we can not get valid tenant config now + // + // If this ddl is a alter tenant stmt + // (1) Need to make sure sys,user,meta tenants all upgraded to 4.1 + const uint64_t tenant_to_check_data_version = (OB_DDL_ADD_TENANT == op || OB_DDL_ADD_TENANT_START == op) + ? OB_SYS_TENANT_ID + : tenant_schema.get_tenant_id(); + if (OB_FAIL(ret)) { + } else if (OB_FAIL(ObShareUtil::check_compat_version_for_arbitration_service( + tenant_to_check_data_version, is_compatible_with_arbitration_service))) { + LOG_WARN("fail to check compat version with arbitration service", KR(ret), K(tenant_to_check_data_version)); + } else if (is_compatible_with_arbitration_service) { + if (OB_FAIL(dml.add_column("arbitration_service_status", tenant_schema.get_arbitration_service_status_str()))) { + LOG_WARN("fail to add arbitration service status column", KR(ret), K(tenant_schema)); + } + } else if (!tenant_schema.get_arbitration_service_status().is_disabled()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("data version is not above 4.1, and arbitration service status is not default value", + KR(ret), K(tenant_schema)); + } +#endif } // insert into __all_tenant if (OB_SUCC(ret)) { diff --git a/src/share/schema/ob_trigger_info.cpp b/src/share/schema/ob_trigger_info.cpp index 46330d2be..9b979477c 100644 --- a/src/share/schema/ob_trigger_info.cpp +++ b/src/share/schema/ob_trigger_info.cpp @@ -394,6 +394,11 @@ int ObTriggerInfo::gen_package_source(const uint64_t tenant_id, OV (T_TG_SOURCE == trigger_source_node->type_, trigger_source_node->type_); OV (OB_NOT_NULL(trigger_define_node = trigger_source_node->children_[1])); if (OB_FAIL(ret)) { +#ifdef OB_BUILD_ORACLE_PL + } else if (lib::is_oracle_mode()) { + OV (5 == trigger_define_node->num_child_); + OV (OB_NOT_NULL(trigger_body_node = trigger_define_node->children_[4])); +#endif } else { OV (4 == trigger_define_node->num_child_); OV (OB_NOT_NULL(trigger_body_node = trigger_define_node->children_[3])); @@ -1020,6 +1025,53 @@ bool ObTriggerInfo::ActionOrderComparator::operator()(const ObTriggerInfo *left, return bool_ret; } +#ifdef OB_BUILD_ORACLE_PL +// for alter trigger rename +int ObTriggerInfo::replace_trigger_name_in_body(ObTriggerInfo &trigger_info, + common::ObIAllocator &alloc, + ObSchemaGetterGuard &schema_guard) +{ + int ret = OB_SUCCESS; + char *buf = NULL; + int64_t buf_len = 0; + int64_t pos = 0; + ObParser parser(alloc, trigger_info.get_sql_mode()); + ParseResult parse_result; + ParseNode *stmt_list_node = NULL; + const ParseNode *trigger_source_node = NULL; + const ObDatabaseSchema *db_schema = NULL; + OZ (schema_guard.get_database_schema(trigger_info.get_tenant_id(), trigger_info.get_database_id(), db_schema)); + OV (OB_NOT_NULL(db_schema)); + OZ (parser.parse(trigger_info.get_trigger_body(), parse_result, TRIGGER_MODE, + false, false, true), + trigger_info.get_trigger_body()); + // stmt list node + OV (OB_NOT_NULL(stmt_list_node = parse_result.result_tree_)); + OV (stmt_list_node->type_ == T_STMT_LIST, OB_ERR_UNEXPECTED, stmt_list_node->type_); + OV (stmt_list_node->num_child_ == 1, OB_ERR_UNEXPECTED, stmt_list_node->num_child_); + OV (OB_NOT_NULL(stmt_list_node->children_)); + // trigger source node + OV (OB_NOT_NULL(trigger_source_node = stmt_list_node->children_[0])); + if (OB_SUCC(ret)) { +#define RENAME_TRIGGER_FMT \ + "TRIGGER \"%.*s\".\"%.*s\" %.*s" + buf_len = STRLEN(RENAME_TRIGGER_FMT) + db_schema->get_database_name_str().length() + + trigger_info.get_trigger_name().length() + trigger_source_node->str_len_; + buf = static_cast(alloc.alloc(buf_len)); + OV (OB_NOT_NULL(buf), OB_ALLOCATE_MEMORY_FAILED); + OZ (BUF_PRINTF(RENAME_TRIGGER_FMT, + db_schema->get_database_name_str().length(), + db_schema->get_database_name_str().ptr(), + trigger_info.get_trigger_name().length(), + trigger_info.get_trigger_name().ptr(), + (int)trigger_source_node->str_len_, + trigger_source_node->str_value_)); + OZ (trigger_info.set_trigger_body(ObString(buf))); +#undef RENAME_TRIGGER_FMT + } + return ret; +} +#endif // for rebuild trigger body due to rename table int ObTriggerInfo::replace_table_name_in_body(ObTriggerInfo &trigger_info, diff --git a/src/share/schema/ob_trigger_info.h b/src/share/schema/ob_trigger_info.h index 53be8b75e..fb10b4b9c 100644 --- a/src/share/schema/ob_trigger_info.h +++ b/src/share/schema/ob_trigger_info.h @@ -501,6 +501,11 @@ public: const ParseNode &parse_node, const ObDataTypeCastParams &dtc_params, ObString &procedure_source); +#ifdef OB_BUILD_ORACLE_PL + static int replace_trigger_name_in_body(ObTriggerInfo &trigger_info, + common::ObIAllocator &alloc, + share::schema::ObSchemaGetterGuard &schema_guard); +#endif static int replace_table_name_in_body(ObTriggerInfo &trigger_info, common::ObIAllocator &alloc, const common::ObString &base_object_database, diff --git a/src/share/schema/ob_udt_info.cpp b/src/share/schema/ob_udt_info.cpp index 7544ad74f..4230010e1 100644 --- a/src/share/schema/ob_udt_info.cpp +++ b/src/share/schema/ob_udt_info.cpp @@ -525,6 +525,16 @@ int ObUDTTypeInfo::transform_to_pl_type(const ObUDTTypeAttr* attr_info, pl::ObPL pl_type.set_user_type_id(pl::ObPLType::PL_RECORD_TYPE, attr_info->get_type_attr_id()); pl_type.set_type_from(pl::ObPLTypeFrom::PL_TYPE_UDT); +#ifdef OB_BUILD_ORACLE_PL + } else if (attr_info->is_coll_type()) { + pl_type.set_user_type_id(pl::ObPLType::PL_NESTED_TABLE_TYPE, + attr_info->get_type_attr_id()); + pl_type.set_type_from(pl::ObPLTypeFrom::PL_TYPE_UDT); + } else if (attr_info->is_opaque_type()) { + pl_type.set_user_type_id(pl::ObPLType::PL_OPAQUE_TYPE, + attr_info->get_type_attr_id()); + pl_type.set_type_from(pl::ObPLTypeFrom::PL_TYPE_UDT); +#endif } else { ret = OB_ERR_UNEXPECTED; LOG_WARN("attr info type is invalid", K(ret), KPC(attr_info)); @@ -538,6 +548,42 @@ int ObUDTTypeInfo::transform_to_pl_type(common::ObIAllocator &allocator, const p void *ptr = NULL; pl::ObUserDefinedType *local_pl_type = NULL; pl_type = NULL; +#ifdef OB_BUILD_ORACLE_PL + if (is_collection()) { + pl::ObCollectionType *table_type = NULL; + pl::ObPLDataType elem_type; + if (OB_ISNULL(ptr = allocator.alloc(is_varray() ? sizeof(pl::ObVArrayType) : sizeof(pl::ObCollectionType)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory for ObNestedTableType", K(ret)); + } else if (OB_FAIL(transform_to_pl_type(coll_info_, elem_type))) { + LOG_WARN("failed get collection elem type", K(ret)); + } else { + if (is_varray()) { + table_type = static_cast(new(ptr)pl::ObVArrayType()); + pl::ObVArrayType *vt = static_cast (ptr); + // + vt->set_capacity(coll_info_->get_upper_bound()); + } else { + table_type = static_cast(new(ptr)pl::ObNestedTableType()); + } + table_type->set_user_type_id(get_type_id()); + table_type->set_type_from(pl::ObPLTypeFrom::PL_TYPE_UDT); + table_type->set_element_type(elem_type); + local_pl_type = table_type; + } + } else if (is_opaque()) { + pl::ObOpaqueType *opaque_type = NULL; + if (OB_ISNULL(ptr = allocator.alloc(sizeof(pl::ObOpaqueType)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory for ObOpaqueType", K(ret)); + } else { + opaque_type = new(ptr)pl::ObOpaqueType(); + opaque_type->set_user_type_id(get_type_id()); + opaque_type->set_type_from(pl::ObPLTypeFrom::PL_TYPE_UDT); + local_pl_type = opaque_type; + } + } else { +#endif pl::ObRecordType *record_type = NULL; if (OB_ISNULL(ptr = allocator.alloc(sizeof(pl::ObRecordType)))) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -565,6 +611,9 @@ int ObUDTTypeInfo::transform_to_pl_type(common::ObIAllocator &allocator, const p local_pl_type = record_type; } } +#ifdef OB_BUILD_ORACLE_PL + } +#endif if (OB_SUCC(ret)) { ObString copy_type_name; if (OB_FAIL(deep_copy_name(allocator, get_type_name(), copy_type_name))) { diff --git a/src/share/system_variable/ob_system_variable.cpp b/src/share/system_variable/ob_system_variable.cpp index 288923922..7d54ac77b 100644 --- a/src/share/system_variable/ob_system_variable.cpp +++ b/src/share/system_variable/ob_system_variable.cpp @@ -35,6 +35,9 @@ #include "sql/engine/expr/ob_expr_plsql_variable.h" #include "share/resource_manager/ob_resource_manager_proxy.h" #include "sql/engine/expr/ob_expr_uuid.h" +#ifdef OB_BUILD_ORACLE_PL +#include "pl/ob_pl_warning.h" +#endif using namespace oceanbase::common; @@ -63,7 +66,12 @@ ObSpecialSysVarValues::ObSpecialSysVarValues() } else if (OB_FAIL(databuff_printf(ObSpecialSysVarValues::version_comment_, ObSpecialSysVarValues::VERSION_COMMENT_MAX_LEN, pos, +#ifdef OB_BUILD_CLOSE_MODULES + "OceanBase %s (r%s) (Built %s %s)", +#else "OceanBase_CE %s (r%s) (Built %s %s)", + +#endif PACKAGE_VERSION, build_version(), build_date(), build_time()))) { LOG_ERROR("fail to print version_comment to buff", K(ret)); @@ -74,7 +82,12 @@ ObSpecialSysVarValues::ObSpecialSysVarValues() } else if (FALSE_IT(pos = 0)) { } else if (OB_FAIL(databuff_printf(ObSpecialSysVarValues::version_, ObSpecialSysVarValues::VERSION_MAX_LEN, +#ifdef OB_BUILD_CLOSE_MODULES + pos, "5.7.25-OceanBase-v%s", PACKAGE_VERSION))) { +#else pos, "5.7.25-OceanBase_CE-v%s", PACKAGE_VERSION))) { + +#endif LOG_ERROR("fail to print version to buff", K(ret)); } @@ -2312,7 +2325,17 @@ int ObSysVarOnCheckFuncs::check_and_convert_plsql_warnings(sql::ObExecContext &c common::ObObj &out_val) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSEDx(ctx, set_var, sys_var, in_val, out_val); +#else + UNUSEDx(ctx, set_var, sys_var); + if (OB_FAIL(pl::PlCompilerWarningCategory::verify_warning_settings(in_val.get_string(), NULL))) { + ret = OB_ERR_PARAM_VALUE_INVALID; + LOG_USER_ERROR(OB_ERR_PARAM_VALUE_INVALID); + } else { + out_val = in_val; + } +#endif return ret; } diff --git a/src/sql/CMakeLists.txt b/src/sql/CMakeLists.txt index 672e9d744..5612627e3 100644 --- a/src/sql/CMakeLists.txt +++ b/src/sql/CMakeLists.txt @@ -212,6 +212,7 @@ ob_set_subtarget(ob_sql engine_cmd engine/cmd/ob_context_executor.cpp engine/cmd/ob_table_direct_insert_ctx.cpp engine/cmd/ob_table_direct_insert_service.cpp + ) ob_set_subtarget(ob_sql engine_dml @@ -1191,7 +1192,6 @@ ob_set_subtarget(ob_sql session session/ob_sess_info_verify.cpp ) - ob_set_subtarget(ob_sql udr udr/ob_udr_struct.cpp udr/ob_udr_sql_service.cpp @@ -1221,7 +1221,6 @@ if (${ARCHITECTURE} STREQUAL "x86_64") endif() target_link_libraries(ob_sql PUBLIC ob_base) -target_compile_definitions(ob_sql PUBLIC SQL_COMPILATION) add_library(ob_sql_static STATIC diff --git a/src/sql/code_generator/ob_dml_cg_service.cpp b/src/sql/code_generator/ob_dml_cg_service.cpp index 038dcab1b..087717fff 100644 --- a/src/sql/code_generator/ob_dml_cg_service.cpp +++ b/src/sql/code_generator/ob_dml_cg_service.cpp @@ -28,6 +28,9 @@ #include "sql/optimizer/ob_log_plan.h" #include "sql/optimizer/ob_log_update.h" #include "sql/engine/expr/ob_expr_calc_partition_id.h" +#ifdef OB_BUILD_TDE_SECURITY +#include "share/ob_master_key_getter.h" +#endif namespace oceanbase { @@ -1325,6 +1328,32 @@ int ObDmlCgService::generate_das_dml_ctdef(ObLogDelUpd &op, das_dml_ctdef.tz_info_ = *session->get_tz_info_wrap().get_time_zone_info(); das_dml_ctdef.is_total_quantity_log_ = (ObBinlogRowImage::FULL == binlog_row_image); } +#ifdef OB_BUILD_TDE_SECURITY + // generate encrypt_meta for table + if (OB_SUCC(ret)) { + ObSchemaGetterGuard *schema_guard = NULL; + const share::schema::ObTableSchema *table_schema = NULL; + if (OB_ISNULL(schema_guard = op.get_plan() + ->get_optimizer_context().get_schema_guard())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("NULL schema guard", K(ret)); + } else if (OB_FAIL(schema_guard->get_table_schema(MTL_ID(), index_tid, table_schema))) { + LOG_WARN("get schema fail", KR(ret), K(index_tid)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_TABLE_NOT_EXIST; + LOG_WARN("table not exist", KR(ret), K(index_tid)); + } else { + ObArray metas; + if (OB_FAIL(init_encrypt_metas_(table_schema, schema_guard, metas))) { + LOG_WARN("init table encrypt meta fail", KR(ret)); + } else if (metas.count() == 0) { + das_dml_ctdef.encrypt_meta_.reset(); + } else if (OB_FAIL(das_dml_ctdef.encrypt_meta_.assign(metas))) { + LOG_WARN("assign encrypt meta fail", KR(ret), K(index_tid), K(metas)); + } + } + } +#endif return ret; } @@ -2802,5 +2831,113 @@ int ObDmlCgService::convert_foreign_keys(ObLogDelUpd &op, return ret; } +#ifdef OB_BUILD_TDE_SECURITY +int ObDmlCgService::init_encrypt_metas_( + const share::schema::ObTableSchema *table_schema, + share::schema::ObSchemaGetterGuard *guard, + ObIArray &meta_array) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(table_schema) || OB_ISNULL(guard)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("table schema is invalid", K(ret)); + } else if (OB_FAIL(init_encrypt_table_meta_(table_schema, guard, meta_array))) { + LOG_WARN("fail to init encrypt_table_meta", KPC(table_schema), K(ret)); + } else if (!table_schema->is_user_table()) { + /*do nothing*/ + } else { + ObSEArray simple_index_infos; + const ObTableSchema *index_schema = nullptr; + if (OB_FAIL(table_schema->get_simple_index_infos(simple_index_infos))) { + LOG_WARN("get simple_index_infos failed", K(ret)); + } + for (int i = 0; i < simple_index_infos.count() && OB_SUCC(ret); ++i) { + if (OB_FAIL(guard->get_table_schema( + table_schema->get_tenant_id(), + simple_index_infos.at(i).table_id_, index_schema))) { + LOG_WARN("fail to get table schema", K(ret)); + } else if (OB_ISNULL(index_schema)) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("index schema not exist", K(simple_index_infos.at(i).table_id_), K(ret)); + } else if (!index_schema->is_index_local_storage()) { + // do nothing + } else if (OB_FAIL(init_encrypt_table_meta_(index_schema, guard, meta_array))) { + LOG_WARN("fail to init encrypt_table_meta", KPC(index_schema), K(ret)); + } + } + } + return ret; +} + +int ObDmlCgService::init_encrypt_table_meta_( + const share::schema::ObTableSchema *table_schema, + share::schema::ObSchemaGetterGuard *guard, + ObIArray&meta_array) +{ + int ret = OB_SUCCESS; + transaction::ObEncryptMetaCache meta_cache; + char master_key[OB_MAX_MASTER_KEY_LENGTH]; + char random_string[OB_CLOG_ENCRYPT_RANDOM_LEN]; + int64_t master_key_length = 0; + if (OB_ISNULL(table_schema) || OB_ISNULL(guard)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("table schema is invalid", K(ret)); + } else if (!(table_schema->need_encrypt() && table_schema->get_master_key_id() > 0 && + table_schema->get_encrypt_key_len() > 0)) { + // do nothing + } else if (OB_FAIL(table_schema->get_encryption_id(meta_cache.meta_.encrypt_algorithm_))) { + LOG_WARN("fail to get encryption id", K(ret)); + } else { + if (table_schema->is_index_local_storage()) { + meta_cache.table_id_ = table_schema->get_data_table_id(); + meta_cache.local_index_id_ = table_schema->get_table_id(); + } else { + meta_cache.table_id_ = table_schema->get_table_id(); + } + meta_cache.meta_.tenant_id_ = table_schema->get_tenant_id(); + meta_cache.meta_.master_key_version_ = table_schema->get_master_key_id(); + if (OB_FAIL(meta_cache.meta_.encrypted_table_key_.set_content( + table_schema->get_encrypt_key()))) { + LOG_WARN("fail to assign encrypt key", K(ret)); + } else if (OB_FAIL(share::ObKeyGenerator::generate_encrypt_key( + random_string, OB_CLOG_ENCRYPT_RANDOM_LEN))) { + LOG_WARN("fail to generate random string", K(ret)); + } else if (OB_FAIL(meta_cache.meta_.random_.set_content(ObString( + OB_CLOG_ENCRYPT_RANDOM_LEN, random_string)))) { + LOG_WARN("fail to assign random string", K(ret)); + } + #ifdef ERRSIM + else if (OB_FAIL(OB_E(EventTable::EN_ENCRYPT_GET_MASTER_KEY_FAILED) OB_SUCCESS)) { + LOG_WARN("ERRSIM, fail to get master key", K(ret)); + } + #endif + else if (OB_FAIL(share::ObMasterKeyGetter::get_master_key(table_schema->get_tenant_id(), + table_schema->get_master_key_id(), + master_key, OB_MAX_MASTER_KEY_LENGTH, master_key_length))) { + LOG_WARN("fail to get master key", K(ret)); + // 如果在cg阶段获取主密钥失败了, 有可能是因为RS执行内部sql没有租户资源引起的. + // 在没有租户资源的情况下获取主密钥, 获取加密租户配置项时会失败 + // 在这种情况下, 我们认为这里是合理的, 缓存中可以不保存主密钥内容 + // cg阶段获取主密钥的任何失败我们可以接受 + // 兜底是执行期再次获取, 再次获取成功了则继续往下走, 失败了则报错出来. + // 见bug + ret = OB_SUCCESS; + } else if (OB_FAIL(meta_cache.meta_.master_key_.set_content(ObString( + master_key_length, master_key)))) { + LOG_WARN("fail to assign master_key", K(ret)); + } else if (OB_FAIL(ObEncryptionUtil::decrypt_table_key(meta_cache.meta_, + meta_cache.meta_.encrypted_table_key_.ptr(), + meta_cache.meta_.encrypted_table_key_.size()))) { + LOG_WARN("failed to decrypt_table_key", K(ret)); + } else {/*do nothing*/} + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(meta_array.push_back(meta_cache))) { + LOG_WARN("fail to push back meta array", K(ret)); + } + } + return ret; +} +#endif } // namespace sql } // namespace oceanbase diff --git a/src/sql/code_generator/ob_dml_cg_service.h b/src/sql/code_generator/ob_dml_cg_service.h index 899c849ad..5016f0a85 100644 --- a/src/sql/code_generator/ob_dml_cg_service.h +++ b/src/sql/code_generator/ob_dml_cg_service.h @@ -256,6 +256,14 @@ private: const ObSQLSessionInfo &session, ObIAllocator &allocator, bool &need_fire); +#ifdef OB_BUILD_TDE_SECURITY + int init_encrypt_metas_(const share::schema::ObTableSchema *table_schema, + share::schema::ObSchemaGetterGuard *guard, + ObIArray&meta_array); + int init_encrypt_table_meta_(const share::schema::ObTableSchema *table_schema, + share::schema::ObSchemaGetterGuard *guard, + ObIArray&meta_array); +#endif int generate_table_loc_meta(const IndexDMLInfo &index_dml_info, ObDASTableLocMeta &loc_meta); private: ObStaticEngineCG &cg_; diff --git a/src/sql/code_generator/ob_static_engine_cg.cpp b/src/sql/code_generator/ob_static_engine_cg.cpp index 32d7c2a12..d91596b2e 100644 --- a/src/sql/code_generator/ob_static_engine_cg.cpp +++ b/src/sql/code_generator/ob_static_engine_cg.cpp @@ -142,6 +142,9 @@ #include "sql/resolver/cmd/ob_load_data_stmt.h" #include "share/stat/ob_stat_define.h" #include "sql/engine/px/p2p_datahub/ob_p2p_dh_mgr.h" +#ifdef OB_BUILD_TDE_SECURITY +#include "share/ob_master_key_getter.h" +#endif namespace oceanbase { @@ -6855,6 +6858,40 @@ int ObStaticEngineCG::set_other_properties(const ObLogPlan &log_plan, ObPhysical } } +#ifdef OB_BUILD_TDE_SECURITY + // set encrypt info in phy plan + if (OB_SUCC(ret)) { + const ObIArray *dependency_table = log_plan.get_stmt()->get_global_dependency_table(); + if (OB_ISNULL(dependency_table)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret)); + } else { + int64_t table_id = OB_INVALID_ID; + const share::schema::ObTableSchema *full_table_schema = NULL; + ObArraymetas; + for (int64_t i = 0; OB_SUCC(ret) && i < dependency_table->count(); i++) { + if (DEPENDENCY_TABLE == dependency_table->at(i).object_type_) { + full_table_schema = NULL; + table_id = dependency_table->at(i).get_object_id(); + if (OB_FAIL(schema_guard->get_table_schema( + MTL_ID(), table_id, full_table_schema))) { + LOG_WARN("fail to get table schema", K(ret), K(table_id)); + } else if (OB_ISNULL(full_table_schema)) { + ret = OB_TABLE_NOT_EXIST; + LOG_WARN("table is not exist", K(ret), K(full_table_schema)); + } else if (OB_FAIL(init_encrypt_metas(full_table_schema, schema_guard, metas))) { + LOG_WARN("fail to init encrypt table meta", K(ret)); + } + } + } + if (OB_SUCC(ret) && metas.count() > 0) { + if (OB_FAIL(phy_plan.get_encrypt_meta_array().assign(metas))) { + LOG_WARN("fail to assgin encrypt meta", K(ret)); + } + } + } + } +#endif if (OB_SUCC(ret)) { if (OB_ISNULL(log_plan.get_stmt())) { ret = OB_ERR_UNEXPECTED; @@ -7206,6 +7243,114 @@ int ObStaticEngineCG::classify_anti_monotone_filter_exprs(const ObIArray &meta_array) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(table_schema) || OB_ISNULL(guard)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("table schema is invalid", K(ret)); + } else if (OB_FAIL(init_encrypt_table_meta(table_schema, guard, meta_array))) { + LOG_WARN("fail to init encrypt_table_meta", KPC(table_schema), K(ret)); + } else if (!table_schema->is_user_table()) { + /*do nothing*/ + } else { + ObSEArray simple_index_infos; + const ObTableSchema *index_schema = nullptr; + if (OB_FAIL(table_schema->get_simple_index_infos(simple_index_infos))) { + LOG_WARN("get simple_index_infos failed", K(ret)); + } + for (int i = 0; i < simple_index_infos.count() && OB_SUCC(ret); ++i) { + if (OB_FAIL(guard->get_table_schema( + MTL_ID(), + simple_index_infos.at(i).table_id_, index_schema))) { + LOG_WARN("fail to get table schema", K(ret)); + } else if (OB_ISNULL(index_schema)) { + ret = OB_SCHEMA_ERROR; + LOG_WARN("index schema not exist", K(simple_index_infos.at(i).table_id_), K(ret)); + } else if (!index_schema->is_index_local_storage()) { + // do nothing + } else if (OB_FAIL(init_encrypt_table_meta(index_schema, guard, meta_array))) { + LOG_WARN("fail to init encrypt_table_meta", KPC(index_schema), K(ret)); + } + } + } + return ret; +} + +int ObStaticEngineCG::init_encrypt_table_meta( + const share::schema::ObTableSchema *table_schema, + share::schema::ObSchemaGetterGuard *guard, + ObIArray&meta_array) +{ + int ret = OB_SUCCESS; + transaction::ObEncryptMetaCache meta_cache; + char master_key[OB_MAX_MASTER_KEY_LENGTH]; + char random_string[OB_CLOG_ENCRYPT_RANDOM_LEN]; + int64_t master_key_length = 0; + if (OB_ISNULL(table_schema) || OB_ISNULL(guard)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("table schema is invalid", K(ret)); + } else if (!(table_schema->need_encrypt() && table_schema->get_master_key_id() > 0 && + table_schema->get_encrypt_key_len() > 0)) { + // do nothing + } else if (OB_FAIL(table_schema->get_encryption_id(meta_cache.meta_.encrypt_algorithm_))) { + LOG_WARN("fail to get encryption id", K(ret)); + } else { + if (table_schema->is_index_local_storage()) { + meta_cache.table_id_ = table_schema->get_data_table_id(); + meta_cache.local_index_id_ = table_schema->get_table_id(); + } else { + meta_cache.table_id_ = table_schema->get_table_id(); + } + meta_cache.meta_.tenant_id_ = table_schema->get_tenant_id(); + meta_cache.meta_.master_key_version_ = table_schema->get_master_key_id(); + if (OB_FAIL(meta_cache.meta_.encrypted_table_key_.set_content( + table_schema->get_encrypt_key()))) { + LOG_WARN("fail to assign encrypt key", K(ret)); + } else if (OB_FAIL(share::ObKeyGenerator::generate_encrypt_key( + random_string, OB_CLOG_ENCRYPT_RANDOM_LEN))) { + LOG_WARN("fail to generate random string", K(ret)); + } else if (OB_FAIL(meta_cache.meta_.random_.set_content(ObString( + OB_CLOG_ENCRYPT_RANDOM_LEN, random_string)))) { + LOG_WARN("fail to assign random string", K(ret)); + } + #ifdef ERRSIM + else if (OB_FAIL(OB_E(EventTable::EN_ENCRYPT_GET_MASTER_KEY_FAILED) OB_SUCCESS)) { + LOG_WARN("ERRSIM, fail to get master key", K(ret)); + } + #endif + else if (OB_FAIL(share::ObMasterKeyGetter::get_master_key(table_schema->get_tenant_id(), + table_schema->get_master_key_id(), + master_key, OB_MAX_MASTER_KEY_LENGTH, master_key_length))) { + LOG_WARN("fail to get master key", K(ret)); + // 如果在cg阶段获取主密钥失败了, 有可能是因为RS执行内部sql没有租户资源引起的. + // 在没有租户资源的情况下获取主密钥, 获取加密租户配置项时会失败 + // 在这种情况下, 我们认为这里是合理的, 缓存中可以不保存主密钥内容 + // cg阶段获取主密钥的任何失败我们可以接受 + // 兜底是执行期再次获取, 再次获取成功了则继续往下走, 失败了则报错出来. + // 见bug + ret = OB_SUCCESS; + } else if (OB_FAIL(meta_cache.meta_.master_key_.set_content(ObString( + master_key_length, master_key)))) { + LOG_WARN("fail to assign master_key", K(ret)); + } else if (OB_FAIL(ObEncryptionUtil::decrypt_table_key(meta_cache.meta_, + meta_cache.meta_.encrypted_table_key_.ptr(), + meta_cache.meta_.encrypted_table_key_.size()))) { + LOG_WARN("failed to decrypt_table_key", K(ret)); + } else {/*do nothing*/} + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(meta_array.push_back(meta_cache))) { + LOG_WARN("fail to push back meta array", K(ret)); + } + } + return ret; +} +#endif int ObStaticEngineCG::map_value_param_index(const ObInsertStmt *insert_stmt, RowParamMap &row_params_map) diff --git a/src/sql/code_generator/ob_static_engine_cg.h b/src/sql/code_generator/ob_static_engine_cg.h index 216aaae56..ec3f1ebbd 100644 --- a/src/sql/code_generator/ob_static_engine_cg.h +++ b/src/sql/code_generator/ob_static_engine_cg.h @@ -192,6 +192,16 @@ public: const static uint8_t IS_JSON_CONSTRAINT_RELAX = 1; const static uint8_t IS_JSON_CONSTRAINT_STRICT = 4; private: +#ifdef OB_BUILD_TDE_SECURITY + int init_encrypt_metas( + const share::schema::ObTableSchema *table_schema, + share::schema::ObSchemaGetterGuard *guard, + ObIArray &meta_array); + + int init_encrypt_table_meta(const share::schema::ObTableSchema *table_schema, + share::schema::ObSchemaGetterGuard *guard, + ObIArray&meta_array); +#endif int classify_anti_monotone_filter_exprs(const common::ObIArray &input_filters, common::ObIArray &non_anti_monotone_filters, diff --git a/src/sql/dblink/ob_dblink_utils.cpp b/src/sql/dblink/ob_dblink_utils.cpp index 9eda84250..f73c6fffe 100644 --- a/src/sql/dblink/ob_dblink_utils.cpp +++ b/src/sql/dblink/ob_dblink_utils.cpp @@ -19,11 +19,37 @@ #include "sql/session/ob_sql_session_info.h" #include "common/ob_smart_call.h" #include "common/object/ob_object.h" +#ifdef OB_BUILD_DBLINK +#include "observer/omt/ob_multi_tenant.h" +#include "share/rc/ob_tenant_base.h" +#include "common/sql_mode/ob_sql_mode_utils.h" +#endif using namespace oceanbase; using namespace oceanbase::sql; using namespace oceanbase::share; +#ifdef OB_BUILD_DBLINK +uint64_t ObDblinkService::get_current_tenant_id() +{ + return MTL_ID(); +} + +oceanbase::common::sqlclient::ObTenantOciEnvs * ObDblinkService::get_tenant_oci_envs() +{ + return MTL(oceanbase::common::sqlclient::ObTenantOciEnvs*); +} + +int ObDblinkService::init_oci_envs_func_ptr() +{ + int ret = common::OB_SUCCESS; + OciEnvironment::get_tenant_id_ = get_current_tenant_id; + OciEnvironment::get_tenant_oci_envs_ = get_tenant_oci_envs; + return ret; +} + +bool g_dblink_oci_func_ptr_inited = ObDblinkService::init_oci_envs_func_ptr(); +#endif int ObDblinkService::check_lob_in_result(common::sqlclient::ObMySQLResult *result, bool &have_lob) { int ret = OB_SUCCESS; diff --git a/src/sql/dblink/ob_dblink_utils.h b/src/sql/dblink/ob_dblink_utils.h index f740dc19d..68e087a04 100644 --- a/src/sql/dblink/ob_dblink_utils.h +++ b/src/sql/dblink/ob_dblink_utils.h @@ -20,6 +20,9 @@ #include "lib/mysqlclient/ob_mysql_connection.h" #include "sql/resolver/dml/ob_select_stmt.h" #include "lib/mysqlclient/ob_mysql_result.h" +#ifdef OB_BUILD_DBLINK +#include "lib/oracleclient/ob_oci_environment.h" +#endif #include "share/rc/ob_tenant_base.h" namespace oceanbase @@ -30,6 +33,11 @@ namespace sql class ObDblinkService { public: +#ifdef OB_BUILD_DBLINK + static uint64_t get_current_tenant_id(); + static common::sqlclient::ObTenantOciEnvs * get_tenant_oci_envs(); + static int init_oci_envs_func_ptr(); +#endif static int check_lob_in_result(common::sqlclient::ObMySQLResult *result, bool &have_lob); static int get_length_from_type_text(ObString &type_text, int32_t &length); static int get_local_session_vars(sql::ObSQLSessionInfo *session_info, diff --git a/src/sql/engine/aggregate/ob_aggregate_processor.cpp b/src/sql/engine/aggregate/ob_aggregate_processor.cpp index 9abd06c7f..eeb470420 100644 --- a/src/sql/engine/aggregate/ob_aggregate_processor.cpp +++ b/src/sql/engine/aggregate/ob_aggregate_processor.cpp @@ -30,6 +30,14 @@ #include "sql/engine/basic/ob_material_op_impl.h" #include "share/stat/ob_hybrid_hist_estimator.h" #include "share/stat/ob_dbms_stats_utils.h" +#ifdef OB_BUILD_ORACLE_XML +#include "lib/xml/ob_xml_util.h" +#include "lib/xml/ob_xml_tree.h" +#include "lib/xml/ob_xml_parser.h" +#include "sql/engine/expr/ob_expr_xml_func_helper.h" +#include "lib/alloc/malloc_hook.h" +#endif +#include "pl/ob_pl_user_type.h" namespace oceanbase { @@ -6641,7 +6649,209 @@ int ObAggregateProcessor::get_ora_xmlagg_result(const ObAggrInfo &aggr_info, ObDatum &concat_result) { int ret = OB_SUCCESS; +#ifdef OB_BUILD_ORACLE_XML + ObString result; + common::ObArenaAllocator tmp_alloc(ObModIds::OB_SQL_AGGR_FUNC, OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); + ObXmlDocument *content = NULL; + ObXmlDocument* doc = NULL; + ObString blob_locator; + ObMulModeNodeType node_type = M_MAX_TYPE; + int64_t row_count = 0; + int64_t is_unparsed = false; + + ObMulModeMemCtx* xml_mem_ctx = nullptr; + lib::ObMallocHookAttrGuard malloc_guard(lib::ObMemAttr(MTL_ID(), "XMLCodeGen")); + if (OB_FAIL(ObXmlUtil::create_mulmode_tree_context(&tmp_alloc, xml_mem_ctx))) { + LOG_WARN("fail to create tree memory context", K(ret)); + } else if (OB_ISNULL(content = OB_NEWx(ObXmlDocument, xml_mem_ctx->allocator_, ObMulModeNodeType::M_CONTENT, xml_mem_ctx))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate row buffer failed at ObXmlDocument", K(ret)); + } else if (OB_FAIL(content->alter_member_sort_policy(false))) { + LOG_WARN("failed to alter sot policy", K(ret)); + } else if (OB_ISNULL(extra) || OB_UNLIKELY(extra->empty())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unpexcted null", K(ret), K(extra)); + } else if (extra->is_iterated() && OB_FAIL(extra->rewind())) { + LOG_WARN("rewind failed", K(extra), K(ret)); + } else if (!extra->is_iterated() && OB_FAIL(extra->finish_add_row())) { + LOG_WARN("finish_add_row failed", KPC(extra), K(ret)); + } else { + const ObChunkDatumStore::StoredRow *stored_row = NULL; + ObObj *tmp_obj = NULL; + bool inited_tmp_obj = false; + char *buf = NULL; + int64_t str_len = 0; + int64_t buf_len = 0; + bool deal_special = false; + int64_t add_time = 0; + + while (OB_SUCC(ret) && OB_SUCC(extra->get_next_row(stored_row))) { + if (OB_ISNULL(stored_row)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret), K(stored_row)); + } else if (!inited_tmp_obj && + OB_ISNULL(tmp_obj = static_cast(tmp_alloc.alloc(sizeof(ObObj) * (stored_row->cnt_))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory", K(ret), K(tmp_obj)); + } else if (!inited_tmp_obj && FALSE_IT(inited_tmp_obj = true)) { + } else if (OB_FAIL(convert_datum_to_obj(aggr_info, *stored_row, tmp_obj, stored_row->cnt_))) { + LOG_WARN("failed to convert datum to obj", K(ret)); + } else if (stored_row->cnt_ < 1) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected column count", K(ret), K(stored_row)); + } else if (tmp_obj->is_null()) { + // do noting for null + } else { + ObIMulModeBase *base; + ObDatum converted_datum; + ObXmlDocument *doc_node = NULL; + converted_datum.set_datum(stored_row->cells()[0]); + ObString cell_string = converted_datum.get_string(); + if (tmp_obj->get_type() == ObObjType::ObExtendType) { + if (pl::PL_OPAQUE_TYPE == tmp_obj->get_meta().get_extend_type()) { + pl::ObPLOpaque *pl_src = reinterpret_cast(converted_datum.get_ext()); + if (OB_ISNULL(pl_src)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("failed to get pl data type info", K(ret)); + } else if (pl_src->get_type() == pl::ObPLOpaqueType::PL_XML_TYPE) { + pl::ObPLXmlType * xmltype = static_cast(pl_src); + ObObj *blob_obj = xmltype->get_data(); + if (OB_ISNULL(blob_obj)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("Unexpected xml data", K(ret), K(*xmltype)); + } else { + cell_string = blob_obj->get_string(); + } + } else if (pl_src->get_type() == pl::ObPLOpaqueType::PL_INVALID) { + cell_string.reset(); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected pl xml to sql udt type", K(ret)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get extend type is not PL_OPAQUE_TYPE", K(ret), K(tmp_obj->get_meta().get_extend_type())); + } + } + + ObString dup_str; + row_count = extra->get_row_count(); + if (OB_FAIL(ret)) { + } else if (OB_FAIL(ObTextStringHelper::read_real_string_data(&tmp_alloc, ObObjType::ObLongTextType, + CS_TYPE_UTF8MB4_BIN, true, cell_string))) { + LOG_WARN("fail to get real data", K(ret), K(cell_string)); + } else if (cell_string.length() <= 0) { + // zero length do noting + } else if (row_count == 1 && OB_FAIL(ObXmlUtil::xml_bin_type(cell_string, node_type))) { + LOG_WARN("xml bin type failed", K(ret), K(cell_string)); + } else if (row_count == 1 && node_type != ObMulModeNodeType::M_DOCUMENT) { + ObString res_str; + deal_special = true; + if (node_type == ObMulModeNodeType::M_CONTENT) { + if (OB_FAIL(ObXMLExprHelper::pack_binary_res(*aggr_info.expr_, eval_ctx_, cell_string, blob_locator))) { + LOG_WARN("pack binary res failed", K(ret)); + } else { + concat_result.set_string(blob_locator.ptr(), blob_locator.length()); + } + } else if (node_type == ObMulModeNodeType::M_UNPARESED_DOC) { + ObXmlDocument *doc = nullptr; + if (OB_FAIL(common::ObMulModeFactory::get_xml_base(xml_mem_ctx, cell_string, + ObNodeMemType::BINARY_TYPE, + ObNodeMemType::TREE_TYPE, + base))) { + LOG_WARN("fail to get xml base", K(ret), K(cell_string)); + } else if (OB_ISNULL(doc = static_cast(base))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get base cast to xmlDocument null", K(ret)); + } else if (OB_FAIL(doc->get_raw_binary(res_str, xml_mem_ctx->allocator_))) { + LOG_WARN("get raw binary failed", K(ret)); + } else { + if (OB_FAIL(ObXMLExprHelper::pack_binary_res(*aggr_info.expr_, eval_ctx_, res_str, blob_locator))) { + LOG_WARN("pack binary res failed", K(ret)); + } else { + concat_result.set_string(blob_locator.ptr(), blob_locator.length()); + } + } + } else if (node_type == ObMulModeNodeType::M_UNPARSED) { + if (OB_FAIL(ObXMLExprHelper::content_unparsed_binary_check_doc(xml_mem_ctx, cell_string, res_str))) { + LOG_WARN("xmlagg content unparsed tag check doc failed", K(ret)); + ret = OB_SUCCESS; + if (OB_FAIL(ObXMLExprHelper::pack_binary_res(*aggr_info.expr_, eval_ctx_, cell_string, blob_locator))) { + LOG_WARN("pack binary res failed", K(ret)); + } else { + concat_result.set_string(blob_locator.ptr(), blob_locator.length()); + } + } else { + if (OB_FAIL(ObXMLExprHelper::pack_binary_res(*aggr_info.expr_, eval_ctx_, res_str, blob_locator))) { + LOG_WARN("pack binary res failed", K(ret)); + } else { + concat_result.set_string(blob_locator.ptr(), blob_locator.length()); + } + } + } + } else { + if (OB_FAIL(deep_copy_ob_string(tmp_alloc, cell_string, dup_str))) { + LOG_WARN("fail copy string", K(ret), K(cell_string.length())); + } else if (OB_FAIL(common::ObMulModeFactory::get_xml_base(xml_mem_ctx, dup_str, + ObNodeMemType::BINARY_TYPE, + ObNodeMemType::TREE_TYPE, + base))) { + LOG_WARN("fail to get xml base", K(ret), K(cell_string)); + } else if (OB_ISNULL(base)) { + ret = OB_BAD_NULL_ERROR; + LOG_WARN("node cast to xmldocument failed", K(ret), K(base)); + } else if (ObMulModeNodeType::M_DOCUMENT == base->type() || + ObMulModeNodeType::M_CONTENT == base->type() || + ObMulModeNodeType::M_UNPARSED == base->type()) { + doc_node = static_cast(base); + for (int32_t j = 0; OB_SUCC(ret) && j < doc_node->size(); j++) { + ObXmlNode *xml_node = doc_node->at(j); + if (OB_ISNULL(xml_node)) { + ret = OB_BAD_NULL_ERROR; + LOG_WARN("doc node get null", K(ret), K(j)); + } else if (OB_XML_TYPE != xml_node->data_type()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("xml node date type unexpect", K(ret), K(j), K(xml_node->data_type())); + } else if (OB_FAIL(content->add_element(xml_node))) { + LOG_WARN("add element failed", K(ret), K(j), K(xml_node)); + } else if (xml_node->get_unparse()) { + is_unparsed = true; + } + } + } + + add_time++; + if (OB_SUCC(ret) && content->get_serialize_size() > OB_MAX_PACKET_LENGTH) { + ret = OB_ERR_TOO_LONG_STRING_IN_CONCAT; + LOG_WARN("result of xmlagg is too long", K(ret), K(add_time), K(content->get_serialize_size()), K(OB_MAX_PACKET_LENGTH)); + } + } + } // end if + } // end of while + if (OB_FAIL(ret) && ret != OB_ITER_END) { + LOG_WARN("fail to get next row", K(ret)); + } else { + ret = OB_SUCCESS; + if (!deal_special) { + ObString res_str; + ObMulModeNodeType target_type = M_MAX_TYPE; + if (node_type == M_DOCUMENT && row_count == 1) { + target_type = M_DOCUMENT; + } else if (is_unparsed) { + target_type = M_UNPARSED; + } else { + target_type = M_CONTENT; + } + if (OB_FAIL(ObXMLExprHelper::pack_xml_res(*aggr_info.expr_, eval_ctx_, concat_result, content, xml_mem_ctx, + target_type, res_str))) { + LOG_WARN("pack document failed", K(ret)); + } + } + } + } +#else ret = OB_NOT_SUPPORTED; +#endif return ret; } diff --git a/src/sql/engine/cmd/ob_alter_system_executor.cpp b/src/sql/engine/cmd/ob_alter_system_executor.cpp index d0c368b88..2b1a15ef4 100644 --- a/src/sql/engine/cmd/ob_alter_system_executor.cpp +++ b/src/sql/engine/cmd/ob_alter_system_executor.cpp @@ -31,6 +31,12 @@ #include "share/rc/ob_context.h" #include "observer/ob_server_struct.h" #include "observer/mysql/ob_mysql_request_manager.h" +#ifdef OB_BUILD_ARBITRATION +#include "share/arbitration_service/ob_arbitration_service_utils.h" //ObArbitrationServiceUtils +#endif +#ifdef OB_BUILD_TDE_SECURITY +#include "share/ob_master_key_getter.h" +#endif #include "share/rc/ob_tenant_base.h" #include "share/scheduler/ob_dag_warning_history_mgr.h" #include "observer/omt/ob_tenant.h" //ObTenant @@ -605,6 +611,13 @@ int ObAdminServerExecutor::execute(ObExecContext &ctx, ObAdminServerStmt &stmt) // check whether all leaders are switched out if (OB_FAIL(wait_leader_switch_out_(*(ctx.get_sql_proxy()), arg.servers_))) { LOG_WARN("fail to wait leader switch out", KR(ret), K(arg)); +#ifdef OB_BUILD_ARBITRATION + // check whether all 2f tenant with arb service finished degration + } else if (OB_FAIL(ObArbitrationServiceUtils::wait_all_2f_tenants_arb_service_degration( + *(ctx.get_sql_proxy()), + arg.servers_))) { + LOG_WARN("fail to wait degration for arb service", KR(ret), K(arg)); +#endif } } } else { @@ -768,6 +781,13 @@ int ObAdminZoneExecutor::execute(ObExecContext &ctx, ObAdminZoneStmt &stmt) } else if (OB_FAIL(wait_leader_switch_out_(*(ctx.get_sql_proxy()), arg))) { // check whether all leaders are switched out LOG_WARN("fail to wait leader switch out", KR(ret), K(arg)); +#ifdef OB_BUILD_ARBITRATION + // check whether all 2f tenant with arb service finished degration + } else if (OB_FAIL(ObArbitrationServiceUtils::wait_all_2f_tenants_arb_service_degration( + *(ctx.get_sql_proxy()), + server_list))) { + LOG_WARN("fail to wait degration for arb service", KR(ret), K(arg), K(server_list)); +#endif } } else {} // force stop, no need to wait leader switch } else if (ObAdminZoneArg::MODIFY == stmt.get_op()) { @@ -1154,30 +1174,115 @@ int ObMigrateUnitExecutor::execute(ObExecContext &ctx, ObMigrateUnitStmt &stmt) int ObAddArbitrationServiceExecutor::execute(ObExecContext &ctx, ObAddArbitrationServiceStmt &stmt) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ARBITRATION UNUSEDx(ctx, stmt); ret = OB_NOT_SUPPORTED; LOG_WARN("not support in CE Version", KR(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "add arbitration service in CE version"); +#else + ObTaskExecutorCtx *task_exec_ctx = GET_TASK_EXECUTOR_CTX(ctx); + obrpc::ObCommonRpcProxy *common_rpc = NULL; + bool is_compatible = false; + if (OB_ISNULL(task_exec_ctx)) { + ret = OB_NOT_INIT; + LOG_WARN("get task executor context failed", KR(ret)); + } else if (OB_FAIL(ObShareUtil::check_compat_version_for_arbitration_service( + OB_SYS_TENANT_ID, is_compatible))) { + LOG_WARN("fail to check compat version with arbitration service", KR(ret)); + } else if (!is_compatible) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("add arbitration service with data version below 4.1 not supported", KR(ret)); + } else if (OB_ISNULL(common_rpc = task_exec_ctx->get_common_rpc())) { + ret = OB_NOT_INIT; + LOG_WARN("get common rpc proxy failed", KR(ret), K(task_exec_ctx)); + } else if (OB_FAIL(common_rpc->admin_add_arbitration_service(stmt.get_rpc_arg()))) { + LOG_WARN("add arbitration service rpc failed", KR(ret), "rpc_arg", stmt.get_rpc_arg()); + } +#endif return ret; } int ObRemoveArbitrationServiceExecutor::execute(ObExecContext &ctx, ObRemoveArbitrationServiceStmt &stmt) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ARBITRATION UNUSEDx(ctx, stmt); ret = OB_NOT_SUPPORTED; LOG_WARN("not support in CE Version", KR(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "remove arbitration service in CE version"); +#else + ObTaskExecutorCtx *task_exec_ctx = GET_TASK_EXECUTOR_CTX(ctx); + obrpc::ObCommonRpcProxy *common_rpc = NULL; + bool is_compatible = false; + if (OB_ISNULL(task_exec_ctx)) { + ret = OB_NOT_INIT; + LOG_WARN("get task executor context failed", KR(ret)); + } else if (OB_FAIL(ObShareUtil::check_compat_version_for_arbitration_service( + OB_SYS_TENANT_ID, is_compatible))) { + LOG_WARN("fail to check compat version with arbitration service", KR(ret)); + } else if (!is_compatible) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("remove arbitration service with data version below 4.1 not supported", KR(ret)); + } else if (OB_ISNULL(common_rpc = task_exec_ctx->get_common_rpc())) { + ret = OB_NOT_INIT; + LOG_WARN("get common rpc proxy failed", KR(ret), K(task_exec_ctx)); + } else if (OB_FAIL(common_rpc->admin_remove_arbitration_service(stmt.get_rpc_arg()))) { + LOG_WARN("remove arbitration service rpc failed", KR(ret), "rpc_arg", stmt.get_rpc_arg()); + } +#endif return ret; } int ObReplaceArbitrationServiceExecutor::execute(ObExecContext &ctx, ObReplaceArbitrationServiceStmt &stmt) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ARBITRATION UNUSEDx(ctx, stmt); ret = OB_NOT_SUPPORTED; LOG_WARN("not support in CE Version", KR(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "replace arbitration service in CE version"); +#else + ObTaskExecutorCtx *task_exec_ctx = GET_TASK_EXECUTOR_CTX(ctx); + obrpc::ObCommonRpcProxy *common_rpc = NULL; + bool is_compatible = false; + if (OB_ISNULL(task_exec_ctx)) { + ret = OB_NOT_INIT; + LOG_WARN("get task executor context failed", KR(ret)); + } else if (OB_FAIL(ObShareUtil::check_compat_version_for_arbitration_service( + OB_SYS_TENANT_ID, is_compatible))) { + LOG_WARN("fail to check compat version with arbitration service", KR(ret)); + } else if (!is_compatible) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("replace arbitration service with data version below 4.1 not supported", KR(ret)); + } else if (OB_ISNULL(common_rpc = task_exec_ctx->get_common_rpc())) { + ret = OB_NOT_INIT; + LOG_WARN("get common rpc proxy failed", KR(ret), K(task_exec_ctx)); + } else if (OB_ISNULL(GCTX.sql_proxy_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", KR(ret), KP(GCTX.sql_proxy_)); + } else if (OB_FAIL(common_rpc->admin_replace_arbitration_service(stmt.get_rpc_arg()))) { + LOG_WARN("replace arbitration service rpc failed", KR(ret), "rpc_arg", stmt.get_rpc_arg()); + } else if (OB_FAIL(ObArbitrationServiceUtils::wait_all_tenant_with_arb_has_arb_member( + *GCTX.sql_proxy_, + stmt.get_rpc_arg().get_arbitration_service(), + stmt.get_rpc_arg().get_previous_arbitration_service()))) { + LOG_WARN("fail to wait all tenant with arb service has expected arb member", + KR(ret), "rpc_arg", stmt.get_rpc_arg()); + } else { + // try clean cluster info from arb server + ObRemoveClusterInfoFromArbServerArg remove_cluster_info_arg; + int tmp_ret = OB_SUCCESS; // for remove_cluster_info operation + if (OB_TMP_FAIL(remove_cluster_info_arg.init(stmt.get_rpc_arg().get_previous_arbitration_service()))) { + LOG_WARN("fail to init a rpc arg", K(tmp_ret), "rpc_arg", stmt.get_rpc_arg()); + } else if (OB_TMP_FAIL(common_rpc->remove_cluster_info_from_arb_server(remove_cluster_info_arg))) { + LOG_WARN("fail to remove cluster info from arb server", K(tmp_ret), K(remove_cluster_info_arg)); + } + if (OB_SUCCESS != tmp_ret) { + LOG_USER_WARN(OB_CLUSTER_INFO_MAYBE_REMAINED, stmt.get_rpc_arg().get_previous_arbitration_service().length(), + stmt.get_rpc_arg().get_previous_arbitration_service().ptr()); + } + } +#endif return ret; } @@ -2146,6 +2251,13 @@ int ObDeletePolicyExecutor::execute(ObExecContext &ctx, ObDeletePolicyStmt &stmt int ObBackupKeyExecutor::execute(ObExecContext &ctx, ObBackupKeyStmt &stmt) { int ret = OB_SUCCESS; +#ifdef OB_BUILD_TDE_SECURITY + if (OB_FAIL(ObMasterKeyUtil::backup_key(stmt.get_tenant_id(), + stmt.get_backup_dest(), + stmt.get_encrypt_key()))) { + LOG_WARN("failed to backup master key", K(ret)); + } +#endif return ret; } diff --git a/src/sql/engine/cmd/ob_restore_executor.cpp b/src/sql/engine/cmd/ob_restore_executor.cpp index ada14b400..810bfc3e8 100644 --- a/src/sql/engine/cmd/ob_restore_executor.cpp +++ b/src/sql/engine/cmd/ob_restore_executor.cpp @@ -90,7 +90,7 @@ int ObPhysicalRestoreTenantExecutor::execute( } } else { - // TODO: fix restore preview later. + // TODO(chongrong.th): fix restore preview in 4.3 ret = OB_NOT_SUPPORTED; LOG_WARN("restore preview is not support now", K(ret)); // if (OB_FAIL(physical_restore_preview(ctx, stmt))) { diff --git a/src/sql/engine/cmd/ob_tablespace_cmd_executor.h b/src/sql/engine/cmd/ob_tablespace_cmd_executor.h deleted file mode 100644 index df4525ec5..000000000 --- a/src/sql/engine/cmd/ob_tablespace_cmd_executor.h +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#ifndef OCEANBASE_SQL_ENGINE_CMD_TABLESPACE_CMD_EXECUTOR_ -#define OCEANBASE_SQL_ENGINE_CMD_TABLESPACE_CMD_EXECUTOR_ -#include "lib/string/ob_string.h" -#include "lib/container/ob_array_serialization.h" -#include "share/schema/ob_schema_struct.h" - -namespace oceanbase -{ -namespace obrpc -{ -class ObCommonRpcProxy; -struct ObTablespaceDDLArg; -} -namespace sql -{ -class ObExecContext; -class ObCreateTablespaceStmt; -class ObDropTablespaceStmt; -class ObAlterTablespaceStmt; -class ObCreateTablespaceExecutor -{ -public: - ObCreateTablespaceExecutor() {} - virtual ~ObCreateTablespaceExecutor() {} - int execute(ObExecContext &ctx, ObCreateTablespaceStmt &stmt); -private: - DISALLOW_COPY_AND_ASSIGN(ObCreateTablespaceExecutor); -}; - - -class ObAlterTablespaceExecutor -{ -public: - ObAlterTablespaceExecutor() {} - virtual ~ObAlterTablespaceExecutor() {} - int execute(ObExecContext &ctx, ObAlterTablespaceStmt &stmt); -private: - DISALLOW_COPY_AND_ASSIGN(ObAlterTablespaceExecutor); -}; - - -class ObDropTablespaceExecutor -{ -public: - ObDropTablespaceExecutor() {} - virtual ~ObDropTablespaceExecutor() {} - int execute(ObExecContext &ctx, ObDropTablespaceStmt &stmt); -private: - DISALLOW_COPY_AND_ASSIGN(ObDropTablespaceExecutor); -}; -} -} -#endif //OCEANBASE_SQL_ENGINE_CMD_USER_CMD_EXECUTOR_ diff --git a/src/sql/engine/cmd/ob_tcl_executor.cpp b/src/sql/engine/cmd/ob_tcl_executor.cpp index 5c9604c4e..8a8bf95f4 100644 --- a/src/sql/engine/cmd/ob_tcl_executor.cpp +++ b/src/sql/engine/cmd/ob_tcl_executor.cpp @@ -22,6 +22,10 @@ #include "sql/resolver/tcl/ob_start_trans_stmt.h" #include "sql/resolver/tcl/ob_savepoint_stmt.h" #include "sql/engine/ob_exec_context.h" +#ifdef OB_BUILD_ORACLE_PL +#include "pl/sys_package/ob_dbms_xa.h" +#include "sql/dblink/ob_tm_service.h" +#endif namespace oceanbase { @@ -44,6 +48,50 @@ int ObEndTransExecutor::end_trans(ObExecContext &ctx, ObEndTransStmt &stmt) LOG_ERROR("session ptr is null", K(ret)); } else if (my_session->is_in_transaction() && my_session->associated_xa()) { +#ifdef OB_BUILD_ORACLE_PL + transaction::ObTxDesc *tx_desc = my_session->get_tx_desc(); + const transaction::ObXATransID xid = my_session->get_xid(); + const transaction::ObGlobalTxType global_tx_type = tx_desc->get_global_tx_type(xid); + if (transaction::ObGlobalTxType::XA_TRANS == global_tx_type) { + if (stmt.get_is_rollback()) { + // rollback can be executed in xa trans + // NOTE that rollback does not finish the xa trans, + // it only rollbacks all actions of the trans + if (OB_FAIL(pl::ObDbmsXA::xa_rollback_origin_savepoint(ctx))) { + LOG_WARN("rollback xa changes failed", K(ret), K(xid), K(global_tx_type)); + } + } else { + // commit is not allowed in xa trans + ret = OB_TRANS_XA_ERR_COMMIT; + LOG_WARN("COMMIT is not allowed in a xa trans", K(ret), K(xid), K(global_tx_type), + KPC(tx_desc)); + } + } else if (transaction::ObGlobalTxType::DBLINK_TRANS == global_tx_type) { + transaction::ObTransID tx_id; + if (stmt.get_is_rollback()) { + if (OB_FAIL(ObTMService::tm_rollback(ctx, tx_id))) { + LOG_WARN("fail to do rollback for dblink trans", K(ret), K(tx_id), K(xid), + K(global_tx_type)); + } + my_session->restore_auto_commit(); + } else { + if (OB_FAIL(ObTMService::tm_commit(ctx, tx_id))) { + LOG_WARN("fail to do commit for dblink trans", K(ret), K(tx_id), K(xid), + K(global_tx_type)); + } + my_session->restore_auto_commit(); + } + const bool force_disconnect = false; + int tmp_ret = OB_SUCCESS; + if (OB_UNLIKELY(OB_SUCCESS != (tmp_ret = my_session->get_dblink_context().clean_dblink_conn(force_disconnect)))) { + LOG_WARN("dblink transaction failed to release dblink connections", K(tmp_ret), K(tx_id), K(xid)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected global trans type", K(ret), K(xid), K(global_tx_type), KPC(tx_desc)); + } + ctx.set_need_disconnect(false); +#endif } else if (OB_FAIL(ObSqlTransControl::explicit_end_trans(ctx, stmt.get_is_rollback(), stmt.get_hint()))) { LOG_WARN("fail end trans", K(ret)); } diff --git a/src/sql/engine/dml/ob_link_dml_op.cpp b/src/sql/engine/dml/ob_link_dml_op.cpp index 094567ce6..2d292ba90 100644 --- a/src/sql/engine/dml/ob_link_dml_op.cpp +++ b/src/sql/engine/dml/ob_link_dml_op.cpp @@ -110,6 +110,8 @@ int ObLinkDmlOp::inner_execute_link_stmt(const char *link_stmt) if (OB_ISNULL(dblink_proxy_) || OB_ISNULL(dblink_conn_) || OB_ISNULL(link_stmt)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("dblink_proxy or link_stmt is NULL", K(ret), KP(dblink_proxy_), KP(link_stmt), KP(dblink_conn_)); + } else if (OB_FAIL(ObTMService::tm_rm_start(ctx_, link_type_, dblink_conn_, tx_id))) { + LOG_WARN("failed to tm_rm_start", K(ret), K(dblink_id_), K(dblink_conn_)); } else if (MY_SPEC.is_reverse_link_ && OB_FAIL(send_reverse_link_info(tx_id))) { LOG_WARN("failed to send reverse link info", K(ret), K(link_stmt)); } else if (OB_FAIL(dblink_proxy_->dblink_write(dblink_conn_, affected_rows_, link_stmt))) { diff --git a/src/sql/engine/expr/ob_datum_cast.cpp b/src/sql/engine/expr/ob_datum_cast.cpp index 1d2a711b1..8c19d5834 100644 --- a/src/sql/engine/expr/ob_datum_cast.cpp +++ b/src/sql/engine/expr/ob_datum_cast.cpp @@ -30,7 +30,12 @@ #include "sql/engine/expr/ob_expr_json_func_helper.h" #include "lib/geo/ob_geometry_cast.h" #include "sql/engine/expr/ob_geo_expr_utils.h" - +#ifdef OB_BUILD_ORACLE_XML +#include "lib/xml/ob_xml_util.h" +#include "sql/engine/expr/ob_expr_xml_func_helper.h" +#endif +#include "pl/ob_pl.h" +#include "pl/ob_pl_user_type.h" namespace oceanbase { namespace sql @@ -7803,7 +7808,57 @@ CAST_FUNC_NAME(string, udt) { EVAL_STRING_ARG() { +#ifdef OB_BUILD_ORACLE_XML + const ObObjMeta &in_obj_meta = expr.args_[0]->obj_meta_; + ObObjType in_type = expr.args_[0]->datum_meta_.type_; + ObObjType out_type = expr.datum_meta_.type_; + ObCollationType in_cs_type; + ObCollationType out_cs_type = expr.datum_meta_.cs_type_; + ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); + common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); + ObString in_str(child_res->len_, child_res->ptr_); + ObDatum t_res_datum; + ObMulModeMemCtx* mem_ctx = nullptr; + ObXmlDocument* doc = nullptr; + ObString xml_plain_text; + if (OB_FAIL(ObXmlUtil::create_mulmode_tree_context(&temp_allocator, mem_ctx))) { + LOG_WARN("fail to create tree memory context", K(ret)); + } else if (in_obj_meta.is_string_type()) { + // first step cs_type transform + in_cs_type = expr.args_[0]->datum_meta_.cs_type_; + if (ObCharset::charset_type_by_coll(in_cs_type) != CHARSET_UTF8MB4) { + bool has_set_res = false; + OZ(common_string_string(expr, in_type, in_cs_type, ObObjType::ObVarcharType, + CS_TYPE_UTF8MB4_BIN, in_str, ctx, t_res_datum, has_set_res)); + } else { + OZ(common_copy_string(expr, in_str, ctx, t_res_datum)); + } + + // second step xmlparse document + xml_plain_text = t_res_datum.get_string(); + ObXmlParser parser(mem_ctx); + if (OB_FAIL(ret)) { + } else if (OB_FAIL(parser.parse_document(xml_plain_text))) { + ret = OB_ERR_XML_PARSE; + LOG_USER_ERROR(OB_ERR_XML_PARSE); + LOG_WARN("parse xml plain text as document failed.", K(xml_plain_text)); + } else { + doc = parser.document(); + } + + if (OB_FAIL(ret)) { + } else if (!doc->get_encoding().empty() || doc->get_encoding_flag()) { + doc->set_encoding(ObXmlUtil::get_charset_name(in_cs_type)); + } + if (OB_SUCC(ret) && OB_FAIL(ObXMLExprHelper::pack_xml_res(expr, ctx, res_datum, doc, mem_ctx, + M_DOCUMENT, + xml_plain_text))) { + LOG_WARN("pack_xml_res failed", K(ret)); + } + } +#else ret = OB_NOT_SUPPORTED; +#endif } return ret; } @@ -7813,7 +7868,49 @@ CAST_FUNC_NAME(udt, string) // udt(xmltype) can be null: select dump(xmlparse(document NULL)) from dual; EVAL_STRING_ARG() { +#ifdef OB_BUILD_ORACLE_XML + const ObObjMeta &in_obj_meta = expr.args_[0]->obj_meta_; + const ObObjMeta &out_obj_meta = expr.obj_meta_; + if (in_obj_meta.is_xml_sql_type()) { + ObString blob_data = child_res->get_string(); + ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); + common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); + ObIMulModeBase *xml_root = NULL; + ObStringBuffer xml_plain_text(&temp_allocator); + ObCollationType session_cs_type = CS_TYPE_UTF8MB4_BIN; + GET_SESSION() { + session_cs_type = session->get_nls_collation(); + } + if (OB_FAIL(ObTextStringHelper::read_real_string_data(&temp_allocator, + ObLongTextType, + CS_TYPE_BINARY, + true, + blob_data))) { + LOG_WARN("fail to get real data.", K(ret), K(blob_data)); + } else if (OB_FAIL(ObXmlUtil::cast_to_string(blob_data, temp_allocator, xml_plain_text, session_cs_type))) { + LOG_WARN("failed to convert xml to string", K(ret), KP(blob_data.ptr()), K(blob_data.length())); + } else { // use clob before xml binary implemented + ObObjType in_type = ObLongTextType; + ObObjType out_type = expr.datum_meta_.type_; + ObCollationType in_cs_type = CS_TYPE_UTF8MB4_BIN; + ObCollationType out_cs_type = expr.datum_meta_.cs_type_; + bool has_set_res = false; + OZ(common_string_string(expr, in_type, in_cs_type, out_type, + out_cs_type, xml_plain_text.string(), ctx, res_datum, has_set_res)); + const ObString res_str = res_datum.get_string(); // res str need deep copy in pl mode + if (OB_SUCC(ret) && OB_FAIL(common_copy_string(expr, res_str, ctx, res_datum))) { + LOG_WARN("fail to deep copy str", K(ret)); + } + } + } else { + ret = OB_ERR_INVALID_TYPE_FOR_OP; + LOG_WARN_RET(OB_ERR_INVALID_TYPE_FOR_OP, "inconsistent datatypes", + "expected", out_obj_meta.get_type(), "got", in_obj_meta.get_type(), + K(in_obj_meta.get_subschema_id())); + } +#else ret = OB_NOT_SUPPORTED; +#endif } return ret; } @@ -7822,7 +7919,69 @@ CAST_FUNC_NAME(pl_extend, string) { EVAL_STRING_ARG() { +#ifdef OB_BUILD_ORACLE_XML + const ObObjMeta &in_obj_meta = expr.args_[0]->obj_meta_; + const ObObjMeta &out_obj_meta = expr.obj_meta_; + if (pl::PL_OPAQUE_TYPE == in_obj_meta.get_extend_type()) { + pl::ObPLOpaque *pl_src = reinterpret_cast(child_res->get_ext()); + if (OB_ISNULL(pl_src)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("failed to get pl data type info", K(ret), K(in_obj_meta)); + } else if (pl_src->get_type() == pl::ObPLOpaqueType::PL_XML_TYPE) { + pl::ObPLXmlType * xmltype = static_cast(pl_src); + ObObj *blob_obj = xmltype->get_data(); + if (OB_ISNULL(blob_obj)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("Unexpected xml data", K(ret), K(*xmltype)); + } else { + ObString blob_data = blob_obj->get_string(); + ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); + common::ObArenaAllocator &temp_allocator = tmp_alloc_g.get_allocator(); + ObStringBuffer xml_plain_text(&temp_allocator); + ObCollationType session_cs_type = CS_TYPE_UTF8MB4_BIN; + GET_SESSION() { + session_cs_type = session->get_nls_collation(); + } + if (OB_FAIL(ObTextStringHelper::read_real_string_data(&temp_allocator, + ObLongTextType, + CS_TYPE_BINARY, + true, + blob_data))) { + LOG_WARN("fail to get real data.", K(ret), K(blob_data)); + } else if (OB_FAIL(ObXmlUtil::cast_to_string(blob_data, temp_allocator, xml_plain_text, session_cs_type))) { + LOG_WARN("failed to convert xml to string", K(ret), KP(blob_data.ptr()), K(blob_data.length())); + } else { // use clob before xml binary implemented + ObObjType in_type = ObLongTextType; + ObObjType out_type = expr.datum_meta_.type_; + ObCollationType in_cs_type = CS_TYPE_UTF8MB4_BIN; + ObCollationType out_cs_type = expr.datum_meta_.cs_type_; + bool has_set_res = false; + OZ(common_string_string(expr, in_type, in_cs_type, out_type, + out_cs_type, xml_plain_text.string(), ctx, res_datum, has_set_res)); + const ObString res_str = res_datum.get_string(); // res str need deep copy in pl mode + if (OB_SUCC(ret) && OB_FAIL(common_copy_string(expr, res_str, ctx, res_datum))) { + LOG_WARN("fail to deep copy str", K(ret)); + } + } + } + } else if (pl_src->get_type() == pl::ObPLOpaqueType::PL_INVALID) { + // possibly an un-initiated pl variable, for example, xml_data and xml_data2 are only declared + // then call this directly: select replace(xml_data,xml_data2 ,'1') into stringval from dual; + res_datum.set_null(); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected type to convert pl format", + K(ret), K(pl_src->get_type()), K(in_obj_meta), K(out_obj_meta)); + } + } else { + ret = OB_ERR_INVALID_TYPE_FOR_OP; + LOG_WARN_RET(OB_ERR_INVALID_TYPE_FOR_OP, "inconsistent datatypes", + "expected", out_obj_meta.get_type(), "got", in_obj_meta.get_type(), + K(in_obj_meta.get_subschema_id())); + } +#else ret = OB_NOT_SUPPORTED; +#endif } return ret; } @@ -7834,7 +7993,62 @@ CAST_FUNC_NAME(sql_udt, pl_extend) // For PL extend type, detaield udt id is stored in accurcy_ before code generation, // then only existed in the data after cg. int ret = OB_SUCCESS; +#ifdef OB_BUILD_ORACLE_XML + ObDatum *child_res = NULL; + if (OB_FAIL(expr.args_[0]->eval(ctx, child_res))) { + LOG_WARN("eval arg failed", K(ret), K(ctx)); + } else { + const ObObjMeta &in_obj_meta = expr.args_[0]->obj_meta_; + const ObObjMeta &out_obj_meta = expr.obj_meta_; + if (in_obj_meta.is_xml_sql_type()) { + pl::ObPLXmlType *xmltype = NULL; + void *ptr = NULL; + ObObj* data = NULL; // obobj for blob; + ObIAllocator &allocator = ctx.exec_ctx_.get_allocator(); + if (OB_ISNULL(ptr = allocator.alloc(sizeof(pl::ObPLXmlType)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("Failed to allocate memory for pl xml data type", K(ret), K(sizeof(pl::ObPLXmlType))); + } else if (FALSE_IT(xmltype = new (ptr)pl::ObPLXmlType())) { + } else if (OB_ISNULL(data = static_cast(allocator.alloc(sizeof(ObObj))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory for pl object", K(ret)); + } else { + ObString xml_data; + + if (!child_res->is_null() && !child_res->get_string().empty()) { + /* + We believe that the memory in child_res is reliable, + and the content in it can be passed to the next layer without copying. + Therefore, there is no deep copy of row-level memory here. + If you find that the memory of xml_data is unstable later, + you can add a deep copy here to copy the data to the memory of expr: get_str_res_mem(). + */ + xml_data = child_res->get_string(); + data->set_string(ObLongTextType, xml_data.ptr(), xml_data.length()); + data->set_has_lob_header(); // must has lob header + data->set_collation_type(CS_TYPE_UTF8MB4_BIN); + } else { + data->set_null(); + } + if (OB_SUCC(ret)) { + ObObj result_obj; + xmltype->set_data(data); + result_obj.set_extend(reinterpret_cast(xmltype), pl::PL_OPAQUE_TYPE); + if (OB_FAIL(res_datum.from_obj(result_obj, expr.obj_datum_map_))) { // check obj_datum_map_ + LOG_WARN("failed to set extend datum from obj", K(ret)); + } + } + } + } else { + ret = OB_ERR_INVALID_TYPE_FOR_OP; + LOG_WARN_RET(OB_ERR_INVALID_TYPE_FOR_OP, "inconsistent datatypes", + "expected", out_obj_meta.get_type(), "got", in_obj_meta.get_type(), + K(in_obj_meta.get_subschema_id())); + } + } +#else ret = OB_NOT_SUPPORTED; +#endif return ret; } @@ -7843,7 +8057,55 @@ CAST_FUNC_NAME(pl_extend, sql_udt) // Convert sql udt type to pl udt type, currently only xmltype is supported EVAL_STRING_ARG() { +#ifdef OB_BUILD_ORACLE_XML + const ObObjMeta &in_obj_meta = expr.args_[0]->obj_meta_; + const ObObjMeta &out_obj_meta = expr.obj_meta_; + if (pl::PL_OPAQUE_TYPE == in_obj_meta.get_extend_type()) { + pl::ObPLOpaque *pl_src = reinterpret_cast(child_res->get_ext()); + if (OB_ISNULL(pl_src)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("failed to get pl data type info", K(ret), K(in_obj_meta)); + } else if (pl_src->get_type() == pl::ObPLOpaqueType::PL_XML_TYPE) { + if(!out_obj_meta.is_xml_sql_type()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected pl xml to sql udt type", K(ret), K(in_obj_meta), K(out_obj_meta)); + } else { + pl::ObPLXmlType * xmltype = static_cast(pl_src); + ObObj *blob_obj = xmltype->get_data(); + if (OB_ISNULL(blob_obj) || blob_obj->is_null()) { + res_datum.set_null(); + } else { + // deep copy here or by the caller ? + ObString xml_data = blob_obj->get_string(); + int64_t xml_data_size = xml_data.length(); + char *xml_data_buff = expr.get_str_res_mem(ctx, xml_data_size); + if (OB_ISNULL(xml_data_buff)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory for xmldata", + K(ret), K(out_obj_meta), K(xml_data_size)); + } else { + MEMCPY(xml_data_buff, xml_data.ptr(), xml_data_size); + res_datum.set_string(ObString(xml_data_size, xml_data_buff)); + // should not destroy input extends + } + } + } + } else if (pl_src->get_type() == pl::ObPLOpaqueType::PL_INVALID) { + // un-initiated pl opaque variable + res_datum.set_null(); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected type to convert pl format", + K(ret), K(pl_src->get_type()), K(in_obj_meta), K(out_obj_meta)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected type to convert pl udt to sql udt format", + K(ret), K(in_obj_meta), K(out_obj_meta)); + } +#else ret = OB_NOT_SUPPORTED; +#endif } return ret; } diff --git a/src/sql/engine/expr/ob_expr_cast.cpp b/src/sql/engine/expr/ob_expr_cast.cpp index 91b4c0180..63a55846e 100644 --- a/src/sql/engine/expr/ob_expr_cast.cpp +++ b/src/sql/engine/expr/ob_expr_cast.cpp @@ -720,8 +720,84 @@ int ObExprCast::construct_collection(const sql::ObExpr &expr, ObSubQueryIterator *subquery_iter) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL ret = OB_NOT_SUPPORTED; LOG_WARN("not support", K(ret)); +#else + pl::ObPLCollection *coll = NULL; + ObSQLSessionInfo *session = ctx.exec_ctx_.get_my_session(); + ObExecContext &exec_ctx = ctx.exec_ctx_; + ObIAllocator &alloc = ctx.exec_ctx_.get_allocator(); + const ObExprCast::CastMultisetExtraInfo *info = + static_cast(expr.extra_info_); + if (OB_ISNULL(info) || OB_ISNULL(session)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (pl::PL_NESTED_TABLE_TYPE == info->pl_type_) { + coll = OB_NEWx(pl::ObPLNestedTable, (&alloc), info->udt_id_); + } else if (pl::PL_VARRAY_TYPE == info->pl_type_) { + coll = OB_NEWx(pl::ObPLVArray, (&alloc), info->udt_id_); + if (OB_NOT_NULL(coll)) { + static_cast(coll)->set_capacity(info->capacity_); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected collection type to construct", K(info->type_), K(ret)); + } + + // set collection property + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(coll)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate coll", K(ret)); + } else { + pl::ObPLINS *ns = NULL; + const pl::ObUserDefinedType *type = NULL; + const pl::ObCollectionType *collection_type = NULL; + pl::ObElemDesc elem_desc; + pl::ObPLPackageGuard package_guard(session->get_effective_tenant_id()); + pl::ObPLResolveCtx resolve_ctx(alloc, + *session, + *(exec_ctx.get_sql_ctx()->schema_guard_), + package_guard, + *(exec_ctx.get_sql_proxy()), + false); + if (OB_FAIL(package_guard.init())) { + LOG_WARN("failed to init package guard", K(ret)); + } else { + ns = &resolve_ctx; + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(ns->get_user_type(info->udt_id_, type))) { + LOG_WARN("failed to get user type", K(ret), K_(info->udt_id)); + } else if (OB_ISNULL(type) || OB_UNLIKELY(!type->is_collection_type()) + || OB_ISNULL(collection_type = static_cast(type))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected type", K(ret), KPC(type)); + } else if (info->elem_type_.get_meta_type().is_ext()) { + int64_t field_cnt = OB_INVALID_COUNT; + elem_desc.set_obj_type(common::ObExtendType); + if (OB_FAIL(collection_type->get_element_type().get_field_count(*ns, field_cnt))) { + LOG_WARN("failed to get field count", K(ret)); + } else { + elem_desc.set_field_count(field_cnt); + } + } else { + elem_desc.set_data_type(info->elem_type_); + elem_desc.set_field_count(1); + } + if (OB_SUCC(ret)) { + coll->set_element_desc(elem_desc); + coll->set_not_null(info->not_null_); + if (OB_FAIL(fill_element(expr, ctx, res_datum, coll, + ns, collection_type, subquery_row, + subquery_ctx, subquery_iter))) { + LOG_WARN("failed to fill element", K(ret)); + } + } + } +#endif return ret; } @@ -736,8 +812,126 @@ int ObExprCast::fill_element(const sql::ObExpr &expr, ObSubQueryIterator *subquery_iter) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL ret = OB_NOT_SUPPORTED; LOG_WARN("not support", K(ret)); +#else + ObIAllocator &alloc = ctx.exec_ctx_.get_allocator(); + ObDatum *subquery_datum = NULL; + ObExecContext &exec_ctx = ctx.exec_ctx_; + ObSEArray data_arr; + ObSQLSessionInfo *session = ctx.exec_ctx_.get_my_session(); + const ObExprCast::CastMultisetExtraInfo *info = + static_cast(expr.extra_info_); + + // fetch the subquery result + while (OB_SUCC(ret) && OB_SUCC(subquery_iter->get_next_row())) { + if (OB_FAIL(subquery_row[0]->eval(*subquery_ctx, subquery_datum))) { + LOG_WARN("failed to eval subquery", K(ret)); + } else { + const ObDatum &d = *subquery_datum; + ObObj v, v2; + if (OB_FAIL(d.to_obj(v, expr.args_[0]->obj_meta_, expr.args_[0]->obj_datum_map_))) { + LOG_WARN("failed to get obj", K(ret), K(d)); + } else if (info->not_null_) { + if (v.is_null()) { + ret = OB_ERR_NUMERIC_OR_VALUE_ERROR; + LOG_WARN("not null check violated", K(ret)); + } else if (info->elem_type_.get_meta_type().is_ext()) { + ObObjParam v1 = v; + if (OB_FAIL(ObSPIService::spi_check_composite_not_null(&v1))) { + LOG_WARN("failed to check not null", K(ret)); + } + } + } + + if (info->elem_type_.get_meta_type().is_ext() && !v.is_null()) { + OZ (pl::ObUserDefinedType::deep_copy_obj(alloc, + v, + v2, + true)) ; + } else { + OZ (deep_copy_obj(alloc, v, v2)); + } + + OZ (data_arr.push_back(v2)); + } + } + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + } + + // put the subquery result into the collection + if (OB_FAIL(ret)) { + } else if (OB_FAIL(ObSPIService::spi_set_collection(session->get_effective_tenant_id(), + ns, + alloc, + *coll, + data_arr.count()))) { + LOG_WARN("failed to set collection", K(ret)); + } else if (0 != data_arr.count()) { + if (OB_ISNULL(coll->get_data()) || + OB_ISNULL(coll->get_allocator()) || + OB_ISNULL(ns) || + OB_ISNULL(collection_type)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), KPC(coll), K(ns), KPC(collection_type)); + } else if (info->elem_type_.get_meta_type().is_ext()) { + for (int64_t i = 0; OB_SUCC(ret) && i < data_arr.count(); ++i) { + ObObj &v = data_arr.at(i); + if (OB_SUCC(ret)) { + if (v.is_null()) { + // construct a null composite obj + ObObj new_composite; + int64_t ptr = 0; + int64_t init_size = OB_INVALID_SIZE; + if (OB_FAIL(collection_type->get_element_type().newx(*coll->get_allocator(), ns, ptr))) { + LOG_WARN("failed to new element", K(ret)); + } else if (OB_FAIL(collection_type->get_element_type().get_size(*ns, + pl::PL_TYPE_INIT_SIZE, + init_size))) { + LOG_WARN("failed to get size", K(ret)); + } else { + new_composite.set_extend(ptr, collection_type->get_element_type().get_type(), init_size); + static_cast(coll->get_data())[i] = new_composite; + } + } else { + static_cast(coll->get_data())[i] = v; + } + } + } + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < data_arr.count(); ++i) { + ObObj &v = data_arr.at(i); + if (OB_FAIL(ObSPIService::spi_pad_char_or_varchar(session, + info->elem_type_.get_obj_type(), + info->elem_type_.get_accuracy(), + coll->get_allocator(), + &v))) { + LOG_WARN("failed to pad", K(ret)); + } else { + static_cast(coll->get_data())[i] = v; + } + } + } + } + + if (OB_SUCC(ret)) { + ObObj result; + result.set_extend(reinterpret_cast(coll), coll->get_type()); + //Collection constructed here must be recorded and destructed at last + if (OB_FAIL(res_datum.from_obj(result, expr.obj_datum_map_))) { + LOG_WARN("failed to from obj", K(ret)); + } else if (OB_ISNULL(exec_ctx.get_pl_ctx()) && OB_FAIL(exec_ctx.init_pl_ctx())) { + LOG_WARN("failed to init pl ctx", K(ret)); + } else if (OB_ISNULL(exec_ctx.get_pl_ctx())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret)); + } else if (OB_FAIL(exec_ctx.get_pl_ctx()->add(result))) { + LOG_WARN("failed to add result", K(ret)); + } + } +#endif return ret; } @@ -746,8 +940,23 @@ int ObExprCast::eval_cast_multiset(const sql::ObExpr &expr, sql::ObDatum &res_datum) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL ret = OB_NOT_SUPPORTED; LOG_USER_ERROR(OB_NOT_SUPPORTED, "eval cast multiset"); +#else + ObObj result; + // ObObj *data_arr = NULL; + ObExpr **subquery_row = NULL; + ObEvalCtx *subquery_ctx = NULL; + ObSubQueryIterator *subquery_iter = NULL; + if (OB_FAIL(get_subquery_iter(expr, ctx, subquery_row, + subquery_ctx, subquery_iter))) { + LOG_WARN("failed to eval subquery", K(ret)); + } else if (OB_FAIL(construct_collection(expr, ctx, res_datum, subquery_row, + subquery_ctx, subquery_iter))) { + LOG_WARN("failed to construct collection", K(ret)); + } +#endif return ret; } @@ -756,8 +965,52 @@ int ObExprCast::cg_cast_multiset(ObExprCGCtx &op_cg_ctx, ObExpr &rt_expr) const { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL ret = OB_NOT_SUPPORTED; LOG_USER_ERROR(OB_NOT_SUPPORTED, "cast multiset"); +#else + ObSchemaChecker schema_checker; + const share::schema::ObUDTTypeInfo *dest_info = NULL; + const int udt_id = raw_expr.get_udt_id(); + const uint64_t dest_tenant_id = pl::get_tenant_id_by_object_id(udt_id); + const pl::ObUserDefinedType *pl_type = NULL; + const pl::ObCollectionType *coll_type = NULL; + ObIAllocator &alloc = *op_cg_ctx.allocator_; + if (OB_ISNULL(op_cg_ctx.schema_guard_) || OB_ISNULL(op_cg_ctx.session_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null", K(ret), K(op_cg_ctx.schema_guard_), K(op_cg_ctx.session_)); + } else if (OB_FAIL(schema_checker.init(*op_cg_ctx.schema_guard_, op_cg_ctx.session_->get_sessid()))) { + LOG_WARN("init schema checker failed", K(ret)); + } else if (OB_FAIL(schema_checker.get_udt_info(dest_tenant_id, udt_id, dest_info))) { + LOG_WARN("failed to get udt info", K(ret)); + } else if (OB_ISNULL(dest_info)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get null udt info", K(ret)); + } else if (OB_FAIL(dest_info->transform_to_pl_type(alloc, pl_type))) { + LOG_WARN("failed to get pl type", K(ret)); + } else if (!pl_type->is_collection_type() || + OB_ISNULL(coll_type = static_cast(pl_type))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected pl type", K(ret), KPC(pl_type)); + } else { + CastMultisetExtraInfo *info = OB_NEWx(CastMultisetExtraInfo, (&alloc), alloc, T_FUN_SYS_CAST); + info->pl_type_ = coll_type->get_type(); + info->not_null_ = coll_type->get_element_type().get_not_null(); + if (coll_type->get_element_type().is_obj_type()) { + CK(OB_NOT_NULL(coll_type->get_element_type().get_data_type())); + OX(info->elem_type_ = *coll_type->get_element_type().get_data_type()); + } else { + info->elem_type_.set_obj_type(ObExtendType); + } + info->capacity_ = coll_type->is_varray_type() ? + static_cast(coll_type)->get_capacity() : + OB_INVALID_SIZE; + info->udt_id_ = udt_id; + rt_expr.eval_func_ = eval_cast_multiset; + rt_expr.eval_batch_func_ = cast_eval_arg_batch; + rt_expr.extra_info_ = info; + } +#endif return ret; } diff --git a/src/sql/engine/expr/ob_expr_collection_construct.cpp b/src/sql/engine/expr/ob_expr_collection_construct.cpp index c39626c29..2e60e2378 100644 --- a/src/sql/engine/expr/ob_expr_collection_construct.cpp +++ b/src/sql/engine/expr/ob_expr_collection_construct.cpp @@ -159,8 +159,195 @@ int ObExprCollectionConstruct::eval_collection_construct(const ObExpr &expr, ObDatum &expr_datum) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL ret = OB_NOT_SUPPORTED; LOG_WARN("not support", K(ret)); +#else + pl::ObPLCollection *coll = NULL; + auto session = ctx.exec_ctx_.get_my_session(); + auto &exec_ctx = ctx.exec_ctx_; + const ExtraInfo *info = static_cast(expr.extra_info_); + CK(NULL != info); + // check types + for (int64_t i = 0; OB_SUCC(ret) && i < expr.arg_cnt_; i++) { + const ObExpr *e = expr.args_[i]; + if (ObNullType != e->datum_meta_.type_ + && e->datum_meta_.type_ != info->elem_type_.get_obj_type()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("check type failed", K(ret), K(*e), K(info->elem_type_)); + } + } + + ObIAllocator &alloc = ctx.exec_ctx_.get_allocator(); + if (OB_SUCC(ret)) { + if (pl::PL_NESTED_TABLE_TYPE == info->type_) { + if (NULL == (coll = + static_cast(alloc.alloc(sizeof(pl::ObPLNestedTable) + 8)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory for pl collection", K(ret), K(coll)); + } else { + coll = new(coll)pl::ObPLNestedTable(info->udt_id_); + } + } else if (pl::PL_VARRAY_TYPE == info->type_) { + if (NULL == (coll = + static_cast(alloc.alloc(sizeof(pl::ObPLVArray) + 8)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory for pl collection", K(ret), K(coll)); + } else { + coll = new(coll)pl::ObPLVArray(info->udt_id_); + static_cast(coll)->set_capacity(info->capacity_); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected collection type to construct", K(info->type_), K(ret)); + } + } + + OZ(expr.eval_param_value(ctx)); + + for (int64_t i = 0; OB_SUCC(ret) && info->not_null_ && i < expr.arg_cnt_; ++i) { + if (expr.args_[i]->locate_expr_datum(ctx).is_null()) { + ret = OB_ERR_NUMERIC_OR_VALUE_ERROR; + LOG_WARN("not null check violated", K(coll), + K(coll->is_not_null()), + K(coll->get_count()), + K(ret)); + } + } + + if (OB_SUCC(ret)) { + const pl::ObUserDefinedType *type = NULL; + const pl::ObCollectionType *collection_type = NULL; + pl::ObElemDesc elem_desc; + pl::ObPLPackageGuard package_guard(session->get_effective_tenant_id()); + pl::ObPLResolveCtx resolve_ctx(alloc, + *session, + *(exec_ctx.get_sql_ctx()->schema_guard_), + package_guard, + *(exec_ctx.get_sql_proxy()), + false); + OZ (package_guard.init()); + pl::ObPLINS *ns = NULL; + if (NULL == session->get_pl_context()) { + ns = &resolve_ctx; + } else { + ns = session->get_pl_context()->get_current_ctx(); + } + + if (info->elem_type_.get_meta_type().is_ext()) { + int64_t field_cnt = OB_INVALID_COUNT; + CK (OB_NOT_NULL(ns)); + OZ (ns->get_user_type(info->udt_id_, type)); + OV (OB_NOT_NULL(type), OB_ERR_UNEXPECTED, K(info->udt_id_)); + CK (type->is_collection_type()); + CK (OB_NOT_NULL(collection_type = static_cast(type))); + OX (elem_desc.set_obj_type(common::ObExtendType)); + OX (elem_desc.set_pl_type(collection_type->get_element_type().get_type())); + OZ (collection_type->get_element_type().get_field_count(*ns, field_cnt)); + OX (elem_desc.set_field_count(field_cnt)); + OX (elem_desc.set_udt_id(collection_type->get_element_type().get_user_type_id())); + } else { + OX (elem_desc.set_data_type(info->elem_type_)); + OX (elem_desc.set_field_count(1)); + } + OX (coll->set_element_desc(elem_desc)); + OX (coll->set_not_null(info->not_null_)); + + // recursive construction is not needed + OZ (ObSPIService::spi_set_collection(session->get_effective_tenant_id(), + ns, + alloc, + *coll, + expr.arg_cnt_)); + if (OB_SUCC(ret)) { + if (info->elem_type_.get_meta_type().is_ext()) { + for (int64_t i = 0; OB_SUCC(ret) && i < expr.arg_cnt_; ++i) { + const ObDatum &d = expr.args_[i]->locate_expr_datum(ctx); + ObObj v; + CK (OB_NOT_NULL(coll->get_data())); + OZ(d.to_obj(v, expr.args_[i]->obj_meta_, expr.args_[i]->obj_datum_map_)); + // check type match first if value is not null + if (OB_SUCC(ret) && !v.is_null()) { + if (pl::PL_OPAQUE_TYPE == v.get_meta().get_extend_type() + || pl::PL_CURSOR_TYPE == v.get_meta().get_extend_type() + || pl::PL_REF_CURSOR_TYPE == v.get_meta().get_extend_type()) { + if (v.get_meta().get_extend_type() != coll->get_element_desc().get_pl_type()) { + ret = OB_ERR_CALL_WRONG_ARG; + LOG_WARN("invalid argument. unexpected composite value", K(ret), K(v), KPC(coll)); + } + } else { + TYPE_CHECK(v, info->elem_type_.get_meta_type().get_type()); + OZ (check_match(v, coll->get_element_desc(), *ns)); + } + } + if (OB_SUCC(ret) && info->not_null_ && !v.is_null()) { + ObObjParam v1 = v; + OZ (ObSPIService::spi_check_composite_not_null(&v1)); + } + if (OB_SUCC(ret)) { + if (v.is_null()) { + ObObj new_composite; + int64_t ptr = 0; + int64_t init_size = OB_INVALID_SIZE; + OZ (collection_type->get_element_type().newx(*coll->get_allocator(), ns, ptr)); + OZ (collection_type->get_element_type().get_size(*ns, pl::PL_TYPE_INIT_SIZE, init_size)); + OX (new_composite.set_extend(ptr, collection_type->get_element_type().get_type(), init_size)); + OX (static_cast(coll->get_data())[i] = new_composite); + } else if (pl::PL_OPAQUE_TYPE == v.get_meta().get_extend_type() + || pl::PL_CURSOR_TYPE == v.get_meta().get_extend_type() + || pl::PL_REF_CURSOR_TYPE == v.get_meta().get_extend_type()) { + OZ (pl::ObUserDefinedType::deep_copy_obj(*coll->get_allocator(), + v, + static_cast(coll->get_data())[i], + true)); + } else { + OZ (pl::ObUserDefinedType::deep_copy_obj(*coll->get_allocator(), + v, + static_cast(coll->get_data())[i], + true)); + } + } + } + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < expr.arg_cnt_; ++i) { + const ObDatum &d = expr.args_[i]->locate_expr_datum(ctx); + ObObj v; + CK (OB_NOT_NULL(coll->get_data())); + OZ(d.to_obj(v, expr.args_[i]->obj_meta_, expr.args_[i]->obj_datum_map_)); + if (OB_SUCC(ret) && !v.is_null()) { + TYPE_CHECK(v, info->elem_type_.get_meta_type().get_type()); + } + OZ (ObSPIService::spi_pad_char_or_varchar(session, + info->elem_type_.get_obj_type(), + info->elem_type_.get_accuracy(), + coll->get_allocator(), + &v)); + OZ (deep_copy_obj(*coll->get_allocator(), + v, + static_cast(coll->get_data())[i])); + } + } + } + + ObObj result; + result.set_extend(reinterpret_cast(coll), coll->get_type()); + OZ(expr_datum.from_obj(result, expr.obj_datum_map_)); + //Collection constructed here must be recorded and destructed at last + if (OB_NOT_NULL(coll->get_allocator())) { + int tmp_ret = OB_SUCCESS; + if (OB_ISNULL(exec_ctx.get_pl_ctx())) { + tmp_ret = exec_ctx.init_pl_ctx(); + } + if (OB_SUCCESS == tmp_ret && OB_NOT_NULL(exec_ctx.get_pl_ctx())) { + tmp_ret = exec_ctx.get_pl_ctx()->add(result); + } + if (OB_SUCCESS != tmp_ret) { + LOG_ERROR("fail to collect pl collection allocator, may be exist memory issue", K(tmp_ret)); + } + ret = OB_SUCCESS == ret ? tmp_ret : ret; + } + } +#endif return ret; } diff --git a/src/sql/engine/expr/ob_expr_extract_value.cpp b/src/sql/engine/expr/ob_expr_extract_value.cpp index d8cc80ff0..cd2d32333 100644 --- a/src/sql/engine/expr/ob_expr_extract_value.cpp +++ b/src/sql/engine/expr/ob_expr_extract_value.cpp @@ -13,8 +13,12 @@ #include "ob_expr_extract_value.h" #include "ob_expr_lob_utils.h" +#ifdef OB_BUILD_ORACLE_XML +#include "lib/xml/ob_xml_parser.h" +#include "lib/xml/ob_xml_util.h" +#include "sql/engine/expr/ob_expr_xml_func_helper.h" +#endif #include "lib/utility/utility.h" -#include "ob_expr_xml_func_helper.h" #include "sql/session/ob_sql_session_info.h" #include "sql/engine/ob_exec_context.h" @@ -82,6 +86,285 @@ int ObExprExtractValue::calc_result_typeN(ObExprResType &type, return ret; } +#ifdef OB_BUILD_ORACLE_XML +int ObExprExtractValue::eval_extract_value(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) +{ + int ret = OB_SUCCESS; + ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); + common::ObArenaAllocator &allocator = tmp_alloc_g.get_allocator(); + ObDatum *xml_datum = NULL; + ObString xpath_str; + ObString namespace_str; + ObIMulModeBase *xml_doc = NULL; + ObPathExprIter xpath_iter(&allocator); + ObString default_ns; + ObPathVarObject *prefix_ns = NULL; + ObNodeMemType expect_type = ObNodeMemType::BINARY_TYPE; + ObString xml_res; + ObCollationType cs_type = CS_TYPE_INVALID; + ObMulModeMemCtx* xml_mem_ctx = nullptr; + lib::ObMallocHookAttrGuard malloc_guard(lib::ObMemAttr(MTL_ID(), "XMLCodeGen")); + + if (OB_FAIL(ObXmlUtil::create_mulmode_tree_context(&allocator, xml_mem_ctx))) { + LOG_WARN("fail to create tree memory context", K(ret)); + } else if (OB_UNLIKELY(expr.arg_cnt_ != 2 && expr.arg_cnt_ != 3)) { + ret = OB_ERR_PARAM_SIZE; + LOG_WARN("invalid arg_cnt_", K(ret), K(expr.arg_cnt_)); + } else if (OB_FAIL(ObXMLExprHelper::get_xmltype_from_expr(expr.args_[0], ctx, xml_datum))) { + LOG_WARN("fail to get xml str", K(ret)); + } else if (ObNullType == expr.args_[1]->datum_meta_.type_) { + ret = OB_ERR_INVALID_XPATH_EXPRESSION; + LOG_WARN("invalid xpath expression", K(ret)); + } else if (OB_FAIL(ObXMLExprHelper::get_str_from_expr(expr.args_[1], ctx, xpath_str, allocator))) { + LOG_WARN("fail to get xpath str", K(ret)); + } else if (expr.arg_cnt_ == 3) { + if (ObNullType == expr.args_[2]->datum_meta_.type_) { + } else if (OB_FAIL(ObXMLExprHelper::get_str_from_expr(expr.args_[2], ctx, namespace_str, allocator))) { + LOG_WARN("fail to get xpath str", K(ret)); + } else if (OB_FAIL(ObXMLExprHelper::construct_namespace_params(namespace_str, default_ns, prefix_ns, allocator))) { + LOG_WARN("fail to construct namespace params", K(ret), K(namespace_str)); + } + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(ObXMLExprHelper::get_xml_base(xml_mem_ctx, xml_datum, cs_type, expect_type, xml_doc))) { + LOG_WARN("fail to parse xml doc", K(ret)); + } else if (OB_FAIL(extract_xpath_result(xml_mem_ctx, xpath_str, default_ns, xml_doc, prefix_ns, xml_res))) { + LOG_WARN("fail to extract xpath result", K(ret), K(xpath_str)); + } else if (xml_res.empty()) { + res.set_null(); + } else if (OB_FAIL(ObXMLExprHelper::set_string_result(expr, ctx, res, xml_res))) { + LOG_WARN("fail to set result datum", K(ret)); + } + return ret; +} + +int ObExprExtractValue::extract_xpath_result(ObMulModeMemCtx *xml_mem_ctx, ObString& xpath_str, ObString& default_ns, + ObIMulModeBase* xml_doc, ObPathVarObject* prefix_ns, ObString &xml_res) +{ + int ret = OB_SUCCESS; + ObPathExprIter xpath_iter(xml_mem_ctx->allocator_); + int64_t valid_node_num = 0; // element ,ns, attribute node + int64_t text_node_num = 0; // text, cdata + int64_t skip_node_num = 0; // pi or comment node + ObIMulModeBase *node = NULL; + ObSEArray result_nodes; + if (OB_FAIL(xpath_iter.init(xml_mem_ctx, xpath_str, default_ns, xml_doc, prefix_ns))) { + LOG_WARN("fail to init xpath iterator", K(xpath_str), K(default_ns), K(ret)); + ObXMLExprHelper::replace_xpath_ret_code(ret); + } else if (OB_FAIL(xpath_iter.open())) { + LOG_WARN("fail to open xpath iterator", K(ret)); + ObXMLExprHelper::replace_xpath_ret_code(ret); + } + + while (valid_node_num < 2 && OB_SUCC(ret)) { + if (OB_FAIL(xpath_iter.get_next_node(node))) { + if (ret != OB_ITER_END) { + LOG_WARN("fail to get next xpath result node", K(ret)); + } + } else if (OB_ISNULL(node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("xpath result node is null", K(ret)); + } else if (M_COMMENT == node->type() || M_INSTRUCT == node->type()) { + skip_node_num++; + } else if (ObXMLExprHelper::is_xml_text_node(node->type())) { + text_node_num++; + if (OB_FAIL(result_nodes.push_back(node))) { + LOG_WARN("fail to push back result node", K(ret)); + } + } else if (ObXMLExprHelper::is_xml_element_node(node->type()) || + ObXMLExprHelper::is_xml_attribute_node(node->type())) { + valid_node_num++; + if (OB_FAIL(result_nodes.push_back(node))) { + LOG_WARN("fail to push back result node", K(ret)); + } + } + } // end while + + if (ret == OB_ITER_END) { + ret = OB_SUCCESS; + } + // check if match the rules + if (OB_FAIL(ret)) { + } else if (valid_node_num > 1 || + (valid_node_num == 1 && (skip_node_num > 0 || text_node_num > 0))) { + ret = OB_ERR_EXTRACTVALUE_MULTI_NODES; + LOG_WARN("EXTRACTVALUE returns value of only one node", K(ret)); + } else if (valid_node_num == 0) { + if (skip_node_num > 0 || text_node_num > 0) { + // classify whether has same parent + bool is_same_parent = false; + if (OB_FAIL(has_same_parent_node(xml_mem_ctx, xpath_str, default_ns, xml_doc, prefix_ns, is_same_parent))) { + LOG_WARN("fail to classify whether has same parent", K(ret)); + } else if (!is_same_parent) { + ret = OB_ERR_EXTRACTVALUE_MULTI_NODES; + LOG_WARN("EXTRACTVALUE returns value of only one node", K(ret)); + } + } + } else { + // do nothing when valid_node_num == 1 && skip_node_num > 0 && text_node_num > 0 + } + + // if match, extract the value + if (OB_FAIL(ret)) { + } else if (result_nodes.count() == 1 && + OB_FAIL(extract_node_value(*xml_mem_ctx->allocator_, result_nodes.at(0), xml_res))) { + LOG_WARN("fail to extract node value", K(ret)); + } else if (result_nodes.count() > 1 && + OB_FAIL(merge_text_nodes_with_same_parent(xml_mem_ctx->allocator_, result_nodes, xml_res))) { + LOG_WARN("fail to merge text nodes", K(ret), K(result_nodes.count())); + } + + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = xpath_iter.close())) { + LOG_WARN("fail to close xpath iter", K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + + return ret; +} + +int ObExprExtractValue::merge_text_nodes_with_same_parent(ObIAllocator *allocator, + ObIArray &result_nodes, + ObString &xml_res) +{ + int ret = OB_SUCCESS; + // text-nodes or skip-nodes has same parent + ObIMulModeBase *child_node = NULL; + ObStringBuffer *buffer = NULL; + if (OB_ISNULL(buffer = OB_NEWx(ObStringBuffer, allocator, allocator))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate buffer", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < result_nodes.count(); i++) { + if (OB_ISNULL(child_node = result_nodes.at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("node is NULL", K(ret)); + } else if (OB_FAIL(append_text_value(*buffer, child_node))) { + LOG_WARN("fail to append text value", K(ret)); + } + } // end for + + if (OB_SUCC(ret)) { + xml_res.assign_ptr(buffer->ptr(), buffer->length()); + } + return ret; +} + + +int ObExprExtractValue::append_text_value(ObStringBuffer &buffer, ObIMulModeBase *node) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("node is NULL", K(ret)); + } else if (ObXMLExprHelper::is_xml_leaf_node(node->type())) { + ObString tmp_res; + if (OB_FAIL(node->get_value(tmp_res))) { + LOG_WARN("fail to get node value", K(ret)); + } else if (OB_FAIL(buffer.append(tmp_res))) { + LOG_WARN("fail to append buffer", K(ret), K(buffer)); + } + } + return ret; +} + + +int ObExprExtractValue::extract_node_value(ObIAllocator &allocator, ObIMulModeBase *node, ObString &xml_res) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("extract node is NULL", K(ret)); + } else if (ObXMLExprHelper::is_xml_leaf_node(node->type())) { + if (OB_FAIL(node->get_value(xml_res))) { + LOG_WARN("fail to get node value", K(ret)); + } + } else if (ObXMLExprHelper::is_xml_element_node(node->type())) { + int64_t child_size = node->size(); + if (child_size == 1 && ObXMLExprHelper::is_xml_element_node(node->at(0)->type())) { + ret = OB_EXTRACTVALUE_NOT_LEAF_NODE; + LOG_WARN("EXTRACTVALUE can only retrieve value of leaf node", K(ret)); + } else { + int64_t ele_node_num = 0; + ObIMulModeBase *child_node = NULL; + ObStringBuffer *buffer = NULL; + if (OB_ISNULL(buffer = OB_NEWx(ObStringBuffer, &allocator, (&allocator)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate buffer", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && i < child_size; i++) { + if (OB_ISNULL(child_node = node->at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("node is NULL", K(ret)); + } else if (OB_FAIL(append_text_value(*buffer, child_node))) { + LOG_WARN("fail to append text value", K(ret)); + } else if (ObXMLExprHelper::is_xml_element_node(child_node->type())) { + ele_node_num++; + } else { + // skip other node type + } + } // end for + if (ele_node_num > 0) { + ret = OB_ERR_EXTRACTVALUE_MULTI_NODES; + LOG_WARN("EXTRACTVALUE returns value of only one node", K(ele_node_num)); + } else { + xml_res.assign_ptr(buffer->ptr(), buffer->length()); + } + } + } // type == M_ELEMENT + return ret; +} + +int ObExprExtractValue::has_same_parent_node(ObMulModeMemCtx *xml_mem_ctx, ObString& xpath_str, ObString& default_ns, + ObIMulModeBase* xml_doc, ObPathVarObject* prefix_ns, bool &is_same_parent) +{ + int ret = OB_SUCCESS; + ObPathExprIter xpath_iter(xml_mem_ctx->allocator_); + ObIMulModeBase *node = NULL; + int64_t node_num = 0; + ObStringBuffer buffer(xml_mem_ctx->allocator_); + ObString parent_xpath; + if (OB_FAIL(buffer.append(xpath_str))) { + LOG_WARN("fail to append buffer", K(ret)); + } else if (OB_FAIL(buffer.append("/parent::node()"))) { + LOG_WARN("fail to append buffer", K(ret)); + } else { + parent_xpath.assign_ptr(buffer.ptr(), buffer.length()); + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(xpath_iter.init(xml_mem_ctx, parent_xpath, default_ns, xml_doc, prefix_ns))) { + LOG_WARN("fail to init xpath iterator", K(parent_xpath), K(default_ns), K(ret)); + ObXMLExprHelper::replace_xpath_ret_code(ret); + } else if (OB_FAIL(xpath_iter.open())) { + LOG_WARN("fail to open xpath iterator", K(ret)); + ObXMLExprHelper::replace_xpath_ret_code(ret); + } + while (OB_SUCC(ret)) { + if (OB_FAIL(xpath_iter.get_next_node(node))) { + if (ret != OB_ITER_END) { + LOG_WARN("fail to get next xpath result node", K(ret)); + } + } else { + node_num++; + } + } + + if (ret == OB_ITER_END) { + ret = OB_SUCCESS; + } + + if (node_num > 1) { // has different parent + is_same_parent = false; + } else { + is_same_parent = true; + } + + return ret; +} + +#endif int ObExprExtractValue::cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) const { diff --git a/src/sql/engine/expr/ob_expr_extract_value.h b/src/sql/engine/expr/ob_expr_extract_value.h index d6a785142..ad5f1f4fb 100644 --- a/src/sql/engine/expr/ob_expr_extract_value.h +++ b/src/sql/engine/expr/ob_expr_extract_value.h @@ -15,6 +15,9 @@ #define OCEANBASE_SQL_ENGINE_EXPR_OB_EXPR_EXRACTVALUE_H #include "sql/engine/expr/ob_expr_operator.h" +#ifdef OB_BUILD_ORACLE_XML +#include "lib/xml/ob_xpath.h" +#endif namespace oceanbase { @@ -31,11 +34,27 @@ class ObExprExtractValue : public ObFuncExprOperator int64_t param_num, common::ObExprTypeCtx &type_ctx) const override; +#ifdef OB_BUILD_ORACLE_XML + static int eval_extract_value(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res); +#else static int eval_extract_value(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) { return OB_NOT_SUPPORTED; } +#endif virtual int cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) const override; +#ifdef OB_BUILD_ORACLE_XML +private: +static int extract_xpath_result(ObMulModeMemCtx *xml_mem_ctx, ObString& xpath_str, ObString& default_ns, + ObIMulModeBase* xml_doc, ObPathVarObject* prefix_ns, ObString &xml_res); +static int extract_node_value(ObIAllocator &allocator, ObIMulModeBase *node, ObString &xml_res); +static int has_same_parent_node(ObMulModeMemCtx *xml_mem_ctx, ObString& xpath_str, ObString& default_ns, + ObIMulModeBase* xml_doc, ObPathVarObject* prefix_ns, bool &is_same_parent); +static int merge_text_nodes_with_same_parent(ObIAllocator *allocator, ObIArray &result_nodes, + ObString &xml_res); +static int append_text_value(ObStringBuffer &buffer, ObIMulModeBase *node); + +#endif private: DISALLOW_COPY_AND_ASSIGN(ObExprExtractValue); }; diff --git a/src/sql/engine/expr/ob_expr_extract_xml.cpp b/src/sql/engine/expr/ob_expr_extract_xml.cpp index 2755c989a..51c38a1b4 100644 --- a/src/sql/engine/expr/ob_expr_extract_xml.cpp +++ b/src/sql/engine/expr/ob_expr_extract_xml.cpp @@ -12,8 +12,12 @@ */ #include "ob_expr_extract_xml.h" +#ifdef OB_BUILD_ORACLE_XML +#include "lib/xml/ob_xml_parser.h" +#include "lib/xml/ob_xml_util.h" +#include "sql/engine/expr/ob_expr_xml_func_helper.h" +#endif #include "lib/utility/utility.h" -#include "ob_expr_xml_func_helper.h" #include "sql/session/ob_sql_session_info.h" #include "sql/engine/ob_exec_context.h" @@ -75,6 +79,208 @@ int ObExprExtractXml::calc_result_typeN(ObExprResType &type, return ret; } +#ifdef OB_BUILD_ORACLE_XML +int ObExprExtractXml::eval_extract_xml(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) +{ + int ret = OB_SUCCESS; + ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); + common::ObArenaAllocator &allocator = tmp_alloc_g.get_allocator(); + ObDatum *xml_datum = NULL; + ObString xpath_str; + ObString namespace_str; + ObNodeMemType expect_type = ObNodeMemType::BINARY_TYPE; + ObIMulModeBase *xml_doc = NULL; + ObPathExprIter xpath_iter(&allocator); + ObString default_ns; + ObPathVarObject *prefix_ns = NULL; + ObString xml_res; + ObXmlDocument *root = nullptr; + ObMulModeNodeType node_type = M_MAX_TYPE; + ObString input_str; + ObCollationType cs_type = CS_TYPE_INVALID; + // eval arg + + ObMulModeMemCtx* mem_ctx = nullptr; + lib::ObMallocHookAttrGuard malloc_guard(lib::ObMemAttr(MTL_ID(), "XMLCodeGen")); + if (OB_FAIL(ObXmlUtil::create_mulmode_tree_context(&allocator, mem_ctx))) { + LOG_WARN("fail to create tree memory context", K(ret)); + } else if (OB_UNLIKELY(expr.arg_cnt_ != 3)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid arg_cnt_", K(ret), K(expr.arg_cnt_)); + } else if (OB_FAIL(ObXMLExprHelper::get_xmltype_from_expr(expr.args_[0], ctx, xml_datum))) { + LOG_WARN("fail to get xmltype value", K(ret)); + } else if (ObNullType == expr.args_[1]->datum_meta_.type_) { + ret = OB_ERR_INVALID_XPATH_EXPRESSION; + LOG_WARN("invalid xpath expression", K(ret)); + } else if (OB_FAIL(ObXMLExprHelper::get_str_from_expr(expr.args_[1], ctx, xpath_str, allocator))) { + LOG_WARN("fail to get xpath str", K(ret)); + } else if (ObNullType == expr.args_[2]->datum_meta_.type_) { + } else if (OB_FAIL(ObXMLExprHelper::get_str_from_expr(expr.args_[2], ctx, namespace_str, allocator))) { + LOG_WARN("fail to get xpath str", K(ret)); + } else if (OB_FAIL(ObXMLExprHelper::construct_namespace_params(namespace_str, default_ns, prefix_ns, allocator))) { + LOG_WARN("fail to construct namespace params", K(ret), K(namespace_str)); + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(ObXMLExprHelper::get_xml_base(mem_ctx, xml_datum, cs_type, expect_type, xml_doc))) { + LOG_WARN("fail to parse xml doc", K(ret)); + } else if (OB_FAIL(xpath_iter.init(mem_ctx, xpath_str, default_ns, xml_doc, prefix_ns))) { + LOG_WARN("fail to init xpath iterator", K(xpath_str), K(default_ns), K(ret)); + ObXMLExprHelper::replace_xpath_ret_code(ret); + } else if (OB_FAIL(concat_xpath_result(xpath_iter, cs_type, root, node_type, mem_ctx))) { + LOG_WARN("fail to concat xpath result", K(ret)); + } else if (OB_ISNULL(root) || root->size() == 0) { + // root is not null and size = 0 if xpath='/.' or '//.' or '.' or 'self::*' and so on + res.set_null(); + } else if (OB_FAIL(ObXMLExprHelper::pack_xml_res(expr, ctx, res, root, mem_ctx, node_type, input_str))) { + LOG_WARN("fail to set result", K(xml_res), K(ret)); + } + return ret; +} + +int ObExprExtractXml::concat_xpath_result(ObPathExprIter &xpath_iter, + ObCollationType cs_type, + ObXmlDocument *&root, + ObMulModeNodeType &node_type, + ObMulModeMemCtx* mem_ctx) +{ + int ret = OB_SUCCESS; + ObStringBuffer buff(mem_ctx->allocator_); + ObIMulModeBase *node = NULL; + int64_t append_node_num = 0; + if (OB_FAIL(xpath_iter.open())) { + LOG_WARN("fail to open xpath iterator", K(ret)); + ObXMLExprHelper::replace_xpath_ret_code(ret); + } + + while (OB_SUCC(ret)) { + ObIMulModeBase* tmp = nullptr; + if (OB_FAIL(xpath_iter.get_next_node(node))) { + if (ret != OB_ITER_END) { + LOG_WARN("fail to get next xml node", K(ret)); + } + } else if (OB_ISNULL(node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("xpath result node is null", K(ret)); + } else if (node->is_binary() && OB_FAIL(ObMulModeFactory::transform(mem_ctx, node, TREE_TYPE, node))) { + LOG_WARN("fail to transform to tree", K(ret)); + } else if (OB_ISNULL(root) && node->type() == M_DOCUMENT) { + // if the xpath return document node, set it as root + root = static_cast(static_cast(node)); + } else { + if (OB_ISNULL(root)) { // if root is NULL, alloc a content node as root + if (OB_ISNULL(root = OB_NEWx(ObXmlDocument, (mem_ctx->allocator_), ObMulModeNodeType::M_CONTENT, (mem_ctx)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to create an xml content node", K(ret)); + } + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(append_node_to_res(*mem_ctx->allocator_, root, node))) { + LOG_WARN("fail to append node text to result buffer", K(ret)); + } else { + append_node_num++; + } + } + } + + if (ret == OB_ITER_END) { + ret = OB_SUCCESS; + if (OB_NOT_NULL(root)) { // res is NULL, do nothing + int element_count = 0; + int text_count = 0; + int cdata_count = 0; + + if (node_type == ObMulModeNodeType::M_CONTENT) { // do nothing + } else if (OB_FAIL(root->get_node_count(ObMulModeNodeType::M_ELEMENT, element_count))) { + LOG_WARN("get element count node failed", K(ret)); + } else if (OB_FAIL(root->get_node_count(ObMulModeNodeType::M_TEXT, text_count))) { + LOG_WARN("get text count node failed", K(ret)); + } else if (OB_FAIL(root->get_node_count(ObMulModeNodeType::M_CDATA, cdata_count))) { + LOG_WARN("get cdata count node failed", K(ret)); + } else if (element_count > 1 || element_count == 0) { + node_type = ObMulModeNodeType::M_CONTENT; + } else if (element_count == 1 && (text_count > 0 || cdata_count > 0)) { + node_type = ObMulModeNodeType::M_CONTENT; + } else if (root->size() == 0) { + // do nothing + } else { + if (append_node_num > 0) { + root->set_has_xml_decl(false); + } + node_type = ObMulModeNodeType::M_DOCUMENT; + } + root->set_xml_type(node_type); + } + } + + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = xpath_iter.close())) { + LOG_WARN("fail to close xpath iter", K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + return ret; +} + +int ObExprExtractXml::append_node_to_res(ObIAllocator &allocator, ObXmlDocument *root, ObIMulModeBase *node) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(node) || OB_ISNULL(root)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("the append node is NULL", K(ret)); + } else if (!node->is_tree()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("node type is not tree type", K(ret)); + } else { + ObXmlNode *xml_node = static_cast(node); + ObMulModeNodeType node_type = xml_node->type(); + ObString tmp_str; + if (node_type == M_TEXT || node_type == M_ATTRIBUTE || node_type == M_NAMESPACE) { + // extract attribute node value + ObXmlText *xml_text = NULL; + if (OB_FAIL(xml_node->get_value(tmp_str))) { + LOG_WARN("fail to get node value", K(ret)); + } else { + int64_t child_size = root->size(); + if (child_size > 0 && root->at(child_size-1)->type() == M_TEXT) { + xml_text = static_cast(root->at(child_size-1)); + ObStringBuffer buff(&allocator); + if (OB_FAIL(buff.append(xml_text->get_text()))) { + LOG_WARN("fail to append the orgin text", K(ret)); + } else if (OB_FAIL(buff.append(tmp_str))) { + LOG_WARN("fail to append the new text", K(ret), K(tmp_str)); + } else { + ObString new_str; + new_str.assign_ptr(buff.ptr(), buff.length()); + xml_text->set_text(new_str); + } + } else { + if (OB_ISNULL(xml_text = OB_NEWx(ObXmlText, (&allocator), ObMulModeNodeType::M_TEXT, (root->get_mem_ctx())))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to create xml text node", K(ret)); + } else if (FALSE_IT(xml_text->set_text(tmp_str))) { + } else if (OB_FAIL(root->append(xml_text))) { + LOG_WARN("fail to append node value to content", K(ret), K(tmp_str)); + } + } + } + } else if (ObXMLExprHelper::is_xml_root_node(node_type)) { + ObXmlDocument *xml_doc = static_cast(xml_node); + for (int64_t i = 0; OB_SUCC(ret) && i < xml_doc->size(); i++) { + if (OB_ISNULL(xml_doc->at(i)) ) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("xml node is null", K(ret), K(i)); + } else if (OB_FAIL(root->append(xml_doc->at(i)))) { + LOG_WARN("fail to append node to content", K(ret), K(i)); + } + } + } else if (OB_FAIL(root->append(node))) { + LOG_WARN("fail to append node to content", K(ret), K(node_type)); + } + } + return ret; +} +#endif int ObExprExtractXml::cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) const { diff --git a/src/sql/engine/expr/ob_expr_extract_xml.h b/src/sql/engine/expr/ob_expr_extract_xml.h index 33d1dcda7..938d9dc6e 100644 --- a/src/sql/engine/expr/ob_expr_extract_xml.h +++ b/src/sql/engine/expr/ob_expr_extract_xml.h @@ -15,6 +15,9 @@ #define OCEANBASE_SQL_ENGINE_EXPR_OB_EXPR_EXRACT_XML_H #include "sql/engine/expr/ob_expr_operator.h" +#ifdef OB_BUILD_ORACLE_XML +#include "lib/xml/ob_xpath.h" +#endif namespace oceanbase { @@ -29,11 +32,29 @@ class ObExprExtractXml : public ObFuncExprOperator ObExprResType *types, int64_t param_num, common::ObExprTypeCtx &type_ctx) const override; +#ifdef OB_BUILD_ORACLE_XML + static int eval_extract_xml(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res); +#else static int eval_extract_xml(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) { return OB_NOT_SUPPORTED; } +#endif virtual int cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) const override; +#ifdef OB_BUILD_ORACLE_XML +private: + static int append_node_to_res(ObIAllocator &allocator,ObXmlDocument *root, ObIMulModeBase *node); + static int concat_xpath_result(ObPathExprIter &xpath_iter, + ObCollationType cs_type, + ObXmlDocument *&root, + ObMulModeNodeType &node_type, + ObMulModeMemCtx* ctx); + static int check_and_set_res(const ObExpr &expr, + ObEvalCtx &ctx, + ObDatum &res, + ObLibTreeNodeVector &node_vector, + ObIAllocator &allocator); +#endif private: DISALLOW_COPY_AND_ASSIGN(ObExprExtractXml); }; diff --git a/src/sql/engine/expr/ob_expr_is.cpp b/src/sql/engine/expr/ob_expr_is.cpp index 774849a2c..3d7dec8dc 100644 --- a/src/sql/engine/expr/ob_expr_is.cpp +++ b/src/sql/engine/expr/ob_expr_is.cpp @@ -99,6 +99,24 @@ int ObExprIs::calc_collection_is_null(const ObExpr &expr, ObEvalCtx &ctx, ObDatu } else { uint64_t ext = param->extend_obj_->get_ext(); switch (param->extend_obj_->get_meta().get_extend_type()) { +#ifdef OB_BUILD_ORACLE_PL + case pl::PL_NESTED_TABLE_TYPE: + case pl::PL_ASSOCIATIVE_ARRAY_TYPE: + case pl::PL_VARRAY_TYPE: { + pl::ObPLCollection *collection = reinterpret_cast(ext); + v = OB_ISNULL(collection) ? true : collection->is_collection_null(); + break; + } + case pl::PL_OPAQUE_TYPE: { + pl::ObPLOpaque *opaque = reinterpret_cast(ext); + v = OB_ISNULL(opaque) ? true : opaque->is_invalid(); + break; + } + case pl::PL_CURSOR_TYPE: + case pl::PL_REF_CURSOR_TYPE: { + v = param->extend_obj_->get_ext() == 0; + } break; +#endif case pl::PL_RECORD_TYPE: { pl::ObPLRecord *rec = reinterpret_cast(ext); v = rec->is_null(); diff --git a/src/sql/engine/expr/ob_expr_multiset.cpp b/src/sql/engine/expr/ob_expr_multiset.cpp index 33b0a20b4..62e948b23 100644 --- a/src/sql/engine/expr/ob_expr_multiset.cpp +++ b/src/sql/engine/expr/ob_expr_multiset.cpp @@ -486,9 +486,127 @@ int ObExprMultiSet::cg_expr(ObExprCGCtx &expr_cg_ctx, int ObExprMultiSet::eval_multiset(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSEDx(expr, ctx, res); ret = OB_NOT_SUPPORTED; LOG_WARN("not support udt", K(ret)); +#else + ObDatum *datum1 = nullptr; + ObDatum *datum2 = nullptr; + ObObj result; + const ObExprMultiSetInfo *info = static_cast(expr.extra_info_); + if (OB_FAIL(expr.eval_param_value(ctx, datum1, datum2))) { + LOG_WARN("failed to eval params", K(ret)); + } else if (lib::is_oracle_mode() + && ObExtendType == expr.args_[0]->datum_meta_.type_ + && ObExtendType == expr.args_[1]->datum_meta_.type_) { + ObObj obj1; + ObObj obj2; + if (OB_FAIL(datum1->to_obj(obj1, expr.args_[0]->obj_meta_))) { + LOG_WARN("failed to convert to obj", K(ret)); + } else if (OB_FAIL(datum2->to_obj(obj2, expr.args_[1]->obj_meta_))) { + LOG_WARN("failed to convert to obj", K(ret)); + } else { + pl::ObPLCollection *c1 = reinterpret_cast(obj1.get_ext()); + pl::ObPLCollection *c2 = reinterpret_cast(obj2.get_ext()); + ObIAllocator &allocator = ctx.exec_ctx_.get_allocator(); + ObIAllocator *collection_allocator = NULL; + ObObj *data_arr = NULL; + int64_t elem_count = -1; + pl::ObPLNestedTable *coll = + static_cast(allocator.alloc(sizeof(pl::ObPLNestedTable))); + if (OB_ISNULL(coll)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc memory failed.", K(ret)); + } else if (OB_ISNULL(c1) || OB_ISNULL(c2)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("union udt failed due to null udt", K(ret), K(obj1), K(obj2)); + } else if (pl::PL_NESTED_TABLE_TYPE != c1->get_type() + || pl::PL_NESTED_TABLE_TYPE != c2->get_type()) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, "udt union except nested table"); + LOG_WARN("not support udt union except nested table", K(ret), K(c1), K(c2)); + } else if (c1->get_element_type().get_obj_type() != c2->get_element_type().get_obj_type()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("udt union failed due to uninited", K(ret), KPC(c1), KPC(c2)); + } else if (!c1->is_inited() || !c2->is_inited()) { + // if has uninit collection, result is uninit, so do nothing ... + } else { + coll = new(coll)pl::ObPLNestedTable(c1->get_id()); + collection_allocator = + static_cast(allocator.alloc(sizeof(pl::ObPLCollAllocator))); + collection_allocator = new(collection_allocator)pl::ObPLCollAllocator(coll); + if (OB_ISNULL(collection_allocator)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc pl collection allocator failed.", K(ret)); + } else { + switch (info->ms_type_) { + case MULTISET_TYPE_UNION: + if (OB_FAIL(calc_ms_union(collection_allocator, + c1, c2, data_arr, elem_count, + info->ms_type_, info->ms_modifier_))) { + LOG_WARN("calc multiset union failed.", K(ret)); + } + break; + case MULTISET_TYPE_INTERSECT: + if (OB_FAIL(calc_ms_intersect(collection_allocator, + c1, c2, data_arr, elem_count, + info->ms_type_, info->ms_modifier_))) { + LOG_WARN("calc multiset union failed.", K(ret)); + } + break; + case MULTISET_TYPE_EXCEPT: + if (OB_FAIL(calc_ms_except(collection_allocator, + c1, c2, data_arr, elem_count, + info->ms_type_, info->ms_modifier_))) { + LOG_WARN("calc multiset union failed.", K(ret)); + } + break; + default: + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unknown multiset operation type", K(info->ms_type_), + K(info->ms_modifier_), K(ret)); + break; + } + } + } + if (OB_FAIL(ret)) { + LOG_WARN("failed to calc multiset operator", K(ret), K(data_arr)); + } else if (elem_count > 0 && OB_ISNULL(data_arr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected result.", K(elem_count), K(data_arr), K(ret)); + } else { + coll->set_allocator(collection_allocator); + coll->set_type(c1->get_type()); + coll->set_id(c1->get_id()); + coll->set_is_null(c1->is_null()); + coll->set_element_type(c1->get_element_type()); + coll->set_column_count(c1->get_column_count()); + coll->set_not_null(c1->is_not_null()); + coll->set_count(elem_count); + coll->set_first(elem_count > 0 ? 1 : OB_INVALID_ID); + coll->set_last(elem_count > 0 ? elem_count : OB_INVALID_ID); + coll->set_data(data_arr); + result.set_extend(reinterpret_cast(coll), coll->get_type()); + OZ(res.from_obj(result, expr.obj_datum_map_)); + //Collection constructed here must be recorded and destructed at last + if (OB_NOT_NULL(coll->get_allocator())) { + int tmp_ret = OB_SUCCESS; + if (OB_ISNULL(ctx.exec_ctx_.get_pl_ctx())) { + tmp_ret = ctx.exec_ctx_.init_pl_ctx(); + } + if (OB_SUCCESS == tmp_ret && OB_NOT_NULL(ctx.exec_ctx_.get_pl_ctx())) { + tmp_ret = ctx.exec_ctx_.get_pl_ctx()->add(result); + } + if (OB_SUCCESS != tmp_ret) { + LOG_ERROR("fail to collect pl collection allocator, may be exist memory issue", K(tmp_ret)); + } + ret = OB_SUCCESS == ret ? tmp_ret : ret; + } + } + } + } +#endif return ret; } diff --git a/src/sql/engine/expr/ob_expr_obj_access.cpp b/src/sql/engine/expr/ob_expr_obj_access.cpp index 7a1506599..6c740b642 100644 --- a/src/sql/engine/expr/ob_expr_obj_access.cpp +++ b/src/sql/engine/expr/ob_expr_obj_access.cpp @@ -408,6 +408,68 @@ int ObExprObjAccess::ExtraInfo::calc(ObObj &result, } } OX(result.set_extend(attr_addr, res_type.get_extend_type(), extend_size)); +#ifdef OB_BUILD_ORACLE_PL + } else if (pl::ObCollectionType::INVALID_PROPERTY != property_type_) { + pl::ObPLCollection *coll = reinterpret_cast(attr_addr); + pl::ObPLAssocArray *assoc = + (OB_NOT_NULL(coll) && coll->is_associative_array()) + ? static_cast(coll) : NULL; + CK (OB_NOT_NULL(coll)); + if (OB_SUCC(ret)) { + switch (property_type_) { + case pl::ObCollectionType::FIRST_PROPERTY: { + OZ (OB_NOT_NULL(assoc) ? assoc->first(result) : coll->first(result)); + } break; + case pl::ObCollectionType::LAST_PROPERTY: { + OZ (OB_NOT_NULL(assoc) ? assoc->last(result) : coll->last(result)); + } break; + case pl::ObCollectionType::PRIOR_PROPERTY: { + CK (param_array.count() > 1); + int64_t idx = param_array.count() - 1; + OZ (OB_NOT_NULL(assoc) + ? assoc->prior(param_array.at(idx), result) : coll->prior(param_array.at(idx), result)); + } break; + case pl::ObCollectionType::NEXT_PROPERTY: { + CK (param_array.count() > 1); + int64_t idx = param_array.count() - 1; + int64_t index_val = param_array.at(idx); + if (OB_SUCC(ret)) { + if(OB_INVALID_INDEX == param_array.at(idx)) { + if (OB_NOT_NULL(assoc)) { + OZ (assoc->next(index_val, result)); + } else { + result.set_null(); + } + } else { + OZ (OB_NOT_NULL(assoc) ? assoc->next(index_val, result) : coll->next(index_val, result)); + } + } + } break; + case pl::ObCollectionType::EXISTS_PROPERTY: { + CK (param_array.count() > 1); + int64_t idx = param_array.count() - 1; + OZ (OB_NOT_NULL(assoc) + ? assoc->exist(param_array.at(idx), result) : coll->exist(param_array.at(idx), result)); + } break; + case pl::ObCollectionType::COUNT_PROPERTY: { + result.set_int(coll->get_actual_count()); + } break; + case pl::ObCollectionType::LIMIT_PROPERTY: { + if (coll->is_varray()) { + pl::ObPLVArray *varr = static_cast(coll); + CK (OB_NOT_NULL(varr)); + OX (result.set_int(varr->get_capacity())); + } else { + result.set_null(); + } + } break; + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid property type", K(ret), K(property_type_)); + } break; + } + } +#endif } else { ObObj *datum = reinterpret_cast(attr_addr); if (ObMaxType == datum->get_type()) { //means has been deleted diff --git a/src/sql/engine/expr/ob_expr_ols_funcs.cpp b/src/sql/engine/expr/ob_expr_ols_funcs.cpp index ed32f3252..ad793f028 100644 --- a/src/sql/engine/expr/ob_expr_ols_funcs.cpp +++ b/src/sql/engine/expr/ob_expr_ols_funcs.cpp @@ -1237,7 +1237,78 @@ int ObExprOLSSessionLabel::cg_expr(ObExprCGCtx &op_cg_ctx, } int ObExprOLSSessionLabel::eval_label(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) { +#ifndef OB_BUILD_LABEL_SECURITY int ret = OB_NOT_IMPLEMENT; +#else + int ret = OB_SUCCESS; + ObSQLSessionInfo *session = NULL; + ObSchemaGetterGuard *schema_guard = NULL; + uint64_t tenant_id = OB_INVALID_ID; + ObDatum *param = nullptr; + + if (OB_ISNULL(session =ctx.exec_ctx_.get_my_session())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(session), K(ret)); + } else if (OB_FAIL(init_phy_plan_timeout(ctx.exec_ctx_, *session))) { + LOG_WARN("fail to init phy plan timeout", K(ret)); + } else if (OB_FAIL(get_schema_guard(&ctx.exec_ctx_, schema_guard))) { + LOG_WARN("schema guard is NULL", K(ret)); + } else if (OB_FAIL(expr.args_[0]->eval(ctx, param))) { + LOG_WARN("failed to eval params", K(ret)); + } else { + tenant_id = session->get_effective_tenant_id(); + } + + ObString policy_name; + + if (OB_SUCC(ret)) { + policy_name = param->get_string(); + if (OB_FAIL(POLICY_NAME_CHECKER.validate_str_length(policy_name))) { + LOG_WARN("check policy name failed", K(ret), K(policy_name)); + } + } + + uint64_t policy_id = OB_INVALID_ID; + + if (OB_SUCC(ret)) { + const ObLabelSePolicySchema *ols_policy_schema = NULL; + + if (OB_FAIL(schema_guard->get_label_se_policy_schema_by_name(tenant_id, + policy_name, + ols_policy_schema))) { + LOG_WARN("fail to get ols policy schema", K(ret)); + } else if (OB_FAIL(POLICY_SCHEMA_CHECKER.validate(ols_policy_schema))) { + LOG_WARN("policy is not exist", K(ret)); + } else { + policy_id = ols_policy_schema->get_label_se_policy_id(); + } + } + + ObLabelSeSessionLabel session_label; + if (OB_SUCC(ret)) { + if (OB_SUCCESS == session->get_session_label(policy_id, session_label)) { + //do nothing + } else { + if (OB_FAIL(ObLabelSeUtil::load_default_session_label(tenant_id, + policy_id, session->get_user_id(), + *schema_guard, session_label))) { + LOG_WARN("fail to load default session label", K(ret)); + } else if (OB_FAIL(session->replace_new_session_label(policy_id, session_label))) { + LOG_WARN("fail to replace new session label", K(ret)); + } + LOG_DEBUG("load default session label", K(ret)); + } + } + + if (OB_SUCC(ret)) { + ObLabelSeLabelTag &read_tag = session_label.get_read_label_tag(); + if (read_tag.is_unauth()) { + res.set_null(); + } else { + res.set_int(read_tag.get_value()); + } + } +#endif return ret; } @@ -1275,7 +1346,78 @@ int ObExprOLSSessionRowLabel::cg_expr(ObExprCGCtx &op_cg_ctx, int ObExprOLSSessionRowLabel::eval_row_label(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) { +#ifndef OB_BUILD_LABEL_SECURITY int ret = OB_NOT_IMPLEMENT; +#else + int ret = OB_SUCCESS; + ObSQLSessionInfo *session = NULL; + ObSchemaGetterGuard *schema_guard = NULL; + uint64_t tenant_id = OB_INVALID_ID; + ObDatum *param = nullptr; + + if (OB_ISNULL(session = ctx.exec_ctx_.get_my_session())) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(session), K(ret)); + } else if (OB_FAIL(init_phy_plan_timeout(ctx.exec_ctx_, *session))) { + LOG_WARN("fail to init phy plan timeout", K(ret)); + } else if (OB_FAIL(get_schema_guard(&ctx.exec_ctx_, schema_guard))) { + LOG_WARN("schema guard is NULL", K(ret)); + } else if (OB_FAIL(expr.args_[0]->eval(ctx, param))) { + LOG_WARN("failed to eval params", K(ret)); + } else { + tenant_id = session->get_effective_tenant_id(); + } + + ObString policy_name; + + if (OB_SUCC(ret)) { + policy_name = param->get_string(); + if (OB_FAIL(POLICY_NAME_CHECKER.validate_str_length(policy_name))) { + LOG_WARN("check policy name failed", K(ret), K(policy_name)); + } + } + + uint64_t policy_id = OB_INVALID_ID; + + if (OB_SUCC(ret)) { + const ObLabelSePolicySchema *ols_policy_schema = NULL; + + if (OB_FAIL(schema_guard->get_label_se_policy_schema_by_name(tenant_id, + policy_name, + ols_policy_schema))) { + LOG_WARN("fail to get ols policy schema", K(ret)); + } else if (OB_FAIL(POLICY_SCHEMA_CHECKER.validate(ols_policy_schema))) { + LOG_WARN("policy is not exist", K(ret)); + } else { + policy_id = ols_policy_schema->get_label_se_policy_id(); + } + } + + ObLabelSeSessionLabel session_label; + if (OB_SUCC(ret)) { + if (OB_SUCCESS == session->get_session_label(policy_id, session_label)) { + //do nothing + } else { + if (OB_FAIL(ObLabelSeUtil::load_default_session_label(tenant_id, + policy_id, session->get_user_id(), + *schema_guard, session_label))) { + LOG_WARN("fail to load default session label", K(ret)); + } else if (OB_FAIL(session->replace_new_session_label(policy_id, session_label))) { + LOG_WARN("fail to replace new session label", K(ret)); + } + LOG_DEBUG("load default session label", K(ret)); + } + } + + if (OB_SUCC(ret)) { + ObLabelSeLabelTag &write_tag = session_label.get_write_label_tag(); + if (write_tag.is_unauth()) { + res.set_null(); + } else { + res.set_int(write_tag.get_value()); + } + } +#endif return ret; } @@ -1321,7 +1463,58 @@ int ObExprOLSLabelCmpLE::cg_expr(ObExprCGCtx &op_cg_ctx, int ObExprOLSLabelCmpLE::eval_cmple(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) { +#ifndef OB_BUILD_LABEL_SECURITY int ret = OB_NOT_IMPLEMENT; +#else + int ret = OB_SUCCESS; + ObSQLSessionInfo *session = NULL; + ObSchemaGetterGuard *schema_guard = NULL; + int64_t label_tag[2] = {-1, -1}; + // uint64_t policy_id[2] = {OB_INVALID_ID, OB_INVALID_ID}; + enum {COLUMN_LABEL = 0, SESSION_LABEL}; + ObDatum *param1 = nullptr; + ObDatum *param2 = nullptr; + if (OB_ISNULL(session = ctx.exec_ctx_.get_my_session())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get session", K(ret)); + } else if (OB_FAIL(expr.eval_param_value(ctx, param1, param2))) { + LOG_WARN("failed to eval parmas", K(ret)); + } else if (OB_FAIL(get_schema_guard(&ctx.exec_ctx_, schema_guard))) { + LOG_WARN("fail to get schema guard", K(ret)); + } else if (param1->is_null()) { + //column label is null, noone can read it, return fail(-1) + res.set_int(-1); + } else if (param2->is_null()) { + //session label is null, column label is not null, can't access return fail(-1) + res.set_int(-1); + } else { + //从参数中获取label tag + if (OB_SUCC(ret)) { + label_tag[COLUMN_LABEL] = param1->get_int(); + label_tag[SESSION_LABEL] = param2->get_int(); + if (OB_FAIL(LABEL_TAG_CHECKER.validate(label_tag[COLUMN_LABEL])) + || OB_FAIL(LABEL_TAG_CHECKER.validate(label_tag[SESSION_LABEL]))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unknown label tag", K(ret), + K(label_tag[COLUMN_LABEL]), K(label_tag[SESSION_LABEL])); + } + } + + //比较component值，返回函数结果 + if (OB_SUCC(ret)) { + int64_t cmp_result = -1; + if (OB_FAIL(ObExprOLSUtil::label_tag_compare(session->get_effective_tenant_id(), + *schema_guard, + label_tag[COLUMN_LABEL], + label_tag[SESSION_LABEL], + cmp_result))) { + LOG_WARN("fail to compare label tag", K(ret)); + } else { + res.set_int(cmp_result); + } + } + } +#endif return ret; } @@ -1365,7 +1558,78 @@ int ObExprOLSLabelCheck::cg_expr(ObExprCGCtx &op_cg_ctx, int ObExprOLSLabelCheck::eval_label_check(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) { +#ifndef OB_BUILD_LABEL_SECURITY int ret = OB_NOT_IMPLEMENT; +#else + int ret = OB_SUCCESS; + ObSQLSessionInfo *session = NULL; + ObSchemaGetterGuard *schema_guard = NULL; + const ObLabelSeLabelSchema *ols_label_schema = NULL; + + ObLabelSeLabelTag label_tag; + + ObDatum *param = nullptr; + if (OB_ISNULL(session = ctx.exec_ctx_.get_my_session())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get session failed", K(ret)); + } else if (OB_FAIL(get_schema_guard(&ctx.exec_ctx_, schema_guard))) { + LOG_WARN("fail to get schema guard", K(ret)); + } else if (OB_FAIL(expr.eval_param_value(ctx, param))) { + LOG_WARN("failed to eval params", K(ret)); + } + + if (OB_SUCC(ret)) { + if (param->is_null()) { + label_tag.set_unauth_value(); + } else if (ObIntType == expr.args_[0]->datum_meta_.type_) { + int64_t label_tag_value = param->get_int(); + uint64_t tenant_id = session->get_effective_tenant_id(); + + if (OB_FAIL(LABEL_TAG_CHECKER.validate(label_tag_value))) { + LOG_WARN("invalid label tag value", K(ret)); + } else if (OB_FAIL(schema_guard->get_label_se_label_by_label_tag(tenant_id, + label_tag_value, + ols_label_schema))) { + LOG_WARN("get label schema failed", K(ret)); + } else if (OB_FAIL(LABEL_SCHEMA_CHECKER.validate(ols_label_schema))) { + LOG_WARN("label schema not exist", K(ret)); + } else if (ols_label_schema->get_flag() != 1) { + ret = OB_ERR_POLICY_WITH_CHECK_OPTION_VIOLATION; + LOG_WARN("data label is false for write dml", K(ret)); + } else { + ObString label_text = ols_label_schema->get_label_str(); + uint64_t policy_id = ols_label_schema->get_label_se_policy_id(); + //TODO [label] get label_nums from ols_label_schema->get_label_num_str() + ObLabelSeDecomposedLabel label_comps; + ObLabelSeLabelCompNums label_nums; + if (OB_FAIL(ObLabelSeResolver::resolve_label_text(label_text, label_comps))) { + LOG_WARN("fail to resolve label", K(ret)); + } else if (OB_FAIL(ObLabelSeUtil::convert_label_comps_name_to_num(tenant_id, policy_id, + *schema_guard, + label_comps, label_nums))) { + LOG_WARN("fail to validate component", K(ret), K(label_comps)); + } else if (OB_FAIL(ObLabelSeUtil::validate_user_auth(tenant_id, policy_id, + session->get_user_id(), + *schema_guard, label_nums, true))) { + LOG_WARN("validate user auth failed", K(ret)); + } else { + label_tag.set_value(label_tag_value); + } + } + } else { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid obj type", K(expr.args_[0]->datum_meta_.type_)); + } + } + + if (OB_SUCC(ret)) { + if (label_tag.is_unauth()) { + res.set_null(); + } else { + res.set_int(label_tag.get_value()); + } + } +#endif return ret; } @@ -1409,7 +1673,49 @@ int ObExprOLSLabelToChar::cg_expr(ObExprCGCtx &op_cg_ctx, int ObExprOLSLabelToChar::eval_label_to_char(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) { +#ifndef OB_BUILD_LABEL_SECURITY int ret = OB_NOT_IMPLEMENT; +#else + int ret = OB_SUCCESS; + ObSQLSessionInfo *session = NULL; + ObSchemaGetterGuard *schema_guard = NULL; + const ObLabelSeLabelSchema *ols_label_schema = NULL; + ObDatum *param = nullptr; + + if (OB_ISNULL(session = ctx.exec_ctx_.get_my_session())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get session", K(ret)); + } else if (OB_FAIL(get_schema_guard(&ctx.exec_ctx_, schema_guard))) { + LOG_WARN("fail to get schema guard", K(ret)); + } else if (OB_FAIL(expr.eval_param_value(ctx, param))) { + LOG_WARN("failed to eval param value", K(ret)); + } + + if (OB_SUCC(ret)) { + if (param->is_null()) { + res.set_null(); + } else if (ObIntType == expr.args_[0]->datum_meta_.type_) { + int64_t label_tag_value = param->get_int(); + + if (OB_FAIL(LABEL_TAG_CHECKER.validate(label_tag_value))) { + LOG_WARN("invalid label tag value", K(ret)); + } else if (OB_FAIL(schema_guard->get_label_se_label_by_label_tag( + session->get_effective_tenant_id(), + label_tag_value, + ols_label_schema))) { + LOG_WARN("get label schema failed", K(ret)); + } else if (OB_FAIL(LABEL_SCHEMA_CHECKER.validate(ols_label_schema))) { + LOG_WARN("label schema not exist", K(ret)); + } else { + ObString label_text = ols_label_schema->get_label_str(); + res.set_string(label_text); + } + } else { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid obj type", K(expr.args_[0]->datum_meta_.type_)); + } + } +#endif return ret; } @@ -1454,7 +1760,108 @@ int ObExprOLSCharToLabel::cg_expr(ObExprCGCtx &op_cg_ctx, } int ObExprOLSCharToLabel::eval_char_to_label(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) { +#ifndef OB_BUILD_LABEL_SECURITY int ret = OB_NOT_IMPLEMENT; +#else + int ret = OB_SUCCESS; + ObString policy_name; + ObString label_text; + + ObSQLSessionInfo *session = NULL; + ObSchemaGetterGuard *schema_guard = NULL; + ObSqlString stmt_param_str; + + uint64_t tenant_id = OB_INVALID_ID; + ObObj obj1; + ObObj obj2; + ObDatum *param1 = nullptr; + ObDatum *param2 = nullptr; + + if (OB_ISNULL(session = ctx.exec_ctx_.get_my_session())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get session failed", K(ret)); + } else if (OB_FAIL(init_phy_plan_timeout(ctx.exec_ctx_, *session))) { + LOG_WARN("fail to init phy plan timeout", K(ret)); + } else if (OB_FAIL(get_schema_guard(&ctx.exec_ctx_, schema_guard))) { + LOG_WARN("schema guard is NULL", K(ret)); + } else if (OB_FAIL(expr.eval_param_value(ctx, param1, param2))) { + LOG_WARN("failed to eval params", K(ret)); + } else if (param1->to_obj(obj1, expr.args_[0]->obj_meta_)) { + LOG_WARN("failed to convert to obj", K(ret)); + } else if (param2->to_obj(obj2, expr.args_[1]->obj_meta_)) { + LOG_WARN("failed to convert to obj", K(ret)); + } else { + tenant_id = session->get_effective_tenant_id(); + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(get_string_from_obj_and_check(&ctx.exec_ctx_.get_allocator(), obj1, + policy_name, POLICY_NAME_CHECKER, + stmt_param_str))) { + LOG_WARN("fail to check input value", K(ret)); + } else if (OB_FAIL(get_string_from_obj_and_check(&ctx.exec_ctx_.get_allocator(), obj2, + label_text, LABEL_TEXT_CHECKER, + stmt_param_str))) { + LOG_WARN("fail to check input value", K(ret)); + } + } + + uint64_t policy_id = OB_INVALID_ID; + + if (OB_SUCC(ret)) { + const ObLabelSePolicySchema *ols_policy_schema = NULL; + + if (OB_FAIL(schema_guard->get_label_se_policy_schema_by_name(tenant_id, + policy_name, + ols_policy_schema))) { + LOG_WARN("fail to get ols policy schema", K(ret)); + } else if (OB_FAIL(POLICY_SCHEMA_CHECKER.validate(ols_policy_schema))) { + LOG_WARN("policy is not exist", K(ret)); + } else { + policy_id = ols_policy_schema->get_label_se_policy_id(); + } + } + + //resolve session label + ObLabelSeDecomposedLabel label_comps; + ObLabelSeLabelCompNums label_nums; + ObString normalized_label_text; + + if (OB_SUCC(ret)) { + if (OB_FAIL(ObLabelSeResolver::resolve_label_text(label_text, label_comps))) { + LOG_WARN("fail to resolve label", K(ret)); + } else if (OB_FAIL(ObLabelSeUtil::convert_label_comps_name_to_num(tenant_id, policy_id, + *schema_guard, + label_comps, label_nums))) { + LOG_WARN("fail to validate component", K(ret), K(label_comps)); + } else if (OB_FAIL(ObLabelSeResolver::construct_label_text(label_comps, + &ctx.exec_ctx_.get_allocator(), + normalized_label_text))) { + LOG_WARN("fail to construct label text", K(ret)); + } + } + + int64_t new_label_tag; + + if (OB_SUCC(ret)) { + const ObLabelSeLabelSchema *ols_label_schema = NULL; + + if (OB_FAIL(schema_guard->get_label_se_label_schema_by_name(tenant_id, + normalized_label_text, + ols_label_schema))) { + LOG_WARN("get label schema failed", K(ret)); + } else if (OB_FAIL(LABEL_SCHEMA_CHECKER.validate(ols_label_schema))) { + //TODO [label] create one + LOG_WARN("label schema not exist", K(ret), K(normalized_label_text)); + } else { + new_label_tag = ols_label_schema->get_label_tag(); + } + } + + if (OB_SUCC(ret)) { + res.set_int(new_label_tag); + } +#endif return ret; } diff --git a/src/sql/engine/expr/ob_expr_pl_associative_index.cpp b/src/sql/engine/expr/ob_expr_pl_associative_index.cpp index f37d29f78..7203e3864 100644 --- a/src/sql/engine/expr/ob_expr_pl_associative_index.cpp +++ b/src/sql/engine/expr/ob_expr_pl_associative_index.cpp @@ -72,6 +72,244 @@ int ObExprPLAssocIndex::calc_result_type2(ObExprResType &type, return ret; } +#ifdef OB_BUILD_ORACLE_PL +int ObExprPLAssocIndex::do_eval_assoc_index(int64_t &assoc_idx, + ObExecContext &exec_ctx, + const Info &info, + pl::ObPLAssocArray &assoc_array_ref, + const common::ObObj &key) +{ + int ret = OB_SUCCESS; + pl::ObPLAssocArray *assoc_array = &assoc_array_ref; + if (assoc_array->get_count() >0 && OB_ISNULL(assoc_array->get_key())) { + //如果下标是int类型，并且是BULK的数据，那么key_为空，这是一个优化 + if (!key.is_integer_type()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid associative array", K(*assoc_array), K(key), K(ret)); + } else if (info.for_write_) { + OZ (reserve_assoc_key(*assoc_array)); + } else { /*do nothing*/ } + } else { /*do nothing*/ } + + int64_t index = OB_INVALID_INDEX; + if (OB_SUCC(ret) && OB_NOT_NULL(assoc_array->get_key())) { + for (int64_t i = 0; OB_SUCC(ret) && OB_INVALID_INDEX == index && i < assoc_array->get_count(); ++i) { + const ObObj &cur_obj = assoc_array->get_key()[i]; + if (!cur_obj.can_compare(key)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("can not compare", K(cur_obj), K(key), K(i), K(ret)); + } else if (cur_obj.is_string_type() && cur_obj.get_collation_type() != key.get_collation_type()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("can not compare", K(cur_obj), K(key), K(i), K(ret)); + } else if (0 == cur_obj.compare(key)) { + index = i; + } else { /*do nothing*/ } + } + } + + if (OB_SUCC(ret)) { + if (info.for_write_) { + if (OB_INVALID_INDEX == index) { + pl::ObPLExecCtx *pl_exec_ctx = exec_ctx.get_my_session()->get_pl_context()->get_current_ctx(); + if (OB_FAIL(ObSPIService::spi_extend_assoc_array(exec_ctx.get_my_session()->get_effective_tenant_id(), + pl_exec_ctx, exec_ctx.get_allocator(), *assoc_array, 1))) { + LOG_WARN("failed to spi_set_collection_data", K(*assoc_array), K(ret)); + } + } + + if (OB_SUCC(ret)) { + int64_t pos = OB_INVALID_INDEX == index ? assoc_array->get_count() : index + 1; //pos是传给ObjAccess使用的下标，从1开始 + assoc_idx = pos; + + if (OB_INVALID_INDEX == index) { //如果是一个新的key，需要插入key和sort + ObObj new_key; + if (OB_FAIL(deep_copy_obj(*assoc_array->get_allocator(), key, new_key))) { + LOG_WARN("failed to copy key", K(pos), K(key), K(*assoc_array), K(ret)); + } else if (OB_FAIL(assoc_array->set_key(pos - 1, new_key))) { + LOG_WARN("failed to set key", K(pos), K(key), K(*assoc_array), K(ret)); + } else { + //找到新插入key的index + int64_t pre_idx = OB_INVALID_INDEX; + int64_t sort_idx = OB_INVALID_INDEX == assoc_array->get_first() ? OB_INVALID_INDEX : assoc_array->get_first() - 1; + while (OB_SUCC(ret) && OB_INVALID_INDEX != sort_idx) { + ObObj *current_key = assoc_array->get_key(sort_idx); + if (!key.can_compare(*current_key)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("key can not be compared", + K(ret), K(assoc_array), K(sort_idx), K(key), K(*current_key), + K(index), K(pos), K(info)); + } else if (key < *current_key) { + break; + } else if (key > *current_key) { + //do nothing + } else{ + ret = OB_ERR_UNEXPECTED; + LOG_WARN("key exists, but not found", K(assoc_array), K(sort_idx), K(key), K(*current_key), K(ret)); + } + pre_idx = sort_idx; + sort_idx = assoc_array->get_sort(sort_idx); + } + + //重排sort数组 + if (OB_SUCC(ret)) { + //如果是首元素，修改first指针，否则修改前项指针 + if (OB_INVALID_INDEX == pre_idx) { + assoc_array->set_first(pos); + } else { + OZ (assoc_array->set_sort(pre_idx, pos - 1), K(pre_idx), K(pos - 1), K(ret)); + } + //修改自己的后项指针 + OZ (assoc_array->set_sort(pos - 1, sort_idx), K(pos - 1), K(sort_idx), K(ret)); + //如果是末元素，修改last指针 + if (OB_SUCC(ret)) { + OB_INVALID_INDEX == sort_idx ? assoc_array->set_last(pos) : (void)NULL; + } + } + } + } else { + // 这儿处理一个场景，a('a') := 1; a('b'):=2; a('c'):=3; a.delete('a'); a('a'):=1; + // 这种场景下，删除会将first，last更新一次，后面的赋值需要在这里重新更新first，last + bool is_deleted = false; + if (OB_FAIL(assoc_array->is_elem_deleted(index, is_deleted))) { + LOG_WARN("failed to test element deleted.", K(ret)); + } else if (is_deleted) { + if (OB_INVALID_INDEX != assoc_array->get_first()) { + const ObObj &cur_obj = assoc_array->get_key()[index]; + const ObObj &first = assoc_array->get_key()[assoc_array->get_first() - 1]; + if (cur_obj < first) { + assoc_array->set_first(index + 1); + } + } else { + assoc_array->set_first(index + 1); + } + if (OB_INVALID_INDEX != assoc_array->get_last()) { + const ObObj &cur_obj = assoc_array->get_key()[index]; + const ObObj &last = assoc_array->get_key()[assoc_array->get_last() - 1]; + if (cur_obj > last) { + assoc_array->set_last(index + 1); + } + } else { + assoc_array->set_last(index + 1); + } + } + } + } + } else { + if (OB_INVALID_INDEX != index) { + assoc_idx = index + 1; + } else if (NULL == assoc_array->get_key() && key.get_int() <= assoc_array->get_count()) { + assoc_idx = key.get_int(); + } else { + if (info.out_of_range_set_err_) { + ret = OB_READ_NOTHING; + LOG_WARN("key of associative not exists", K(key), K(*assoc_array), K(ret)); + } else { + // pl collection的构建有时候会走到这里，out of range的话默认属性是赋值为NULL + // 例如 aa('e') ===> NULL, 如果aa是关联数组，e不是一个key + // 假设a的索引是 b, c, g, 那么a.exists('e')和a.next('e')不能返回相同的值。 + // next和prior需要返回三种不同的值，一是小于first，返回-1， 二是大于next，返回-2， + // 三是中间值，但不存在，这个时候需要看prior和next返回最接近的前一个值索引位置。 + /*enum parent_expr_type { + EXPR_UNKNOWN = -1, + EXPR_PRIOR, + EXPR_NEXT, + EXPR_EXISTS, + }; */ + const pl::parent_expr_type type = info.parent_expr_type_; + int64_t cnt = 0, index = 0; + if (pl::parent_expr_type::EXPR_EXISTS == type) { + assoc_idx = OB_INVALID_INDEX; + } else if (pl::parent_expr_type::EXPR_NEXT == type + && OB_INVALID_INDEX == assoc_array->get_last()) { + assoc_idx = pl::ObPLCollection::IndexRangeType::LARGE_THAN_LAST; + } else if (pl::parent_expr_type::EXPR_PRIOR == type + && OB_INVALID_INDEX == assoc_array->get_first()) { + assoc_idx = pl::ObPLCollection::IndexRangeType::LESS_THAN_FIRST; + } else if (pl::parent_expr_type::EXPR_NEXT == type + || pl::parent_expr_type::EXPR_PRIOR == type) { + int64_t fidx = assoc_array->get_first() - 1; + int64_t lidx = assoc_array->get_last() - 1; + const ObObj &first = assoc_array->get_key()[fidx]; + const ObObj &last = assoc_array->get_key()[lidx]; + if (key < first) { + assoc_idx = pl::ObPLCollection::IndexRangeType::LESS_THAN_FIRST; + } else if (key > last) { + assoc_idx = pl::ObPLCollection::IndexRangeType::LARGE_THAN_LAST; + } else if (first != last) { // 相等时, 上面的两个分中必然会进入一个，因为first和las相等 + ObObj head; + ObObj next; + index = fidx; + // 确定中间值的索引 + do { + head = assoc_array->get_key()[index]; + next = assoc_array->get_key()[assoc_array->get_sort()[index]]; + if (key >= first && key <= next) { + if (pl::parent_expr_type::EXPR_NEXT == type) { + // do nothing + } else { + index = assoc_array->get_sort()[index]; + } + break; + } + index = assoc_array->get_sort()[index]; + cnt++; + } while (cnt < assoc_array->get_count() - 1 && OB_INVALID_INDEX != index); + if (cnt < assoc_array->get_count() - 1) { + assoc_idx = index + 1; + } else { + ret = OB_ERROR_OUT_OF_RANGE; + LOG_WARN("index out of range, associative array", K(key), K(*assoc_array), K(ret)); + } + } else { + // do nothing + } + } + } + } + } + } + return ret; +} + +int ObExprPLAssocIndex::reserve_assoc_key(pl::ObPLAssocArray &assoc_array) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(assoc_array.get_allocator()) || 0 == assoc_array.get_count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid associative array", K(assoc_array), K(ret)); + } else if (NULL != assoc_array.get_key() || NULL != assoc_array.get_sort()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid associative array to reserve", K(assoc_array), K(ret)); + } else { /*do nothing*/ } + +#define RESERVE_ASSOC_ARRAY(TYPE, PROPERTY) \ + do { \ + if (OB_SUCC(ret)) { \ + TYPE *addr = static_cast(assoc_array.get_allocator()->alloc(sizeof(TYPE) * assoc_array.get_count())); \ + if (OB_ISNULL(addr)) { \ + ret = OB_ALLOCATE_MEMORY_FAILED; \ + LOG_WARN("alloc failed", K(assoc_array), K(assoc_array.get_count()), K(sizeof(TYPE)), K(ret)); \ + } else { \ + assoc_array.set_##PROPERTY(addr); \ + } \ + } \ + } while(0) + + RESERVE_ASSOC_ARRAY(ObObj, key); + + RESERVE_ASSOC_ARRAY(int64_t, sort); + + for (int64_t i = 0; OB_SUCC(ret) && i < assoc_array.get_count(); ++i) { + assoc_array.get_key(i)->set_int32(i + 1); + assoc_array.set_sort(i, i + 1); + } + assoc_array.set_sort(assoc_array.get_count() - 1, OB_INVALID_INDEX); + assoc_array.set_first(1); //first和last从1开始 + assoc_array.set_last(assoc_array.get_count()); + + return ret; +} +#endif int ObExprPLAssocIndex::cg_expr(ObExprCGCtx &op_cg_ctx, const ObRawExpr &raw_expr, @@ -99,7 +337,49 @@ int ObExprPLAssocIndex::eval_assoc_idx(const ObExpr &expr, ObDatum &expr_datum) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSEDx(expr, ctx, expr_datum); +#else + ObDatum *array; + ObDatum *key; + ObObj key_obj; + Info info; + info.v_ = expr.extra_; + + OZ(expr.eval_param_value(ctx, array, key)); + OZ(key->to_obj(key_obj, expr.args_[1]->obj_meta_, expr.args_[1]->obj_datum_map_)); + + if (OB_SUCC(ret) + && key_obj.is_null() + && pl::parent_expr_type::EXPR_UNKNOWN == info.parent_expr_type_) { + ObString msg("NULL index table key value"); + ret = OB_ERR_NUMERIC_OR_VALUE_ERROR; + LOG_USER_ERROR(OB_ERR_NUMERIC_OR_VALUE_ERROR, msg.length(), msg.ptr()); + } + if (OB_FAIL(ret)) { + } else if (key_obj.is_null()) { + if (pl::parent_expr_type::EXPR_UNKNOWN == info.parent_expr_type_) { + ObString msg("NULL index table key value"); + ret = OB_ERR_NUMERIC_OR_VALUE_ERROR; + LOG_USER_ERROR(OB_ERR_NUMERIC_OR_VALUE_ERROR, msg.length(), msg.ptr()); + } else if (pl::parent_expr_type::EXPR_NEXT != info.parent_expr_type_) { + expr_datum.set_int(pl::ObPLCollection::IndexRangeType::LESS_THAN_FIRST); + } else if (expr.args_[1]->obj_meta_.is_null() || !info.is_index_by_varchar_) { // null + expr_datum.set_int(pl::ObPLCollection::IndexRangeType::LARGE_THAN_LAST); + } else { // '' empty string + expr_datum.set_int(pl::ObPLCollection::IndexRangeType::LESS_THAN_FIRST); + } + } else { + pl::ObPLAssocArray *assoc_array = reinterpret_cast( + array->extend_obj_->get_ext()); + CK(NULL != assoc_array); + int64_t assoc_idx = 0; + OZ(do_eval_assoc_index(assoc_idx, ctx.exec_ctx_, info, *assoc_array, key_obj)); + if (OB_SUCC(ret)) { + expr_datum.set_int(assoc_idx); + } + } +#endif return ret; } diff --git a/src/sql/engine/expr/ob_expr_pl_associative_index.h b/src/sql/engine/expr/ob_expr_pl_associative_index.h index 3c7ad0fa0..377e3c7fd 100644 --- a/src/sql/engine/expr/ob_expr_pl_associative_index.h +++ b/src/sql/engine/expr/ob_expr_pl_associative_index.h @@ -77,6 +77,15 @@ private: static_assert(sizeof(Info) == 8, "unexpected size"); +#ifdef OB_BUILD_ORACLE_PL + static int do_eval_assoc_index(int64_t &assoc_idx, + ObExecContext &exec_ctx, + const Info &info, + pl::ObPLAssocArray &assoc_array, + const common::ObObj &key); + + static int reserve_assoc_key(pl::ObPLAssocArray &assoc_array); +#endif DISALLOW_COPY_AND_ASSIGN(ObExprPLAssocIndex); private: Info info_; diff --git a/src/sql/engine/expr/ob_expr_priv_xml_binary.cpp b/src/sql/engine/expr/ob_expr_priv_xml_binary.cpp index a684e1c54..e539eb999 100644 --- a/src/sql/engine/expr/ob_expr_priv_xml_binary.cpp +++ b/src/sql/engine/expr/ob_expr_priv_xml_binary.cpp @@ -14,7 +14,11 @@ #define USING_LOG_PREFIX SQL_ENG #include "ob_expr_priv_xml_binary.h" #include "ob_expr_lob_utils.h" -#include "ob_expr_xml_func_helper.h" +#ifdef OB_BUILD_ORACLE_XML +#include "sql/engine/expr/ob_expr_xml_func_helper.h" +#include "lib/xml/ob_xml_util.h" +#include "sql/engine/ob_exec_context.h" +#endif using namespace oceanbase::common; using namespace oceanbase::sql; @@ -66,6 +70,120 @@ int ObExprPrivXmlBinary::calc_result_typeN(ObExprResType& type, return ret; } +#ifdef OB_BUILD_ORACLE_XML +int ObExprPrivXmlBinary::eval_priv_xml_binary(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) +{ + int ret = OB_SUCCESS; + ObDatum *schema_id = NULL; + ObDatum *xml_datum = NULL; + ObObjParam old_expr_type; + if (expr.args_[1]->type_ == T_QUESTIONMARK) { + const ParamStore ¶m_store = ctx.exec_ctx_.get_physical_plan_ctx()->get_param_store(); + int64_t param_idx = expr.args_[1]->extra_; + if (param_idx < 0 || param_idx >= param_store.count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("param_idx is invalid", K(ret), K(param_idx)); + } else { + old_expr_type = param_store.at(param_idx); + if (ObLongTextType == old_expr_type.get_type()) { + ret = OB_ERR_INVALID_TYPE_FOR_OP; + LOG_WARN("old_expr_type invalid ObLongTextType type", K(ret)); + } + } + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(expr.args_[0]->eval(ctx, schema_id))) { + LOG_WARN("eval geo arg failed", K(ret)); + } else if (OB_FAIL(expr.args_[1]->eval(ctx, xml_datum))) { + LOG_WARN("eval sird arg failed", K(ret)); + } else if (xml_datum->is_null()) { + res.set_null(); + } else { + ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); + common::ObArenaAllocator &tmp_allocator = tmp_alloc_g.get_allocator(); + ObString xml_plain_text = xml_datum->get_string(); + ObIMulModeBase *xml_root = NULL; + ObStringBuffer jbuf(&tmp_allocator); + ObCollationType cs_type = expr.args_[1]->obj_meta_.is_character_type() + ? expr.args_[1]->obj_meta_.get_collation_type() + : CS_TYPE_BINARY; + ObObjType in_type = expr.args_[1]->obj_meta_.is_character_type() + ? expr.args_[1]->obj_meta_.get_type() + : ObLongTextType; + ObMulModeMemCtx* mem_ctx = nullptr; + lib::ObMallocHookAttrGuard malloc_guard(lib::ObMemAttr(MTL_ID(), "XMLCodeGen")); + if (OB_FAIL(ObXmlUtil::create_mulmode_tree_context(&tmp_allocator, mem_ctx))) { + LOG_WARN("fail to create tree memory context", K(ret)); + } else if (OB_FAIL(ObTextStringHelper::read_real_string_data(&tmp_allocator, + in_type, + cs_type, + true, + xml_plain_text))) { + LOG_WARN("get xml plain text failed", K(ret), K(xml_datum)); + } else if (xml_plain_text.empty()) { + res.set_null(); + } else if (expr.args_[1]->obj_meta_.is_character_type()) { + // remove later, use xmlparse in rewrite, input arg can be restricted to xmltype + ObXmlDocument *xml_doc = NULL; + if (OB_FAIL(ObXmlParserUtils::parse_document_text(mem_ctx, xml_plain_text, xml_doc))) { + LOG_WARN("parse xml plain text as document failed.", K(ret), K(xml_plain_text)); + ret = OB_ERR_XML_PARSE; + LOG_USER_ERROR(OB_ERR_XML_PARSE); + } else if (OB_ISNULL(xml_doc)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get null xml document.", K(ret)); + } else { + xml_root = static_cast(xml_doc); + ObString res_string; + if (OB_FAIL(xml_root->get_raw_binary(res_string, &tmp_allocator))) { + LOG_WARN("failed to get xml binary", K(ret), K(xml_plain_text)); + } else { + ObTextStringDatumResult str_result(expr.datum_meta_.type_, &expr, &ctx, &res); + if (OB_FAIL(str_result.init(res_string.length()))) { + LOG_WARN("init lob result failed"); + } else if (OB_FAIL(str_result.append(res_string.ptr(), res_string.length()))) { + LOG_WARN("append lob result failed"); + } else { + str_result.set_result(); + } + } + } + } else { // must be xmlsql type + // Todo: xml schema validation + ObMulModeNodeType type = M_NULL; + if (OB_FAIL(ObXmlUtil::xml_bin_type(xml_plain_text, type))) { + } else if (type == M_UNPARESED_DOC) { + ObString res_string; + if (OB_FAIL(common::ObMulModeFactory::get_xml_base(mem_ctx, xml_plain_text, + ObNodeMemType::BINARY_TYPE, + ObNodeMemType::BINARY_TYPE, + xml_root))) { + LOG_WARN("get xml base failed", K(ret)); + } else if (OB_FAIL(xml_root->get_raw_binary(res_string, mem_ctx->allocator_))) { + LOG_WARN("get raw binary failed", K(ret)); + } else { + ObTextStringDatumResult str_result(expr.datum_meta_.type_, &expr, &ctx, &res); + if (OB_FAIL(str_result.init(res_string.length()))) { + LOG_WARN("init lob result failed"); + } else if (OB_FAIL(str_result.append(res_string.ptr(), res_string.length()))) { + LOG_WARN("append lob result failed"); + } else { + str_result.set_result(); + } + } + } else if (type != M_DOCUMENT && type != M_UNPARESED_DOC) { + ret = OB_XML_INSERT_FRAGMENT; + LOG_WARN("can only insert xml document", K(ret), K(type)); + } else { + res.set_string(xml_datum->get_string()); + } + } + } + + return ret; +} +#endif int ObExprPrivXmlBinary::cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) const { diff --git a/src/sql/engine/expr/ob_expr_priv_xml_binary.h b/src/sql/engine/expr/ob_expr_priv_xml_binary.h index 5c7e58589..c8677776a 100644 --- a/src/sql/engine/expr/ob_expr_priv_xml_binary.h +++ b/src/sql/engine/expr/ob_expr_priv_xml_binary.h @@ -30,7 +30,11 @@ public: int64_t param_num, common::ObExprTypeCtx& type_ctx) const override; +#ifdef OB_BUILD_ORACLE_XML + static int eval_priv_xml_binary(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res); +#else static int eval_priv_xml_binary(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) { return OB_NOT_SUPPORTED; } +#endif virtual int cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) const override; diff --git a/src/sql/engine/expr/ob_expr_set.cpp b/src/sql/engine/expr/ob_expr_set.cpp index 819ad0002..4d12f9b1d 100644 --- a/src/sql/engine/expr/ob_expr_set.cpp +++ b/src/sql/engine/expr/ob_expr_set.cpp @@ -79,6 +79,69 @@ int ObExprSet::calc_result_type1(ObExprResType &type, return ret; } +#ifdef OB_BUILD_ORACLE_PL +int ObExprSet::eval_coll(const ObObj &obj, ObExecContext &ctx, pl::ObPLNestedTable *&coll) +{ + int ret = OB_SUCCESS; + pl::ObPLCollection *c1 = reinterpret_cast(obj.get_ext()); + ObIAllocator &allocator = ctx.get_allocator(); + ObIAllocator *collection_allocator = NULL; + ObObj *data_arr = NULL; + int64_t elem_count = 0; + coll = static_cast(allocator.alloc(sizeof(pl::ObPLNestedTable))); + if (OB_ISNULL(coll)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc memory failed.", K(ret)); + } else if (OB_ISNULL(c1)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("union udt failed due to null udt", K(ret), K(obj)); + } else if (pl::PL_NESTED_TABLE_TYPE != c1->get_type()) { + ret = OB_ERR_WRONG_TYPE_FOR_VAR; + LOG_WARN("PLS-00306: wrong number or types of arguments in call stmt", + K(ret), K(c1->get_type())); + } else if (0 > c1->get_count() || !(c1->is_inited())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("union udt failed due to null udt", K(ret), K(c1->get_count()), K(c1->is_inited())); + } else { + coll = new(coll)pl::ObPLNestedTable(c1->get_id()); + collection_allocator = + static_cast(allocator.alloc(sizeof(pl::ObPLCollAllocator))); + collection_allocator = new(collection_allocator)pl::ObPLCollAllocator(coll); + if (OB_ISNULL(collection_allocator)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc pl collection allocator failed.", K(ret)); + } else { + if (OB_FAIL(ObExprMultiSet::calc_ms_one_distinct(collection_allocator, + reinterpret_cast(c1->get_data()), + c1->get_count(), + data_arr, + elem_count))) { + LOG_WARN("failed to distinct nest table", K(ret)); + } + } + } + if (OB_FAIL(ret)) { + LOG_WARN("failed to calc set operator", K(ret), K(data_arr)); + } + else if (0 != elem_count && OB_ISNULL(data_arr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected result.", K(elem_count), K(data_arr), K(ret)); + } else { + coll->set_allocator(collection_allocator); + coll->set_type(c1->get_type()); + coll->set_id(c1->get_id()); + coll->set_is_null(c1->is_null()); + coll->set_element_type(c1->get_element_type()); + coll->set_column_count(c1->get_column_count()); + coll->set_not_null(c1->is_not_null()); + coll->set_count(elem_count); + coll->set_first(elem_count > 0 ? 1 : OB_INVALID_ID); + coll->set_last(elem_count > 0 ? elem_count : OB_INVALID_ID); + coll->set_data(data_arr); + } + return ret; +} +#endif int ObExprSet::cg_expr( ObExprCGCtx &op_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) const @@ -94,7 +157,25 @@ int ObExprSet::cg_expr( int ObExprSet::calc_set(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &expr_datum) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSEDx(expr, ctx, expr_datum); +#else + if (lib::is_oracle_mode() && ObExtendType == expr.args_[0]->datum_meta_.type_) { + ObDatum *datum = NULL; + pl::ObPLNestedTable *coll = NULL; + ObObj *obj = NULL; + ObObjMeta obj_meta; + obj_meta.set_ext(); + CK (OB_NOT_NULL(obj = static_cast( + ctx.exec_ctx_.get_allocator().alloc(sizeof(ObObj))))); + OZ (expr.args_[0]->eval(ctx, datum)); + OX (datum->to_obj(*obj, obj_meta)); + OZ (eval_coll(*obj, ctx.exec_ctx_, coll)); + CK (OB_NOT_NULL(coll)); + OX (obj->set_extend(reinterpret_cast(coll), coll->get_type())); + OZ (expr_datum.from_obj(*obj)); + } +#endif return ret; } diff --git a/src/sql/engine/expr/ob_expr_set.h b/src/sql/engine/expr/ob_expr_set.h index b31b544a5..cc7499ec0 100644 --- a/src/sql/engine/expr/ob_expr_set.h +++ b/src/sql/engine/expr/ob_expr_set.h @@ -32,6 +32,9 @@ public: const ObRawExpr &raw_expr, ObExpr &rt_expr) const override; static int calc_set(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &expr_datum); +#ifdef OB_BUILD_ORACLE_PL + static int eval_coll(const common::ObObj &obj, ObExecContext &ctx, pl::ObPLNestedTable *&coll); +#endif private: DISALLOW_COPY_AND_ASSIGN(ObExprSet); diff --git a/src/sql/engine/expr/ob_expr_subquery_ref.cpp b/src/sql/engine/expr/ob_expr_subquery_ref.cpp index 41e6798a7..e862c320f 100644 --- a/src/sql/engine/expr/ob_expr_subquery_ref.cpp +++ b/src/sql/engine/expr/ob_expr_subquery_ref.cpp @@ -317,6 +317,72 @@ int ObExprSubQueryRef::expr_eval( LOG_WARN("filter to rewind subquery iterator", K(ret)); } if (OB_FAIL(ret)) { +#ifdef OB_BUILD_ORACLE_PL + } else if (extra_info->is_cursor_) { + const pl::ObRefCursorType pl_type; + int64_t param_size = 0; + char *data = NULL; + ObObj *obj = NULL; + pl::ObPLCursorInfo *cursor = NULL; + ObSPICursor* spi_cursor = NULL; + uint64_t size = 0; + ObNewRow row; + ObSQLSessionInfo *session = ctx.exec_ctx_.get_my_session(); + CK (OB_NOT_NULL(ctx.exec_ctx_.get_sql_ctx())); + CK (OB_NOT_NULL(ctx.exec_ctx_.get_sql_ctx()->schema_guard_)); + OZ (pl_type.get_size(pl::ObPLUDTNS(*ctx.exec_ctx_.get_sql_ctx()->schema_guard_), + pl::PL_TYPE_INIT_SIZE, param_size)); + CK (OB_NOT_NULL(data = static_cast( + ctx.exec_ctx_.get_allocator().alloc(param_size)))); + CK (OB_NOT_NULL(obj = static_cast( + ctx.exec_ctx_.get_allocator().alloc(sizeof(ObObj))))); + if (OB_SUCC(ret)) { + MEMSET(data, 0, param_size); + new(data) pl::ObPLCursorInfo(&ctx.exec_ctx_.get_allocator()); + obj->set_extend(reinterpret_cast(data), pl::PL_CURSOR_TYPE); + expr_datum.from_obj(*obj); + OX (cursor = reinterpret_cast(obj->get_ext())); + } + CK (OB_NOT_NULL(cursor)); + CK (OB_NOT_NULL(session)); + OZ (session->get_tmp_table_size(size)); + OZ (cursor->prepare_spi_cursor(spi_cursor, + session->get_effective_tenant_id(), + size)); + OZ (spi_cursor->row_desc_.assign(extra_info->row_desc_)); + while (OB_SUCC(ret)) { + if (OB_FAIL(iter->get_next_row())) { + //do nothing + } else if (OB_FAIL(convert_datum_to_obj(ctx, *iter, ctx.exec_ctx_.get_allocator(), row))) { + LOG_WARN("failed to convert datum to obj", K(ret)); + } else if (OB_FAIL(spi_cursor->row_store_.add_row(row))) { + LOG_WARN("failed to add row to row store", K(ret)); + } else { /*do nothing*/ } + } + if (OB_LIKELY(OB_ITER_END == ret)) { + ret = OB_SUCCESS; + } else { + LOG_WARN("get next row from row iterator failed", K(ret)); + } + if (OB_SUCC(ret)) { + OX (spi_cursor->row_store_.finish_add_row()); + OX (cursor->open(spi_cursor)); + } + if (OB_SUCC(ret) && lib::is_oracle_mode()) { + transaction::ObTxReadSnapshot &snapshot = ctx.exec_ctx_.get_das_ctx().get_snapshot(); + OZ (cursor->set_and_register_snapshot(snapshot)); + } + if (OB_SUCC(ret)) { + ObExprSubQueryRefCtx *sub_query_ctx = NULL; + uint64_t ctx_id = static_cast(expr.expr_ctx_id_); + if (NULL == (sub_query_ctx = static_cast + (ctx.exec_ctx_.get_expr_op_ctx(ctx_id)))) { + OZ (ctx.exec_ctx_.create_expr_op_ctx(ctx_id, sub_query_ctx)); + } + CK (OB_NOT_NULL(sub_query_ctx)); + OZ (sub_query_ctx->add_cursor_info(cursor, ctx.exec_ctx_.get_my_session())); + } +#endif } else if (extra.is_scalar_) { if (1 != iter->get_output().count()) { //not a scalar obj diff --git a/src/sql/engine/expr/ob_expr_sys_makexml.cpp b/src/sql/engine/expr/ob_expr_sys_makexml.cpp index c9ccc3de7..78cb3f544 100644 --- a/src/sql/engine/expr/ob_expr_sys_makexml.cpp +++ b/src/sql/engine/expr/ob_expr_sys_makexml.cpp @@ -13,8 +13,9 @@ #define USING_LOG_PREFIX SQL_ENG #include "ob_expr_sys_makexml.h" -#include "ob_expr_xml_func_helper.h" - +#ifdef OB_BUILD_ORACLE_XML +#include "sql/engine/expr/ob_expr_xml_func_helper.h" +#endif using namespace oceanbase::common; using namespace oceanbase::sql; @@ -78,6 +79,94 @@ int ObExprSysMakeXML::calc_result_typeN(ObExprResType& type, return ret; } +#ifdef OB_BUILD_ORACLE_XML +int ObExprSysMakeXML::eval_sys_makexml(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) +{ + int ret = OB_SUCCESS; + ObDatum *xml_datum = NULL; + ObExpr *xml_arg = NULL; + if (expr.arg_cnt_ == 1) { + xml_arg = expr.args_[0]; + } else if (expr.arg_cnt_ == 2) { + xml_arg = expr.args_[1]; + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("only binary xml without schema is supported", K(ret), K(expr.arg_cnt_), K(expr)); + } + + bool is_null_result = false; + ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); + common::ObArenaAllocator &allocator = tmp_alloc_g.get_allocator(); + ObString full_xml_data; + ObMulModeMemCtx* mem_ctx = nullptr; + lib::ObMallocHookAttrGuard malloc_guard(lib::ObMemAttr(MTL_ID(), "XMLCodeGen")); + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(ObXmlUtil::create_mulmode_tree_context(&allocator, mem_ctx))) { + LOG_WARN("fail to create tree memory context", K(ret)); + } else if (xml_arg->datum_meta_.type_ != ObLongTextType + || (xml_arg->datum_meta_.cs_type_ != CS_TYPE_BINARY + && xml_arg->datum_meta_.cs_type_ != CS_TYPE_UTF8MB4_BIN)) { + ret = OB_ERR_INVALID_TYPE_FOR_OP; + LOG_WARN("input type error", K(xml_arg->datum_meta_)); + } else if (OB_FAIL(xml_arg->eval(ctx, xml_datum))) { + LOG_WARN("eval xml arg failed", K(ret)); + } else if (xml_datum->is_null()) { + res.set_null(); + } else if (xml_arg->datum_meta_.cs_type_ == CS_TYPE_UTF8MB4_BIN) { + // incase this function used directly with clob + ObXmlDocument *xml_doc = NULL; + ObIMulModeBase *xml_root = NULL; + ObString xml_plain_text; + if (OB_FAIL(ObTextStringHelper::get_string(expr, allocator, 1, xml_datum, xml_plain_text))) { + LOG_WARN("get xml plain text failed", K(ret), K(xml_datum)); + } else if (xml_plain_text.empty()) { + res.set_null(); + } else if (OB_FAIL(ObXmlParserUtils::parse_document_text(mem_ctx, xml_plain_text, xml_doc))) { + LOG_WARN("parse xml plain text as document failed.", K(ret), K(xml_plain_text)); + ret = OB_ERR_XML_PARSE; + LOG_USER_ERROR(OB_ERR_XML_PARSE); + } else if (OB_ISNULL(xml_doc)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get null xml document.", K(ret)); + } else { + xml_root = static_cast(xml_doc); + ObString res_string; + if (OB_FAIL(xml_root->get_raw_binary(res_string, &allocator))) { + LOG_WARN("failed to get xml binary", K(ret), K(xml_plain_text)); + } else { + ObExprStrResAlloc expr_res_alloc(expr, ctx); + ObTextStringResult blob_res(ObLongTextType, true, &expr_res_alloc); + + if (OB_FAIL(blob_res.init(res_string.length()))) { + LOG_WARN("init lob result failed"); + } else if (OB_FAIL(blob_res.append(res_string))) { + LOG_WARN("append lob result failed"); + } else { + ObString blob_locator; + blob_res.get_result_buffer(blob_locator); + res.set_string(blob_locator.ptr(), blob_locator.length()); + } + } + } + } else { // CS_TYPE_BINARY + // call by internal rewrite or used direclty with blob, need validation ? + // check is xml binary + ObMulModeNodeType node_type; + ObXmlBinaryType binary_head_type; + ObString val = xml_datum->get_string(); + if (OB_FAIL(ObTextStringHelper::read_real_string_data(&allocator, ObObjType::ObLongTextType, + CS_TYPE_UTF8MB4_BIN, true, val))) { + LOG_WARN("fail to get real data", K(ret), K(val)); + } else if (val.length() == 0) { // do nothing + res.set_string(xml_datum->get_string()); + } else { + res.set_string(xml_datum->get_string()); + } + } + return ret; +} +#endif int ObExprSysMakeXML::cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) const diff --git a/src/sql/engine/expr/ob_expr_sys_makexml.h b/src/sql/engine/expr/ob_expr_sys_makexml.h index 32d695eac..fd814cae7 100644 --- a/src/sql/engine/expr/ob_expr_sys_makexml.h +++ b/src/sql/engine/expr/ob_expr_sys_makexml.h @@ -33,7 +33,11 @@ public: ObExprResType* types, int64_t param_num, common::ObExprTypeCtx& type_ctx) const override; +#ifdef OB_BUILD_ORACLE_XML + static int eval_sys_makexml(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res); +#else static int eval_sys_makexml(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) { return OB_NOT_SUPPORTED; } +#endif virtual int cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) const override; diff --git a/src/sql/engine/expr/ob_expr_treat.cpp b/src/sql/engine/expr/ob_expr_treat.cpp index 541233a87..04ace185b 100644 --- a/src/sql/engine/expr/ob_expr_treat.cpp +++ b/src/sql/engine/expr/ob_expr_treat.cpp @@ -20,6 +20,10 @@ #include "lib/json_type/ob_json_parse.h" #include "lib/json_type/ob_json_base.h" #include "sql/engine/expr/ob_expr_json_func_helper.h" +#ifdef OB_BUILD_ORACLE_PL +#include "pl/sys_package/ob_json_pl_utils.h" +#include "pl/ob_pl_user_type.h" +#endif namespace oceanbase { namespace sql @@ -60,6 +64,12 @@ int ObExprTreat::calc_result_type2(ObExprResType &type, type.set_length(OB_MAX_SQL_LENGTH); type.set_length_semantics(LS_CHAR); type.set_calc_type(ObJsonType); +#ifdef OB_BUILD_ORACLE_PL + } else if(ob_is_extend(in_type) && pl::ObPlJsonUtil::is_pl_jsontype(type1.get_udt_id()) + && ob_is_extend(as_type) && pl::ObPlJsonUtil::is_pl_jsontype(type2.get_udt_id())){ + type.set_type(ObExtendType); + type.set_udt_id(type2.get_udt_id()); +#endif } else { ret = OB_ERR_INVALID_TYPE_FOR_OP; LOG_WARN("target type not json", K(ret), K(type1), K(type2)); @@ -83,6 +93,37 @@ int ObExprTreat::cg_expr(ObExprCGCtx &expr_cg_ctx, return ret; } +#ifdef OB_BUILD_ORACLE_PL + +static int treat_as_json_udt(const ObExpr &expr, ObEvalCtx &ctx, common::ObIAllocator &temp_allocator, + pl::ObPLOpaque *opaque, ObDatum &res) { + INIT_SUCC(ret); + ObJsonNode *json_doc = nullptr; + pl::ObPLJsonBaseType *jsontype = nullptr; + pl::ObPLJsonBaseType *new_jsontype = nullptr; + ObObj res_obj; + + if(OB_ISNULL(jsontype = static_cast(opaque))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("cast to json type is null", K(ret), K(opaque)); + } else if(OB_ISNULL(json_doc = jsontype->get_data())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get json doc is null", K(ret), K(jsontype)); + } else { + if (OB_FAIL(pl::ObPlJsonUtil::transform_JsonBase_2_PLJsonType(ctx.exec_ctx_, json_doc, new_jsontype))) { + LOG_WARN("failed to transfrom ObJsonNode to ObPLJsonBaseType", K(ret)); + } else if(OB_ISNULL(new_jsontype)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to transfrom ObJsonNode to ObPLJsonBaseType", K(ret)); + } else { + res_obj.set_extend(reinterpret_cast(new_jsontype), pl::PL_OPAQUE_TYPE); + res.from_obj(res_obj); + } + } + return ret; +} + +#endif int ObExprTreat::eval_treat(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) { INIT_SUCC(ret); @@ -102,8 +143,24 @@ int ObExprTreat::eval_treat(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) { } else if (OB_ISNULL(child_res) || child_res->is_null()) { res.set_null(); } else if (ob_is_extend(in_type)) { + #ifdef OB_BUILD_ORACLE_PL + pl::ObPLOpaque *opaque = reinterpret_cast(child_res->get_ext()); + if(OB_ISNULL(opaque)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("not json type", K(ret), K(in_type), K(in_cs_type)); + } else if(opaque->is_json_type()) { + // res = *child_res; + if(OB_FAIL(treat_as_json_udt(expr, ctx, temp_allocator, opaque, res))) { + LOG_WARN("as json udt fail", K(ret)); + } + } else { + ret = OB_ERR_INVALID_TYPE_FOR_OP; + LOG_WARN("opaque type unexpected", K(ret), K(opaque->get_type())); + } + #else ret = OB_NOT_SUPPORTED; LOG_WARN("not surpport udt type", K(ret)); + #endif } else if (ob_is_raw(in_type)) { if(OB_FAIL(ObDatumHexUtils::rawtohex(expr, child_res->get_string(), ctx, res))) { LOG_WARN("fail raw to hex", K(ret), K(in_type), K(child_res->get_string())); diff --git a/src/sql/engine/expr/ob_expr_update_xml.cpp b/src/sql/engine/expr/ob_expr_update_xml.cpp index 795206209..0bc405ca6 100644 --- a/src/sql/engine/expr/ob_expr_update_xml.cpp +++ b/src/sql/engine/expr/ob_expr_update_xml.cpp @@ -12,7 +12,11 @@ */ #include "ob_expr_update_xml.h" -#include "ob_expr_xml_func_helper.h" +#ifdef OB_BUILD_ORACLE_XML +#include "sql/engine/expr/ob_expr_xml_func_helper.h" +#include "lib/xml/ob_xml_parser.h" +#include "lib/xml/ob_xml_util.h" +#endif #include "sql/session/ob_sql_session_info.h" #include "sql/engine/ob_exec_context.h" #include "sql/session/ob_sql_session_info.h" @@ -98,6 +102,845 @@ int ObExprUpdateXml::calc_result_typeN(ObExprResType &type, return ret; } +#ifdef OB_BUILD_ORACLE_XML +int ObExprUpdateXml::eval_update_xml(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) +{ + int ret = OB_SUCCESS; + ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); + common::ObArenaAllocator &allocator = tmp_alloc_g.get_allocator(); + ObDatum *xml_datum = NULL; + ObString namespace_str; + ObIMulModeBase *xml_tree = NULL; + bool has_namespace_str = false; + int64_t num_child = expr.arg_cnt_; + ObPathVarObject *prefix_ns = NULL; + ObString default_ns; + ObCollationType cs_type = CS_TYPE_INVALID; + ObMulModeMemCtx* xml_mem_ctx = nullptr; + bool input_is_doc = false; + ObMulModeNodeType node_type = M_MAX_TYPE; + lib::ObMallocHookAttrGuard malloc_guard(lib::ObMemAttr(MTL_ID(), "XMLCodeGen")); + if (OB_FAIL(ObXmlUtil::create_mulmode_tree_context(&allocator, xml_mem_ctx))) { + LOG_WARN("fail to create tree memory context", K(ret)); + } else if (num_child < 3) { + ret = OB_ERR_PARAM_SIZE; + LOG_WARN("invalid param number", K(ret), K(num_child)); + } else if (OB_FAIL(ObXMLExprHelper::get_xmltype_from_expr(expr.args_[0], ctx, xml_datum))) { + LOG_WARN("fail to get xmltype", K(ret)); + } else if (FALSE_IT(has_namespace_str = ((num_child - 1) % 2 != 0))) { + } else if (has_namespace_str) { + // namespace can be NULL + if (ObNullType == expr.args_[num_child - 1]->datum_meta_.type_) { + } else if (!ob_is_string_type(expr.args_[num_child - 1]->datum_meta_.type_)) { + ret = OB_ERR_INVALID_XPATH_EXPRESSION; + } else if (OB_FAIL(ObXMLExprHelper::get_str_from_expr(expr.args_[num_child - 1], ctx, namespace_str, allocator))) { + LOG_WARN("fail to get namespace string", K(ret)); + } else if (OB_FAIL(ObXMLExprHelper::construct_namespace_params(namespace_str, default_ns, prefix_ns, allocator))) { + LOG_WARN("fail to construct namespace params", K(ret), K(namespace_str)); + } + } + + if (OB_SUCC(ret)) { + int64_t xpath_value_size = has_namespace_str ? num_child - 1 : num_child; + if (OB_FAIL(ObXMLExprHelper::get_xml_base(xml_mem_ctx, xml_datum, cs_type, ObNodeMemType::TREE_TYPE, xml_tree, node_type, true))) { + LOG_WARN("fail to get xml base", K(ret)); + } + // do update xml + for (int64_t i = 1; i < xpath_value_size && OB_SUCC(ret); i+=2) { + ObString xpath_str; + if (ObNullType == expr.args_[i]->datum_meta_.type_) { + ret = OB_ERR_INVALID_XPATH_EXPRESSION; + LOG_WARN("invalid xpath expression", K(ret)); + } else if (!ob_is_string_type(expr.args_[i]->datum_meta_.type_)) { + } else if (OB_FAIL(ObXMLExprHelper::get_str_from_expr(expr.args_[i], ctx, xpath_str, allocator))) { + LOG_WARN("fail to get xpath string", K(ret), K(i)); + } else if (xpath_str.empty()) { + ret = OB_ERR_INVALID_XPATH_EXPRESSION; + LOG_WARN("xpath is empty", K(ret)); + } else if (OB_FAIL(update_xml_tree(xml_mem_ctx, expr.args_[i+1], ctx, xpath_str, default_ns, prefix_ns, xml_tree))) { + LOG_WARN("fail to do update in xml tree", K(ret), K(xml_tree), K(xpath_str), K(default_ns), K(prefix_ns), K(i+1)); + } + } + // set result + if (OB_SUCC(ret)) { + ObXmlDocument *xml_doc = static_cast(xml_tree); + ObStringBuffer buff(&allocator); + ObString xml_plain_text; + if (OB_ISNULL(xml_doc)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate xml doc failed", K(ret)); + } else if (OB_FAIL(xml_doc->print_document(buff, CS_TYPE_INVALID, ObXmlFormatType::NO_FORMAT))) { + LOG_WARN("fail to print xml tree", K(ret)); + } else if (FALSE_IT(xml_plain_text.assign_ptr(buff.ptr(), buff.length()))) { + } else if (node_type == ObMulModeNodeType::M_DOCUMENT && + OB_FAIL(ObXmlParserUtils::parse_document_text(xml_mem_ctx, xml_plain_text, xml_doc))) { + if (ret == OB_ERR_PARSER_SYNTAX) ret = OB_ERR_XML_PARSE; + LOG_WARN("parsing document failed", K(ret), K(xml_plain_text)); + } else if (node_type != ObMulModeNodeType::M_DOCUMENT && OB_FAIL(ObXmlParserUtils::parse_content_text(xml_mem_ctx, xml_plain_text, xml_doc))) { + if (ret == OB_ERR_PARSER_SYNTAX) ret = OB_ERR_XML_PARSE; + LOG_WARN("parsing content failed", K(ret), K(xml_plain_text)); + } + if (OB_FAIL(ret)) { + } else if (xml_doc->count() == 0) { + res.set_null(); + } else if (OB_FAIL(ObXMLExprHelper::pack_xml_res(expr, ctx, res, xml_doc, + xml_mem_ctx, + node_type == ObMulModeNodeType::M_DOCUMENT ? + ObMulModeNodeType::M_DOCUMENT : ObMulModeNodeType::M_CONTENT, + xml_plain_text))) { + LOG_WARN("fail to pack xml res", K(ret), K(xml_doc), K(xml_plain_text)); + } + } + } + return ret; +} + +int ObExprUpdateXml::update_xml_tree(ObMulModeMemCtx* xml_mem_ctx, + const ObExpr *expr, + ObEvalCtx &ctx, + ObString &xpath_str, + ObString &default_ns, + ObPathVarObject *prefix_ns, + ObIMulModeBase *xml_tree) +{ + int ret = OB_SUCCESS; + ObPathExprIter xpath_iter((static_cast(xml_tree))->get_mem_ctx()->allocator_); + ObIMulModeBase *node = NULL; + if (OB_FAIL(xpath_iter.init((static_cast(xml_tree))->get_mem_ctx(), xpath_str, default_ns, xml_tree, prefix_ns))) { + LOG_WARN("fail to init xpath iterator", K(xpath_str), K(default_ns), K(ret)); + ObXMLExprHelper::replace_xpath_ret_code(ret); + } else if (OB_FAIL(xpath_iter.open())) { + LOG_WARN("fail to open xpath iterator", K(ret)); + ObXMLExprHelper::replace_xpath_ret_code(ret); + } + + ObArray res_array; + while (OB_SUCC(ret)) { + if (OB_FAIL(xpath_iter.get_next_node(node))) { + if (ret != OB_ITER_END) { + LOG_WARN("fail to get next xml node", K(ret)); + } + } else if (OB_FAIL(res_array.push_back(node))) { + LOG_WARN("fail to push xml node", K(ret)); + } + } + + if (ret == OB_ITER_END) { + ret = OB_SUCCESS; + } + + for (int i = 0; i < res_array.size() && OB_SUCC(ret); ++i) { + ObIMulModeBase* update_node = res_array[i]; + if (OB_ISNULL(update_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("xpath result node is null", K(ret)); + } else if (OB_FAIL(update_xml_node(xml_mem_ctx, expr, ctx, update_node))) { + LOG_WARN("fail to update xml node", K(ret), K(node)); + } + } + + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = xpath_iter.close())) { + LOG_WARN("fail to close xpath iter", K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + return ret; +} + +int ObExprUpdateXml::update_xml_node(ObMulModeMemCtx* xml_mem_ctx, + const ObExpr *expr, + ObEvalCtx &ctx, + ObIMulModeBase *node) +{ + int ret = OB_SUCCESS; + ObXmlNode *xml_node = static_cast(node); + ObMulModeNodeType xml_type = xml_node->type(); + bool is_empty_content = false; + if (OB_ISNULL(xml_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("update node is NULL", K(ret)); + } else if (xml_type == M_DOCUMENT) { + int64_t child_size = xml_node->size(); + bool is_found = false; + for (int64_t i = 0; OB_SUCC(ret) && !is_found && i < child_size ; i++) { + ObXmlNode *child_node = xml_node->at(i); + if (OB_ISNULL(child_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("child node is NULL", K(ret)); + } else if (child_node->type() == M_ELEMENT) { + xml_node = child_node; + is_found = true; + } + } + if (!is_found) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("fail to found the root element node", K(ret)); + } + } else if (xml_type == M_CONTENT) { + if (xml_node->size() > 0) { + xml_node = xml_node->at(xml_node->size() - 1); + } else { + is_empty_content = true; + } + } + if (!is_empty_content) { + switch (xml_node->type()) { + case M_TEXT: { + if (OB_FAIL(update_text_or_attribute_node(xml_mem_ctx, xml_node, expr, ctx, true))) { + LOG_WARN("fail to update text node", K(ret), K(xml_type)); + } + break; + } + case M_ATTRIBUTE: { + if (OB_FAIL(update_text_or_attribute_node(xml_mem_ctx, xml_node, expr, ctx, false))) { + LOG_WARN("fail to update attribute node", K(ret)); + } + break; + } + case M_NAMESPACE: { + if (OB_FAIL(update_namespace_node(xml_mem_ctx, xml_node, expr, ctx))) { + LOG_WARN("fail to update namespace node", K(ret)); + } + break; + } + case M_COMMENT: + case M_CDATA: { + if (OB_FAIL(update_cdata_and_comment_node(xml_mem_ctx, xml_node, expr, ctx))) { + LOG_WARN("fail to update cdata node", K(ret), K(xml_type)); + } + break; + } + case M_ELEMENT: { + if (OB_FAIL(update_element_node(xml_mem_ctx, xml_node, expr, ctx))) { + LOG_WARN("fail to update element node", K(ret), K(xml_type)); + } + break; + } + case M_INSTRUCT: { + if (OB_FAIL(update_pi_node(xml_mem_ctx, xml_node, expr, ctx))) { + LOG_WARN("fail to pi node", K(ret), K(xml_type)); + } + break; + } + default: { + ret = OB_NOT_SUPPORTED; + LOG_WARN("unsupported xml node type", K(ret), K(xml_type)); + break; + } + } + } + return ret; +} + +int ObExprUpdateXml::update_pi_node(ObMulModeMemCtx* xml_mem_ctx, + ObXmlNode *xml_node, + const ObExpr *expr, + ObEvalCtx &ctx) +{ + int ret = OB_SUCCESS; + ObObjType val_type = expr->datum_meta_.type_; + uint16_t sub_schema_id = expr->obj_meta_.get_subschema_id(); + ObDatum *datum = NULL; + if (OB_FAIL(expr->eval(ctx, datum))) { + LOG_WARN("fail to eval datum", K(ret)); + } else { + if (val_type == ObNullType) { + ret = OB_ERR_UPDATE_XML_WITH_INVALID_NODE; + LOG_WARN("XML nodes must be updated with valid nodes and of the same type", K(ret)); + } else if (ob_is_string_type(val_type)) { + ObXmlDocument *xml_doc = NULL; + ObString value_str; + if (OB_FAIL(ObXMLExprHelper::get_str_from_expr(expr, ctx, value_str, *xml_mem_ctx->allocator_))) { + LOG_WARN("fail to get value str", K(ret), K(value_str)); + } else if (OB_FAIL(ObXmlParserUtils::parse_content_text(xml_mem_ctx, value_str, xml_doc))) { + if (ret == OB_ERR_PARSER_SYNTAX) { + ret = OB_ERR_UPDATE_XML_WITH_INVALID_NODE; + } + LOG_WARN("fail to parse xml str", K(ret)); + } else if (xml_doc->size() > 0 && xml_doc->at(0)->type() == M_INSTRUCT) { + if (OB_FAIL(update_xml_child_node(*xml_mem_ctx->allocator_, xml_node, xml_doc))) { + LOG_WARN("fail to update pi node with content node", K(ret)); + } + } else { + ret = OB_ERR_UPDATE_XML_WITH_INVALID_NODE; + LOG_WARN("update pi node with invalid node", K(ret), K(value_str)); + } + } else if (ob_is_xml_sql_type(val_type, sub_schema_id)) { + ObIMulModeBase *update_node = NULL; + ObXmlNode *update_xml_node = NULL; + ObCollationType cs_type = CS_TYPE_INVALID; + if (OB_FAIL(ObXMLExprHelper::get_xml_base(xml_mem_ctx, datum, cs_type, ObNodeMemType::TREE_TYPE, update_node))) { + LOG_WARN("fail to get update xml node", K(ret)); + } else if (OB_ISNULL(update_xml_node = static_cast(update_node))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("update node is NULL", K(ret)); + } else if (update_xml_node->size() == 0) { + ret = OB_ERR_UPDATE_XML_WITH_INVALID_NODE; + LOG_WARN("update pi node with invalid node", K(ret)); + } else if (OB_FAIL(update_xml_child_node(*xml_mem_ctx->allocator_, xml_node, update_xml_node))) { + LOG_WARN("fail to update xml content node ", K(ret)); + } + } + } + return ret; +} + +int ObExprUpdateXml::update_text_or_attribute_node(ObMulModeMemCtx* xml_mem_ctx, + ObXmlNode *xml_node, + const ObExpr *expr, + ObEvalCtx &ctx, + bool is_text) +{ + int ret = OB_SUCCESS; + ObObjType val_type = expr->datum_meta_.type_; + ObCollationType cs_type = expr->datum_meta_.cs_type_; + uint16_t sub_schema_id = expr->obj_meta_.get_subschema_id(); + ObDatum *datum = NULL; + if (OB_FAIL(expr->eval(ctx, datum))) { + LOG_WARN("fail to eval datum", K(ret)); + } else { + if (val_type == ObNullType) { + xml_node->set_value(ObString::make_empty_string()); + } else if (is_text && ob_is_clob(val_type, cs_type)) { + ObXmlDocument *xml_doc = NULL; + ObString value_str; + if (OB_FAIL(ObXMLExprHelper::get_str_from_expr(expr, ctx, value_str, *xml_mem_ctx->allocator_))) { + LOG_WARN("fail to get value str", K(ret), K(value_str)); + } else if (value_str.empty()) { + ret = OB_LOB_VALUE_NOT_EXIST; + LOG_WARN("LOB value is empty", K(ret), K(value_str)); + } else if (OB_FAIL(ObXmlParserUtils::parse_content_text(xml_mem_ctx, value_str, xml_doc))) { + if (ret == OB_ERR_PARSER_SYNTAX) { + ret = OB_ERR_XML_PARSE; + } + LOG_WARN("fail to parse xml str", K(ret)); + } else if (OB_FAIL(update_xml_child_node(*xml_mem_ctx->allocator_, xml_node, xml_doc))) { + LOG_WARN("fail to update xml element node", K(ret)); + } + } else if (ob_is_string_type(val_type)) { + ObStringBuffer buff(xml_mem_ctx->allocator_); + ObString value_str; + if (OB_FAIL(ObXMLExprHelper::get_str_from_expr(expr, ctx, value_str, *xml_mem_ctx->allocator_))) { + LOG_WARN("fail to get value str", K(ret)); + } else { + xml_node->set_value(value_str); + } + } else if (ob_is_xml_sql_type(val_type, sub_schema_id)) { + ObIMulModeBase *update_node = NULL; + ObXmlNode *update_xml_node = NULL; + ObCollationType cs_type = CS_TYPE_INVALID; + if (OB_FAIL(ObXMLExprHelper::get_xml_base(xml_mem_ctx, datum, cs_type, ObNodeMemType::TREE_TYPE, update_node))) { + LOG_WARN("fail to get update xml node", K(ret)); + } else if (OB_ISNULL(update_xml_node = static_cast(update_node))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("update node is NULL", K(ret)); + } else if (is_text && OB_FAIL(update_xml_child_node(*xml_mem_ctx->allocator_, xml_node, update_xml_node))) { + LOG_WARN("fail to update xml node ", K(ret)); + } else if (!is_text && OB_FAIL(update_attribute_xml_node(xml_node, update_xml_node))) { + LOG_WARN("fail to update xml node ", K(ret)); + } + } + } + return ret; +} + +int ObExprUpdateXml::update_namespace_node(ObMulModeMemCtx* xml_mem_ctx, + ObXmlNode *xml_node, + const ObExpr *expr, + ObEvalCtx &ctx) +{ + int ret = OB_SUCCESS; + ObObjType val_type = expr->datum_meta_.type_; + uint16_t sub_schema_id = expr->obj_meta_.get_subschema_id(); + ObDatum *datum = NULL; + if (OB_FAIL(expr->eval(ctx, datum))) { + LOG_WARN("fail to eval datum", K(ret)); + } else { + if (val_type == ObNullType) { + ret = OB_ERR_XML_PARSE; + LOG_WARN("update namespace node value to be NULL is unsupported", K(ret)); + } else if (ob_is_string_type(val_type)) { + ObStringBuffer buff(xml_mem_ctx->allocator_); + ObString value_str; + if (OB_FAIL(ObXMLExprHelper::get_str_from_expr(expr, ctx, value_str, *xml_mem_ctx->allocator_))) { + LOG_WARN("fail to get value str", K(ret)); + } else if (OB_FAIL(update_namespace_value(*xml_mem_ctx->allocator_, xml_node, value_str))) { + LOG_WARN("fail to update namespace node value", K(ret), K(value_str)); + } + } else if (ob_is_xml_sql_type(val_type, sub_schema_id)) { + ObIMulModeBase *update_node = NULL; + ObXmlNode *update_xml_node = NULL; + ObCollationType cs_type = CS_TYPE_INVALID; + if (OB_FAIL(ObXMLExprHelper::get_xml_base(xml_mem_ctx, datum, cs_type, ObNodeMemType::TREE_TYPE, update_node))) { + LOG_WARN("fail to get update xml node", K(ret)); + } else if (OB_ISNULL(update_xml_node = static_cast(update_node))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("update node is NULL", K(ret)); + } else if (OB_FAIL(update_namespace_xml_node(*xml_mem_ctx->allocator_, xml_node, update_xml_node))) { + LOG_WARN("fail to update xml node ", K(ret)); + } + } + } + return ret; +} + +int ObExprUpdateXml::update_element_node(ObMulModeMemCtx* xml_mem_ctx, + ObXmlNode *xml_node, + const ObExpr *expr, + ObEvalCtx &ctx) +{ + int ret = OB_SUCCESS; + ObObjType val_type = expr->datum_meta_.type_; + uint16_t sub_schema_id = expr->obj_meta_.get_subschema_id(); + ObDatum *datum = NULL; + if (OB_FAIL(expr->eval(ctx, datum))) { + LOG_WARN("fail to eval datum", K(ret)); + } else { + ObXmlElement *ele = static_cast(xml_node); + if (val_type == ObNullType) { + ObMulModeNodeType node_type = ele->type(); + if (OB_FAIL(clear_element_child_node(ele))) { + LOG_WARN("fail to clear child node", K(ret)); + } + } else if (ob_is_string_type(val_type)) { + ObXmlDocument *xml_doc = NULL; + ObString value_str; + if (OB_FAIL(ObXMLExprHelper::get_str_from_expr(expr, ctx, value_str, *xml_mem_ctx->allocator_))) { + LOG_WARN("fail to get value str", K(ret), K(value_str)); + } else if (value_str.trim().empty()) { + if (OB_FAIL(clear_element_child_node(ele))) { + LOG_WARN("fail to clear child node", K(ret)); + } + } else if (OB_FAIL(ObXmlParserUtils::parse_content_text(xml_mem_ctx, value_str, xml_doc))) { + if (ret == OB_ERR_PARSER_SYNTAX) { + ret = OB_ERR_XML_PARSE; + } + LOG_WARN("fail to parse xml str", K(ret)); + } else if (OB_FAIL(update_xml_child_node(*xml_mem_ctx->allocator_, ele, xml_doc))) { + LOG_WARN("fail to update xml element node", K(ret)); + } + } else if (ob_is_xml_sql_type(val_type, sub_schema_id)) { + ObIMulModeBase *update_node = NULL; + ObXmlNode *update_xml_node = NULL; + ObCollationType cs_type = CS_TYPE_INVALID; + if (OB_FAIL(ObXMLExprHelper::get_xml_base(xml_mem_ctx, datum, cs_type, ObNodeMemType::TREE_TYPE, update_node))) { + LOG_WARN("fail to get update xml node", K(ret)); + } else if (OB_ISNULL(update_xml_node = static_cast(update_node))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("update node is NULL", K(ret)); + } else if (update_xml_node->count() == 0) { + if (OB_FAIL(clear_element_child_node(ele))) { + LOG_WARN("fail to clear child node", K(ret)); + } + } else if (OB_FAIL(update_xml_child_node(*xml_mem_ctx->allocator_, xml_node, update_xml_node))) { + LOG_WARN("fail to update xml node ", K(ret)); + } + } + } + return ret; +} + +int ObExprUpdateXml::update_cdata_and_comment_node(ObMulModeMemCtx* xml_mem_ctx, + ObXmlNode *xml_node, + const ObExpr *expr, + ObEvalCtx &ctx) +{ + int ret = OB_SUCCESS; + ObObjType val_type = expr->datum_meta_.type_; + uint16_t sub_schema_id = expr->obj_meta_.get_subschema_id(); + ObDatum *datum = NULL; + if (OB_FAIL(expr->eval(ctx, datum))) { + LOG_WARN("fail to eval datum", K(ret)); + } else { + if (val_type == ObNullType) { + xml_node->set_value(ObString::make_empty_string()); + } else if (ob_is_string_type(val_type)) { + ObXmlDocument *xml_doc = NULL; + ObString value_str; + if (OB_FAIL(ObXMLExprHelper::get_str_from_expr(expr, ctx, value_str, *xml_mem_ctx->allocator_))) { + LOG_WARN("fail to get value str", K(ret), K(value_str)); + } else if (OB_FAIL(ObXmlParserUtils::parse_content_text(xml_mem_ctx, value_str, xml_doc))) { + if (ret == OB_ERR_PARSER_SYNTAX) { + ret = OB_ERR_XML_PARSE; + } + LOG_WARN("fail to parse xml str", K(ret)); + } else if (OB_FAIL(update_xml_child_node(*xml_mem_ctx->allocator_, xml_node, xml_doc))) { + LOG_WARN("fail to update xml element node", K(ret)); + } + } else if (ob_is_xml_sql_type(val_type, sub_schema_id)) { + ObIMulModeBase *update_node = NULL; + ObXmlNode *update_xml_node = NULL; + ObCollationType cs_type = CS_TYPE_INVALID; + if (OB_FAIL(ObXMLExprHelper::get_xml_base(xml_mem_ctx, datum, cs_type, ObNodeMemType::TREE_TYPE, update_node))) { + LOG_WARN("fail to get update xml node", K(ret)); + } else if (OB_ISNULL(update_xml_node = static_cast(update_node))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("update node is NULL", K(ret)); + } else if (OB_FAIL(update_xml_child_node(*xml_mem_ctx->allocator_, xml_node, update_xml_node))) { + LOG_WARN("fail to update xml node ", K(ret)); + } + } + } + return ret; +} + +int ObExprUpdateXml::update_namespace_value(ObIAllocator &allocator, ObXmlNode *xml_node, const ObString &ns_value) +{ + int ret = OB_SUCCESS; + ObXmlElement *parent = NULL; + ObXmlAttribute *ns = NULL; + ObXmlAttribute *new_ns = NULL; + ObString key; + if (OB_ISNULL(xml_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("xml node is NULL", K(ret)); + } else if (xml_node->type() != M_NAMESPACE) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("xml node type is not expected", K(ret), K(xml_node->type())); + } else { + ns = static_cast(xml_node); + parent = static_cast(xml_node->get_parent()); + key = ns->get_key(); + } + if (OB_FAIL(ret)) { + } else if (0 == key.compare(ObXmlConstants::XMLNS_STRING)) { + ret = OB_ERR_XML_PARSE; + LOG_WARN("defaul namespace is not allowed to update value", K(ret)); + } else if (OB_FAIL(update_exist_nodes_ns(parent, ns))) { + LOG_WARN("fail to udpate exist node ns", K(ret)); + } else if (OB_ISNULL(new_ns = OB_NEWx(ObXmlAttribute, (&allocator), ObMulModeNodeType::M_NAMESPACE, parent->get_mem_ctx()))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc failed", K(ret)); + } else { + int64_t pos = -1; + new_ns->set_value(ns_value); + new_ns->set_prefix(ObXmlConstants::XMLNS_STRING); + new_ns->set_key(key); + if (OB_FAIL(parent->get_attribute_pos(ObMulModeNodeType::M_NAMESPACE, key, pos))) { + LOG_WARN("fail to get namespace node pos", K(ret)); + } else if (OB_FAIL(parent->remove_namespace(pos))) { + LOG_WARN("fail to remove namespace node", K(ret)); + } else if (OB_FAIL(parent->add_attribute(new_ns, false, 0))) { + LOG_WARN("fail to add new namespace node", K(ret)); + } + } + + return ret; +} + +int ObExprUpdateXml::clear_element_child_node(ObXmlElement *ele_node) +{ + int ret = OB_SUCCESS; + // 1. first clear child node + if (OB_ISNULL(ele_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("xml node is NULL", K(ret)); + } else { + // remove all child node + int64_t child_size = ele_node->size(); + for (int64_t i = 0; OB_SUCC(ret) && i < child_size; i++) { + if (OB_FAIL(ele_node->remove_element(ele_node->at(0)))) { + LOG_WARN("fail to remove element node", K(ret)); + } + } + // remove all attributes and namespaces not use + int64_t attr_size = ele_node->attribute_size(); // attributes size will change after add or remove + for (int64_t i = 0; OB_SUCC(ret) && i < attr_size; i++) { + ObXmlAttribute *attr = NULL; + // use as a queue, always remove the first node + if (OB_FAIL(ele_node->get_attribute(attr, 0))) { + LOG_WARN("fail to get attribute", K(ret), K(i)); + } else if (attr->type() == M_NAMESPACE) { + if (OB_FAIL(ele_node->remove_namespace(0))) { + LOG_WARN("fail to remove namespace", K(ret)); + } + } else if (OB_FAIL(ele_node->remove_attribute(0))) { + LOG_WARN("fail to remove attribute node", K(ret)); + } + } + // add namespace if use + ObXmlAttribute *ns = ele_node->get_ns(); + if (OB_SUCC(ret) && OB_NOT_NULL(ns)) { + if (ele_node->get_prefix().empty()) { + ObXmlAttribute *default_ns = NULL; + if (OB_FAIL(get_valid_default_ns_from_parent(ele_node->get_parent(), default_ns))) { + LOG_WARN("fail to get valid default ns", K(ret)); + } else if (OB_NOT_NULL(default_ns) + && default_ns->get_value().compare(ns->get_value()) == 0) { + // do nothing + } else if (OB_FAIL(ele_node->add_attribute(ns, false, 0))) { + LOG_WARN("fail to add default namespace", K(ret)); + } + } else if (OB_FAIL(ele_node->add_attribute(ns, false, 0))) { + LOG_WARN("fail to add prefix namespace", K(ret)); + } + } + } + return ret; +} + +int ObExprUpdateXml::update_attribute_xml_node(ObXmlNode *old_node, ObXmlNode *update_node) +{ + int ret = OB_SUCCESS; + int64_t pos = -1; + ObXmlNode *parent = NULL; + ObString key; + ObXmlElement *ele_node = NULL; + if (OB_ISNULL(old_node) || OB_ISNULL(update_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("node is NULL", K(ret), K(old_node), K(update_node)); + } else if (OB_ISNULL(parent = old_node->get_parent())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("attribute parent node is NULL", K(ret)); + } else if (FALSE_IT(key = old_node->get_key())) { + } else if (parent->type() != M_ELEMENT) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("parent of attribute node is not an element node", K(ret), K(parent->type())); + } else if (FALSE_IT(ele_node = static_cast(parent))) { + } else if (OB_FAIL(ele_node->get_attribute_pos(old_node->type(), key, pos))) { + LOG_WARN("fail to get attribute pos", K(ret), K(key), K(old_node->type())); + } else if (OB_FAIL(ele_node->remove_attribute(pos))) { // remove attribute + LOG_WARN("fail to remove attribute", K(ret), K(pos), K(key)); + } else if (OB_FAIL(remove_and_insert_element_node(ele_node, update_node, 0, false))) { + LOG_WARN("fail to update element node", K(ret)); + } + return ret; +} + +int ObExprUpdateXml::update_namespace_xml_node(ObIAllocator &allocator, ObXmlNode *old_node, ObXmlNode *update_node) +{ + int ret = OB_SUCCESS; + int64_t pos = -1; + ObXmlNode *parent = NULL; + ObXmlElement *ele_node = NULL; + ObXmlAttribute *ns_node = NULL; + ObString key; + bool is_default_ns = false; + if (OB_ISNULL(old_node) || OB_ISNULL(update_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("node is NULL", K(ret), K(old_node), K(update_node)); + } else if (FALSE_IT(ns_node = static_cast(old_node))) { + } else if (OB_ISNULL(parent = ns_node->get_parent())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("attribute parent node is NULL", K(ret)); + } else if (FALSE_IT(key = ns_node->get_key())) { + } else if (FALSE_IT(is_default_ns = 0 == key.compare(ObXmlConstants::XMLNS_STRING))) { + } else if (parent->type() != M_ELEMENT) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("parent of namespace node is not an element node", K(ret), K(parent->type())); + } else if (FALSE_IT(ele_node = static_cast(parent))) { + } else if (OB_FAIL(ele_node->get_attribute_pos(ns_node->type(), key, pos))) { + LOG_WARN("fail to get attribute pos", K(ret), K(key), K(ns_node->type())); + } else if (!is_default_ns && OB_FAIL(update_exist_nodes_ns(ele_node, ns_node))) { + LOG_WARN("fail to update exist node ns", K(ret)); + } else if (OB_FAIL(update_new_nodes_ns(allocator, ele_node, update_node))) { + LOG_WARN("fail to update new node ns", K(ret)); + } else { + // remove prefix ns: not default ns && ns of element is not this prefix && attr of element not use, remove the prefix xmlns + if (!is_default_ns && ele_node->get_ns() != ns_node && !ele_node->has_attribute_with_ns(ns_node)) { + if (OB_FAIL(ele_node->remove_namespace(pos))) { + LOG_WARN("fail to remove prefix namespace", K(ret), K(key)); + } + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(remove_and_insert_element_node(ele_node, update_node, 0, false))) { + LOG_WARN("fail to update element node", K(ret), K(pos)); + } + } + return ret; +} + +// update the descendent node prefix ns when need to remove parent node prefix ns +int ObExprUpdateXml::update_exist_nodes_ns(ObXmlElement *parent, ObXmlAttribute *prefix_ns) +{ + int ret = OB_SUCCESS; + ObXmlAttribute *ns = NULL; + if (OB_ISNULL(parent) || OB_ISNULL(prefix_ns)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("node is NULL", K(ret), K(parent), K(prefix_ns)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < parent->size(); i++) { + if (OB_FAIL(set_ns_recrusively(parent->at(i), prefix_ns))) { + LOG_WARN("fail to set exist nodes ns", K(ret)); + } + } + } + return ret; +} + +// update the new node default ns to empty when the parent node has default ns +int ObExprUpdateXml::update_new_nodes_ns(ObIAllocator &allocator, ObXmlNode *parent, ObXmlNode *update_node) +{ + int ret = OB_SUCCESS; + ObXmlAttribute *empty_ns = NULL; + ObXmlAttribute *default_ns = NULL; + if (OB_ISNULL(parent) || OB_ISNULL(update_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("node is NULL", K(ret), K(parent), K(update_node)); + } else if (OB_FAIL(get_valid_default_ns_from_parent(parent, default_ns))) { + LOG_WARN("unexpected error in find default ns from parent", K(ret)); + } else if (OB_NOT_NULL(default_ns) && !default_ns->get_value().empty()) { + // need to update the new node default ns with empty default ns + if (OB_ISNULL(empty_ns = OB_NEWx(ObXmlAttribute, (&allocator), ObMulModeNodeType::M_NAMESPACE, parent->get_mem_ctx()))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc failed", K(ret)); + } else { + empty_ns->set_key(ObXmlConstants::XMLNS_STRING); + empty_ns->set_value(ObString::make_empty_string()); + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(set_ns_recrusively(update_node, empty_ns))) { + LOG_WARN("fail to set empty default ns recrusively", K(ret)); + } + } + return ret; +} + +// found valid default ns from down to top +int ObExprUpdateXml::get_valid_default_ns_from_parent(ObXmlNode *cur_node, ObXmlAttribute* &default_ns) +{ + int ret = OB_SUCCESS; + ObXmlNode* t_node = NULL; + bool is_found = false; + if (OB_ISNULL(cur_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("update node is NULL", K(ret)); + } else if (!ObXMLExprHelper::is_xml_element_node(cur_node->type())) { + t_node = cur_node->get_parent(); + } else { + t_node = cur_node; + } + + while(!is_found && OB_SUCC(ret) && OB_NOT_NULL(t_node)) { + ObXmlElement *t_element = static_cast(t_node); + ObArray attr_list; + if (OB_FAIL(t_element->get_namespace_list(attr_list))) { + LOG_WARN("fail to get namespace list", K(ret)); + } + for (int i = 0; !is_found && OB_SUCC(ret) && i < attr_list.size(); i ++) { + ObXmlAttribute *attr = static_cast(attr_list.at(i)); + if (attr->get_key().compare(ObXmlConstants::XMLNS_STRING) == 0) { + is_found = true; + default_ns = attr; + } + } + t_node = t_node->get_parent(); + } + return ret; +} + +int ObExprUpdateXml::set_ns_recrusively(ObXmlNode *update_node, ObXmlAttribute *ns) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(update_node) || OB_ISNULL(ns)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("update node is NULL", K(ret), K(update_node), K(ns)); + } else if (!ObXMLExprHelper::is_xml_element_node(update_node->type())) { + // no need to set default ns + } else { + bool is_stop = false; + ObXmlElement *ele_node = static_cast(update_node); + ObString key = ns->get_key(); + if (ele_node->type() != M_ELEMENT) { + // skip + } else if (key.compare(ObXmlConstants::XMLNS_STRING) == 0) { + // update default ns + if (ele_node->get_prefix().empty()) { + // this condition mean: has no ns || has non-empty default ns + is_stop = true; + if (OB_ISNULL(ele_node->get_ns())) { + ele_node->add_attribute(ns, false, 0); + ele_node->set_ns(ns); + } else { /* has non-empty default ns, skip and stop find */ } + } + } else { // has prefix + ObXmlAttribute *tmp_ns = NULL; + if (ele_node->get_ns() == ns || + ele_node->has_attribute_with_ns(ns) || + OB_NOT_NULL(tmp_ns = ele_node->get_ns_by_name(key))) { + // match condition below will stop recrusive + // element use this prefix ns || attributes of element use this prefix ns || this prefix in attributes + is_stop = true; + if (OB_NOT_NULL(tmp_ns)) { // if the prefix not in attributes + } else if (OB_FAIL(ele_node->add_attribute(ns, false, 0))) { + LOG_WARN("fail to add namespace node", K(ret), K(key)); + } + } + } + + if (!is_stop) { + // find its child node recrusivle when no need to set default ns + for (int64_t i = 0; OB_SUCC(ret) && i < ele_node->size(); i++) { + if (OB_FAIL(SMART_CALL(set_ns_recrusively(ele_node->at(i), ns)))) { + LOG_WARN("fail set default ns in origin tree recursively", K(ret)); + } + } // end for + } // end is_stop + } + return ret; +} + +// for xml nodes other than xmlattribute(including attribute and namespace) +int ObExprUpdateXml::update_xml_child_node(ObIAllocator &allocator, ObXmlNode *old_node, ObXmlNode *update_node) +{ + int ret = OB_SUCCESS; + ObXmlNode *parent = NULL; + ObXmlElement *ele_node = NULL; + int64_t pos = -1; + if (OB_ISNULL(old_node) || OB_ISNULL(update_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("node is NULL", K(ret), K(old_node), K(update_node)); + } else if (OB_ISNULL(parent = old_node->get_parent())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("parent node is NULL", K(ret)); + } else { + ele_node = static_cast(parent); + pos = old_node->get_index(); + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(update_new_nodes_ns(allocator, ele_node, update_node))) { + LOG_WARN("fail to update new node ns", K(ret)); + } else if (OB_FAIL(remove_and_insert_element_node(ele_node, update_node, pos, true))) { + LOG_WARN("fail to update element node", K(ret)); + } + + return ret; +} + +int ObExprUpdateXml::remove_and_insert_element_node(ObXmlElement *ele_node, ObXmlNode *update_node, int64_t pos, bool is_remove) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(ele_node) || OB_ISNULL(update_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("node is NULL", K(ret), K(ele_node), K(update_node)); + } else if (pos < 0 || pos > ele_node->count()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("pos is invalid", K(ret), K(pos)); + } else if (ObXMLExprHelper::is_xml_root_node(update_node->type())) { + if ((is_remove && ele_node->count() == 0) || update_node->count() == 0) { + // skip and do nothing + } else if (is_remove && OB_FAIL(ele_node->remove_element(ele_node->at(pos)))) { // remove the node + LOG_WARN("fail to remove element node", K(ret), K(pos)); + } else { + ObXmlDocument *xml_doc = static_cast(update_node); + for (int64_t i = 0; OB_SUCC(ret) && i < xml_doc->size(); i++) { + if (OB_ISNULL(xml_doc->at(i)) ) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("xml node is null", K(ret), K(i)); + } else if (OB_FAIL(ele_node->add_element(xml_doc->at(i), false, pos + i))) { // insert the element node + LOG_WARN("fail to add element node", K(ret), K(i)); + } + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected node type", K(ret), K(update_node->type())); + } + return ret; +} + +#endif int ObExprUpdateXml::cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) const { diff --git a/src/sql/engine/expr/ob_expr_update_xml.h b/src/sql/engine/expr/ob_expr_update_xml.h index 3bf535810..ce24b7c60 100644 --- a/src/sql/engine/expr/ob_expr_update_xml.h +++ b/src/sql/engine/expr/ob_expr_update_xml.h @@ -15,6 +15,11 @@ #define OCEANBASE_SQL_ENGINE_EXPR_OB_EXPR_UPDATE_XML_H #include "sql/engine/expr/ob_expr_operator.h" +#ifdef OB_BUILD_ORACLE_XML +#include "lib/xml/ob_multi_mode_interface.h" +#include "lib/xml/ob_xml_tree.h" +#include "lib/xml/ob_xpath.h" +#endif namespace oceanbase { @@ -31,11 +36,51 @@ public: ObExprResType *types, int64_t param_num, common::ObExprTypeCtx &type_ctx) const override; +#ifdef OB_BUILD_ORACLE_XML + static int eval_update_xml(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res); +#else static int eval_update_xml(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) { return OB_NOT_SUPPORTED; } +#endif virtual int cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) const override; +#ifdef OB_BUILD_ORACLE_XML +private: + static int update_xml_tree(ObMulModeMemCtx* xml_mem_ctx, + const ObExpr *expr, + ObEvalCtx &ctx, + ObString &xpath_str, + ObString &default_ns, + ObPathVarObject *prefix_ns, + ObIMulModeBase *xml_tree); + static int update_xml_node(ObMulModeMemCtx* xml_mem_ctx, const ObExpr *expr, ObEvalCtx &ctx, ObIMulModeBase *node); + // for text node and attribute + static int update_text_or_attribute_node(ObMulModeMemCtx* xml_mem_ctx, + ObXmlNode *xml_node, + const ObExpr *expr, + ObEvalCtx &ctx, + bool is_text); + static int update_attribute_node(ObIAllocator &allocator, ObXmlNode *xml_node, const ObExpr *expr, ObEvalCtx &ctx); + static int update_attribute_xml_node(ObXmlNode *old_node, ObXmlNode *update_node); + // for cdata and comment + static int update_cdata_and_comment_node(ObMulModeMemCtx* xml_mem_ctx, ObXmlNode *xml_node, const ObExpr *expr, ObEvalCtx &ctx); + // for namespace + static int update_namespace_node(ObMulModeMemCtx* xml_mem_ctx, ObXmlNode *xml_node, const ObExpr *expr, ObEvalCtx &ctx); + static int update_namespace_xml_node(ObIAllocator &allocator, ObXmlNode *old_node, ObXmlNode *update_node); + static int update_namespace_value(ObIAllocator &allocator, ObXmlNode *xml_node, const ObString &ns_value); + static int get_valid_default_ns_from_parent(ObXmlNode *cur_node, ObXmlAttribute* &default_ns); + static int set_ns_recrusively(ObXmlNode *update_node, ObXmlAttribute *ns); + static int update_new_nodes_ns(ObIAllocator &allocator, ObXmlNode *parent, ObXmlNode *update_node); + static int update_exist_nodes_ns(ObXmlElement *parent, ObXmlAttribute *prefix_ns); + // for element + static int update_element_node(ObMulModeMemCtx* xml_mem_ctx, ObXmlNode *xml_node, const ObExpr *expr, ObEvalCtx &ctx); + static int clear_element_child_node(ObXmlElement *ele_node); + static int update_xml_child_node(ObIAllocator &allocator, ObXmlNode *old_node, ObXmlNode *update_node); + static int remove_and_insert_element_node(ObXmlElement *ele_node, ObXmlNode *update_node, int64_t pos, bool is_remove); + // for pi node + static int update_pi_node(ObMulModeMemCtx* xml_mem_ctx, ObXmlNode *xml_node, const ObExpr *expr, ObEvalCtx &ctx); +#endif private: DISALLOW_COPY_AND_ASSIGN(ObExprUpdateXml); }; diff --git a/src/sql/engine/expr/ob_expr_xml_attributes.cpp b/src/sql/engine/expr/ob_expr_xml_attributes.cpp index 090f096c0..0b176edd3 100644 --- a/src/sql/engine/expr/ob_expr_xml_attributes.cpp +++ b/src/sql/engine/expr/ob_expr_xml_attributes.cpp @@ -14,8 +14,9 @@ #define USING_LOG_PREFIX SQL_ENG #include "ob_expr_xml_attributes.h" #include "sql/engine/expr/ob_expr_json_func_helper.h" // may need json for kv pairs -#include "ob_expr_xml_func_helper.h" - +#ifdef OB_BUILD_ORACLE_XML +#include "sql/engine/expr/ob_expr_xml_func_helper.h" +#endif using namespace oceanbase::common; using namespace oceanbase::sql; @@ -81,6 +82,75 @@ int ObExprXmlAttributes::calc_result_typeN(ObExprResType &type, return ret; } +#ifdef OB_BUILD_ORACLE_XML +int ObExprXmlAttributes::eval_xml_attributes(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) +{ + INIT_SUCC(ret); + ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); + common::ObArenaAllocator &tmp_allocator = tmp_alloc_g.get_allocator(); + int attributes_escape = 1; + int has_schema_check = 0; + ObDatum *datum = NULL; + int param_num = expr.arg_cnt_; + if (param_num < 3) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("attributes arg cnt invalid", K(ret), K(param_num)); + } else if (OB_FAIL(expr.args_[0]->eval(ctx, datum))) { + LOG_WARN("get escape value failed", K(ret)); + } else if (FALSE_IT(attributes_escape = datum->get_int())) { + } else if (OB_FAIL(expr.args_[1]->eval(ctx, datum))) { + LOG_WARN("get schema check value failed", K(ret)); + } else if (FALSE_IT(has_schema_check = datum->get_int())) { + } else { + ObJsonArray j_arr(&tmp_allocator); + ObIJsonBase *j_base = &j_arr; + for (int i = 2; OB_SUCC(ret) && i < param_num; i++) { + ObIJsonBase *j_val; + ObStringBuffer content(&tmp_allocator); + if (OB_FAIL(ObJsonExprHelper::get_json_val(expr, ctx, &tmp_allocator, i, j_val))) { + LOG_WARN("failed: get_json_val failed", K(ret)); + } else if (0 == i % 2 && attributes_escape) { // TODO Subsequent considerations to pass attributes with element + // content + if ((j_val->json_type() != ObJsonNodeType::J_NULL) && + OB_FAIL(ObXmlParserUtils::escape_xml_text(ObString(j_val->get_data_length(), j_val->get_data()), content))) { + LOG_WARN("escape xml attributes failed", K(ret), K(attributes_escape)); + } else { + void *node = tmp_allocator.alloc(sizeof(ObJsonString)); + if (OB_ISNULL(node)){ + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc jsonString failed", K(ret)); + } else { + ObJsonString *jstr = new (node) ObJsonString(content.ptr(), content.length()); + if (OB_FAIL(j_base->array_append(jstr))) { + LOG_WARN("json_keys array append failed", K(ret), K(content)); + } + } + } + } else if (OB_FAIL(j_base->array_append(j_val))) { + LOG_WARN("failed: json array append json value", K(ret)); + } + } + // set result(json bin) + if (OB_SUCC(ret)) { + ObString raw_bin; + if (OB_FAIL(j_base->get_raw_binary(raw_bin, &tmp_allocator))) { + LOG_WARN("failed: get json raw binary", K(ret)); + } else { + uint64_t length = raw_bin.length(); + char *buf = expr.get_str_res_mem(ctx, length); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed: alloc memory for json array result", K(ret), K(length)); + } else { + MEMMOVE(buf, raw_bin.ptr(), length); + res.set_string(buf, length); + } + } + } + } + return ret; +} +#endif int ObExprXmlAttributes::cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) const { diff --git a/src/sql/engine/expr/ob_expr_xml_attributes.h b/src/sql/engine/expr/ob_expr_xml_attributes.h index 6487c785c..52679c157 100644 --- a/src/sql/engine/expr/ob_expr_xml_attributes.h +++ b/src/sql/engine/expr/ob_expr_xml_attributes.h @@ -32,7 +32,11 @@ public: int64_t param_num, common::ObExprTypeCtx& type_ctx) const override; +#ifdef OB_BUILD_ORACLE_XML + static int eval_xml_attributes(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res); +#else static int eval_xml_attributes(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) { return OB_NOT_SUPPORTED; } +#endif virtual int cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) const override; diff --git a/src/sql/engine/expr/ob_expr_xml_element.cpp b/src/sql/engine/expr/ob_expr_xml_element.cpp index 34fc9901e..7a338a8a0 100644 --- a/src/sql/engine/expr/ob_expr_xml_element.cpp +++ b/src/sql/engine/expr/ob_expr_xml_element.cpp @@ -13,7 +13,10 @@ #define USING_LOG_PREFIX SQL_ENG #include "ob_expr_xml_element.h" -#include "ob_expr_xml_func_helper.h" +#ifdef OB_BUILD_ORACLE_XML +#include "lib/xml/ob_xml_util.h" +#include "sql/engine/expr/ob_expr_xml_func_helper.h" +#endif using namespace oceanbase::common; using namespace oceanbase::sql; @@ -96,6 +99,354 @@ int ObExprXmlElement::calc_result_typeN(ObExprResType& type, return ret; } +#ifdef OB_BUILD_ORACLE_XML +int ObExprXmlElement::eval_xml_element(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) +{ + INIT_SUCC(ret); + ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); + common::ObArenaAllocator &tmp_allocator = tmp_alloc_g.get_allocator(); + ObDatum *datum = NULL; + int num_args = expr.arg_cnt_; + ObString name_tag; + int need_escape = 0; + int is_name = 0; + ObVector value_vec; + const ObIJsonBase *attr_json = NULL; + ObString binary_str; + ObString blob_locator; + bool has_attribute = false; + ObXmlElement *element = NULL; + ObXmlDocument *res_doc = NULL; + ObMulModeMemCtx* mem_ctx = nullptr; + lib::ObMallocHookAttrGuard malloc_guard(lib::ObMemAttr(MTL_ID(), "XMLCodeGen")); + if (OB_FAIL(ObXmlUtil::create_mulmode_tree_context(&tmp_allocator, mem_ctx))) { + LOG_WARN("fail to create tree memory context", K(ret)); + } else if (OB_UNLIKELY(num_args < 3)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid args num", K(ret), K(num_args)); + } else { + expr.args_[0]->eval(ctx, datum); + need_escape = datum->get_int(); + expr.args_[1]->eval(ctx, datum); + is_name = datum->get_int(); + if (need_escape != 0 && need_escape != 1) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid escaping opt", K(ret), K(need_escape)); + } else if (need_escape == 0 && is_name == 0) { + ret = OB_ERR_XML_MISSING_COMMA; + LOG_WARN("xml element param invalid", K(ret)); + } else if (OB_FAIL(expr.args_[2]->eval(ctx, datum))) { + LOG_WARN("expr args 1 failed", K(ret), K(expr.args_[2])); + } else if(expr.args_[2]->datum_meta_.type_ == ObNumberType) { + ret = OB_ERR_INVALID_XML_DATATYPE; + LOG_USER_ERROR(OB_ERR_INVALID_XML_DATATYPE, "Character", "-"); + LOG_WARN("Unsupport for string typee with binary charset input.", K(ret), K(expr.args_[2]->datum_meta_.type_)); + } else if (OB_FAIL(ObTextStringHelper::get_string(expr, tmp_allocator, 2, datum, name_tag))) { + LOG_WARN("get xml plain text failed", K(ret)); + } + } + for (int i = 3; OB_SUCC(ret) && i < num_args && OB_SUCC(ret); i++) { + ObObjType val_type = expr.args_[i]->datum_meta_.type_; + ObItemType item_type = expr.args_[i]->type_; + if (OB_FAIL(expr.args_[i]->eval(ctx, datum))) { + LOG_WARN("expr args failed", K(ret), K(i)); + } else if (ob_is_json(val_type) && i == 3) { // result from attribute + has_attribute = true; + void *buf = NULL; + if (OB_ISNULL(buf = tmp_allocator.alloc(sizeof(ObJsonBin)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to alloc memory", K(ret), K(sizeof(ObJsonBin))); + } else { + ObJsonBin *j_bin = new (buf) ObJsonBin(datum->get_string().ptr(), datum->get_string().length(), &tmp_allocator); + attr_json = static_cast(j_bin); + if (OB_FAIL(j_bin->reset_iter())) { + LOG_WARN("fail to reset iter", K(ret)); + } else if (attr_json->json_type() != ObJsonNodeType::J_ARRAY) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid json type", K(ret), K(attr_json->json_type())); + } + } + } else { // element value + ObString xml_value_data = datum->get_string(); + ObExpr *xml_arg = expr.args_[i]; + bool validity = false; + if (val_type == ObUserDefinedSQLType) { // xmltype + if (OB_FAIL(ObTextStringHelper::read_real_string_data(&tmp_allocator, + ObObjType::ObLongTextType, + xml_arg->datum_meta_.cs_type_, + true, xml_value_data))) { + LOG_WARN("fail to get real data.", K(ret), K(xml_value_data)); + // } else if (OB_FAIL(ObXMLExprHelper::check_xml_document_unparsed(mem_ctx, xml_value_data, validity))) { + // LOG_WARN("check document unparsed failed", K(ret), K(xml_value_data)); + // } else if (!validity) { + // ret = OB_ERR_XML_PARSE; + // LOG_WARN("input a unparsed document and parsing failed", K(ret), K(i), K(xml_value_data)); + } else { + ObObj temp_value; + temp_value.set_string(ObUserDefinedSQLType, xml_value_data); + value_vec.push_back(temp_value); + } + } else if (OB_FAIL(ObTextStringHelper::read_real_string_data(tmp_allocator, *datum, + xml_arg->datum_meta_, + xml_arg->obj_meta_.has_lob_header(), + xml_value_data))) { + LOG_WARN("fail to get real data.", K(ret), K(xml_value_data)); + } else if (expr.args_[i]->datum_meta_.cs_type_ == CS_TYPE_BINARY) { // binary + const ObObjMeta obj_meta = expr.args_[i]->obj_meta_; + ObObj obj; + ObObj tmp_result; + ObCastCtx cast_ctx(&tmp_allocator, NULL, CM_NONE, CS_TYPE_INVALID); + obj.set_string(ObVarcharType, xml_value_data); + if (OB_FAIL(ObHexUtils::rawtohex(obj, cast_ctx, tmp_result))) { + LOG_WARN("fail to check xml binary syntax", K(ret)); + } else if (OB_FAIL(construct_value_array(tmp_allocator, + tmp_result.get_string(), + value_vec))) { + LOG_WARN("construct value array failed", K(ret)); + } + } else { // varchar + if (need_escape) { + ObStringBuffer *escape_value = nullptr; + if (OB_ISNULL(escape_value = OB_NEWx(ObStringBuffer, &tmp_allocator, (&tmp_allocator)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate buffer", K(ret)); + } else if (OB_FAIL(ObXmlParserUtils::escape_xml_text(xml_value_data, *escape_value))) { + LOG_WARN("escape xml value failed", K(ret), K(need_escape)); + } else if (OB_FAIL(construct_value_array(tmp_allocator, + ObString(escape_value->length(), escape_value->ptr()), + value_vec))) { + LOG_WARN("construct value array failed", K(ret)); + } + } else if (OB_FAIL(construct_value_array(tmp_allocator, + xml_value_data, + value_vec))) { + LOG_WARN("construct value array failed", K(ret)); + } + } + } + } + + ObStringBuffer tmp_buff(&tmp_allocator); + ObString text_xml; + ObXmlDocument *xml_doc = nullptr; + + bool tag_validity = false; + bool doc_validity = true; + ObXmlText *tag_name_start; + ObXmlText *tag_name_end; + ObString start_tag("<>"); + ObString end_tag(""); + if (OB_FAIL(ret)) { + } else if (!has_attribute && name_tag.empty()) { + if (OB_ISNULL(tag_name_start = OB_NEWx(ObXmlText, mem_ctx->allocator_, ObMulModeNodeType::M_TEXT, mem_ctx))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("new xml text failed", K(ret)); + } else if (OB_FALSE_IT(tag_name_start->set_text(start_tag))) { + } else if (OB_ISNULL(tag_name_end = OB_NEWx(ObXmlText, mem_ctx->allocator_, ObMulModeNodeType::M_TEXT, mem_ctx))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("new xml text failed", K(ret)); + } else if (OB_FALSE_IT(tag_name_end->set_text(end_tag))) { + } + } + + if (OB_FAIL(ret)) { + LOG_WARN("ret failed", K(ret)); + } else if (OB_ISNULL(element = OB_NEWx(ObXmlElement, (mem_ctx->allocator_), ObMulModeNodeType::M_ELEMENT, mem_ctx))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate mem failed", K(ret)); + } else if (!has_attribute && name_tag.empty() && OB_FAIL(element->add_element(tag_name_start))) { + LOG_WARN("element add start element failed", K(ret)); + } else if (OB_FAIL(construct_element(mem_ctx, name_tag, value_vec, attr_json, element, tag_validity))) { + LOG_WARN("construct_element failed", K(ret)); + } else if (!has_attribute && name_tag.empty() && OB_FAIL(element->add_element(tag_name_end))) { + LOG_WARN("element add end element failed", K(ret)); + } else if (OB_ISNULL(res_doc = OB_NEWx(ObXmlDocument, (mem_ctx->allocator_), ObMulModeNodeType::M_UNPARSED, mem_ctx))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("allocate mem failed", K(ret)); + } else if (OB_FAIL(res_doc->add_element(element))) { + LOG_WARN("res doc add element failed", K(ret)); + } else if (tag_validity && element->get_unparse() && OB_FAIL(ObXMLExprHelper::check_doc_validity(mem_ctx, res_doc, doc_validity))) { // && element.get_unparse() + LOG_WARN("check doc validity failed", K(ret), K(tag_validity)); + } else if ((!tag_validity || !doc_validity)) { + res_doc->set_xml_type(ObMulModeNodeType::M_UNPARSED); + } else if (tag_validity && doc_validity) { + static_cast(res_doc->at(0))->set_unparse(0); + res_doc->set_xml_type(ObMulModeNodeType::M_DOCUMENT); + } + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(res_doc)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN(" failed to pack result, as res_doc is nullptr", K(ret)); + } else if (OB_FAIL(res_doc->get_raw_binary(binary_str, &tmp_allocator))) { + LOG_WARN("get raw binary failed", K(ret)); + } else if (OB_FAIL(ObXMLExprHelper::pack_binary_res(expr, ctx, binary_str, blob_locator))) { + LOG_WARN("pack binary res failed", K(ret), K(binary_str)); + } else { + res.set_string(blob_locator.ptr(), blob_locator.length()); + } + return ret; +} + +int ObExprXmlElement::construct_value_array(ObIAllocator &allocator, const ObString &value, ObVector &res_value) +{ + INIT_SUCC(ret); + if (value.empty()) { + // donothing + } else { + ObObj temp_value; + uint32_t vec_size = res_value.size(); + if (res_value.size() == 0) { + temp_value.set_string(ObVarcharType, value); + } else { + if (res_value[vec_size - 1].get_type() == ObUserDefinedSQLType) { + temp_value.set_string(ObVarcharType, value); + } else { + char *new_value = NULL; + ObString temp_from = res_value[vec_size - 1].get_string(); + if (OB_ISNULL(new_value = static_cast(allocator.alloc(temp_from.length() + value.length())))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("new value allocator failed", K(ret)); + } else if (OB_FAIL(res_value.remove(res_value.last()))) { + LOG_WARN("res value remove failed", K(ret)); + } else { + MEMCPY(new_value, temp_from.ptr(), temp_from.length()); + MEMCPY(new_value + temp_from.length(), value.ptr(), value.length()); + temp_value.set_string(ObVarcharType, new_value, temp_from.length() + value.length()); + } + } + } + if (OB_SUCC(ret)) { + res_value.push_back(temp_value); + } + } + return ret; +} +int ObExprXmlElement::construct_attribute(ObMulModeMemCtx* mem_ctx, const ObIJsonBase *attr, ObXmlElement *&element) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(attr)) { // do nothing + } else if (attr->json_type() != ObJsonNodeType::J_ARRAY) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid json type", K(ret), K(attr->json_type())); + } else if (attr->element_count() % 2 != 0) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("attribute element count invalid", K(ret), K(attr->element_count())); + } else { + for (uint64_t i = 0; OB_SUCC(ret) && i < attr->element_count(); i += 2) { + ObIJsonBase *jb_name = NULL; + ObIJsonBase *jb_value = NULL; + ObString value_str = NULL; + ObString key_str = NULL; + ObXmlAttribute *attribute = NULL; + if (OB_FAIL(attr->get_array_element(i, jb_value))) { + LOG_WARN("get attribute value failed", K(ret), K(i)); + } else if (OB_FAIL(attr->get_array_element(i + 1, jb_name))) { + LOG_WARN("get attribute name failed", K(ret), K(i)); + } else if (OB_ISNULL(jb_name) || OB_ISNULL(jb_value)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("attribute name or value is null", K(ret), K(i), K(jb_name), K(jb_value)); + } else if (jb_name->json_type() == ObJsonNodeType::J_NULL || + jb_value->json_type() == ObJsonNodeType::J_NULL) { + LOG_DEBUG("name or content is null", K(jb_name->json_type()), K(jb_value->json_type())); + } else if (OB_FAIL(ob_write_string(*mem_ctx->allocator_, ObString(jb_value->get_data_length(), jb_value->get_data()), value_str, false))) { + LOG_WARN("write string value to string failed", K(ret), K(i), K(jb_name), K(jb_value)); + } else if (OB_FAIL(ob_write_string(*mem_ctx->allocator_, ObString(jb_name->get_data_length(), jb_name->get_data()), key_str, false))) { + LOG_WARN("write string key to string failed", K(ret), K(i), K(jb_name), K(jb_value)); + } else if (OB_FAIL(element->add_attr_by_str(key_str, value_str, ObMulModeNodeType::M_ATTRIBUTE))) { + LOG_WARN("add element failed", K(ret)); + } + } + } + return ret; +} + +int ObExprXmlElement::construct_element_children(ObMulModeMemCtx* mem_ctx, + ObVector &value_vec, + ObXmlElement *&element, + ObXmlElement *valid_ele) +{ + int ret = OB_SUCCESS; + ObXmlText *tag_value = NULL; + if (OB_ISNULL(element)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("element node is NULL", K(ret)); + } + // build xml text + for (int i = 0; OB_SUCC(ret) && i < value_vec.size(); i++) { + ObObj value = value_vec[i]; + ObIMulModeBase *node = NULL; + ObXmlDocument *doc_node = NULL; + if (value.get_type() == ObUserDefinedSQLType) { + if (OB_FAIL(ObXMLExprHelper::add_binary_to_element(mem_ctx, value.get_string(), *element))) { + LOG_WARN("add binary to element failed", K(ret), K(i)); + } + } else { + if (OB_ISNULL(tag_value = OB_NEWx(ObXmlText, mem_ctx->allocator_, ObMulModeNodeType::M_TEXT, mem_ctx))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("new xml text failed", K(ret)); + } else if (OB_FALSE_IT(tag_value->set_text(value.get_string()))) { + } else if (OB_FAIL(element->add_element(tag_value))) { + LOG_WARN("element add element failed", K(ret)); + } else { + element->set_unparse(1); + } + } + } + + // if the constructed element is not unparsed, use valid_ele to replace the element + if (!element->get_unparse()) { + if (OB_ISNULL(valid_ele)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("valid element is NULL, but validity is true", K(ret)); + } + ObXmlNode *xml_node = NULL; + for (int64_t i = 0; OB_SUCC(ret) && i < element->size(); i++) { + if (OB_ISNULL(xml_node = element->at(i))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("xml node is NULL", K(ret)); + } else if (OB_FAIL(valid_ele->add_element(xml_node))) { + LOG_WARN("fail to add element", K(ret)); + } + } // end for + + if (OB_SUCC(ret)) { + element = valid_ele; + } + } + return ret; +} + +int ObExprXmlElement::construct_element(ObMulModeMemCtx* mem_ctx, + const ObString &name, + ObVector &value_vec, + const ObIJsonBase *attr, + ObXmlElement *&element, + bool &validity) +{ + INIT_SUCC(ret); + ObXmlElement *valid_ele = NULL; + if (OB_ISNULL(element)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("element node is NULL", K(ret)); + } else if (OB_FAIL(element->init())) { + LOG_WARN("element init failed", K(ret)); + } else if (FALSE_IT(element->set_key(name))) { + } else if (OB_FAIL(element->alter_member_sort_policy(false))) { + LOG_WARN("fail to sort child element", K(ret)); + } else if (OB_FAIL(construct_attribute(mem_ctx, attr, element))) { + LOG_WARN("fail to construct attribute", K(ret)); + } else if (OB_FAIL(ObXMLExprHelper::check_element_validity(mem_ctx, element, valid_ele, validity))) { + LOG_WARN("check element validity failed", K(ret)); + } else if (OB_FAIL(construct_element_children(mem_ctx, value_vec, element, valid_ele))) { + LOG_WARN("fail to construct element chidren", K(ret)); + } + // set sort flag + if (OB_SUCC(ret) && OB_FAIL(element->alter_member_sort_policy(true))) { + LOG_WARN("fail to sort child element", K(ret)); + } + return ret; +} +#endif int ObExprXmlElement::cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, diff --git a/src/sql/engine/expr/ob_expr_xml_element.h b/src/sql/engine/expr/ob_expr_xml_element.h index ec41f6e9d..8d3ba3629 100644 --- a/src/sql/engine/expr/ob_expr_xml_element.h +++ b/src/sql/engine/expr/ob_expr_xml_element.h @@ -19,6 +19,9 @@ #include "lib/json_type/ob_json_bin.h" // for ObJsonBin #include "sql/engine/expr/ob_expr_lob_utils.h" #include "lib/container/ob_vector.h" +#ifdef OB_BUILD_ORACLE_XML +#include "lib/xml/ob_xml_util.h" +#endif using namespace oceanbase::common; @@ -37,11 +40,36 @@ public: int64_t param_num, common::ObExprTypeCtx& type_ctx) const override; +#ifdef OB_BUILD_ORACLE_XML + static int eval_xml_element(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res); +#else static int eval_xml_element(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) { return OB_NOT_SUPPORTED; } +#endif virtual int cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) const override; private: +#ifdef OB_BUILD_ORACLE_XML + static int get_keys_from_wrapper(ObIJsonBase *json_doc, + ObIAllocator *allocator, + ObString &str); + static int construct_element(ObMulModeMemCtx* mem_ctx, + const ObString &name, + ObVector &value_vec, + const ObIJsonBase *attr, + ObXmlElement *&element, + bool &validity); + static int construct_attribute(ObMulModeMemCtx* mem_ctx, + const ObIJsonBase *attr, + ObXmlElement *&element); + static int construct_element_children(ObMulModeMemCtx* mem_ctx, + ObVector &value_vec, + ObXmlElement *&element, + ObXmlElement *valid_ele); + static int construct_value_array(ObIAllocator &allocator, + const ObString &value, + ObVector &res_value); +#endif DISALLOW_COPY_AND_ASSIGN(ObExprXmlElement); }; diff --git a/src/sql/engine/expr/ob_expr_xml_func_helper.h b/src/sql/engine/expr/ob_expr_xml_func_helper.h deleted file mode 100644 index 49dfb3c4d..000000000 --- a/src/sql/engine/expr/ob_expr_xml_func_helper.h +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - * This file is for define of func xml expr helper - */ - -#ifndef OCEANBASE_SQL_OB_EXPR_XML_FUNC_HELPER_H_ -#define OCEANBASE_SQL_OB_EXPR_XML_FUNC_HELPER_H_ - - -#endif // OCEANBASE_SQL_OB_EXPR_XML_FUNC_HELPER_H_ \ No newline at end of file diff --git a/src/sql/engine/expr/ob_expr_xml_serialize.cpp b/src/sql/engine/expr/ob_expr_xml_serialize.cpp index e3dbe1b72..14a141028 100644 --- a/src/sql/engine/expr/ob_expr_xml_serialize.cpp +++ b/src/sql/engine/expr/ob_expr_xml_serialize.cpp @@ -13,7 +13,11 @@ #define USING_LOG_PREFIX SQL_ENG #include "ob_expr_xml_serialize.h" -#include "ob_expr_xml_func_helper.h" +#ifdef OB_BUILD_ORACLE_XML +#include "sql/engine/expr/ob_expr_xml_func_helper.h" +#include "lib/xml/ob_xml_parser.h" +#include "lib/xml/ob_xml_util.h" +#endif #include "sql/session/ob_sql_session_info.h" #include "sql/engine/ob_exec_context.h" @@ -112,6 +116,326 @@ int ObExprXmlSerialize::get_dest_type(const ObExprResType as_type, ObExprResType } +#ifdef OB_BUILD_ORACLE_XML +int ObExprXmlSerialize::eval_xml_serialize(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) +{ + int ret = OB_SUCCESS; + ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); + common::ObArenaAllocator &allocator = tmp_alloc_g.get_allocator(); + int64_t xml_doc_type = OB_XML_DOC_TYPE_IMPLICIT; + int64_t opt_encoding_type = OB_XML_NONE_ENCODING; + int64_t opt_version_type = OB_XML_NONE_VERSION; + int64_t opt_indent_type = OB_XML_INDENT_IMPLICT; + int64_t opt_indent_size = OB_XML_INDENT_INVALID; + int64_t opt_defaults = OB_XML_DEFAULTS_IMPLICIT; + ObString opt_version_str; + ObString opt_encoding_spec; + ObObjType return_type = expr.datum_meta_.type_; + ObCollationType return_cs_type = expr.datum_meta_.cs_type_; + ObDatum *xml_datum = NULL; + + ObMulModeMemCtx* mem_ctx = nullptr; + lib::ObMallocHookAttrGuard malloc_guard(lib::ObMemAttr(MTL_ID(), "XMLCodeGen")); + if (OB_FAIL(ObXmlUtil::create_mulmode_tree_context(&allocator, mem_ctx))) { + LOG_WARN("fail to create tree memory context", K(ret)); + } else if (OB_UNLIKELY(expr.arg_cnt_ != 10)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arg_cnt_", K(ret), K(expr.arg_cnt_)); + } else if (OB_FAIL(get_and_check_int_from_expr(expr.args_[0], ctx, OB_XML_DOCUMENT, OB_XML_CONTENT, xml_doc_type))) { + LOG_WARN("fail to get xml doc type", K(ret)); + } else if (!is_supported_return_type(return_type, return_cs_type)) { + ret = OB_ERR_INVALID_DATATYPE; + LOG_WARN("invalid return datatype", K(ret), K(return_type), K(return_cs_type)); + } else if (OB_FAIL(get_and_check_int_from_expr(expr.args_[3], ctx, 0, 1, opt_encoding_type))) { + LOG_WARN("fail to get indent size", K(ret)); + } else if (opt_encoding_type == 1 && + OB_FAIL(get_and_check_encoding_spec(expr.args_[4], ctx, return_type, return_cs_type, opt_encoding_spec))) { + LOG_WARN("fail to get encoding spec", K(ret)); + } else if (OB_FAIL(get_and_check_int_from_expr(expr.args_[5], ctx, 0, 1, opt_version_type))) { + LOG_WARN("fail to get indent size", K(ret)); + } else if (opt_version_type == 1 && + OB_FAIL(ObXMLExprHelper::get_str_from_expr(expr.args_[6], ctx, opt_version_str, allocator))) { + LOG_WARN("fail to get version", K(ret)); + } else if (OB_FAIL(get_and_check_int_from_expr(expr.args_[7], ctx, OB_XML_NO_INDENT, OB_XML_INDENT_IMPLICT, opt_indent_type))) { + LOG_WARN("fail to get indent size", K(ret)); + } else if (opt_indent_type != OB_XML_INDENT_IMPLICT && + OB_FAIL(get_and_check_int_from_expr(expr.args_[8], ctx, OB_XML_INDENT_MIN, 255, opt_indent_size))) { + LOG_WARN("fail to get indent size", K(ret)); + } else if (OB_FAIL(get_and_check_int_from_expr(expr.args_[9], ctx, OB_XML_DEFAULTS_IMPLICIT, OB_XML_SHOW_DEFAULTS, opt_defaults))) { + LOG_WARN("fail to get defaults option", K(ret)); + } else if (OB_FAIL(ObXMLExprHelper::get_xmltype_from_expr(expr.args_[1], ctx, xml_datum))) { + // according to xmlserialize behavior in oracle: it will check the xmltype data type after the checks IMPLICIT done + LOG_WARN("fail to get xmltype data", K(ret)); + } else if (xml_datum->is_null()) { + res.set_null(); + } else { + ObIMulModeBase *xml_node = NULL; + bool is_xml_doc = xml_doc_type == OB_XML_DOCUMENT; + bool with_encoding = opt_encoding_type == 1; + bool with_version = opt_version_type == 1; + bool is_format = is_xml_doc || opt_indent_type != OB_XML_INDENT_IMPLICT || + with_encoding || + with_version || + opt_defaults == OB_XML_SHOW_DEFAULTS; + + ObParameterPrint print_params; + print_params.encode = opt_encoding_spec; + print_params.version = opt_version_str; + print_params.indent = opt_indent_type != OB_XML_INDENT_IMPLICT ? opt_indent_size : 2; + ObString xml_serialize_str; + ObCollationType cs_type = ctx.exec_ctx_.get_my_session()->get_nls_collation(); + ObCharsetType charset_type = CHARSET_INVALID; + if (OB_FAIL(get_xml_base_by_doc_type(mem_ctx, is_xml_doc, cs_type, xml_datum, xml_node))) { + LOG_WARN("fail to parse xml doc", K(ret), K(is_xml_doc)); + } else { + uint32_t xml_format_type = (opt_indent_type == OB_XML_NO_INDENT) ? ObXmlFormatType::MERGE_EMPTY_TAG : ObXmlFormatType::WITH_FORMAT; + if (opt_indent_type == OB_XML_INDENT_SIZE && opt_indent_size > OB_XML_INDENT_MAX) { + ret = OB_INVALID_PRINT_OPTION; // error code: ora-31188 + LOG_WARN("invalid specified print option ", K(opt_indent_size)); + } else if (with_encoding && (CHARSET_INVALID == (charset_type = ObXmlUtil::check_support_charset(opt_encoding_spec)))) { + ret = OB_ERR_UTL_ENCODE_CHARSET_INVALID; + LOG_WARN("invalid charset", K(opt_encoding_spec)); + } else if (OB_FAIL(print_format_xml_text(allocator, + ctx, + xml_node, + with_encoding, + with_version, + xml_format_type, + print_params, + xml_serialize_str))) { + LOG_WARN("fail to print format xml text", K(ret)); + } else if (ob_is_blob(return_type, return_cs_type)) { + ObCollationType new_cs_type = with_encoding ? ObCharset::get_default_collation_oracle(charset_type) : cs_type; + if (new_cs_type != CS_TYPE_UTF8MB4_BIN && + OB_FAIL(ObCharset::charset_convert(allocator, xml_serialize_str, CS_TYPE_UTF8MB4_BIN, new_cs_type, xml_serialize_str))) { + LOG_WARN("charset convertion failed", K(ret), K(xml_serialize_str), K(cs_type), K(new_cs_type)); + } + } + } + + if (OB_FAIL(ret)) { + } else if (OB_FAIL(transform_to_dst_type(expr, xml_serialize_str))) { + LOG_WARN("fail to transfrom result to dest datatype", K(ret)); + } else if (OB_FAIL(ObXMLExprHelper::set_string_result(expr, ctx, res, xml_serialize_str))) { + LOG_WARN("fail to set result", K(ret), K(xml_serialize_str)); + } + } + return ret; +} + +bool ObExprXmlSerialize::is_supported_return_type(ObObjType val_type, ObCollationType cs_type) +{ + return ob_is_varchar_type(val_type, cs_type) || + ob_is_blob(val_type, cs_type) || + ob_is_clob(val_type, cs_type) || + ob_is_nvarchar2(val_type); +} + +int ObExprXmlSerialize::print_format_xml_text(ObIAllocator &allocator, + ObEvalCtx &ctx, + ObIMulModeBase *xml_node, + bool with_encoding, + bool with_version, + uint32_t format_flag, + ObParameterPrint &print_params, + ObString &res) +{ + int ret = OB_SUCCESS; + ObStringBuffer buff(&allocator); + + ObString version = xml_node->get_version(); + ObString encoding = xml_node->get_encoding(); + + if (with_encoding || with_version) { + // if has encoding or vesion clause, do not show standalone attribute + xml_node->set_standalone(0); + } + + with_version = with_version && !print_params.version.empty(); + // set output prolog + if (with_encoding && with_version) { + } else if (with_encoding) { + if (xml_node->has_flags(XML_DECL_FLAG) && !version.empty()) { + print_params.version = version; + } else { + print_params.version = ObString::make_string("1.0"); + } + with_version = true; + } else if (with_version) { + if (!encoding.empty() || xml_node->has_flags(XML_ENCODING_EMPTY_FLAG)) { + with_encoding = true; + print_params.encode = ObString(ObXmlUtil::get_charset_name(ctx.exec_ctx_.get_my_session()->get_local_collation_connection())); + } + } else { + if (xml_node->has_flags(XML_DECL_FLAG)) { + with_version = true; + print_params.version = version; + } + if (!encoding.empty() || xml_node->has_flags(XML_ENCODING_EMPTY_FLAG)) { + with_encoding = true; + print_params.encode = ObString(ObXmlUtil::get_charset_name(ctx.exec_ctx_.get_my_session()->get_local_collation_connection())); + } + } + + if (OB_FAIL(xml_node->print_content(buff, with_encoding, with_version, format_flag, print_params))) { + LOG_WARN("fail to print xml content", K(ret), K(with_encoding), K(with_version), K(format_flag)); + } + if (OB_SUCC(ret)) { + res.assign_ptr(buff.ptr(), buff.length()); + } + return ret; +} + +int ObExprXmlSerialize::get_and_check_int_from_expr(const ObExpr *expr, + ObEvalCtx &ctx, + int64_t start, + int64_t end, + int64_t &res) +{ + int ret = OB_SUCCESS; + ObObjType val_type = expr->datum_meta_.type_; + ObDatum *datum = NULL; + if (OB_FAIL(expr->eval(ctx, datum))) { + LOG_WARN("eval xml arg failed", K(ret)); + } else if (val_type != ObIntType) { + ret = OB_ERR_REQUIRE_INTEGER; + LOG_WARN("input type error", K(val_type)); + } else { + int64_t value = datum->get_int(); + if (value < start || value > end) { + ret = OB_NUMERIC_SCALE_OUT_RANGE; + LOG_WARN("input value error", K(value)); + } else { + res = value; + } + } + + return ret; +} + +int ObExprXmlSerialize::get_and_check_encoding_spec(const ObExpr *expr, + ObEvalCtx &ctx, + ObObjType return_type, + ObCollationType return_cs_type, + ObString &encoding_spec) +{ + int ret = OB_SUCCESS; + ObDatum *datum = NULL; + ObObjType val_type = expr->datum_meta_.type_; + if (OB_FAIL(expr->eval(ctx, datum))) { + LOG_WARN("eval json arg failed", K(ret)); + } else if (!ob_is_string_type(val_type)) { + ret = OB_ERR_INVALID_TYPE_FOR_OP; + LOG_WARN("invalid encoding spec type ", K(val_type)); + } else if (!ob_is_blob(return_type, return_cs_type)) { + ret = OB_ERR_INVALID_DATATYPE; + LOG_WARN("invalid data type", K(ret), K(encoding_spec), K(return_type), K(return_cs_type)); + } else { + encoding_spec = datum->get_string(); + char *ptr = str_toupper(encoding_spec.ptr(), encoding_spec.length()); + } + return ret; +} + +int ObExprXmlSerialize::transform_to_dst_type(const ObExpr &expr, ObString &xml_format_str) +{ + int ret = OB_SUCCESS; + ObObjType obj_type = expr.datum_meta_.type_; + ObCollationType cs_type = expr.datum_meta_.cs_type_; + ObLength max_length = expr.max_length_; + if (ob_is_varchar_type(obj_type, cs_type) || ob_is_nvarchar2(obj_type)) { + ObObj obj; + obj.set_string(ObVarcharType, xml_format_str); + if (OB_FAIL(string_length_check_only(max_length, cs_type, obj))) { + LOG_WARN("fail to check string len", K(ret), K(max_length), K(obj), K(cs_type)); + } + } else if (ob_is_clob(obj_type, cs_type) || ob_is_blob(obj_type, cs_type)) { + /* do nothing */ + } else { + ret = OB_ERR_INVALID_DATATYPE; + LOG_WARN("invalid return datatype", K(ret), K(obj_type), K(cs_type)); + } + + return ret; +} + +int ObExprXmlSerialize::string_length_check_only(const ObLength max_len_char, + const ObCollationType cs_type, + const ObObj &obj) +{ + int ret = OB_SUCCESS; + const char *str = obj.get_string_ptr(); + const int32_t str_len_byte = obj.get_string_len(); + const int32_t str_len_char = static_cast(ObCharset::strlen_char(cs_type, str, str_len_byte)); + if (OB_UNLIKELY(max_len_char <= 0)) { + if (OB_UNLIKELY(0 == max_len_char && str_len_byte > 0)) { + ret = OB_XML_CHAR_LEN_TOO_SMALL; + LOG_WARN("char type length is too long", K(obj), K(max_len_char), K(str_len_char)); + } + } else if (OB_UNLIKELY(str_len_char > max_len_char)) { + ret = OB_XML_CHAR_LEN_TOO_SMALL; + LOG_WARN("string length is too long", K(max_len_char), K(str_len_char), K(obj)); + } + return ret; +} + +int ObExprXmlSerialize::get_xml_base_by_doc_type(ObMulModeMemCtx* mem_ctx, + bool is_xml_doc, + ObCollationType cs_type, + ObDatum *xml_datum, + ObIMulModeBase *&node) +{ + int ret = OB_SUCCESS; + // temporary use for xml clob + ObString xml_text; + ObDatumMeta xml_meta; + xml_meta.type_ = ObLongTextType; + xml_meta.cs_type_ = CS_TYPE_UTF8MB4_BIN; + ObXmlDocument *xml_node = NULL; + ObIMulModeBase *tmp_node = NULL; + ObNodeMemType expect_type = ObNodeMemType::BINARY_TYPE; + + if (OB_ISNULL(xml_datum)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("xml datum is NULL", K(ret)); + } else if (OB_FAIL(ObXMLExprHelper::get_xml_base(mem_ctx, xml_datum, CS_TYPE_INVALID, expect_type, node))) { + LOG_WARN("fail to get xml base", K(ret), K(cs_type), K(expect_type)); + } else if (is_xml_doc) {// document + ObString check_xml; + ObStringBuffer buff(mem_ctx->allocator_); + ObXmlDocument *xml_doc = nullptr; + if (OB_FAIL(node->print_document(buff, CS_TYPE_INVALID, ObXmlFormatType::NO_FORMAT))) { + LOG_WARN("fail to serialize unparse string", K(ret)); + } else { + check_xml.assign_ptr(buff.ptr(), buff.length()); + } + if (OB_FAIL(ret)) { + } else if (check_xml.empty()) { + /* return an empty xml content node*/ + if (OB_ISNULL(xml_doc = OB_NEWx(ObXmlDocument, (mem_ctx->allocator_), ObMulModeNodeType::M_CONTENT, (mem_ctx)))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to create an empty xml content node", K(ret), K(xml_text)); + } else { + node = xml_doc; + } + } else if (OB_FAIL(ObXmlParserUtils::parse_document_text(mem_ctx, check_xml, xml_doc))) { + if (ret == OB_ERR_PARSER_SYNTAX) { + ret = OB_ERR_XML_FRAMENT_CONVERT; + LOG_USER_ERROR(OB_ERR_XML_FRAMENT_CONVERT); + } + LOG_WARN("fail to parse xml doc", K(check_xml), K(ret)); + } else if (node->type() == M_CONTENT) { + // fix issue-49263291 + // case like select xmlserialize(DOCUMENT xmlparse(CONTENT 'aaa') ) as res from dual; + if (node->size() > 1) { + ret = OB_ERR_XML_FRAMENT_CONVERT; + LOG_USER_ERROR(OB_ERR_XML_FRAMENT_CONVERT); + } + } + } + return ret; +} +#endif int ObExprXmlSerialize::cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, diff --git a/src/sql/engine/expr/ob_expr_xml_serialize.h b/src/sql/engine/expr/ob_expr_xml_serialize.h index 1998c57c5..6b9f0064e 100644 --- a/src/sql/engine/expr/ob_expr_xml_serialize.h +++ b/src/sql/engine/expr/ob_expr_xml_serialize.h @@ -15,6 +15,10 @@ #define OCEANBASE_SQL_OB_EXPR_XML_SERIALIZE_H_ #include "sql/engine/expr/ob_expr_operator.h" +#ifdef OB_BUILD_ORACLE_XML +#include "lib/xml/ob_multi_mode_interface.h" +#include "lib/xml/ob_xml_tree.h" +#endif using namespace oceanbase::common; namespace oceanbase @@ -31,11 +35,72 @@ public: int64_t param_num, common::ObExprTypeCtx& type_ctx) const override; +#ifdef OB_BUILD_ORACLE_XML + static int eval_xml_serialize(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res); +#else static int eval_xml_serialize(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) { return OB_NOT_SUPPORTED; } +#endif virtual int cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) const override; int get_dest_type(const ObExprResType as_type, ObExprResType &dst_type) const; +#ifdef OB_BUILD_ORACLE_XML +private: + static bool is_supported_return_type(ObObjType val_type, ObCollationType cs_type); + + static int get_xml_base_by_doc_type(ObMulModeMemCtx* mem_ctx, + bool is_xml_doc, + ObCollationType cs_type, + ObDatum *xml_datum, + ObIMulModeBase *&node); + static int get_and_check_int_from_expr(const ObExpr *expr, + ObEvalCtx &ctx, + int64_t start, + int64_t end, + int64_t &res); + static int transform_to_dst_type(const ObExpr &expr, ObString &xml_format_str); + static int string_length_check_only(const ObLength max_length_char, + const ObCollationType cs_type, + const ObObj &obj); + static int get_and_check_encoding_spec(const ObExpr *expr, + ObEvalCtx &ctx, + ObObjType return_type, + ObCollationType return_cs_type, + ObString &encoding_spec); + static int print_format_xml_text(ObIAllocator &allocator, + ObEvalCtx &ctx, + ObIMulModeBase *xml_doc, + bool with_encoding, + bool with_version, + uint32_t format_flag, + ObParameterPrint &print_params, + ObString &res); + +private: + // xml_doc_type + const static int64_t OB_XML_DOCUMENT = 0; + const static int64_t OB_XML_CONTENT = 1; + const static int64_t OB_XML_DOC_TYPE_IMPLICIT = 2; + // encoding type + const static int64_t OB_XML_NONE_ENCODING = 0; + const static int64_t OB_XML_WITH_ENCODING = 1; + // version type + const static int64_t OB_XML_NONE_VERSION = 0; + const static int64_t OB_XML_WITH_VERSION = 1; + // indent type + const static int64_t OB_XML_NO_INDENT = 1; + const static int64_t OB_XML_INDENT = 2; + const static int64_t OB_XML_INDENT_SIZE = 3; + const static int64_t OB_XML_INDENT_IMPLICT = 4; + // indent size + const static int64_t OB_XML_INDENT_INVALID = -1; + const static int64_t OB_XML_INDENT_MIN = 0; + const static int64_t OB_XML_INDENT_MAX = 12; + // default type + const static int64_t OB_XML_DEFAULTS_IMPLICIT = 0; + const static int64_t OB_XML_HIDE_DEFAULTS = 1; + const static int64_t OB_XML_SHOW_DEFAULTS = 2; +#endif private: DISALLOW_COPY_AND_ASSIGN(ObExprXmlSerialize); }; diff --git a/src/sql/engine/expr/ob_expr_xmlcast.cpp b/src/sql/engine/expr/ob_expr_xmlcast.cpp index 6589f9eda..b7ef07838 100644 --- a/src/sql/engine/expr/ob_expr_xmlcast.cpp +++ b/src/sql/engine/expr/ob_expr_xmlcast.cpp @@ -15,7 +15,13 @@ #include "share/object/ob_obj_cast.h" #include "sql/engine/ob_exec_context.h" #include "sql/session/ob_sql_session_info.h" -#include "ob_expr_xml_func_helper.h" +#ifdef OB_BUILD_ORACLE_XML +#include "lib/xml/ob_xml_parser.h" +#include "lib/xml/ob_xml_tree.h" +#include "lib/xml/ob_xml_util.h" +#include "lib/xml/ob_xpath.h" +#include "sql/engine/expr/ob_expr_xml_func_helper.h" +#endif #include "sql/session/ob_sql_session_info.h" #include "sql/engine/ob_exec_context.h" #include "sql/ob_spi.h" @@ -141,5 +147,183 @@ int ObExprXmlcast::cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, return OB_SUCCESS; } +#ifdef OB_BUILD_ORACLE_XML +int ObExprXmlcast::eval_xmlcast(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) +{ + int ret = OB_SUCCESS; + ObExpr *xml_expr = NULL; + ObDatum *xml_datum = NULL; + ObIMulModeBase *xml_doc = NULL; + ObString xml_res_str; + ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); + common::ObArenaAllocator &allocator = tmp_alloc_g.get_allocator(); + ObCollationType cs_type = CS_TYPE_INVALID; + + ObMulModeMemCtx* mem_ctx = nullptr; + lib::ObMallocHookAttrGuard malloc_guard(lib::ObMemAttr(MTL_ID(), "XMLCodeGen")); + if (OB_FAIL(ObXmlUtil::create_mulmode_tree_context(&allocator, mem_ctx))) { + LOG_WARN("fail to create tree memory context", K(ret)); + } else if (OB_UNLIKELY(expr.arg_cnt_ != 2)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid arg_cnt_", K(ret), K(expr.arg_cnt_)); + } else if (OB_FAIL(ObXMLExprHelper::get_xmltype_from_expr(expr.args_[0], ctx, xml_datum))) { + // temporally use + LOG_WARN("fail to get xml str", K(ret)); + } else if (OB_FAIL(ObXMLExprHelper::get_xml_base(mem_ctx, xml_datum, cs_type, ObNodeMemType::BINARY_TYPE, xml_doc))) { + LOG_WARN("fail to parse xml doc", K(ret)); + } else if (OB_FAIL(extract_xml_text_node(mem_ctx, xml_doc, xml_res_str))) { + LOG_WARN("fail to extract xml text node", K(ret), K(xml_res_str)); + } else if (OB_FAIL(cast_to_res(allocator, xml_res_str, expr, ctx, res))) { + LOG_WARN("fail to cast to res", K(ret), K(xml_res_str)); + } + return ret; +} + +int ObExprXmlcast::cast_to_res(ObIAllocator &allocator, ObString &xml_content, const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) +{ + int ret = OB_SUCCESS; + ObCastMode def_cm = CM_NONE; + ObSQLSessionInfo *session = NULL; + ObObj src_obj,dst_obj, buf_obj; + const ObObj *res_obj = NULL; + ObAccuracy out_acc; + if (xml_content.empty()) { + res.set_null(); + } else { + src_obj.set_string(ObVarcharType, xml_content); + src_obj.set_collation_type(CS_TYPE_UTF8MB4_BIN); + // to type + if (OB_ISNULL(session = ctx.exec_ctx_.get_my_session())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sessioninfo is NULL"); + } else if (OB_FAIL(ObSQLUtils::get_default_cast_mode(session->get_stmt_type(), + session, def_cm))) { + LOG_WARN("get_default_cast_mode failed", K(ret)); + } else { + ObObjType obj_type = expr.datum_meta_.type_; + ObCollationType cs_type = expr.datum_meta_.cs_type_; + ObPhysicalPlanCtx *phy_plan_ctx = ctx.exec_ctx_.get_physical_plan_ctx(); + const ObDataTypeCastParams dtc_params = ObBasicSessionInfo::create_dtc_params(session); + ObCastCtx cast_ctx(&allocator, &dtc_params, get_cur_time(phy_plan_ctx), def_cm, + cs_type, NULL, NULL); + if (OB_FAIL(ObObjCaster::to_type(obj_type, cs_type, cast_ctx, src_obj, dst_obj))) { + LOG_WARN("failed to cast object to ", K(ret), K(src_obj), K(obj_type)); + } else if (FALSE_IT(get_accuracy_from_expr(expr, out_acc))) { + } else if (FALSE_IT(res_obj = &dst_obj)) { + } else if (OB_FAIL(obj_accuracy_check(cast_ctx, out_acc, cs_type, dst_obj, buf_obj, res_obj))) { + if ((ob_is_varchar_or_char(obj_type, cs_type) || ob_is_nchar(obj_type)) && ret == OB_ERR_DATA_TOO_LONG) { + ObLengthSemantics ls = lib::is_oracle_mode() ? + expr.datum_meta_.length_semantics_ : LS_CHAR; + const char* str = dst_obj.get_string_ptr(); + int32_t str_len_byte = dst_obj.get_string_len(); + int64_t char_len = 0; + int32_t trunc_len_byte = 0; + trunc_len_byte = (ls == LS_BYTE ? + ObCharset::max_bytes_charpos(cs_type, str, str_len_byte, + expr.max_length_, char_len): + ObCharset::charpos(cs_type, str, str_len_byte, expr.max_length_)); + if (trunc_len_byte == 0) { + (const_cast(res_obj))->set_null(); + } else { + (const_cast(res_obj))->set_common_value(ObString(trunc_len_byte, str)); + } + ret = OB_SUCCESS; + } else { + LOG_WARN("accuracy check failed", K(ret), K(out_acc), K(res_obj)); + } + } else if (OB_FAIL(ObSPIService::spi_pad_char_or_varchar(session, obj_type, out_acc, &allocator, const_cast(res_obj)))) { + LOG_WARN("fail to pad char", K(ret), K(*res_obj)); + } + + if (OB_SUCC(ret)) { + if (OB_NOT_NULL(res_obj)) { + res.from_obj(*res_obj); + ObExprStrResAlloc res_alloc(expr, ctx); + if (OB_FAIL(res.deep_copy(res, res_alloc))) { + LOG_WARN("fail to deep copy for res datum", K(ret), KPC(res_obj), K(res)); + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("res obj is NULL", K(ret)); + } + } + } + } + return ret; +} + + +int ObExprXmlcast::extract_xml_text_node(ObMulModeMemCtx* mem_ctx, ObIMulModeBase *xml_doc, ObString &res) +{ + int ret = OB_SUCCESS; + ObStringBuffer buff(mem_ctx->allocator_); + ObPathExprIter xpath_iter(mem_ctx->allocator_); + ObString xpath_str = ObString::make_string("//node()"); + ObString default_ns; // unused + ObIMulModeBase *xml_node = NULL; + bool is_xml_document = false; + bool is_head_comment = true; + if (OB_ISNULL(xml_doc)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("xml doc node is NULL", K(ret)); + } else if (FALSE_IT(is_xml_document = xml_doc->type() == M_DOCUMENT)) { + } else if (OB_FAIL(xpath_iter.init(mem_ctx, xpath_str, default_ns, xml_doc, NULL))) { + LOG_WARN("fail to init xpath iterator", K(xpath_str), K(default_ns), K(ret)); + } else if (OB_FAIL(xpath_iter.open())) { + LOG_WARN("fail to open xpath iterator", K(ret)); + } + + while (OB_SUCC(ret)) { + ObString content; + if (OB_FAIL(xpath_iter.get_next_node(xml_node))) { + if (ret != OB_ITER_END) { + LOG_WARN("fail to get next xml node", K(ret)); + } + } else if (OB_ISNULL(xml_node)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("xpath result node is null", K(ret)); + } else { + ObMulModeNodeType node_type = xml_node->type(); + if (node_type != M_TEXT && + node_type != M_CDATA && + node_type != M_COMMENT && + node_type != M_INSTRUCT) { + is_head_comment = false; + } else if ((node_type == M_COMMENT || node_type == M_INSTRUCT) && + (is_xml_document || (!is_xml_document && !is_head_comment))) { + /* filter the comment node */ + } else if (OB_FAIL(xml_node->get_value(content))) { + LOG_WARN("fail to get text node content", K(ret)); + } else if (OB_FAIL(buff.append(content))) { + LOG_WARN("fail to append text node content", K(ret), K(content)); + } + } + } + + if (ret == OB_ITER_END) { + res.assign_ptr(buff.ptr(), buff.length()); + ret = OB_SUCCESS; + } + + int tmp_ret = OB_SUCCESS; + if (OB_SUCCESS != (tmp_ret = xpath_iter.close())) { + LOG_WARN("fail to close xpath iter", K(tmp_ret)); + ret = COVER_SUCC(tmp_ret); + } + return ret; +} + +void ObExprXmlcast::get_accuracy_from_expr(const ObExpr &expr, ObAccuracy &accuracy) +{ + accuracy.set_length(expr.max_length_); + accuracy.set_scale(expr.datum_meta_.scale_); + const ObObjTypeClass &dst_tc = ob_obj_type_class(expr.datum_meta_.type_); + if (ObStringTC == dst_tc || ObTextTC == dst_tc) { + accuracy.set_length_semantics(expr.datum_meta_.length_semantics_); + } else { + accuracy.set_precision(expr.datum_meta_.precision_); + } +} +#endif } // sql } // oceanbase \ No newline at end of file diff --git a/src/sql/engine/expr/ob_expr_xmlcast.h b/src/sql/engine/expr/ob_expr_xmlcast.h index c92c30d74..508d761a9 100644 --- a/src/sql/engine/expr/ob_expr_xmlcast.h +++ b/src/sql/engine/expr/ob_expr_xmlcast.h @@ -15,6 +15,9 @@ #define OCEANBASE_SQL_ENGINE_EXPR_OB_EXPR_XMLCAST_H #include "sql/engine/expr/ob_expr_operator.h" +#ifdef OB_BUILD_ORACLE_XML +#include "lib/xml/ob_multi_mode_interface.h" +#endif namespace oceanbase { @@ -31,13 +34,22 @@ class ObExprXmlcast : public ObFuncExprOperator ObExprResType &type2, common::ObExprTypeCtx &type_ctx) const; +#ifdef OB_BUILD_ORACLE_XML + static int eval_xmlcast(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res); +#else static int eval_xmlcast(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) { return OB_NOT_SUPPORTED; } +#endif virtual int cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) const override; private: int set_dest_type(ObExprResType ¶m_type, ObExprResType &dst_type, ObExprTypeCtx &type_ctx) const; +#ifdef OB_BUILD_ORACLE_XML + static int extract_xml_text_node(ObMulModeMemCtx* mem_ctx, ObIMulModeBase *xml_doc, ObString &res); + static int cast_to_res(ObIAllocator &allocator, ObString &xml_content, const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res); + static void get_accuracy_from_expr(const ObExpr &expr, ObAccuracy &acc); +#endif private: DISALLOW_COPY_AND_ASSIGN(ObExprXmlcast); }; diff --git a/src/sql/engine/expr/ob_expr_xmlparse.cpp b/src/sql/engine/expr/ob_expr_xmlparse.cpp index 988f095c9..ff12b4dab 100644 --- a/src/sql/engine/expr/ob_expr_xmlparse.cpp +++ b/src/sql/engine/expr/ob_expr_xmlparse.cpp @@ -13,7 +13,11 @@ #include "ob_expr_xmlparse.h" #include "sql/engine/ob_exec_context.h" -#include "ob_expr_xml_func_helper.h" +#ifdef OB_BUILD_ORACLE_XML +#include "sql/engine/expr/ob_expr_xml_func_helper.h" +#include "lib/xml/ob_xml_tree.h" +#include "lib/xml/ob_xml_util.h" +#endif #define USING_LOG_PREFIX SQL_ENG @@ -82,6 +86,104 @@ int ObExprXmlparse::calc_result_typeN(ObExprResType &type, return ret; } +#ifdef OB_BUILD_ORACLE_XML +int ObExprXmlparse::eval_xmlparse(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) +{ + INIT_SUCC(ret); + ObDatum *xml_datum = NULL; + ObString xml_plain_text; + ObMulModeNodeType res_type; + uint8_t xml_type = OB_XML_DOC_TYPE_IMPLICIT; + uint8_t is_wellformed = OB_WELLFORMED_IMPLICIT; + bool is_xml_text_null = false; + bool need_format = false; + + ObEvalCtx::TempAllocGuard tmp_alloc_g(ctx); + common::ObArenaAllocator &tmp_allocator = tmp_alloc_g.get_allocator(); + ObMulModeMemCtx* mem_ctx = nullptr; + lib::ObMallocHookAttrGuard malloc_guard(lib::ObMemAttr(MTL_ID(), "XMLCodeGen")); + + if (OB_FAIL(ObXmlUtil::create_mulmode_tree_context(&tmp_allocator, mem_ctx))) { + LOG_WARN("fail to create tree memory context", K(ret)); + } else if (OB_UNLIKELY(expr.arg_cnt_ != 4)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("invalid arg_cnt_", K(expr.arg_cnt_)); + } else if (OB_FAIL(expr.args_[3]->eval(ctx, xml_datum))) { + LOG_WARN("get extra para failed", K(ret)); + } else if (FALSE_IT(need_format = (xml_datum->get_int() & 0x08) != 0)) { + } else if (OB_FAIL(get_clause_opt(expr, ctx, 0, xml_type, OB_XML_DOC_TYPE_COUNT))) { + LOG_WARN("get document/context error", K(ret), K(xml_type)); + } else if (OB_FAIL(get_clause_opt(expr, ctx, 2, is_wellformed, OB_WELLFORMED_COUNT))) { + LOG_WARN("get wellformed error", K(ret), K(is_wellformed)); + } else if (OB_FAIL(expr.args_[1]->eval(ctx, xml_datum))) { + LOG_WARN("get extra para failed", K(ret)); + } else if (expr.args_[1]->datum_meta_.type_ == ObNullType || xml_datum->is_null()) { + is_xml_text_null = true; + } else if (OB_FAIL(ObTextStringHelper::get_string(expr, tmp_allocator, 1, xml_datum, xml_plain_text))) { + LOG_WARN("get xml plain text failed", K(ret), K(is_xml_text_null)); + } + + bool is_unparsed_res = false; + if(OB_FAIL(ret)) { + } else if (is_xml_text_null) { + res.set_null(); + } else { + ObXmlDocument* doc = nullptr; + ObXmlParser parser(mem_ctx); + + if (xml_type == OB_XML_DOCUMENT) { + if (is_wellformed) { + res_type = M_UNPARESED_DOC; + if(OB_FAIL(ObMulModeFactory::add_unparsed_text_into_doc(mem_ctx, xml_plain_text, doc))) { + LOG_WARN("add text failed", K(ret), K(xml_plain_text)); + } + } else if (OB_FAIL(parser.parse_document(xml_plain_text))) { + ret = OB_ERR_XML_PARSE; + LOG_USER_ERROR(OB_ERR_XML_PARSE); + LOG_WARN("parse xml plain text as document failed.", K(xml_plain_text)); + } else { + res_type = M_DOCUMENT; + doc = parser.document(); + } + } else if (xml_type == OB_XML_CONTENT) { + if (!is_wellformed) { + if (OB_FAIL(parser.parse_content(xml_plain_text))) { + ret = OB_ERR_XML_PARSE; + LOG_USER_ERROR(OB_ERR_XML_PARSE); + LOG_WARN("parse xml plain text as content failed.", K(xml_plain_text)); + } else { + res_type = M_CONTENT; + doc = parser.document(); + } + } else { + is_unparsed_res = true; + res_type = M_UNPARSED; + if (OB_ISNULL(doc = OB_NEWx(ObXmlDocument, mem_ctx->allocator_, ObMulModeNodeType::M_DOCUMENT, mem_ctx))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to create document", K(ret)); + } else if(OB_FAIL(doc->append_unparse_text(xml_plain_text))) { + LOG_WARN("fail to add unparse text to doc", K(ret)); + } + } + } + + + if (OB_SUCC(ret)) { + ObCollationType session_cs_type = CS_TYPE_UTF8MB4_BIN; + if (!doc->get_encoding().empty() || doc->get_encoding_flag()) { + doc->set_encoding(ObXmlUtil::get_charset_name(session_cs_type)); + } + if (OB_FAIL(ObXMLExprHelper::pack_xml_res(expr, ctx, res, doc, mem_ctx, + res_type, + xml_plain_text))) { + LOG_WARN("pack_xml_res failed", K(ret)); + } + } + } + + return ret; +} +#endif int ObExprXmlparse::get_clause_opt(const ObExpr &expr, ObEvalCtx &ctx, diff --git a/src/sql/engine/expr/ob_expr_xmlparse.h b/src/sql/engine/expr/ob_expr_xmlparse.h index 968ac5088..9d7416403 100644 --- a/src/sql/engine/expr/ob_expr_xmlparse.h +++ b/src/sql/engine/expr/ob_expr_xmlparse.h @@ -33,7 +33,11 @@ class ObExprXmlparse : public ObFuncExprOperator int64_t param_num, common::ObExprTypeCtx& type_ctx) const override; +#ifdef OB_BUILD_ORACLE_XML + static int eval_xmlparse(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res); +#else static int eval_xmlparse(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &res) { return OB_NOT_SUPPORTED; } +#endif virtual int cg_expr(ObExprCGCtx &expr_cg_ctx, const ObRawExpr &raw_expr, ObExpr &rt_expr) @@ -67,4 +71,4 @@ private: } // oceanbase -#endif // OCEANBASE_SRC_SQL_ENGINE_EXPR_OB_EXPR_XMLPARSE_H \ No newline at end of file +#endif // OCEANBASE_SRC_SQL_ENGINE_EXPR_OB_EXPR_XMLPARSE_H diff --git a/src/sql/engine/ob_exec_context.cpp b/src/sql/engine/ob_exec_context.cpp index 2e14f0c35..ba89dea68 100644 --- a/src/sql/engine/ob_exec_context.cpp +++ b/src/sql/engine/ob_exec_context.cpp @@ -25,6 +25,9 @@ #include "share/interrupt/ob_global_interrupt_call.h" #include "ob_operator.h" #include "observer/ob_server.h" +#ifdef OB_BUILD_SPM +#include "sql/spm/ob_spm_controller.h" +#endif namespace oceanbase { @@ -774,6 +777,13 @@ int ObExecContext::init_physical_plan_ctx(const ObPhysicalPlan &plan) phy_plan_ctx_->set_foreign_key_checks(0 != foreign_key_checks); phy_plan_ctx_->set_table_row_count_list_capacity(plan.get_access_table_num()); THIS_WORKER.set_timeout_ts(phy_plan_ctx_->get_timeout_timestamp()); +#ifdef OB_BUILD_SPM + if (sql_ctx_ != NULL && sql_ctx_->spm_ctx_.need_spm_timeout_) { + phy_plan_ctx_->set_spm_timeout_timestamp( + ObSpmController::calc_spm_timeout_us(start_time + plan_timeout, + sql_ctx_->spm_ctx_.baseline_exec_time_)); + } +#endif } } if (OB_SUCC(ret)) { diff --git a/src/sql/engine/table/ob_link_scan_op.cpp b/src/sql/engine/table/ob_link_scan_op.cpp index a6f20dbe1..fbd2eb8e8 100644 --- a/src/sql/engine/table/ob_link_scan_op.cpp +++ b/src/sql/engine/table/ob_link_scan_op.cpp @@ -24,6 +24,9 @@ #include "lib/string/ob_hex_utils_base.h" #include "sql/session/ob_sql_session_mgr.h" #include "sql/engine/expr/ob_expr_lob_utils.h" +#ifdef OB_BUILD_DBLINK +#include "lib/oracleclient/ob_oci_connection.h" +#endif namespace oceanbase { using namespace common; @@ -76,8 +79,67 @@ int ObLinkScanOp::init_tz_info(const ObTimeZoneInfo *tz_info) } return ret; } +#ifdef OB_BUILD_DBLINK +int ObLinkScanOp::init_conn_snapshot(bool &new_snapshot) +{ + int ret = OB_SUCCESS; + new_snapshot = false; + void *snapshot = NULL; + bool snapshot_inited = false; + if (NULL == dblink_conn_ || DBLINK_DRV_OCI != link_type_) { + // do nothing. + } else { + hash::ObHashMap &dblink_snapshot_map = ctx_.get_dblink_snapshot_map(); + ObOciConnection *ob_oci_conn = static_cast(dblink_conn_); + if (!dblink_snapshot_map.created() && OB_FAIL(dblink_snapshot_map.create(8, "dblinksnapshot"))) { + LOG_WARN("create hash map failed", K(ret)); + } else if (OB_FAIL(dblink_snapshot_map.get_refactored(dblink_id_, snapshot))) { + if (OB_HASH_NOT_EXIST != ret) { + LOG_WARN("get dblink snapshot failed", K(ret)); + } else if (OB_FAIL(ob_oci_conn->create_snapshot(snapshot))) { + LOG_WARN("init snapshot failed", K(ret)); + } else if (OB_ISNULL(snapshot)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("create null snapshot", K(ret), K(get_spec().get_id()), K(link_type_)); + } else { + snapshot_created_ = snapshot; + new_snapshot = true; + snapshot_inited = false; + LOG_TRACE("init conn snapshot, create one", K(get_spec().get_id()), K(ob_oci_conn), + K(ob_oci_conn->get_oci_connection()), K(snapshot)); + } + } else { + snapshot_inited = true; + LOG_TRACE("init conn snapshot, fetch one", K(get_spec().get_id()), K(ob_oci_conn), + K(ob_oci_conn->get_oci_connection()), K(snapshot)); + } + if (OB_SUCC(ret) && OB_FAIL(ob_oci_conn->set_conn_snapshot(snapshot, snapshot_inited))) { + LOG_ERROR("set conn snapshot failed", K(ret)); + } + } + return ret; +} - +int ObLinkScanOp::free_snapshot() +{ + int ret = OB_SUCCESS; + if (NULL == snapshot_created_) { + // do nothing. + } else if (OB_UNLIKELY(NULL == dblink_conn_ || DBLINK_DRV_OCI != link_type_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("dblink conn is null or unexpected dblink type", K(ret), K(dblink_conn_), K(link_type_)); + } else { + ObOciConnection *ob_oci_conn = static_cast(dblink_conn_); + void *snapshot = NULL; + bool snapshot_inited = false; + if (OB_FAIL(ob_oci_conn->free_snapshot(snapshot_created_))) { + LOG_WARN("set conn snapshot failed", K(ret)); + } + snapshot_created_ = NULL; + } + return ret; +} +#endif int ObLinkScanOp::inner_execute_link_stmt(const char *link_stmt) { int ret = OB_SUCCESS; @@ -107,9 +169,26 @@ int ObLinkScanOp::inner_execute_link_stmt(const char *link_stmt) } else if (OB_ISNULL(dblink_proxy_) || OB_ISNULL(my_session)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected NULL", K(ret), KP(dblink_proxy_), KP(my_session)); + } else if (need_tx(my_session) && OB_FAIL(ObTMService::tm_rm_start(ctx_, link_type_, dblink_conn_, tx_id))) { + LOG_WARN("failed to tm_rm_start", K(ret), K(dblink_id_), K(dblink_conn_), K(sessid_)); +#ifdef OB_BUILD_DBLINK + } else if (OB_FAIL(init_conn_snapshot(new_snapshot))) { + LOG_WARN("init conn snapshot failed", K(ret)); +#endif } else { if (OB_FAIL(dblink_proxy_->dblink_read(dblink_conn_, res_, link_stmt))) { LOG_WARN("read failed", K(ret), K(link_type_), K(dblink_conn_), K(link_stmt)); +#ifdef OB_BUILD_DBLINK + } else if (new_snapshot && OB_FAIL(ctx_.get_dblink_snapshot_map().set_refactored(dblink_id_, snapshot_created_))) { + LOG_WARN("set snapshot failed", K(ret)); + } + if (OB_FAIL(ret) && new_snapshot) { + int tmp_ret = OB_SUCCESS; + // when fail and have created new snapshot before, need free this snapshot. + if (OB_UNLIKELY(OB_SUCCESS != (tmp_ret = free_snapshot()))) { + LOG_ERROR("free dblink snapshot failed", K(tmp_ret)); + } +#endif } } if (OB_FAIL(ret)) { @@ -136,6 +215,21 @@ int ObLinkScanOp::inner_execute_link_stmt(const char *link_stmt) void ObLinkScanOp::reset_dblink() { int tmp_ret = OB_SUCCESS; +#ifdef OB_BUILD_DBLINK + if (DBLINK_DRV_OCI == link_type_ && + NULL != dblink_conn_) { + if (OB_SUCCESS != (tmp_ret = static_cast(dblink_conn_)->free_oci_stmt())) { + LOG_WARN_RET(tmp_ret, "failed to close oci result", K(tmp_ret)); + } + if (OB_UNLIKELY(OB_SUCCESS != (tmp_ret = free_snapshot()))) { + LOG_WARN_RET(tmp_ret, "free dblink snapshot failed"); + } + } else if (NULL != tm_rm_connection_ && + DblinkDriverProto::DBLINK_DRV_OCI == tm_rm_connection_->get_dblink_driver_proto() && + OB_SUCCESS != (tmp_ret = static_cast(tm_rm_connection_)->free_oci_stmt())) { + LOG_WARN_RET(tmp_ret, "failed to close oci result", K(tmp_ret)); + } +#endif if (OB_NOT_NULL(dblink_proxy_) && OB_NOT_NULL(dblink_conn_) && !in_xa_trascaction_ && OB_SUCCESS != (tmp_ret = dblink_proxy_->release_dblink(link_type_, dblink_conn_))) { LOG_WARN_RET(tmp_ret, "failed to release connection", K(tmp_ret)); @@ -368,6 +462,14 @@ int ObLinkScanOp::inner_rescan() iter_end_ = false; iterated_rows_ = -1; int tmp_ret = OB_SUCCESS; +#ifdef OB_BUILD_DBLINK + if (DBLINK_DRV_OCI == link_type_ && + NULL != dblink_conn_) { + if (OB_SUCCESS != (tmp_ret = static_cast(dblink_conn_)->free_oci_stmt())) { + LOG_WARN_RET(tmp_ret, "failed to close oci result", K(tmp_ret)); + } + } +#endif return ObOperator::inner_rescan(); } diff --git a/src/sql/executor/ob_cmd_executor.cpp b/src/sql/executor/ob_cmd_executor.cpp index 99f23d8e0..ab1becf8e 100644 --- a/src/sql/executor/ob_cmd_executor.cpp +++ b/src/sql/executor/ob_cmd_executor.cpp @@ -142,6 +142,19 @@ #include "sql/resolver/ddl/ob_create_context_resolver.h" #include "sql/resolver/ddl/ob_drop_context_resolver.h" #include "sql/engine/cmd/ob_context_executor.h" +#ifdef OB_BUILD_TDE_SECURITY +#include "sql/resolver/ddl/ob_create_keystore_stmt.h" +#include "sql/resolver/ddl/ob_alter_keystore_stmt.h" +#include "sql/resolver/ddl/ob_create_tablespace_stmt.h" +#include "sql/resolver/ddl/ob_alter_tablespace_stmt.h" +#include "sql/resolver/ddl/ob_drop_tablespace_stmt.h" +#include "sql/engine/cmd/ob_keystore_cmd_executor.h" +#include "sql/engine/cmd/ob_tablespace_cmd_executor.h" +#endif +#ifdef OB_BUILD_AUDIT_SECURITY +#include "sql/resolver/ddl/ob_audit_stmt.h" +#include "sql/engine/cmd/ob_audit_executor.h" +#endif namespace oceanbase { @@ -263,6 +276,12 @@ int ObCmdExecutor::execute(ObExecContext &ctx, ObICmd &cmd) DEFINE_EXECUTE_CMD(ObCreateTenantStmt, ObCreateStandbyTenantExecutor); break; } +#ifdef OB_BUILD_AUDIT_SECURITY + case stmt::T_AUDIT: { + DEFINE_EXECUTE_CMD(ObAuditStmt, ObAuditExecutor); + break; + } +#endif case stmt::T_DROP_TENANT: { DEFINE_EXECUTE_CMD(ObDropTenantStmt, ObDropTenantExecutor); break; @@ -695,6 +714,16 @@ int ObCmdExecutor::execute(ObExecContext &ctx, ObICmd &cmd) sql_text = ObString::make_empty_string(); // do not record break; } +#ifdef OB_BUILD_ORACLE_PL + case stmt::T_CREATE_TYPE: { + DEFINE_EXECUTE_CMD(ObCreateUDTStmt, ObCreateUDTExecutor); + break; + } + case stmt::T_DROP_TYPE: { + DEFINE_EXECUTE_CMD(ObDropUDTStmt, ObDropUDTExecutor); + break; + } +#endif case stmt::T_CREATE_PACKAGE: { DEFINE_EXECUTE_CMD(ObCreatePackageStmt, ObCreatePackageExecutor); break; @@ -837,6 +866,28 @@ int ObCmdExecutor::execute(ObExecContext &ctx, ObICmd &cmd) DEFINE_EXECUTE_CMD(ObSetRoutineStmt, ObSetRoleExecutor); break; }*/ +#ifdef OB_BUILD_TDE_SECURITY + case stmt::T_CREATE_KEYSTORE: { + DEFINE_EXECUTE_CMD(ObCreateKeystoreStmt, ObCreateKeystoreExecutor); + break; + } + case stmt::T_ALTER_KEYSTORE: { + DEFINE_EXECUTE_CMD(ObAlterKeystoreStmt, ObAlterKeystoreExecutor); + break; + } + case stmt::T_CREATE_TABLESPACE: { + DEFINE_EXECUTE_CMD(ObCreateTablespaceStmt, ObCreateTablespaceExecutor); + break; + } + case stmt::T_ALTER_TABLESPACE: { + DEFINE_EXECUTE_CMD(ObAlterTablespaceStmt, ObAlterTablespaceExecutor); + break; + } + case stmt::T_DROP_TABLESPACE: { + DEFINE_EXECUTE_CMD(ObDropTablespaceStmt, ObDropTablespaceExecutor); + break; + } +#endif case stmt::T_CREATE_PROFILE: case stmt::T_ALTER_PROFILE: case stmt::T_DROP_PROFILE: { diff --git a/src/sql/monitor/ob_security_audit.h b/src/sql/monitor/ob_security_audit.h index 4cdc785b8..cdc6567ec 100644 --- a/src/sql/monitor/ob_security_audit.h +++ b/src/sql/monitor/ob_security_audit.h @@ -23,6 +23,185 @@ namespace oceanbase { namespace sql { +#ifdef OB_BUILD_AUDIT_SECURITY +//Audit file /data/log1/oracle/oracle/admin/orcl/adump/orcl_ora_998570_20181225191358365255143795.aud +//Oracle Database 12c Enterprise Edition Release 12.2.0.1.0 - 64bit Production +//Build label: RDBMS_12.2.0.1.0_LINUX.X64_170125 +//ORACLE_HOME: /data/log1/oracle/oracle/product/12.2.0 +//System name: Linux +//Node name: OceanBase004065.sqa.ztt +//Release: 3.10.0-327.ali2000.alios7.x86_64 +//Version: #1 SMP Tue Dec 29 19:54:05 CST 2015 +//Machine: x86_64 +//Instance name: orcl +//Redo thread mounted by this instance: 1 +//Oracle process number: 165 +//Unix process pid: 998570, image: oracle@OceanBase004065.sqa.ztt +// +//Tue Dec 25 19:13:58 2018 +08:00 +//LENGTH : '336' +//ACTION :[7] 'CONNECT' +//DATABASE USER:[3] 'SYS' +//PRIVILEGE :[6] 'SYSDBA' +//CLIENT USER:[9] 'xiaoyi.xy' +//CLIENT TERMINAL:[5] 'pts/7' +//STATUS:[1] '0' +//DBID:[10] '1492669708' +//SESSIONID:[10] '4294967295' +//USERHOST:[23] 'OceanBase224012.sqa.bja' +//CLIENT ADDRESS:[56] '(ADDRESS=(PROTOCOL=tcp)(HOST=10.125.224.12)(PORT=53119))' +//ACTION NUMBER:[3] '100' +class ObSecurityAuditData : public common::ObBasebLogPrint +{ +public: + ObSecurityAuditData() + : ObBasebLogPrint(), is_const_filled_(false), length_(0), svr_port_(0), tenant_id_(0), + user_id_(0), effective_user_id_(0), proxy_session_id_(0), session_id_(0), + entry_id_(0), statement_id_(0), commit_version_(0), trace_id_{0}, db_id_(0), cur_db_id_(0), + sql_timestamp_us_(0), record_timestamp_us_(0), audit_id_(0), audit_type_(0), + operation_type_(0), action_id_(0), return_code_(0), + logoff_logical_read_(0), logoff_physical_read_(0), logoff_logical_write_(0), + logoff_lock_count_(0), logoff_cpu_time_us_(0), logoff_exec_time_us_(0), + logoff_alive_time_us_(0) + {} + + virtual ~ObSecurityAuditData() {} + void reset() { new (this) ObSecurityAuditData(); } + + TO_STRING_KV(K_(is_const_filled), K_(svr_ip), K_(svr_port), K_(tenant_id), K_(tenant_name), + K_(user_id), K_(user_name), K_(effective_user_id), K_(effective_user_name), + K_(client_ip), K_(user_client_ip), K_(proxy_session_id), K_(session_id), + K_(entry_id), K_(statement_id), K_(trans_id), K_(commit_version), + K(trace_id_[0]), K(trace_id_[1]), K_(cur_db_id), K_(cur_db_name), + K_(db_id), K_(db_name), K_(sql_timestamp_us), K_(record_timestamp_us), + K_(audit_id), K_(audit_type), K_(operation_type), K_(action_id), + K_(return_code), K_(sql_text), K_(auth_privileges), K_(auth_grantee)); + + void calc_total_length(); + virtual int64_t get_data_length() const { return length_; } + virtual int64_t get_timestamp() const { return record_timestamp_us_; } + virtual int print_data(char *buf, int64_t buf_len, int64_t &pos) const; + + inline static uint64_t get_next_entry_id() + { + static uint64_t next_entry_id = 1; + return ATOMIC_FAA(&next_entry_id, 1); + } + +public: + bool is_const_filled_; + //refers to the total number of bytes used in this audit record. + //This number includes the trailing newline bytes (\n), if any, + //at the end of the audit record. + int64_t length_; + + //SERVER ADDR + common::ObString svr_ip_; + int32_t svr_port_; + + //TENANTID + uint64_t tenant_id_; + + //CLIENT TENANT + common::ObString tenant_name_; + + //USERID + uint64_t user_id_; + common::ObString user_name_; + uint64_t effective_user_id_; + common::ObString effective_user_name_; + + //Client host machine name + //CLIENT ADDRESS + common::ObString client_ip_; + + //PROXY_CLIENT ADDRESS + common::ObString user_client_ip_; + + //OB_PROXY_SESSIONID + uint64_t proxy_session_id_; + + //Numeric ID for each Oracle session. Each user session gets a unique session ID. + //SESSIONID + uint64_t session_id_; + + + //indicates the current audit entry number, assigned to each audit trail record. + //The audit ENTRYID sequence number is shared between fine-grained audit records + //and regular audit records + //ENTRYID + uint64_t entry_id_; + + //nth statement in the user session. The first SQL statement gets a value of 1 and + //the value is incremented for each subsequent SQL statement. + //Note that one SQL statement can create more than one audit trail entry + //(for example, when more than one object is audited from the same SQL statement), + //and in this case the statement ID remains the same for that statement + //and the entry ID increases for each audit trail entry created by the statement. + //STATEMENT + uint64_t statement_id_; + + common::ObString trans_id_; + + int64_t commit_version_; + + uint64_t trace_id_[2]; + + //is a database identifier calculated when the database is created. + //It corresponds to the DBID column of the V$DATABASE data dictionary view. + uint64_t db_id_; + + uint64_t cur_db_id_; + + //DATABASE USER + common::ObString db_name_; + common::ObString cur_db_name_; + + int64_t sql_timestamp_us_; + + //Date and time of the creation of the audit trail entry in the local database session time zone + int64_t record_timestamp_us_; + + uint64_t audit_id_; + uint64_t audit_type_; + uint64_t operation_type_; + + //is a numeric value representing the action the user performed. + //The corresponding name of the action type is in the AUDIT_ACTIONS table. + //For example, action 100 refers to LOGON + //ACTION_NUMBER + uint64_t action_id_; + + //indicates if the audited action was successful. 0 indicates success. + //If the action fails, the return code lists the Oracle Database error number. + //For example, if you try to drop a non-existent table, the error number + //is ORA-00903 invalid table name, which in turn translates to 903 in the RETURNCODE setting. + //RETURNCODE + int return_code_; + + common::ObString obj_owner_name_; + common::ObString obj_name_; + common::ObString new_obj_owner_name_; + common::ObString new_obj_name_; + + common::ObString auth_privileges_; + common::ObString auth_grantee_; + + uint64_t logoff_logical_read_; + uint64_t logoff_physical_read_; + uint64_t logoff_logical_write_; + uint64_t logoff_lock_count_; + common::ObString logoff_dead_lock_; + uint64_t logoff_cpu_time_us_; + uint64_t logoff_exec_time_us_; + uint64_t logoff_alive_time_us_; + + + common::ObString comment_text_; + common::ObString sql_bind_; + common::ObString sql_text_; +}; +#endif } //namespace sql } //namespace oceanbase #endif diff --git a/src/sql/monitor/ob_security_audit_utils.h b/src/sql/monitor/ob_security_audit_utils.h index 5e56da512..4016783a1 100644 --- a/src/sql/monitor/ob_security_audit_utils.h +++ b/src/sql/monitor/ob_security_audit_utils.h @@ -21,6 +21,7 @@ namespace oceanbase { +#ifndef OB_BUILD_AUDIT_SECURITY namespace sql { enum class ObAuditTrailType{ @@ -46,6 +47,285 @@ public: int64_t &pos); }; } +#else +namespace share +{ +namespace schema +{ +enum ObSAuditOperationType : uint64_t; +enum ObSAuditType : uint64_t; +enum class ObObjectType; +struct ObObjectStruct; +class ObSchemaGetterGuard; +} +} +namespace sql +{ +class ObStmt; +class ObPhysicalPlan; +class ObSQLSessionInfo; +class ObSecurityAuditData; +class ObResultSet; + +enum class ObAuditTrailType{ + INVALID = 0, + NONE, + OS, + DB, + DB_EXTENDED, +}; + +ObAuditTrailType get_audit_trail_type_from_string(const common::ObString &string); + +// 定义可以被审计的一个元素 +// 记录一个对象和对其进行的操作，如 (SELECT, TABLE T1) +// 或者一个没有对应对象的操作，如 (LOGIN, NULL) +struct ObAuditUnit +{ + ObAuditUnit() : stmt_type_(sql::stmt::T_NONE), + obj_type_(share::schema::ObObjectType::INVALID), + obj_id_(common::OB_INVALID_ID), + obj_idx_(-1), + stmt_operation_type_(share::schema::AUDIT_OP_INVALID), + obj_operation_type_(share::schema::AUDIT_OP_INVALID) {} + ObAuditUnit( + sql::stmt::StmtType stmt_type, + share::schema::ObObjectType obj_type, + uint64_t obj_id, + int64_t obj_idx, + share::schema::ObSAuditOperationType stmt_operation_type, + share::schema::ObSAuditOperationType obj_operation_type) + : stmt_type_(stmt_type), obj_type_(obj_type), obj_id_(obj_id), obj_idx_(obj_idx), + stmt_operation_type_(stmt_operation_type), obj_operation_type_(obj_operation_type) {} + + ~ObAuditUnit() {}; + + sql::stmt::StmtType stmt_type_; + share::schema::ObObjectType obj_type_; + uint64_t obj_id_; + int64_t obj_idx_; + share::schema::ObSAuditOperationType stmt_operation_type_; + share::schema::ObSAuditOperationType obj_operation_type_; + TO_STRING_KV(K_(stmt_type), K_(obj_type), K_(obj_id), K_(obj_idx), K_(stmt_operation_type), + K_(obj_operation_type)); +}; + +class ObSecurityAuditUtils final +{ +public: + struct AuditDataParam { + AuditDataParam() : audit_id_(0), audit_type_(0), + operation_type_(0), action_id_(0), + db_id_(0), action_sql_(), comment_text_(), return_code_(0), + sys_privilege_(0), grant_option_(0), + audit_option_(share::schema::AUDIT_OP_INVALID), grantee_(), + stmt_type_(stmt::StmtType::T_NONE), stmt_(NULL) + { + MEMSET(client_ip_buf_, 0, sizeof(client_ip_buf_)); + MEMSET(server_ip_buf_, 0, sizeof(server_ip_buf_)); + MEMSET(trans_id_buf_, 0, sizeof(trans_id_buf_)); + MEMSET(auth_privileges_buf_, 0, sizeof(auth_privileges_buf_)); + } + int64_t to_string(char *buf, const int64_t buf_len) const + { + int64_t pos = 0; + J_OBJ_START(); + J_KV(K_(client_ip_buf), + K_(server_ip_buf), + K_(trans_id_buf), + K_(auth_privileges_buf), + K_(audit_id), + K_(audit_type), + K_(operation_type), + K_(db_id), + K_(action_sql), + K_(comment_text), + K_(return_code), + K_(sys_privilege), + K_(grant_option), + K_(audit_option), + K_(grantee), + K_(stmt_type), + KP_(stmt)); + J_OBJ_END(); + return pos; + } + + char client_ip_buf_[common::OB_IP_STR_BUFF]; + char server_ip_buf_[common::OB_IP_STR_BUFF]; + char trans_id_buf_[common::OB_MAX_TRANS_ID_BUFFER_SIZE]; + char auth_privileges_buf_[common::MAX_COLUMN_PRIVILEGE_LENGTH]; + uint64_t audit_id_; + uint64_t audit_type_; + uint64_t operation_type_; + int64_t action_id_; + uint64_t db_id_; + common::ObString action_sql_; + common::ObString comment_text_; + int return_code_; + uint64_t sys_privilege_; + uint64_t grant_option_; + share::schema::ObSAuditOperationType audit_option_; + common::ObString grantee_; + stmt::StmtType stmt_type_; + const sql::ObStmt *stmt_; + }; + + struct AuditActionTypeTransform + { + public: + AuditActionTypeTransform() + : audit_action_type_from_stmt_() + { + for (int j = 0; j < ARRAYSIZEOF(audit_action_type_from_stmt_); j++) { + audit_action_type_from_stmt_[j] = audit::ACTION_TYPE_UNKNOWN; + } + #define OB_STMT_TYPE_DEF(stmt_type, priv_check_func, id, action_type) audit_action_type_from_stmt_[stmt::stmt_type] = audit::action_type; + #include "sql/resolver/ob_stmt_type.h" + #undef OB_STMT_TYPE_DEF + } + audit::AuditActionType audit_action_type_from_stmt_[stmt::T_MAX + 1]; + }; + + typedef int (*ObCheckAllowAuditFunc) (ObSQLSessionInfo &session, + share::schema::ObSchemaGetterGuard *schema_guard, + const ObAuditUnit &audit_unit, + AuditDataParam &filled_param, + bool &is_allow_audit); + +public: + ObSecurityAuditUtils() {} + + static int handle_security_audit(ObSQLSessionInfo &session, + const stmt::StmtType stmt_type, + const common::ObString &action_sql, + const common::ObString &comment_text, + const int return_code); + static int handle_security_audit(ObResultSet &result, + share::schema::ObSchemaGetterGuard *schema_guard, + const sql::ObStmt *stmt, + const common::ObString &comment_text, + const int return_code); + static int check_allow_audit(ObSQLSessionInfo &session, bool &allow_audit); + static int check_allow_audit(ObSQLSessionInfo &session, ObAuditTrailType &at_type); + static int do_security_audit_record(ObSQLSessionInfo &session, + share::schema::ObSchemaGetterGuard *schema_guard, + const sql::ObStmt *stmt, + ObResultSet *result, + AuditDataParam &filled_param, + const ObAuditTrailType at_type); + static int gen_audit_records(ObSQLSessionInfo &session, + share::schema::ObSchemaGetterGuard &schema_guard, + AuditDataParam ¶m, + const sql::ObStmt *stmt, + const ObAuditUnit &audit_unit, + const ObAuditTrailType at_type); + static int gen_single_audit_record(ObSQLSessionInfo &session, + share::schema::ObSchemaGetterGuard &schema_guard, + const AuditDataParam ¶m, + const sql::ObStmt *stmt, + const ObAuditUnit &audit_unit, + const ObAuditTrailType at_type); + static int fill_audit_data(ObSQLSessionInfo &session, + const AuditDataParam ¶m, + const sql::ObStmt *stmt, + ObSecurityAuditData &security_audit_data); + static int fill_audit_obj_name(const AuditDataParam ¶m, + share::schema::ObSchemaGetterGuard &schema_guard, + const sql::ObStmt *stmt, + const ObAuditUnit &audit_unit, + ObSecurityAuditData &saudit_data); + static int fill_audit_obj_name_from_stmt(const AuditDataParam ¶m, + share::schema::ObSchemaGetterGuard &schema_guard, + const sql::ObStmt *stmt, + const ObAuditUnit &audit_unit, + ObSecurityAuditData &saudit_data); + static int record_audit_data(const ObAuditTrailType at_type, + ObSecurityAuditData &security_audit_data); + static int record_audit_data_into_table(ObSecurityAuditData &security_audit_data, + const bool need_record_sql); + + static int get_action_sql(const stmt::StmtType stmt_type, const sql::ObStmt *stmt, + ObSQLSessionInfo &session, common::ObString &action_sql); + static int get_audit_units(const stmt::StmtType stmt_type, + const sql::ObStmt *basic_stmt, + common::ObIArray &audit_units); + static int get_audit_units_in_subquery(const sql::ObDMLStmt *basic_stmt, + common::ObIArray &audit_units); + static int get_stmt_operation_type_from_stmt(const stmt::StmtType stmt_type, + const sql::ObStmt *stmt, share::schema::ObSAuditOperationType &operation_type); + static int get_object_operation_type_from_stmt(const stmt::StmtType stmt_type, + sql::ObStmt *stmt, + share::schema::ObSAuditOperationType &operation_type, + common::ObIArray &object_ids); + static int get_dml_objects(const sql::ObDMLStmt *stmt, + common::ObIArray &object_ids); + static int get_uniq_sequences_in_dml(const sql::ObDMLStmt *stmt, + common::hash::ObHashSet &object_ids); + // 下面一系列函数各种检查一种场景的审计规则，符合ObCheckAllowAuditFunc接口 + // 检查是否命中指定用户的语句审计规则 + static int check_allow_stmt_operation(ObSQLSessionInfo &session, + share::schema::ObSchemaGetterGuard *schema_guard, + const ObAuditUnit &audit_unit, + AuditDataParam &filled_param, + bool &is_allow_audit); + // 检查是否命中不指定用户的语句审计规则 + static int check_allow_stmt_all_user_operation(ObSQLSessionInfo &session, + share::schema::ObSchemaGetterGuard *schema_guard, + const ObAuditUnit &audit_unit, + AuditDataParam &filled_param, + bool &is_allow_audit); + // 检查是否命中not exist规则，此类规则根据返回码判断，和其他语句审计规则根据stmt_type判断的逻辑不同 + static int check_allow_not_exist_operation(ObSQLSessionInfo &session, + share::schema::ObSchemaGetterGuard *schema_guard, + const ObAuditUnit &audit_unit, + AuditDataParam &filled_param, + bool &is_allow_audit); + // 检查是否命中对象审计规则 + static int check_allow_obj_operation(ObSQLSessionInfo &session, + share::schema::ObSchemaGetterGuard *schema_guard, + const ObAuditUnit &audit_unit, + AuditDataParam &filled_param, + bool &is_allow_audit); + // 新增审计规则时，如果这条语句自身满足新增的规则，则也需要被审计 + static int check_allow_self_audit_operation(ObSQLSessionInfo &session, + share::schema::ObSchemaGetterGuard *schema_guard, + const ObAuditUnit &audit_unit, + AuditDataParam &filled_param, + bool &is_allow_audit); + static share::schema::ObObjectType get_object_type_from_audit_type( + const share::schema::ObSAuditType type); + static share::schema::ObSAuditType get_audit_type_from_object_type( + const share::schema::ObObjectType type); + static bool is_not_exist_errno(const int errcode); + static int print_obj_privs_to_buff(char *buf, + const int64_t buf_len, + int64_t &pos, + const stmt::StmtType stmt_type, + const share::ObRawObjPrivArray &obj_priv_array, + const uint64_t grant_option); + static int print_admin_option_to_buff(char *buf, + const int64_t buf_len, + int64_t &pos, + const stmt::StmtType stmt_type, + uint64_t option); + + static int get_audit_file_name(char *buf, + const int64_t buf_len, + int64_t &pos); + static int get_action_type_from_stmt_type(const stmt::StmtType stmt_type, + const sql::ObStmt *stmt, + audit::AuditActionType &action_type); + static common::ObString get_action_type_string(const audit::AuditActionType action_type); + static const common::ObString audit_action_type_string[]; + +private: + static const char* priv_names[]; + static const ObCheckAllowAuditFunc check_allow_audit_funcs_[]; + static const int check_allow_funcs_nums_; +}; +} //namespace sql +#endif } //namespace oceanbase #endif diff --git a/src/sql/monitor/ob_security_audit_utils_os.cpp b/src/sql/monitor/ob_security_audit_utils_os.cpp index 37f679333..83f7c1785 100644 --- a/src/sql/monitor/ob_security_audit_utils_os.cpp +++ b/src/sql/monitor/ob_security_audit_utils_os.cpp @@ -9,7 +9,7 @@ * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. * See the Mulan PubL v2 for more details. */ - +#ifndef OB_BUILD_AUDIT_SECURITY #define USING_LOG_PREFIX SQL_MONITOR #include "sql/monitor/ob_security_audit_utils.h" @@ -50,4 +50,4 @@ int ObSecurityAuditUtils::get_audit_file_name(char *buf, const int64_t buf_len, } //namespace sql } //namespace oceanbase - +#endif \ No newline at end of file diff --git a/src/sql/ob_result_set.cpp b/src/sql/ob_result_set.cpp index 021ec5967..d1444941d 100644 --- a/src/sql/ob_result_set.cpp +++ b/src/sql/ob_result_set.cpp @@ -310,6 +310,12 @@ int ObResultSet::start_stmt() } else { if (-1 != phy_plan->tx_id_) { const transaction::ObTransID tx_id(phy_plan->tx_id_); + if (OB_FAIL(sql::ObTMService::recover_tx_for_callback(tx_id, get_exec_context()))) { + LOG_WARN("failed to recover dblink xa transaction", K(ret), K(tx_id)); + } else { + need_revert_tx_ = true; + LOG_DEBUG("succ to recover dblink xa transaction", K(tx_id)); + } } else { LOG_DEBUG("recover dblink xa skip", K(phy_plan->tx_id_)); } @@ -366,6 +372,9 @@ int ObResultSet::end_stmt(const bool is_rollback) // do nothing } if (need_revert_tx_) { // ignore ret + int tmp_ret = sql::ObTMService::revert_tx_for_callback(get_exec_context()); + need_revert_tx_ = false; + LOG_DEBUG("revert tx for callback", K(tmp_ret)); } NG_TRACE(end_stmt); return ret; @@ -976,6 +985,23 @@ OB_INLINE int ObResultSet::auto_end_plan_trans(ObPhysicalPlan& plan, async = false; } else { // 外部SQL请求，走异步接口 +#ifdef OB_BUILD_AUDIT_SECURITY + { + // do security audit before async end trans to avoid deadlock of query_lock. + // + // don't need to set ret + ObSqlCtx *sql_ctx = get_exec_context().get_sql_ctx(); + if (OB_ISNULL(sql_ctx)) { + LOG_WARN("sql_ctx is null when handle security audit"); + } else { + ObSecurityAuditUtils::handle_security_audit(*this, + sql_ctx->schema_guard_, + sql_ctx->cur_stmt_, + ObString::make_empty_string(), + ret); + } + } +#endif int save_ret = ret; my_session_.get_end_trans_cb().set_last_error(ret); ret = ObSqlTransControl::implicit_end_trans(get_exec_context(), diff --git a/src/sql/ob_spi.cpp b/src/sql/ob_spi.cpp index 470805d43..56450221c 100644 --- a/src/sql/ob_spi.cpp +++ b/src/sql/ob_spi.cpp @@ -832,6 +832,16 @@ int ObSPIService::spi_calc_expr(ObPLExecCtx *ctx, (tmp_ret = ObSqlTransControl::rollback_savepoint(*ctx->exec_ctx_, expr_savepoint_name))) { LOG_WARN("failed to rollback current pl to implicit savepoint", K(ret), K(tmp_ret)); } +#ifdef OB_BUILD_ORACLE_PL + } else if (ctx->exec_ctx_->get_my_session()->associated_xa()) { + if (OB_TRANS_XA_BRANCH_FAIL != ret) { + tmp_ret = ObDbmsXA::xa_rollback_savepoint(*ctx->exec_ctx_); + if (OB_SUCCESS != tmp_ret) { + LOG_WARN("xa trans roll back to save point failed", + K(tmp_ret), KPC(ctx->exec_ctx_->get_my_session()->get_tx_desc())); + } + } +#endif } else if (ctx->exec_ctx_->get_my_session()->get_in_transaction()) { tmp_ret = ObPLContext::implicit_end_trans(*ctx->exec_ctx_->get_my_session(), *ctx->exec_ctx_, true); } @@ -1361,8 +1371,37 @@ int ObSPIService::spi_end_trans(ObPLExecCtx *ctx, const char *sql, bool is_rollb if (OB_SUCC(ret)) { if (my_session->is_in_transaction() && my_session->get_tx_desc()->is_xa_trans()) { +#ifndef OB_BUILD_ORACLE_PL ret = OB_ERR_UNEXPECTED; LOG_WARN("not support ObDbmsXA", K(ret)); +#else + transaction::ObXATransID xid = my_session->get_xid(); + transaction::ObGlobalTxType global_tx_type = my_session->get_tx_desc()->get_global_tx_type(xid); + if (is_rollback) { + //Rollback can be executed in the xa transaction, + //the role is to roll back all modifications, + //but does not end the xa transaction + if (transaction::ObGlobalTxType::DBLINK_TRANS == global_tx_type) { + transaction::ObTransID tx_id; + if (OB_FAIL(ObTMService::tm_rollback(*ctx->exec_ctx_, tx_id))) { + LOG_WARN("fail to do rollback for dblink trans", K(ret), K(tx_id), K(xid), K(global_tx_type)); + } + const bool force_disconnect = false; + int tmp_ret = OB_SUCCESS; + if (OB_UNLIKELY(OB_SUCCESS != (tmp_ret = + my_session->get_dblink_context().clean_dblink_conn(force_disconnect)))) { + LOG_WARN("dblink transaction failed to release dblink connections", K(tmp_ret), K(tx_id), K(xid)); + } + } else if (OB_FAIL(pl::ObDbmsXA::xa_rollback_origin_savepoint(*(ctx->exec_ctx_)))) { + LOG_WARN("rollback xa changes failed", K(ret), K(xid)); + } + } else { + //Commit is prohibited in xa transaction + ret = OB_TRANS_XA_ERR_COMMIT; + LOG_WARN("COMMIT is not allowed in a xa trans", K(ret), K(xid)); + } + ctx->exec_ctx_->set_need_disconnect(false); +#endif } else { // PL内部的提交使用同步提交 OZ (sql::ObSqlTransControl::end_trans(*ctx->exec_ctx_, is_rollback, true)); @@ -4393,7 +4432,42 @@ int ObSPIService::spi_pipe_row_to_result(pl::ObPLExecCtx *ctx, ObObjParam *single_row) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSEDx(ctx, single_row); +#else + ObPLNestedTable *coll = NULL; + CK (OB_NOT_NULL(ctx)); + CK (OB_NOT_NULL(ctx->result_)); + CK (OB_NOT_NULL(ctx->allocator_)); + CK (OB_NOT_NULL(single_row)); + CK (ctx->result_->is_ext()); + CK (OB_NOT_NULL(coll = reinterpret_cast(ctx->result_->get_ext()))); + OX (coll->is_inited() ? (void)NULL : coll->set_inited()); + OZ (spi_set_collection(ctx->exec_ctx_->get_my_session()->get_effective_tenant_id(), + ctx, *(ctx->allocator_), *coll, 1, true)); + CK (coll->get_count() >= 1); + CK (coll->get_column_count() > 0); + if (OB_SUCC(ret)) { + ObObj copied; + OZ (spi_copy_datum(ctx, coll->get_allocator(), single_row, &copied, + const_cast(&coll->get_element_type()), + OB_INVALID_ID /* package_id */)); + OZ (coll->set_row(copied, coll->get_count() - 1)); + } + + if (OB_SUCC(ret) && NULL != coll->get_allocator()) { + // use _pipelined_table_function_memory_limit to limit pipe row + omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID())); + if (OB_LIKELY(tenant_config.is_valid())) { + if ((static_cast(coll->get_allocator()))->get_used() + > tenant_config->_pipelined_table_function_memory_limit) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("pipe row use too much memory", K(ret)); + } + } + } + +#endif return ret; } @@ -4405,7 +4479,64 @@ int ObSPIService::spi_alloc_complex_var(pl::ObPLExecCtx *ctx, int64_t *addr) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSEDx(ctx, type, id, var_idx, init_size, addr); +#else + void *ptr = NULL; + CK (OB_NOT_NULL(ctx)); + CK (OB_NOT_NULL(ctx->allocator_)); + CK (OB_NOT_NULL(ctx->params_)); + CK (OB_NOT_NULL(ctx->result_)); + CK ((var_idx >= 0 && var_idx < ctx->params_->count()) + || OB_INVALID_INDEX == var_idx); // OB_INVALID_ID代表初始化函数返回值 + if (OB_SUCC(ret)) { + ptr = ctx->allocator_->alloc(init_size); + LOG_DEBUG("debug for spi alloc complex var", + K(var_idx), K(init_size), + K(sizeof(ObPLCollection)), K(sizeof(ObPLAssocArray)), + K(sizeof(ObPLNestedTable)), K(sizeof(ObPLVArray))); + if (OB_ISNULL(ptr)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate memory for complex var", K(ret)); + } else { + MEMSET(ptr, 0, init_size); + if (NULL != addr) { + *addr = reinterpret_cast(ptr); + } + +#define CONSTRUCT_COLLECTION(type, class) \ + case type: { \ + new(ptr)class(id); \ + break; \ + } + ObPLType complex_type = (ObPLType)type; + switch (complex_type) { + CONSTRUCT_COLLECTION(PL_NESTED_TABLE_TYPE, ObPLNestedTable); + CONSTRUCT_COLLECTION(PL_ASSOCIATIVE_ARRAY_TYPE, ObPLAssocArray); + CONSTRUCT_COLLECTION(PL_VARRAY_TYPE, ObPLVArray); + case PL_RECORD_TYPE: { + new (ptr) ObPLRecord(id, 0); + } break; + case PL_OPAQUE_TYPE: { + new (ptr) ObPLOpaque(); + } break; + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected complex type", K(ret), K(complex_type)); + } + } +#undef CONSTRUCT_COLLECTION + + if (OB_FAIL(ret)) { + } else if (OB_INVALID_INDEX != var_idx) { + ctx->params_->at(var_idx).set_extend(reinterpret_cast(ptr), type, init_size); + ctx->params_->at(var_idx).set_param_meta(); + } else if (OB_INVALID_INDEX == var_idx && NULL == addr) { + ctx->result_->set_extend(reinterpret_cast(ptr), type, init_size); + } else { /*do nothing*/ } + } + } +#endif return ret; } @@ -4783,7 +4914,56 @@ int ObSPIService::spi_trim_collection(pl::ObPLExecCtx *ctx, const ObSqlExpression *n_expr) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSEDx(ctx, collection_expr, row_size, n_expr); +#else + UNUSED(row_size); + CK (OB_NOT_NULL(ctx), + OB_NOT_NULL(ctx->exec_ctx_)); + if (OB_SUCC(ret)) { + ObObjParam result; + ObPLCollection *table = NULL; + int64_t n = OB_INVALID_INDEX; + + CK (OB_NOT_NULL(collection_expr)); + OZ (spi_calc_expr(ctx, collection_expr, OB_INVALID_INDEX, &result)); + CK (OB_LIKELY(ObExtendType == result.get_type())); + CK (OB_LIKELY(result.get_ext() != 0)); + CK (OB_NOT_NULL(table = reinterpret_cast(result.get_ext()))); + CK (OB_LIKELY(table->get_column_count() != 0)); + + if (OB_SUCC(ret) && !table->is_inited()) { + ret = OB_NOT_INIT; + LOG_WARN("ORA-06531: Reference to uninitialized collection", K(ret)); + } + if (OB_SUCC(ret) && table->is_associative_array()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("associative array is not support trim operation", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "associative array trim operation"); + } + if (OB_SUCC(ret)) { + if (OB_NOT_NULL(n_expr)) { + OZ (spi_calc_expr(ctx, n_expr, OB_INVALID_INDEX, &result)); + if (OB_SUCC(ret)) { + GET_INTEGER_FROM_OBJ(result, n); + } + } else { + // coll.trim/ coll.trim() 无参，删除最后一个 + n = 1; + } + if (OB_SUCC(ret)) { + if (0 > n) { + ret = OB_ERR_NUMERIC_OR_VALUE_ERROR; + } else if (n > table->get_count()) { + // raise exception + ret = OB_ERR_SUBSCRIPT_BEYOND_COUNT; + } else { + OZ (table->trim_collection_elem(n)); + } + } + } + } +#endif return ret; } @@ -4794,28 +4974,325 @@ int ObSPIService::spi_delete_collection(pl::ObPLExecCtx *ctx, const ObSqlExpression *n_expr) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSEDx(ctx, collection_expr, row_size, m_expr, n_expr); +#else + UNUSED(row_size); + CK (OB_NOT_NULL(ctx), + OB_NOT_NULL(ctx->exec_ctx_)); + if (OB_SUCC(ret)) { + ObObjParam result; + ObPLCollection *table = NULL; + int64_t m = OB_INVALID_INDEX; + int64_t n = OB_INVALID_INDEX; + + OZ (spi_calc_expr(ctx, collection_expr, OB_INVALID_INDEX, &result)); + CK (OB_LIKELY(result.get_type() == ObExtendType)); + CK (OB_LIKELY(result.get_ext() != 0)); + CK (OB_NOT_NULL(table = reinterpret_cast(result.get_ext()))); + CK (OB_LIKELY(table->get_column_count() != 0)); + + CK (!(OB_ISNULL(m_expr) && OB_NOT_NULL(n_expr))); + if (OB_SUCC(ret) && !table->is_inited()) { + ret = OB_NOT_INIT; + LOG_WARN("ORA-06531: Reference to uninitialized collection", K(ret)); + } + if (OB_SUCC(ret)) { + if (OB_ISNULL(m_expr) && OB_ISNULL(n_expr)) { + // delete all data + if (table->get_count() != 0) { + for (int64_t i = 0; OB_SUCC(ret) && i < table->get_count(); ++i) { + OZ (ObUserDefinedType::destruct_obj(table->get_data()[i], NULL)); + } + CK (OB_NOT_NULL(dynamic_cast(table->get_allocator()))); + OX (table->get_allocator()->reset()); + OX (table->set_count(0)); + OX (table->set_data(NULL)); + OX (table->set_first(OB_INVALID_INDEX)); + OX (table->set_last(OB_INVALID_INDEX)); + if (OB_SUCC(ret) && PL_ASSOCIATIVE_ARRAY_TYPE == table->get_type()) { + (static_cast(table))->set_key(NULL); + (static_cast(table))->set_sort(NULL); + } + } + } else if (OB_NOT_NULL(m_expr) && OB_ISNULL(n_expr)) { + // delete table(m) + OZ (spi_calc_expr(ctx, m_expr, OB_INVALID_INDEX, &result)); + if (OB_SUCC(ret) && 0 < table->get_count() && !result.is_null()) { + if (table->is_nested_table()) { + GET_INTEGER_FROM_OBJ(result, m); + if (OB_SUCC(ret)) { + if (m > table->get_count() || m <= 0) { + /* + * From an associative array or nested table (but not a varray): + * – DELETE(n) deletes the element whose index is n, if that element exists; + * otherwise, it does nothing. + * – DELETE(m,n) deletes all elements whose indexes are in the range m..n, if + * both m and n exist and m <= n; otherwise, it does nothing. + * */ + } else { + OX (table->delete_collection_elem(m - 1)); + } + } + } else if (table->is_varray()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("varray type is not able to delete with index", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "delete varray type with index"); + } else if (table->is_associative_array()) { + ObObj *key1 = static_cast(&result); + ObPLAssocArray *atable = static_cast(table); + ObObj *key = atable->get_key(); + // 到这儿count必然大于0，可以直接拿来比较 + OV (key!=NULL ? key[0].can_compare(*key1) : key1->is_integer_type(), + OB_ERR_UNEXPECTED, K(key[0]), KPC(key1)); + // key 不可能重复，如果有重复是一个bug + for (int64_t i = 0; OB_SUCC(ret) && i < atable->get_count(); ++i) { + if (key!=NULL ? key[i]==*key1 : (result.get_int32() == i)) { + OX (atable->delete_collection_elem(i)); + if (atable->get_first() - 1 == i) { + atable->update_first(); + } + if (atable->get_last() - 1 == i) { + atable->update_last(); + } + break; + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("illegal table type detected.", K(*table)); + } + } + } else if (OB_NOT_NULL(m_expr) && OB_NOT_NULL(n_expr)) { + // delete table(m,n) + ObObjParam result1; + OZ (spi_calc_expr(ctx, m_expr, OB_INVALID_INDEX, &result)); + OZ (spi_calc_expr(ctx, n_expr, OB_INVALID_INDEX, &result1)); + if (OB_SUCC(ret) && 0 < table->get_count() && !result.is_null() && !result1.is_null()) { + if (table->is_nested_table()) { + GET_INTEGER_FROM_OBJ(result, m); + GET_INTEGER_FROM_OBJ(result1, n); + if (OB_SUCC(ret)) { + m = m < 0 ? 0 : m > table->get_count() ? table->get_count() + 1 : m; + n = n < 0 ? 0 : n > table->get_count() ? table->get_count() + 1 : n; + if (m < n || ((m == n && m != 0 && m != table->get_count()))) { + m = 0 == m ? 1 : m; + n = (table->get_count() + 1) == n ? table->get_count() : n; + for (int64_t i = m - 1; OB_SUCC(ret) && i <= n - 1; ++i) { + OZ (table->delete_collection_elem(i)); + } + } + } + } else if (table->is_varray()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("varray type is not able to delete with index", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "delete varray type with index"); + } else if (table->is_associative_array()) { + + ObObj *key1 = static_cast(&result); + ObObj *key2 = static_cast(&result1); + ObPLAssocArray *atable = static_cast(table); + ObObj *key = atable->get_key(); + CK (OB_NOT_NULL(key1)); + CK (OB_NOT_NULL(key2)); + CK (key!=NULL ? key1->can_compare(*key2) && key[0].can_compare(*key1) + : (key1->is_integer_type() && key2->is_integer_type())); + if (OB_SUCC(ret)) { + if (*key1 <= *key2) { + for (int64_t i = 0; OB_SUCC(ret) && i < atable->get_count(); ++i) { + bool flag = key!=NULL ? (key[i] >= *key1 && key[i] <= *key2) + : (i >= key1->get_int32() && i <= key2->get_int32()); + if (flag) { + OZ (atable->delete_collection_elem(i)); + if (atable->get_first() - 1 == i) { + atable->update_first(); + } + if (atable->get_last() - 1 == i) { + atable->update_last(); + } + } + } + } else { + // do nothing; + } + } + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("illegal table type detected.", K(*table)); + } + } + } + } + } + SET_SPI_STATUS; +#endif return ret; } #undef GET_INTEGER_FROM_OBJ int ObSPIService::spi_destruct_collection(ObPLExecCtx *ctx, int64_t idx) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSEDx(ctx, idx); +#else + if (OB_ISNULL(ctx) + || OB_ISNULL(ctx->exec_ctx_) + || OB_ISNULL(ctx->exec_ctx_->get_my_session()) + || OB_INVALID_INDEX == idx) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Argument passed in is NULL", K(ctx), K(idx), K(ret)); + } else { + ParamStore *params = ctx->params_; + if (OB_ISNULL(params)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Argument in pl context is NULL", K(params), K(ret)); + } else if (OB_FAIL(ObUserDefinedType::destruct_obj(params->at(idx), ctx->exec_ctx_->get_my_session()))) { + LOG_WARN("collection is NULL", K(idx), K(params->at(idx).get_int()), K(ret)); + } else { /*do nothing*/ } + } + SET_SPI_STATUS; +#endif return ret; } int ObSPIService::spi_sub_nestedtable(ObPLExecCtx *ctx, int64_t src_idx, int64_t dst_idx, int32_t lower, int32_t upper) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSEDx(ctx, src_idx, dst_idx, lower, upper); +#else + CK (OB_NOT_NULL(ctx)); + CK (OB_NOT_NULL(ctx->params_)); + CK (OB_LIKELY(src_idx >= 0 && src_idx < ctx->params_->count())); + CK (OB_LIKELY(dst_idx >= 0 && dst_idx < ctx->params_->count())); + CK (OB_LIKELY(lower <= upper)); + if (OB_SUCC(ret)) { + ParamStore *params = ctx->params_; + ObObjParam& src_obj = params->at(src_idx); + ObObjParam& dst_obj = params->at(dst_idx); + ObPLCollection *src_coll = reinterpret_cast(src_obj.get_ext()); + CK (OB_LIKELY(src_obj.is_ext())); + CK (OB_LIKELY(dst_obj.is_ext() || dst_obj.is_null())); + CK (OB_NOT_NULL(src_coll)); + CK (OB_NOT_NULL(ctx->allocator_)); + if (OB_SUCC(ret)) { + ObPLCollection *dst_coll = NULL; + if (lower <= 0 // 检查是否越界, lower,upper从1开始 + || lower > src_coll->get_count() + || upper > src_coll->get_count()) { + ret = OB_ARRAY_OUT_OF_RANGE; + LOG_WARN("ORA-22160: element at index does not exist", + K(lower), K(upper), K(src_coll->get_count()), K(ret)); + } else if (OB_ISNULL(dst_coll = static_cast(ctx->allocator_->alloc(sizeof(ObPLCollection))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc dst collection memory", K(ret)); + } else { + ObObj *data_ptr = src_coll->get_data(); + CK (OB_NOT_NULL(data_ptr)); + if (OB_SUCC(ret)) { + int64_t count = upper - lower + 1; + data_ptr = data_ptr + (lower - 1); + dst_coll->set_data(data_ptr); + dst_coll->set_type(PL_NESTED_TABLE_TYPE); + dst_coll->set_element_type(src_coll->get_element_type()); + dst_coll->set_column_count(src_coll->get_column_count()); + dst_coll->set_first(OB_INVALID_INDEX); + dst_coll->set_last(OB_INVALID_INDEX); + dst_coll->set_allocator(NULL); + dst_coll->set_count(count); + dst_coll->set_element_type(src_coll->get_element_type()); + dst_obj.set_extend(reinterpret_cast(dst_coll), dst_coll->get_type()); + dst_obj.set_param_meta(); + } + } + } + } +#endif return ret; } int ObSPIService::spi_init_collection(ObPLExecCtx *ctx, ObPLCollection *src, ObPLCollection *dest, int64_t row_size, uint64_t package_id) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSEDx(ctx, src, dest, row_size, package_id); +#else + if (OB_ISNULL(ctx) + || OB_ISNULL(ctx->exec_ctx_) + || OB_ISNULL(ctx->exec_ctx_->get_my_session()) + || OB_ISNULL(src) + || OB_ISNULL(dest) + || row_size <= 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Argument passed in is NULL", K(ctx), K(src), K(dest), K(row_size), K(ret)); + } else if (!src->is_inited()) { // 如果源collection未初始化, 将目标也设置为未初始化 + dest->set_count(OB_INVALID_COUNT); + dest->set_first(OB_INVALID_INDEX); + dest->set_last(OB_INVALID_INDEX); + dest->set_data(NULL); + } else { + ObIAllocator *allocator = NULL; + if (package_id != OB_INVALID_ID) { + OZ (spi_get_package_allocator(ctx, package_id, allocator)); + } else { + allocator = ctx->allocator_; + } + CK (OB_NOT_NULL(allocator)); + OZ (spi_set_collection(ctx->exec_ctx_->get_my_session()->get_effective_tenant_id(), + ctx, *allocator, *dest, src->get_count()), src->get_count()); + CK (OB_NOT_NULL(dest->get_allocator())); + if (OB_SUCC(ret)) { + switch (src->get_type()) { + case PL_NESTED_TABLE_TYPE: { + // do nothing + } + break; + case PL_ASSOCIATIVE_ARRAY_TYPE: { + ObObj *key = NULL; + int64_t *sort = NULL; + ObPLAssocArray *src_aa = static_cast(src); + ObPLAssocArray *dest_aa = static_cast(dest); + CK (OB_NOT_NULL(src_aa)); + CK (OB_NOT_NULL(dest_aa)); + if (OB_SUCC(ret) && src->get_count() > 0) { + if (NULL != src_aa->get_sort() && NULL != src_aa->get_key()) { + key = static_cast(dest->get_allocator()->alloc(src->get_count() * sizeof(ObObj))); + sort = static_cast(dest->get_allocator()->alloc(src->get_count() * sizeof(int64_t))); + CK (OB_NOT_NULL(key)); + CK (OB_NOT_NULL(sort)); + for (int64_t i = 0; OB_SUCC(ret) && i < src->get_count(); ++i) { + OZ (deep_copy_obj(*dest->get_allocator(), *src_aa->get_key(i), key[i])); + OX (sort[i] = src_aa->get_sort(i)); + } + } else if (NULL == src_aa->get_sort() && NULL == src_aa->get_key()) { + //Associative array的优化 + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Unexpected associative array", K(*src), K(*dest), K(row_size), K(package_id), K(ret)); + } + } + OX (dest_aa->set_key(key)); + OX (dest_aa->set_sort(sort)); + } + break; + case PL_VARRAY_TYPE: { + ObPLVArray *src_va = static_cast(src); + ObPLVArray *dest_va = static_cast(dest); + CK (OB_NOT_NULL(src_va)); + CK (OB_NOT_NULL(dest_va)); + OX (dest_va->set_capacity(src_va->get_capacity())); + } + break; + default: { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("src is not a collection", K(ctx), K(*src), K(*dest), K(row_size), K(package_id), K(ret)); + } + break; + } + } + } + SET_SPI_STATUS; +#endif return ret; } @@ -4827,7 +5304,192 @@ int ObSPIService::spi_set_collection(int64_t tenant_id, bool extend_mode) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSEDx(tenant_id, ns, allocator, coll, n, extend_mode); +#else +#define CHECK_VARRAY_CAPACITY \ + do { \ + if (OB_SUCC(ret)) { \ + if (coll.is_varray()) { \ + ObPLVArray &va = static_cast(coll); \ + if (va.get_capacity() < va.get_count() + n) { \ + ret = OB_SIZE_OVERFLOW; \ + LOG_WARN("varray size overflow, ORA-06532: Subscript outside of limit", K(va), K(va.get_capacity()), K(n), K(coll.get_count()), K(ret)); \ + } \ + } \ + } \ + } while (0) + +#define CHECK_NOT_NULL \ + do { \ + if (OB_SUCC(ret)) { \ + if (coll.is_not_null()) { \ + ret = OB_ERR_NUMERIC_OR_VALUE_ERROR; \ + LOG_WARN("not null check violated", K(coll), K(n), K(extend_mode), K(coll.is_not_null()), K(coll.get_count()), K(ret)); \ + } \ + } \ + } while (0) + +#define SET_COLLECTION_INFO \ + do { \ + if (OB_SUCC(ret)) { \ + coll.set_data(reinterpret_cast(data)); \ + coll.set_count(coll.get_count() + n); \ + if (coll.is_associative_array()) { \ + /*associative array的first和last不在这里设置，需要根据数据计算*/ \ + } else { \ + OB_INVALID_INDEX == coll.get_first() ? coll.set_first(1) : (void)NULL; /*first和last从1开始*/ \ + coll.set_last(coll.get_count()); \ + } \ + } \ + } while (0) + + UNUSED(tenant_id); + common::ObIAllocator *collection_allocator = NULL; + void *data = NULL; + bool set_data = false; + if (extend_mode && !coll.is_inited()) { + ret = OB_NOT_INIT; + LOG_WARN("ORA-06531: Reference to uninitialized collection", K(ret)); + } else if (NULL == coll.get_allocator()) { + collection_allocator = static_cast(allocator.alloc(sizeof(ObPLCollAllocator))); + if (OB_ISNULL(collection_allocator)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("get a invalud obj", K(ret), K(collection_allocator)); + } else { + collection_allocator = new(collection_allocator)ObPLCollAllocator(&coll); + coll.set_allocator(collection_allocator); + coll.set_count(0); + coll.set_first(OB_INVALID_INDEX); + coll.set_last(OB_INVALID_INDEX); + coll.set_data(NULL); +// set_data = true; + } + } + if (OB_SUCC(ret)) { + if (extend_mode && n > 0) { + + CHECK_VARRAY_CAPACITY; + + // CHECK_NOT_NULL; + + if (OB_SUCC(ret)) { + data = coll.get_allocator()->alloc(sizeof(ObObj) * (coll.get_count() + n)); + if (OB_ISNULL(data)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc failed", + K(sizeof(ObObj) * (coll.get_count() + n)), + K(n), + K(coll.get_count()), + K(ret)); + } else { + ObObj *obj_data = reinterpret_cast(data); + for (int64_t i = coll.get_count(); OB_SUCC(ret) && i < coll.get_count() + n; ++i) { + new (obj_data + i) ObObj(); + } + } + OX (OB_NOT_NULL(coll.get_data()) ? MEMCPY(data, coll.get_data(), sizeof(ObObj) * coll.get_count()) : (void*)(NULL)); + OX (OB_NOT_NULL(coll.get_data()) ? coll.get_allocator()->free(coll.get_data()) : void(NULL)); + LOG_DEBUG("PL/SQL realloc collection memory", + K(coll.get_data()), + K(coll.get_count()), + K(n), + K(coll.get_allocator()->total()), + K(coll.get_allocator()->used())); + + CK (coll.get_column_count() > 0); + for (int64_t i = 0; OB_SUCC(ret) && i < n; ++i) { + /* + * extend模式需要把下层结构递归分配出来，因为在下面这个case里extend之后下面就可以直接访问内存了 + * declare + * TYPE rec_type IS RECORD(a int, b varchar(10)); + * TYPE tab_type IS TABLE OF rec_type; + * v_tab1 tab_type := tab_type(); + * begin + * v_tab1.extend; + * v_tab1(1).a := 1; + * v_tab1(1).b := 'a'; + * end; + */ + ObObj* row = &(reinterpret_cast(data)[(coll.get_count() + i)]); + if (!coll.get_element_desc().is_composite_type() || NULL == ns) { + new (row) ObObj(ObNullType); + } else { + const ObUserDefinedType *type = NULL; + const ObCollectionType *collection_type = NULL; + int64_t ptr = 0; + int64_t init_size = OB_INVALID_SIZE; + CK (OB_NOT_NULL(ns)); + OZ (ns->get_user_type(coll.get_id(), type)); + CK (OB_NOT_NULL(type)); + CK (type->is_collection_type()); + CK (OB_NOT_NULL(collection_type = static_cast(type))); + OZ (collection_type->get_element_type().newx(*coll.get_allocator(), ns, ptr)); + OZ (collection_type->get_element_type().get_size(*ns, PL_TYPE_INIT_SIZE, init_size)); + OX (row->set_extend(ptr, collection_type->get_element_type().get_type(), init_size)); + } + } + SET_COLLECTION_INFO; + } + //only extend data in this call, key and sort will be extended by caller. + //do not shrink here, may core cause key and sort not equal to data. + //also, Associative array will shrink by caller. + if (OB_SUCC(ret) && !coll.is_associative_array()) { + OZ (coll.shrink()); + } + } else { + for (int64_t cnt = 0; OB_SUCC(ret) && cnt < coll.get_count(); ++cnt) { + ObObj &old_obj = reinterpret_cast(coll.get_data())[cnt]; + if (old_obj.get_meta().get_extend_type() != PL_CURSOR_TYPE) { + OZ (ObUserDefinedType::destruct_obj(old_obj)); + } + } + coll.set_count(0); + coll.set_first(OB_INVALID_INDEX); + coll.set_last(OB_INVALID_INDEX); + coll.set_data(NULL); + if (coll.is_associative_array()) { + ObPLAssocArray &aa = static_cast(coll); + if (NULL != aa.get_key()) { + aa.set_key(NULL); + } + if (NULL != aa.get_sort()) { + aa.set_sort(NULL); + } + } + collection_allocator = dynamic_cast(coll.get_allocator()); + if (NULL != collection_allocator) { + collection_allocator->reset(); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("Allocator unexpected", K(ret)); + } + set_data = true; + } + } + + if (OB_SUCC(ret) && set_data && n > 0) { + CHECK_VARRAY_CAPACITY; + + // CHECK_NOT_NULL; + + if (OB_SUCC(ret)) { + data = coll.get_allocator()->alloc(sizeof(ObObj) * n); + if (OB_ISNULL(data)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc failed", K(sizeof(ObObj) * n), K(n), K(coll), K(ret)); + } else { + CK (coll.get_column_count() > 0); + //非extend模式仅扩展出n个Obj结构并初始化为NULL，下层结构不分配 + for (int64_t i = 0; OB_SUCC(ret) && i < n; ++i) { + new (&reinterpret_cast(data)[i])ObObj(ObNullType); + } + + SET_COLLECTION_INFO; + } + } + } +#endif return ret; } @@ -4866,6 +5528,105 @@ int ObSPIService::spi_reset_collection(ObPLCollection *coll) return ret; } +#ifdef OB_BUILD_ORACLE_PL +int ObSPIService::spi_extend_assoc_array(int64_t tenant_id, + const ObPLINS *ns, + ObIAllocator &allocator, + ObPLAssocArray &assoc_array, + int64_t n) +{ + int ret = OB_SUCCESS; + assoc_array.is_inited() ? (void)NULL : assoc_array.set_inited(); + if (OB_FAIL(spi_set_collection(tenant_id, ns, allocator, assoc_array, n, true))) { + LOG_WARN("failed to spi_reset_collection", K(assoc_array), K(ret)); + } else if (OB_ISNULL(assoc_array.get_allocator())) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("allocator is NULL", K(assoc_array), K(n), K(ret)); + } else if (NULL == dynamic_cast(assoc_array.get_allocator())) { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("Unexpected allocator used in Collection", K(assoc_array), K(n), K(ret)); + } else if (NULL == assoc_array.get_key()) { + if (NULL != assoc_array.get_sort()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("key is NULL, but sort is not NULL", K(assoc_array), K(n), K(ret)); + } + +#define ALLOC_ASSOC_ARRAY(TYPE, PROPERTY) \ + do { \ + if (OB_SUCC(ret)) { \ + TYPE *addr = NULL; \ + ObPLAllocator *allocator = static_cast(assoc_array.get_allocator()); \ + if (OB_ISNULL(allocator)) { \ + ret = OB_ERR_UNEXPECTED; \ + LOG_WARN("failed to static cast pl allocator", \ + K(ret), K(allocator), K(assoc_array.get_allocator())); \ + } else if (OB_ISNULL(allocator->get_allocator())) { \ + ret = OB_ERR_UNEXPECTED; \ + LOG_WARN("allocator is null", K(ret), K(allocator), K(assoc_array.get_allocator())); \ + } else if (OB_ISNULL(addr = static_cast(allocator->get_allocator()->alloc(sizeof(TYPE) * n)))) { \ + ret = OB_ALLOCATE_MEMORY_FAILED; \ + LOG_WARN("alloc failed", K(assoc_array), K(n), K(sizeof(TYPE)), K(ret)); \ + } else { \ + assoc_array.set_##PROPERTY(addr); \ + } \ + } \ + } while(0) + + ALLOC_ASSOC_ARRAY(ObObj, key); + + ALLOC_ASSOC_ARRAY(int64_t, sort); + + // NOTE: init extend key avoid to deep copy core!!! + if (OB_SUCC(ret)) { + for (int64_t i = 0; i < assoc_array.get_count(); ++i) { + ObObj *key = assoc_array.get_key(i); + key = new(key)ObObj(); + } + } + + } else { + +#define REALLOC_ASSOC_ARRAY(TYPE, PROPERTY) \ + do { \ + if (OB_SUCC(ret)) { \ + TYPE *addr = NULL; \ + ObPLAllocator *allocator = static_cast(assoc_array.get_allocator()); \ + if (OB_ISNULL(allocator)) { \ + ret = OB_ERR_UNEXPECTED; \ + LOG_WARN("failed to static cast pl allocator", \ + K(ret), K(allocator), K(assoc_array.get_allocator())); \ + } else if (OB_ISNULL(allocator->get_allocator())) { \ + ret = OB_ERR_UNEXPECTED; \ + LOG_WARN("allocator is null", K(ret), K(allocator), K(assoc_array.get_allocator())); \ + } else if (OB_ISNULL(addr = reinterpret_cast(allocator->get_allocator()->alloc(sizeof(TYPE) * assoc_array.get_count())))) { \ + ret = OB_ALLOCATE_MEMORY_FAILED; \ + LOG_WARN("alloc failed", K(ret), K(assoc_array), K(n), K(sizeof(TYPE) * (assoc_array.get_count() - n)), K(sizeof(TYPE) * assoc_array.get_count())); \ + } \ + OX (OB_NOT_NULL(assoc_array.get_##PROPERTY()) ? MEMCPY(addr, assoc_array.get_##PROPERTY(), sizeof(TYPE) * (assoc_array.get_count() - n)) \ + : (void*)(NULL)); \ + OX (OB_NOT_NULL(assoc_array.get_##PROPERTY()) ? assoc_array.get_allocator()->free(assoc_array.get_##PROPERTY()) : void(NULL)); \ + OX (assoc_array.set_##PROPERTY(addr)); \ + } \ + } while(0) + + REALLOC_ASSOC_ARRAY(ObObj, key); + + REALLOC_ASSOC_ARRAY(int64_t, sort); + + // NOTE: init extend key avoid to deep copy core!!! + if (OB_SUCC(ret)) { + for (int64_t i = assoc_array.get_count() - 1; i >= assoc_array.get_count() - n; --i) { + ObObj *key = assoc_array.get_key(i); + key = new(key)ObObj(); + } + } + OZ (assoc_array.shrink()); + + LOG_DEBUG("spi extend assoc array", K(ret), K(assoc_array)); + } + return ret; +} +#endif int ObSPIService::spi_get_package_allocator( ObPLExecCtx *ctx, uint64_t package_id, ObIAllocator *&allocator) @@ -4955,7 +5716,34 @@ int ObSPIService::spi_handle_ref_cursor_refcount(ObPLExecCtx *ctx, int64_t addend) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSEDx(ctx, package_id, routine_id, index, addend); +#else + ObPLCursorInfo *cursor_info = NULL; + ObObjParam cursor_var; + ObCusorDeclareLoc odc; + LOG_DEBUG("spi handle ref cursor", K(package_id), K(routine_id), K(index), K(addend)); + OZ (spi_get_cursor_info(ctx, package_id, routine_id, index, cursor_info, cursor_var, odc)); + if (OB_NOT_NULL(cursor_info)) { + OZ (spi_add_ref_cursor_refcount(ctx, &cursor_var, addend)); + } else { + // do nothing + // for example: + /* + * declare + cur1 SYS_REFCURSOR; + begin + end; + cur1 is not open, so it is null; + + declare + cursor cur is select 1 from dual; + begin + null; + end; + */ + } +#endif return ret; } @@ -4967,7 +5755,52 @@ int ObSPIService::spi_copy_ref_cursor(ObPLExecCtx *ctx, uint64_t package_id) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSEDx(ctx, allocator, src, dest, dest_type, package_id); +#else + UNUSEDx(allocator, dest_type); + CK (OB_INVALID_ID == package_id); // ref cursor can't define inside package + CK (OB_NOT_NULL(src)); + CK (OB_NOT_NULL(dest)); + if (OB_SUCC(ret)) { + bool need_inc_ref_cnt = false; + // see if null cursor + // case 1: dest is not null, add dest cursor ref count + // case 2: src is not null, dec src cursor ref count, if equals to 0, close cursor, + ObPLCursorInfo *src_cursor = reinterpret_cast(src->get_ext()); + ObPLCursorInfo *dest_cursor = reinterpret_cast(dest->get_ext()); + if (OB_NOT_NULL(src_cursor) && src_cursor->isopen()) { + LOG_DEBUG("copy ref cursor, src ref count: ", K(src_cursor->get_ref_count())); + need_inc_ref_cnt = (0 == src_cursor->get_ref_count() && src_cursor->get_is_returning()); + CK (0 < src_cursor->get_ref_count() || need_inc_ref_cnt); + // 到了这里先把returning状态重置，ref count先不加，等赋值成功再加 + OX (src_cursor->set_is_returning(false)); + } + if (src_cursor == dest_cursor) { + // 说明指向同一个内存，或者都为null， 这个时候应该不干，除了一个特殊的场景，即return场景，需要将count+1 + // 例如: c1 := c2; c1 := c2; c2 may null or not null; + if (need_inc_ref_cnt) { + OX (src_cursor->inc_ref_count()); + } + } else { + if (OB_NOT_NULL(dest_cursor) && dest_cursor->isopen()) { + LOG_DEBUG("copy ref cursor, dest ref count: ",K(*dest_cursor), + K(dest_cursor->get_ref_count())); + CK (0 < dest_cursor->get_ref_count()); + OX (dest_cursor->dec_ref_count()); + if (0 == dest_cursor->get_ref_count()) { + OZ (cursor_close_impl(ctx, dest_cursor, true, OB_INVALID_ID, OB_INVALID_ID, true)); + } + } + OX (*dest = *src); + if (OB_NOT_NULL(src_cursor)) { + OX (src_cursor->inc_ref_count()); + LOG_DEBUG("copy ref cursor, src ref count: ",K(*src_cursor), + K(src_cursor->get_ref_count())); + } + } + } +#endif return ret; } @@ -6123,7 +6956,28 @@ int ObSPIService::get_result(ObPLExecCtx *ctx, table->set_first(OB_INVALID_INDEX); table->set_last(OB_INVALID_INDEX); table->set_data(NULL); +#ifdef OB_BUILD_ORACLE_PL + if (table->is_associative_array()) { + ObPLAssocArray &aa = static_cast(*table); + if (NULL != aa.get_key()) { + aa.set_key(NULL); + } + if (NULL != aa.get_sort()) { + aa.set_sort(NULL); + } + } + } else if (table->is_associative_array()) { + ObPLAssocArray* assoc = static_cast(table); + CK (OB_NOT_NULL(assoc)); + if (OB_SUCC(ret) && assoc->get_key() != NULL) { + ret = OB_NOT_SUPPORTED; + LOG_USER_ERROR(OB_NOT_SUPPORTED, + "Implementation restriction, bulk collect with associative array has key values"); + } } +#else + } +#endif } } } @@ -6685,6 +7539,19 @@ int ObSPIService::check_and_copy_composite( if (PL_OBJ_TYPE == type || PL_INTEGER_TYPE == type) { ret = OB_ERR_EXPRESSION_WRONG_TYPE; LOG_WARN("expr is wrong type", K(ret)); +#ifdef OB_BUILD_ORACLE_PL + } else if (PL_OPAQUE_TYPE == type) { + if (src.get_meta().get_extend_type() != PL_OPAQUE_TYPE) { + ret = OB_ERR_EXPRESSION_WRONG_TYPE; + LOG_WARN("expr is wrong type", K(ret), K(src), K(type), K(dst_udt_id)); + } else { + ObPLOpaque* opaque = reinterpret_cast(src.get_ext()); + if (opaque->get_type() != ObPLOpaque::get_type(dst_udt_id)) { + ret = OB_ERR_EXPRESSION_WRONG_TYPE; + LOG_WARN("expr is wrong type", K(ret)); + } + } +#endif } else { ObPLComposite *composite = reinterpret_cast(src.get_ext()); if (NULL == composite) { @@ -6869,6 +7736,18 @@ int ObSPIService::store_datums(ObObj &dest_addr, const ObIArray &obj_arra if (PL_OPAQUE_TYPE == dest_addr.get_meta().get_extend_type()) { CK (1 == obj_array.count()); OX (current_datum = reinterpret_cast(&dest_addr)); +#ifdef OB_BUILD_ORACLE_PL + if (OB_SUCC(ret) && obj_array.at(0).is_pl_extend()) { + is_opaque = true; + ObPLOpaque *src = reinterpret_cast(obj_array.at(0).get_ext()); + ObPLOpaque *dst = reinterpret_cast(dest_addr.get_ext()); + if (OB_FAIL(ObSPIService::spi_copy_opaque(NULL, NULL, *src, dst))) { // dst buffer has enough size? + LOG_WARN("opaque data store datum failed ", KPC(src), K(dest_addr), K(obj_array), K(ret)); + } else if (OB_NOT_NULL(src)) { + src->~ObPLOpaque(); + } + } +#endif } else { ObPLComposite *composite = reinterpret_cast(dest_addr.get_ext()); ObPLRecord *record = NULL; @@ -7424,6 +8303,47 @@ int ObSPIService::spi_update_package_change_info( return ret; } +#ifdef OB_BUILD_ORACLE_PL +int ObSPIService::spi_copy_opaque( + pl::ObPLExecCtx *ctx, ObIAllocator *allocator, + ObPLOpaque &src, ObPLOpaque *&dest, uint64_t package_id) +{ + int ret = OB_SUCCESS; + UNUSEDx(ctx, package_id); + if (NULL == dest) { + CK (OB_NOT_NULL(allocator)); + if (OB_SUCC(ret) && OB_ISNULL(dest = reinterpret_cast(allocator->alloc(src.get_init_size())))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for dest opaque", K(ret), K(src.get_init_size())); + } + OX (new (dest)ObPLOpaque()); + } + if (OB_SUCC(ret)) { + switch (src.get_type()) { + case ObPLOpaqueType::PL_ANY_TYPE: { + ObPLAnyType &anytype = static_cast(src); + OZ (anytype.deep_copy(dest)); + } break; + case ObPLOpaqueType::PL_ANY_DATA: { + ObPLAnyData &anydata = static_cast(src); + OZ (anydata.deep_copy(dest)); + } break; + case ObPLOpaqueType::PL_XML_TYPE: { + ObPLXmlType &xml = static_cast(src); + OZ (xml.deep_copy(dest)); + } break; + case ObPLOpaqueType::PL_JSON_TYPE: { + ObPLJsonBaseType &jsn = static_cast(src); + OZ (jsn.deep_copy(dest)); + } break; + default: { + OZ (src.deep_copy(dest)); + } break; + } + } + return ret; +} +#endif int ObSPIService::spi_check_composite_not_null(ObObjParam *v) { diff --git a/src/sql/ob_spi.h b/src/sql/ob_spi.h index cb11929e2..160af28e9 100644 --- a/src/sql/ob_spi.h +++ b/src/sql/ob_spi.h @@ -565,6 +565,13 @@ public: static int spi_sub_nestedtable(pl::ObPLExecCtx *ctx, int64_t src_idx, int64_t dst_idx, int32_t lower, int32_t upper); +#ifdef OB_BUILD_ORACLE_PL + static int spi_extend_assoc_array(int64_t tenant_id, + const pl::ObPLINS *ns, + ObIAllocator &allocator, + pl::ObPLAssocArray &assoc_array, + int64_t n); +#endif static int spi_get_package_allocator(pl::ObPLExecCtx *ctx, uint64_t package_id, ObIAllocator *&allocator); static int spi_copy_datum(pl::ObPLExecCtx *ctx, @@ -674,6 +681,11 @@ public: static int spi_update_package_change_info( pl::ObPLExecCtx *ctx, uint64_t package_id, uint64_t var_idx); +#ifdef OB_BUILD_ORACLE_PL + static int spi_copy_opaque( + pl::ObPLExecCtx *ctx, ObIAllocator *allocator, + pl::ObPLOpaque &src, pl::ObPLOpaque *&dest, uint64_t package_id = OB_INVALID_ID); +#endif static int spi_check_composite_not_null(ObObjParam *v); static int spi_update_location(pl::ObPLExecCtx *ctx, uint64_t location); diff --git a/src/sql/ob_sql.cpp b/src/sql/ob_sql.cpp index 82e3389a6..254ca29e6 100644 --- a/src/sql/ob_sql.cpp +++ b/src/sql/ob_sql.cpp @@ -81,6 +81,13 @@ #include "sql/ob_optimizer_trace_impl.h" #include "share/resource_manager/ob_resource_manager.h" #include "share/resource_manager/ob_cgroup_ctrl.h" +#ifdef OB_BUILD_SPM +#include "sql/spm/ob_spm_controller.h" +#include "sql/spm/ob_spm_define.h" +#endif +#ifdef OB_BUILD_ORACLE_PL +#include "pl/sys_package/ob_json_pl_utils.h" +#endif #include "sql/ob_optimizer_trace_impl.h" #include "sql/monitor/ob_sql_plan.h" #include "sql/optimizer/ob_explain_log_plan.h" @@ -655,6 +662,12 @@ int ObSql::fill_select_result_set(ObResultSet &result_set, ObSqlCtx *context, co } else if (NULL == context->secondary_namespace_ // pl resolve && NULL == context->session_info_->get_pl_context()) { // pl execute ret = OB_NOT_SUPPORTED; +#ifdef OB_BUILD_ORACLE_PL + // error code compiltable with oracle + if (pl::ObPlJsonUtil::is_pl_jsontype(expr->get_result_type().get_udt_id())) { + ret = OB_ERR_PL_JSONTYPE_USAGE; + } +#endif LOG_WARN("composite type use in pure sql context not supported!"); } } @@ -1541,6 +1554,14 @@ int ObSql::handle_pl_execute(const ObString &sql, } } +#ifdef OB_BUILD_AUDIT_SECURITY + (void)ObSecurityAuditUtils::handle_security_audit(result, + context.schema_guard_, + context.cur_stmt_, + ObString::make_string("pl/sql"), + ret); + +#endif if (OB_SUCC(ret) && session.get_in_transaction()) { if (ObStmt::is_dml_write_stmt(result.get_stmt_type()) || ObStmt::is_savepoint_stmt(result.get_stmt_type())) { @@ -2427,7 +2448,11 @@ OB_INLINE int ObSql::handle_text_query(const ObString &stmt, ObSqlCtx &context, const uint64_t tenant_id = session.get_effective_tenant_id(); ObExecContext& ectx = result.get_exec_context(); int get_plan_err = OB_SUCCESS; //used for judge whether add plan to plan cache +#ifndef OB_BUILD_SPM bool use_plan_cache = session.get_local_ob_enable_plan_cache(); +#else + bool use_plan_cache = session.get_local_ob_enable_plan_cache() && !context.spm_ctx_.is_retry_for_spm_; +#endif ObPlanCacheCtx *pc_ctx = NULL; bool is_begin_commit_stmt = false; if (OB_FAIL(ret)) { @@ -2466,7 +2491,11 @@ OB_INLINE int ObSql::handle_text_query(const ObString &stmt, ObSqlCtx &context, database_id)) { // do nothing } else if (!use_plan_cache) { +#ifndef OB_BUILD_SPM if (context.multi_stmt_item_.is_batched_multi_stmt()) { +#else + if (context.multi_stmt_item_.is_batched_multi_stmt() && !context.spm_ctx_.is_retry_for_spm_) { +#endif ret = OB_BATCHED_MULTI_STMT_ROLLBACK; LOG_WARN("batched multi_stmt needs rollback"); } @@ -2481,6 +2510,9 @@ OB_INLINE int ObSql::handle_text_query(const ObString &stmt, ObSqlCtx &context, int tmp_ret = ret; if (!is_begin_commit_stmt && 0 == context.multi_stmt_item_.get_seq_num() /* only first item of a multi stmt, or single stmt */ +#ifdef OB_BUILD_SPM + && !context.spm_ctx_.is_retry_for_spm_ +#endif && OB_FAIL(handle_large_query(tmp_ret, result, ectx.get_need_disconnect_for_update(), @@ -2572,6 +2604,14 @@ OB_NOINLINE int ObSql::handle_large_query(int tmp_ret, LOG_INFO("compile time is too long, need delay", K(elapsed_time), K(ret)); } } +#ifdef OB_BUILD_SPM + if (OB_SUCC(ret) && !is_large_query) { + result.get_session().reset_spm_select_plan_type(); + } else if (OB_EAGAIN == ret || is_large_query) { + ObSpmCacheCtx& spm_ctx = exec_ctx.get_sql_ctx()->spm_ctx_; + result.get_session().set_spm_select_plan_type(spm_ctx.select_plan_type_); + } +#endif } return ret; @@ -2997,6 +3037,15 @@ int ObSql::generate_plan(ParseResult &parse_result, LOG_WARN("query ctx is null", K(ret)); } else if (OB_FAIL(fill_result_set(result, &sql_ctx, mode, *basic_stmt))) { LOG_WARN("Failed to fill result set", K(ret)); +#ifdef OB_BUILD_AUDIT_SECURITY + } else if (OB_FAIL(ObSecurityAuditUtils::check_allow_audit( + *sql_ctx.session_info_, allow_audit))) { + LOG_WARN("Failed to check allow audit", K(ret)); + } else if (allow_audit && + OB_FAIL(ObSecurityAuditUtils::get_audit_units( + basic_stmt->get_stmt_type(), basic_stmt, audit_units))) { + LOG_WARN("Failed to get audit units", K(ret)); +#endif } else if (OB_FAIL(sql_ctx.session_info_->get_sys_variable( share::SYS_VAR__AGGREGATION_OPTIMIZATION_SETTINGS, aggregate_setting))) { @@ -3005,6 +3054,9 @@ int ObSql::generate_plan(ParseResult &parse_result, ObDMLStmt *stmt = static_cast(basic_stmt); SQL_LOG(DEBUG, "stmt", "stmt", *stmt); SQL_LOG(DEBUG, "stmt success", "query", SJ(*stmt)); +#ifdef OB_BUILD_SPM + stmt->get_query_ctx()->is_spm_evolution_ = sql_ctx.spm_ctx_.is_retry_for_spm_; +#endif stmt->get_query_ctx()->root_stmt_ = stmt; const ObGlobalHint &global_hint = stmt->get_query_ctx()->get_global_hint(); sql_ctx.session_info_->set_early_lock_release(global_hint.enable_lock_early_release_); @@ -3873,6 +3925,17 @@ int ObSql::get_outline_data(ObSqlCtx &context, //no outline is available when database name of session is not specified, just keep the stmt } else if (pc_ctx.is_begin_commit_stmt()) { /* do nothing */ +#ifdef OB_BUILD_SPM + } else if (pc_ctx.sql_ctx_.spm_ctx_.is_retry_for_spm_) { + ObPlanBaselineItem* baseline_item = + static_cast(pc_ctx.sql_ctx_.spm_ctx_.baseline_guard_.get_cache_obj()); + if (OB_ISNULL(baseline_item)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get unexpected null", K(ret)); + } else { + outline_content = baseline_item->get_outline_data_str(); + } +#endif } else if (OB_FAIL(get_outline_data(pc_ctx, signature_sql, outline_state, outline_content))) { LOG_WARN("failed to get outline data", K(ret)); } @@ -3887,6 +3950,11 @@ int ObSql::get_outline_data(ObSqlCtx &context, } else if (OB_FAIL(ob_write_string(pc_ctx.allocator_, sql_helper.string(), temp_outline_sql))) { LOG_WARN("failed to write string", K(outline_content), K(ret)); } else if (OB_FAIL(parser.parse(temp_outline_sql, outline_parse_result))) { +#ifdef OB_BUILD_SPM + if (pc_ctx.sql_ctx_.spm_ctx_.is_retry_for_spm_) { + LOG_WARN("failed to parse outline data result for spm", K(ret), K(temp_outline_sql)); + } else +#endif { LOG_WARN("failed to parse outline data result", K(ret), K(temp_outline_sql)); outline_state.reset(); @@ -4079,7 +4147,11 @@ int ObSql::parser_and_check(const ObString &outlined_stmt, LOG_WARN("Invalid plan cache", K(ret)); } else { plan_cache->inc_access_cnt(); +#ifndef OB_BUILD_SPM if (OB_SQL_PC_NOT_EXIST == get_plan_err) { +#else + if (OB_SQL_PC_NOT_EXIST == get_plan_err || pc_ctx.sql_ctx_.spm_ctx_.is_retry_for_spm_) { +#endif add_plan_to_pc = true; } else { add_plan_to_pc = false; @@ -4599,6 +4671,9 @@ OB_NOINLINE int ObSql::handle_physical_plan(const ObString &trimed_stmt, pc_ctx.neg_param_index_.reset(); bool plan_added = false; bool need_get_baseline = false; +#ifdef OB_BUILD_SPM + spm_ctx.bl_key_.sql_cs_type_ = session.get_local_collation_connection(); +#endif LOG_DEBUG("gen plan info", K(spm_ctx.bl_key_), K(get_plan_err)); // for batched multi stmt, we only parse and optimize the first statement // only in multi_query, need do this @@ -4635,6 +4710,7 @@ OB_NOINLINE int ObSql::handle_physical_plan(const ObString &trimed_stmt, LOG_WARN("batched multi_stmt needs rollback", K(ret)); } generate_sql_id(pc_ctx, add_plan_to_pc, parse_result, signature_sql, ret); +#ifndef OB_BUILD_SPM if (OB_FAIL(ret)) { // do nothing } else if (OB_FAIL(get_outline_data(context, pc_ctx, signature_sql, @@ -4669,6 +4745,116 @@ OB_NOINLINE int ObSql::handle_physical_plan(const ObString &trimed_stmt, } else if (!is_match_udr && OB_FAIL(pc_add_plan(pc_ctx, result, outline_state, plan_cache, plan_added))) { LOG_WARN("fail to add plan to plan cache", K(ret)); } +#else + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_FAIL(get_outline_data(context, pc_ctx, signature_sql, + outline_state, outline_parse_result))) { + LOG_WARN("failed to get outline data for query", K(ret)); + } else if (OB_FAIL(generate_physical_plan(parse_result, + &pc_ctx, + context, + result, + pc_ctx.is_begin_commit_stmt(), + mode, + &outline_parse_result))) { + if (OB_ERR_PROXY_REROUTE == ret) { + LOG_DEBUG("Failed to generate plan", K(ret)); + } else if (OB_OUTLINE_NOT_REPRODUCIBLE == ret && spm_ctx.is_retry_for_spm_) { + LOG_TRACE("spm need get baseline due to generate plan failed"); + need_get_baseline = true; + ret = OB_SUCCESS; + } else { + LOG_WARN("Failed to generate plan", K(ret), K(result.get_exec_context().need_disconnect())); + } + } else if (OB_FALSE_IT(backup_recovery_guard.recovery())) { + } else if (OB_FAIL(need_add_plan(pc_ctx, + result, + use_plan_cache, + add_plan_to_pc))) { //加入多表分布式计划的判断，判断是否还需需要add plan + LOG_WARN("get need_add_plan failed", K(ret)); + } else if (!add_plan_to_pc && context.is_batch_params_execute()) { + ret = OB_BATCHED_MULTI_STMT_ROLLBACK; + LOG_WARN("add_plan_to_pc is false so batched multi_stmt rollback", K(ret)); + } else if (spm_ctx.is_retry_for_spm_) { + // handle spm baseline plan + ObPlanBaselineItem* baseline_item = static_cast(spm_ctx.baseline_guard_.get_cache_obj()); + if (OB_ISNULL(result.get_physical_plan()) || OB_ISNULL(baseline_item)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get null physical plan", K(ret), K(result.get_physical_plan()), K(baseline_item)); + } else if (result.get_physical_plan()->get_plan_hash_value() == baseline_item->get_plan_hash_value()) { + pc_ctx.need_evolution_ = true; + if (spm_ctx.cur_baseline_not_enable_ || !spm_ctx.capture_baseline_) { + spm_ctx.spm_stat_ = ObSpmCacheCtx::SpmStat::STAT_ACCEPT_BASELINE_PLAN; + } else { + spm_ctx.spm_stat_ = ObSpmCacheCtx::SpmStat::STAT_ADD_BASELINE_PLAN; + } + if (OB_FAIL(pc_add_plan(pc_ctx, result, outline_state, plan_cache, plan_added))) { + LOG_WARN("fail to add plan to plan cache", K(ret)); + } else if (!plan_added && ObSpmCacheCtx::SpmStat::STAT_ADD_BASELINE_PLAN == spm_ctx.spm_stat_) { + if (spm_ctx.evolution_task_in_two_plan_set_) { + // now baseline plan and evolving plan may be added to differnet plan set due to different + // constrain. In this situation, we can't compare two plan. So just keep try next baseline. + spm_ctx.evolution_task_in_two_plan_set_ = false; + need_get_baseline = true; + } else { + // add baseline plan failed, need evict unaccepted baseline in baseline cache. + (void) ObSpmController::deny_new_plan_as_baseline(spm_ctx); + } + } else if (plan_added && ObSpmCacheCtx::SpmStat::STAT_ADD_BASELINE_PLAN == spm_ctx.spm_stat_) { + spm_ctx.spm_force_disable_ = true; + spm_ctx.spm_stat_ = ObSpmCacheCtx::STAT_FIRST_EXECUTE_PLAN; + spm_ctx.is_retry_for_spm_ = false; + ret = OB_SQL_RETRY_SPM; + } + } else { + LOG_TRACE("spm need get baseline due to plan hash value not equal"); + need_get_baseline = true; + } + } else { + // handle spm evolution plan or not spm plan + if (add_plan_to_pc) { + bool baseline_enable = false; + bool baseline_exists = true; + bool baseline_find = false; + if (DEPENDENCY_OUTLINE == outline_state.outline_version_.object_type_ || is_match_udr) { + // outline has higher priority than baseline + } else if (OB_FAIL(ObSpmController::check_baseline_enable(pc_ctx, result.get_physical_plan(), baseline_enable))) { + LOG_WARN("failed to check need capture baseline", K(ret)); + } else if (!baseline_enable) { + // do nothing + } else if (OB_FAIL(ObSpmController::check_baseline_exists(pc_ctx, result.get_physical_plan(), baseline_exists))) { + LOG_WARN("failed to check baseline exists", K(ret)); + } else if (!baseline_exists) { + pc_ctx.need_evolution_ = true; + spm_ctx.spm_stat_ = ObSpmCacheCtx::SpmStat::STAT_ADD_EVOLUTION_PLAN; + } + if (OB_SUCC(ret)) { + if (is_match_udr && OB_FAIL(pc_add_udr_plan(item_guard, + pc_ctx, + result, + outline_state, + plan_added))) { + LOG_WARN("fail to add plan to plan cache", K(ret)); + } else if (!is_match_udr && OB_FAIL(pc_add_plan(pc_ctx, result, outline_state, plan_cache, plan_added))) { + LOG_WARN("fail to add plan to plan cache", K(ret)); + } else if (!plan_added) { + // plan not add to plan cache, do not check if need evolution. + if (spm_ctx.check_execute_status_) { + (void) ObSpmController::deny_new_plan_as_baseline(spm_ctx); + } + } else if (baseline_enable && !baseline_exists) { + need_get_baseline = true; + } + } + } + } + if (OB_SUCC(ret) && need_get_baseline) { + ObSpmController::get_next_baseline_outline(spm_ctx); + ret = OB_SQL_RETRY_SPM; + LOG_TRACE("spm get next baeline outline", K(spm_ctx.baseline_guard_.get_cache_obj()), K(ret)); + } +#endif //if the error code is ob_timeout, we add more error info msg for dml query. if (OB_TIMEOUT == ret && parse_result.result_tree_ != NULL && diff --git a/src/sql/ob_sql_context.h b/src/sql/ob_sql_context.h index 439a3378b..1f9729c08 100644 --- a/src/sql/ob_sql_context.h +++ b/src/sql/ob_sql_context.h @@ -25,6 +25,9 @@ #include "observer/omt/ob_tenant_config_mgr.h" #include "share/client_feedback/ob_feedback_partition_struct.h" #include "sql/dblink/ob_dblink_utils.h" +#ifdef OB_BUILD_SPM +#include "sql/spm/ob_spm_define.h" +#endif namespace oceanbase { @@ -422,6 +425,7 @@ private: common::hash::ObHashMap dblink_scn_; }; +#ifndef OB_BUILD_SPM struct ObBaselineKey { ObBaselineKey() @@ -457,6 +461,7 @@ struct ObSpmCacheCtx inline void reset() { bl_key_.reset(); } ObBaselineKey bl_key_; }; +#endif struct ObSqlCtx { @@ -662,6 +667,9 @@ public: reference_obj_tables_(), is_table_gen_col_with_udf_(false), query_hint_(), +#ifdef OB_BUILD_SPM + is_spm_evolution_(false), +#endif literal_stmt_type_(stmt::T_NONE), sql_stmt_(), sql_stmt_coll_type_(CS_TYPE_INVALID), @@ -703,6 +711,9 @@ public: reference_obj_tables_.reset(); is_table_gen_col_with_udf_ = false; query_hint_.reset(); +#ifdef OB_BUILD_SPM + is_spm_evolution_ = false; +#endif literal_stmt_type_ = stmt::T_NONE; sql_stmt_.reset(); sql_stmt_coll_type_ = CS_TYPE_INVALID; @@ -780,6 +791,9 @@ public: share::schema::ObReferenceObjTable reference_obj_tables_; bool is_table_gen_col_with_udf_; // for data consistent check ObQueryHint query_hint_; +#ifdef OB_BUILD_SPM + bool is_spm_evolution_; +#endif stmt::StmtType literal_stmt_type_; common::ObString sql_stmt_; common::ObCollationType sql_stmt_coll_type_; diff --git a/src/sql/ob_sql_init.h b/src/sql/ob_sql_init.h index eed976c4b..422dd4f01 100644 --- a/src/sql/ob_sql_init.h +++ b/src/sql/ob_sql_init.h @@ -31,6 +31,9 @@ #include "lib/alloc/ob_malloc_allocator.h" #include "share/object/ob_obj_cast.h" #include "engine/ob_serializable_function.h" +#ifdef OB_BUILD_ORACLE_PL +#include "pl/debug/ob_pl_debugger_manager.h" +#endif namespace oceanbase { @@ -85,6 +88,10 @@ inline int init_sql_executor_singletons() int ret = common::OB_SUCCESS; if (OB_FAIL(ObTaskRunnerNotifierService::build_instance())) { SQL_LOG(ERROR, "fail to build ObTaskRunnerNotifierService instance", K(ret)); +#ifdef OB_BUILD_ORACLE_PL + } else if (OB_FAIL(pl::ObPDBManager::build_instance())) { + SQL_LOG(ERROR, "fail to build ObPDBManager instance", K(ret)); +#endif } else { ObFuncSerialization::init(); } diff --git a/src/sql/ob_sql_utils.cpp b/src/sql/ob_sql_utils.cpp index e5f519df5..1baca4f7e 100644 --- a/src/sql/ob_sql_utils.cpp +++ b/src/sql/ob_sql_utils.cpp @@ -53,6 +53,9 @@ #include "lib/charset/ob_charset.h" #include "pl/ob_pl_user_type.h" #include "sql/engine/expr/ob_expr_lob_utils.h" +#ifdef OB_BUILD_SPM +#include "sql/spm/ob_spm_controller.h" +#endif #include "observer/omt/ob_tenant_srs_mgr.h" #include "sql/executor/ob_maintain_dependency_info_task.h" #include "sql/resolver/ddl/ob_create_view_resolver.h" @@ -2997,8 +3000,23 @@ int ObSQLUtils::get_ext_obj_data_type(const ObObjParam &obj, ObDataType &data_ty data_type = array_params->element_; } } else if (obj.is_pl_extend()) { +#ifndef OB_BUILD_ORACLE_PL ret = OB_NOT_SUPPORTED; LOG_WARN("not support", K(ret)); +#else + if (obj.get_meta().get_extend_type() == pl::PL_NESTED_TABLE_TYPE + || obj.get_meta().get_extend_type() == pl::PL_ASSOCIATIVE_ARRAY_TYPE + || obj.get_meta().get_extend_type() == pl::PL_VARRAY_TYPE) { + const pl::ObPLNestedTable *nested_table = + reinterpret_cast(obj.get_ext()); + if (OB_ISNULL(nested_table)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("nested table is null", K(ret)); + } else { + data_type = nested_table->get_element_type(); + } + } +#endif } else { ret = OB_INVALID_ARGUMENT; LOG_WARN("obj is not extend type", K(ret), K(obj)); @@ -4818,6 +4836,28 @@ int ObSQLUtils::check_location_access_priv(const ObString &location, ObSQLSessio return ret; } +#ifdef OB_BUILD_SPM +int ObSQLUtils::handle_plan_baseline(const ObAuditRecordData &audit_record, + ObPhysicalPlan *plan, + const int ret_code, + ObSqlCtx &sql_ctx) +{ + int ret = OB_SUCCESS; + if (OB_NOT_NULL(plan) && OB_UNLIKELY(OB_SUCCESS != ret_code && plan->get_evolution())) { + plan->update_plan_error_cnt(); + } + if (OB_LIKELY(!sql_ctx.spm_ctx_.check_execute_status_)) { + /*do nothing*/ + } else if (OB_SUCCESS == ret_code) { + if (OB_FAIL(ObSpmController::accept_new_plan_as_baseline(sql_ctx.spm_ctx_, audit_record))) { + LOG_WARN("failed to accept new plan as baseline", K(ret)); + } + } else if (OB_FAIL(ObSpmController::deny_new_plan_as_baseline(sql_ctx.spm_ctx_))) { + LOG_WARN("failed to deny new plan as baseline", K(ret)); + } + return ret; +} +#endif int ObSQLUtils::async_recompile_view(const share::schema::ObTableSchema &old_view_schema, ObSelectStmt *select_stmt, diff --git a/src/sql/ob_sql_utils.h b/src/sql/ob_sql_utils.h index 0e9d2b076..756dcccd1 100644 --- a/src/sql/ob_sql_utils.h +++ b/src/sql/ob_sql_utils.h @@ -602,6 +602,12 @@ public: static bool is_external_files_on_local_disk(const common::ObString &url); static int check_location_access_priv(const common::ObString &location, ObSQLSessionInfo *session); +#ifdef OB_BUILD_SPM + static int handle_plan_baseline(const ObAuditRecordData &audit_record, + ObPhysicalPlan *plan, + const int ret_code, + ObSqlCtx &sql_ctx); +#endif static int async_recompile_view(const share::schema::ObTableSchema &old_view_schema, ObSelectStmt *select_stmt, bool reset_column_infos, diff --git a/src/sql/optimizer/ob_log_plan.cpp b/src/sql/optimizer/ob_log_plan.cpp index 1c2385f4e..84df3f80b 100644 --- a/src/sql/optimizer/ob_log_plan.cpp +++ b/src/sql/optimizer/ob_log_plan.cpp @@ -74,6 +74,9 @@ #include "sql/ob_optimizer_trace_impl.h" #include "sql/optimizer/ob_explain_note.h" #include "share/ob_lob_access_utils.h" +#ifdef OB_BUILD_SPM +#include "sql/spm/ob_spm_define.h" +#endif using namespace oceanbase; using namespace sql; @@ -10857,6 +10860,9 @@ int ObLogPlan::check_enable_plan_expiration(bool &enable) const { int ret = OB_SUCCESS; ObSQLSessionInfo *session = NULL; +#ifdef OB_BUILD_SPM + bool use_spm = false; +#endif enable = false; if (OB_ISNULL(get_stmt()) || OB_ISNULL(session = optimizer_context_.get_session_info())) { @@ -10864,6 +10870,12 @@ int ObLogPlan::check_enable_plan_expiration(bool &enable) const LOG_WARN("stmt is null", K(ret)); } else if (!get_stmt()->is_select_stmt()) { // do nothing +#ifdef OB_BUILD_SPM + } else if (OB_FAIL(session->get_use_plan_baseline(use_spm))) { + LOG_WARN("failed to check is spm enabled", K(ret)); + } else if (use_spm) { + // do nothing +#endif } else if (optimizer_context_.get_phy_plan_type() != OB_PHY_PLAN_LOCAL && optimizer_context_.get_phy_plan_type() != OB_PHY_PLAN_DISTRIBUTED) { // do nothing @@ -11185,9 +11197,16 @@ int ObLogPlan::init_plan_info() } else if (OB_FAIL(ObOptimizerUtil::compute_const_exprs(get_stmt()->get_condition_exprs(), get_const_exprs()))) { LOG_WARN("failed to compute const equivalent exprs", K(ret)); +#ifndef OB_BUILD_SPM } else if (OB_FAIL(log_plan_hint_.init_log_plan_hint(*schema_guard, *get_stmt(), query_ctx->get_query_hint()))) { LOG_WARN("failed to init log plan hint", K(ret)); +#else + } else if (OB_FAIL(log_plan_hint_.init_log_plan_hint(*schema_guard, *get_stmt(), + query_ctx->get_query_hint(), + query_ctx->is_spm_evolution_))) { + LOG_WARN("failed to init log plan hint", K(ret)); +#endif } else if (OB_FAIL(init_onetime_subquery_info())) { LOG_WARN("failed to extract onetime_exprs", K(ret)); } diff --git a/src/sql/parser/CMakeLists.txt b/src/sql/parser/CMakeLists.txt index dc04b62e9..64c550e08 100644 --- a/src/sql/parser/CMakeLists.txt +++ b/src/sql/parser/CMakeLists.txt @@ -1,5 +1,7 @@ # charset objects used for proxy_parser + +if (OB_BUILD_OPENSOURCE) set(ob_sql_parser_charset_object_list ob_ctype_bin_os.cc ob_ctype_gb18030_os.cc @@ -12,10 +14,34 @@ set(ob_sql_parser_charset_object_list ob_ctype_utf8_os.cc ob_dtoa_os.cc ) +endif() list(TRANSFORM ob_sql_parser_charset_object_list PREPEND ${PROJECT_SOURCE_DIR}/deps/oblib/src/lib/charset/) +if(OB_BUILD_FULL_CHARSET) + set(ob_sql_parser_full_charset_object_list + ob_ctype_bin.cc + ob_ctype.cc + ob_ctype_gbk.cc + ob_ctype_latin1.cc + ob_ctype_mb.cc + ob_ctype_simple.cc + ob_ctype_utf8.cc + ob_dtoa.cc + ob_ctype_utf16.cc + ob_ctype_uca.cc + ob_ctype_gb18030.cc + uca900_ja_tbls.cc + uca900_zh_tbls.cc + uca900_zh2_tbls.cc + uca900_zh3_tbls.cc + ) +endif() + +list(TRANSFORM ob_sql_parser_full_charset_object_list + PREPEND ${PROJECT_SOURCE_DIR}/close_modules/charset/deps/oblib/src/lib/charset/) + # hash objects used for proxy parser set(ob_sql_parser_hash_object_list murmur_hash.h @@ -28,10 +54,42 @@ list(TRANSFORM ob_sql_parser_hash_object_list add_library(ob_sql_parser_base INTERFACE) target_include_directories(ob_sql_parser_base INTERFACE ${PROJECT_SOURCE_DIR}/deps/oblib/src/common/sql_mode/ - ${PROJECT_SOURCE_DIR}/src/objit/include/objit/common) + ${PROJECT_SOURCE_DIR}/src/objit/include/objit/common + ${CMAKE_SOURCE_DIR}/deps/oblib/src/lib/charset/ + ) # parser objects for proxy parser +if (OB_BUILD_ORACLE_PARSER) + set(ob_inner_sql_parser_object_list + sql_parser_oracle_utf8_mode_lex.c + sql_parser_oracle_utf8_mode_lex.h + sql_parser_oracle_utf8_mode_tab.c + sql_parser_oracle_utf8_mode_tab.h + sql_parser_oracle_gbk_mode_lex.c + sql_parser_oracle_gbk_mode_lex.h + sql_parser_oracle_gbk_mode_tab.c + sql_parser_oracle_gbk_mode_tab.h + sql_parser_oracle_latin1_mode_lex.c + sql_parser_oracle_latin1_mode_lex.h + sql_parser_oracle_latin1_mode_tab.c + sql_parser_oracle_latin1_mode_tab.h + ) + + set(ob_inner_sql_parser_object_list + ${ob_inner_sql_parser_object_list} + non_reserved_keywords_oracle_utf8_mode.c + non_reserved_keywords_oracle_gbk_mode.c + non_reserved_keywords_oracle_latin1_mode.c + ) +endif() + +# common set(ob_inner_sql_parser_object_list + ${ob_inner_sql_parser_object_list} + sql_parser_mysql_mode_lex.c + sql_parser_mysql_mode_lex.h + sql_parser_mysql_mode_tab.c + sql_parser_mysql_mode_tab.h non_reserved_keywords_mysql_mode.c ob_item_type_str.h ob_non_reserved_keywords.c @@ -51,10 +109,6 @@ set(ob_inner_sql_parser_object_list sql_parser_base.c sql_parser_base.h sql_parser_base.h - sql_parser_mysql_mode_lex.c - sql_parser_mysql_mode_lex.h - sql_parser_mysql_mode_tab.c - sql_parser_mysql_mode_tab.h parser_utility.cpp parser_utility.h type_name.c @@ -70,11 +124,19 @@ set(ob_extra_sql_parser_object_list ) # ob_sql_proxy_parser_objects is the static library for proxy, it does not link against observer -add_library(ob_sql_proxy_parser_objects OBJECT - ${ob_inner_sql_parser_object_list} - ${ob_sql_parser_charset_object_list} - ${ob_sql_parser_hash_object_list} -) +if (OB_BUILD_FULL_CHARSET) + add_library(ob_sql_proxy_parser_objects OBJECT + ${ob_inner_sql_parser_object_list} + ${ob_sql_parser_hash_object_list} + ${ob_sql_parser_full_charset_object_list} + ) +else() + add_library(ob_sql_proxy_parser_objects OBJECT + ${ob_inner_sql_parser_object_list} + ${ob_sql_parser_hash_object_list} + ${ob_sql_parser_charset_object_list} + ) +endif() # ob_sql_server_parser_object is the static library for observer add_library(ob_sql_server_parser_objects OBJECT diff --git a/src/sql/parser/gen_parser.sh b/src/sql/parser/gen_parser.sh index 3322c278b..051f0e02c 100755 --- a/src/sql/parser/gen_parser.sh +++ b/src/sql/parser/gen_parser.sh @@ -42,5 +42,170 @@ sed "/Setup the input buffer state to scan the given bytes/,/}/{/for ( i = 0; i sed "/Setup the input buffer state to scan the given bytes/,/}/{s/\tbuf\[i\] = yybytes\[i\]/memcpy(buf, yybytes, _yybytes_len)/g}" -i ../../../src/sql/parser/sql_parser_mysql_mode_lex.c +if [ -d "../../../close_modules/oracle_parser/sql/parser" ]; then + +ln -sf ../../../close_modules/oracle_parser/sql/parser/sql_parser_oracle_mode.y ../../../src/sql/parser/sql_parser_oracle_mode.y +ln -sf ../../../close_modules/oracle_parser/sql/parser/sql_parser_oracle_mode.l ../../../src/sql/parser/sql_parser_oracle_mode.l + +# generate oracle latin1 sql_parser(do not support multi_byte_space、multi_byte_comma、multi_byte_left_parenthesis、multi_byte_right_parenthesis) +##1.copy lex and yacc files +cat ../../../src/sql/parser/sql_parser_oracle_mode.y > ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y +cat ../../../src/sql/parser/sql_parser_oracle_mode.l > ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l +##2.replace name +sed "s/obsql_oracle_yy/obsql_oracle_latin1_yy/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y +sed "s/obsql_oracle_yy/obsql_oracle_latin1_yy/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l +sed "s/sql_parser_oracle_mode/sql_parser_oracle_latin1_mode/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y +sed "s/sql_parser_oracle_mode/sql_parser_oracle_latin1_mode/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l +sed "s/obsql_oracle_parser_fatal_error/obsql_oracle_latin1_parser_fatal_error/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y +sed "s/obsql_oracle_parser_fatal_error/obsql_oracle_latin1_parser_fatal_error/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l +sed "s/obsql_oracle_fast_parse/obsql_oracle_latin1_fast_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y +sed "s/obsql_oracle_multi_fast_parse/obsql_oracle_latin1_multi_fast_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y +sed "s/obsql_oracle_multi_values_parse/obsql_oracle_latin1_multi_values_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y +##3.do not need to replace multi_byte_space、multi_byte_comma、multi_byte_left_parenthesis、multi_byte_right_parenthesis code +sed "s/multi_byte_space \[\\\u3000\]/multi_byte_space \[\\\x20]/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l +sed "s/multi_byte_comma \[\\\uff0c\]/multi_byte_comma \[\\\x2c]/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l +sed "s/multi_byte_left_parenthesis \[\\\uff08\]/multi_byte_left_parenthesis \[\\\x28]/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l +sed "s/multi_byte_right_parenthesis \[\\\uff09\]/multi_byte_right_parenthesis \[\\\x29]/g" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l +echo "LATIN1_CHAR [\x80-\xFF]" > ../../../src/sql/parser/latin1.txt +sed '/following character status will be rewrite by gen_parse.sh according to connection character/d' -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l +sed '/multi_byte_connect_char \/\*According to connection character to set by gen_parse.sh\*\//r ../../../src/sql/parser/latin1.txt' -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l +sed '/multi_byte_connect_char \/\*According to connection character to set by gen_parse.sh\*\//d' -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l +sed 's/multi_byte_connect_char/LATIN1_CHAR/g' -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l +sed -i '/{multi_byte_space}/,+5d' sql_parser_oracle_latin1_mode.l +sed -i '/{multi_byte_comma}/,+35d' sql_parser_oracle_latin1_mode.l +sed -i '/{multi_byte_comma}/,+23d' sql_parser_oracle_latin1_mode.l +sed -i '/{multi_byte_space}/,+4d' sql_parser_oracle_latin1_mode.l +##4.generate oracle latin1 parser files +bison_parser ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y ../../../src/sql/parser/sql_parser_oracle_latin1_mode_tab.c +flex -o ../../../src/sql/parser/sql_parser_oracle_latin1_mode_lex.c ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l ../../../src/sql/parser/sql_parser_oracle_latin1_mode_tab.h +##5.replace other info +sed "/Setup the input buffer state to scan the given bytes/,/}/{/int i/d}" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode_lex.c +sed "/Setup the input buffer state to scan the given bytes/,/}/{/for ( i = 0; i < _yybytes_len; ++i )/d}" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode_lex.c +sed "/Setup the input buffer state to scan the given bytes/,/}/{s/\tbuf\[i\] = yybytes\[i\]/memcpy(buf, yybytes, _yybytes_len)/g}" -i ../../../src/sql/parser/sql_parser_oracle_latin1_mode_lex.c +cat ../../../src/sql/parser/non_reserved_keywords_oracle_mode.c > ../../../src/sql/parser/non_reserved_keywords_oracle_latin1_mode.c +sed '/#include "ob_non_reserved_keywords.h"/a\#include "sql/parser/sql_parser_oracle_latin1_mode_tab.h\"' -i ../../../src/sql/parser/non_reserved_keywords_oracle_latin1_mode.c +sed "s/non_reserved_keywords_oracle_mode.c is for …/non_reserved_keywords_oracle_latin1_mode.c is auto generated by gen_parser.sh/g" -i ../../../src/sql/parser/non_reserved_keywords_oracle_latin1_mode.c +##6.clean useless files +rm -f ../../../src/sql/parser/latin1.txt +rm -f ../../../src/sql/parser/sql_parser_oracle_latin1_mode.l +rm -f ../../../src/sql/parser/sql_parser_oracle_latin1_mode.y + +# generate oracle utf8 sql_parser(support multi_byte_space、multi_byte_comma、multi_byte_left_parenthesis、multi_byte_right_parenthesis) +##1.copy lex and yacc files +cat ../../../src/sql/parser/sql_parser_oracle_mode.y > ../../../src/sql/parser/sql_parser_oracle_utf8_mode.y +cat ../../../src/sql/parser/sql_parser_oracle_mode.l > ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l +##2.replace name +sed "s/obsql_oracle_yy/obsql_oracle_utf8_yy/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.y +sed "s/obsql_oracle_yy/obsql_oracle_utf8_yy/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l +sed "s/sql_parser_oracle_mode/sql_parser_oracle_utf8_mode/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.y +sed "s/sql_parser_oracle_mode/sql_parser_oracle_utf8_mode/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l +sed "s/obsql_oracle_parser_fatal_error/obsql_oracle_utf8_parser_fatal_error/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.y +sed "s/obsql_oracle_parser_fatal_error/obsql_oracle_utf8_parser_fatal_error/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l +sed "s/obsql_oracle_fast_parse/obsql_oracle_utf8_fast_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.y +sed "s/obsql_oracle_multi_fast_parse/obsql_oracle_utf8_multi_fast_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.y +sed "s/obsql_oracle_multi_values_parse/obsql_oracle_utf8_multi_values_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.y +##3.add multi_byte_space、multi_byte_comma、multi_byte_left_parenthesis、multi_byte_right_parenthesis code. +sed "s/multi_byte_space \[\\\u3000\]/multi_byte_space ([\\\xe3\][\\\x80\][\\\x80])/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l +sed "s/multi_byte_comma \[\\\uff0c\]/multi_byte_comma ([\\\xef\][\\\xbc\][\\\x8c])/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l +sed "s/multi_byte_left_parenthesis \[\\\uff08\]/multi_byte_left_parenthesis ([\\\xef\][\\\xbc\][\\\x88])/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l +sed "s/multi_byte_right_parenthesis \[\\\uff09\]/multi_byte_right_parenthesis ([\\\xef\][\\\xbc\][\\\x89])/g" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l +echo "U [\x80-\xbf] +U_1_1 [\x80] +U_1_2 [\x81-\xbf] +U_1_3 [\x80-\xbb] +U_1_4 [\xbc] +U_1_5 [\xbd-\xbf] +U_1_6 [\x80-\x87] +U_1_7 [\x8a-\x8b] +U_1_8 [\x8d-\xbf] +U_2 [\xc2-\xdf] +U_3 [\xe0-\xe2] +U_3_1 [\xe3] +U_3_2 [\xe4-\xee] +U_3_3 [\xef] +U_4 [\xf0-\xf4] +u_except_space ({U_3_1}{U_1_2}{U}|{U_3_1}{U_1_1}{U_1_2}) +u_except_comma_parenthesis ({U_3_3}{U_1_3}{U}|{U_3_3}{U_1_4}{U_1_6}|{U_3_3}{U_1_4}{U_1_7}|{U_3_3}{U_1_4}{U_1_8}|{U_3_3}{U_1_5}{U}) +UTF8_CHAR ({U_2}{U}|{U_3}{U}{U}|{u_except_space}|{U_3_2}{U}{U}|{u_except_comma_parenthesis}|{U_4}{U}{U}{U})" > ../../../src/sql/parser/utf8.txt +sed '/following character status will be rewrite by gen_parse.sh according to connection character/d' -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l +sed '/multi_byte_connect_char \/\*According to connection character to set by gen_parse.sh\*\//r ../../../src/sql/parser/utf8.txt' -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l +sed '/multi_byte_connect_char \/\*According to connection character to set by gen_parse.sh\*\//d' -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l +sed 's/space \[ \\t\\n\\r\\f\]/space (\[ \\t\\n\\r\\f\]|{multi_byte_space})/g' -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l +sed 's/multi_byte_connect_char/UTF8_CHAR/g' -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l +##4.generate oracle utf8 parser files +bison_parser ../../../src/sql/parser/sql_parser_oracle_utf8_mode.y ../../../src/sql/parser/sql_parser_oracle_utf8_mode_tab.c +flex -o ../../../src/sql/parser/sql_parser_oracle_utf8_mode_lex.c ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l ../../../src/sql/parser/sql_parser_oracle_utf8_mode_tab.h +##5.replace other info +sed "/Setup the input buffer state to scan the given bytes/,/}/{/int i/d}" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode_lex.c +sed "/Setup the input buffer state to scan the given bytes/,/}/{/for ( i = 0; i < _yybytes_len; ++i )/d}" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode_lex.c +sed "/Setup the input buffer state to scan the given bytes/,/}/{s/\tbuf\[i\] = yybytes\[i\]/memcpy(buf, yybytes, _yybytes_len)/g}" -i ../../../src/sql/parser/sql_parser_oracle_utf8_mode_lex.c +cat ../../../src/sql/parser/non_reserved_keywords_oracle_mode.c > ../../../src/sql/parser/non_reserved_keywords_oracle_utf8_mode.c +sed '/#include "ob_non_reserved_keywords.h"/a\#include "sql/parser/sql_parser_oracle_utf8_mode_tab.h\"' -i ../../../src/sql/parser/non_reserved_keywords_oracle_utf8_mode.c +sed "s/non_reserved_keywords_oracle_mode.c is for …/non_reserved_keywords_oracle_utf8_mode.c is auto generated by gen_parser.sh/g" -i ../../../src/sql/parser/non_reserved_keywords_oracle_utf8_mode.c +##6.clean useless files +rm -f ../../../src/sql/parser/utf8.txt +rm -f ../../../src/sql/parser/sql_parser_oracle_utf8_mode.l +rm -f ../../../src/sql/parser/sql_parser_oracle_utf8_mode.y + +# generate oracle gbk sql_parser(support multi_byte_space、multi_byte_comma、multi_byte_left_parenthesis、multi_byte_right_parenthesis) +##1.copy lex and yacc files +cat ../../../src/sql/parser/sql_parser_oracle_mode.y > ../../../src/sql/parser/sql_parser_oracle_gbk_mode.y +cat ../../../src/sql/parser/sql_parser_oracle_mode.l > ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l +##2.replace name +sed "s/obsql_oracle_yy/obsql_oracle_gbk_yy/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.y +sed "s/obsql_oracle_yy/obsql_oracle_gbk_yy/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l +sed "s/sql_parser_oracle_mode/sql_parser_oracle_gbk_mode/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.y +sed "s/sql_parser_oracle_mode/sql_parser_oracle_gbk_mode/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l +sed "s/obsql_oracle_parser_fatal_error/obsql_oracle_gbk_parser_fatal_error/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.y +sed "s/obsql_oracle_parser_fatal_error/obsql_oracle_gbk_parser_fatal_error/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l +sed "s/obsql_oracle_fast_parse/obsql_oracle_gbk_fast_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.y +sed "s/obsql_oracle_multi_fast_parse/obsql_oracle_gbk_multi_fast_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.y +sed "s/obsql_oracle_multi_values_parse/obsql_oracle_gbk_multi_values_parse/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.y +##3.add multi_byte_space、multi_byte_comma、multi_byte_left_parenthesis、multi_byte_right_parenthesis code. +sed "s/multi_byte_space \[\\\u3000\]/multi_byte_space ([\\\xa1][\\\xa1])/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l +sed "s/multi_byte_comma \[\\\uff0c\]/multi_byte_comma ([\\\xa3][\\\xac])/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l +sed "s/multi_byte_left_parenthesis \[\\\uff08\]/multi_byte_left_parenthesis ([\\\xa3][\\\xa8])/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l +sed "s/multi_byte_right_parenthesis \[\\\uff09\]/multi_byte_right_parenthesis ([\\\xa3][\\\xa9])/g" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l +echo "GB_1 [\x81-\xfe] +GB_1_1 [\x81-\xa0] +GB_1_2 [\xa1] +GB_1_3 [\xa2] +GB_1_4 [\xa3] +GB_1_5 [\xa4-\xfe] +GB_2 [\x40-\xfe] +GB_2_1 [\x40-\xa0] +GB_2_2 [\xa2-\xfe] +GB_2_3 [\x40-\xa7] +GB_2_4 [\xaa-\xab] +GB_2_5 [\xad-\xfe] +GB_3 [\x30-\x39] +g_except_space ({GB_1_2}{GB_2_1}|{GB_1_2}{GB_2_2}) +g_except_comma_parenthesis ({GB_1_4}{GB_2_3}|{GB_1_4}{GB_2_4}|{GB_1_4}{GB_2_5}) +GB_CHAR ({GB_1_1}{GB_2}|{g_except_space}|{GB_1_3}{GB_2}|{g_except_comma_parenthesis}|{GB_1_5}{GB_2}|{GB_1}{GB_3}{GB_1}{GB_3})" > ../../../src/sql/parser/gbk.txt +sed '/following character status will be rewrite by gen_parse.sh according to connection character/d' -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l +sed '/multi_byte_connect_char \/\*According to connection character to set by gen_parse.sh\*\//r ../../../src/sql/parser/gbk.txt' -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l +sed '/multi_byte_connect_char \/\*According to connection character to set by gen_parse.sh\*\//d' -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l +sed 's/space \[ \\t\\n\\r\\f\]/space (\[ \\t\\n\\r\\f\]|{multi_byte_space})/g' -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l +sed 's/multi_byte_connect_char/GB_CHAR/g' -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l +##4.generate oracle gbk parser files +bison_parser ../../../src/sql/parser/sql_parser_oracle_gbk_mode.y ../../../src/sql/parser/sql_parser_oracle_gbk_mode_tab.c +flex -o ../../../src/sql/parser/sql_parser_oracle_gbk_mode_lex.c ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l ../../../src/sql/parser/sql_parser_oracle_gbk_mode_tab.h +##5.replace other info +sed "/Setup the input buffer state to scan the given bytes/,/}/{/int i/d}" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode_lex.c +sed "/Setup the input buffer state to scan the given bytes/,/}/{/for ( i = 0; i < _yybytes_len; ++i )/d}" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode_lex.c +sed "/Setup the input buffer state to scan the given bytes/,/}/{s/\tbuf\[i\] = yybytes\[i\]/memcpy(buf, yybytes, _yybytes_len)/g}" -i ../../../src/sql/parser/sql_parser_oracle_gbk_mode_lex.c +cat ../../../src/sql/parser/non_reserved_keywords_oracle_mode.c > ../../../src/sql/parser/non_reserved_keywords_oracle_gbk_mode.c +sed '/#include "ob_non_reserved_keywords.h"/a\#include "sql/parser/sql_parser_oracle_gbk_mode_tab.h\"' -i ../../../src/sql/parser/non_reserved_keywords_oracle_gbk_mode.c +sed "s/non_reserved_keywords_oracle_mode.c is for …/non_reserved_keywords_oracle_gbk_mode.c is auto generated by gen_parser.sh/g" -i ../../../src/sql/parser/non_reserved_keywords_oracle_gbk_mode.c +##6.clean useless files +rm -f ../../../src/sql/parser/gbk.txt +rm -f ../../../src/sql/parser/sql_parser_oracle_gbk_mode.l +rm -f ../../../src/sql/parser/sql_parser_oracle_gbk_mode.y + +rm -rf ../../../src/sql/parser/sql_parser_oracle_mode.y +rm -rf ../../../src/sql/parser/sql_parser_oracle_mode.l + +fi + # generate type name ./gen_type_name.sh ../../../src/objit/include/objit/common/ob_item_type.h > type_name.c diff --git a/src/sql/parser/non_reserved_keywords_oracle_mode.c b/src/sql/parser/non_reserved_keywords_oracle_mode.c deleted file mode 100644 index 0b48e3cac..000000000 --- a/src/sql/parser/non_reserved_keywords_oracle_mode.c +++ /dev/null @@ -1,1436 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#include -#include "lib/alloc/alloc_assist.h" -#include "ob_non_reserved_keywords.h" - -#define WINDOW_FUNCTION_NUM 41 - -static t_node *oracle_keywords_root = NULL; - -/* List of non-reserved keywords */ -/*一开始会对这些word建立一颗trie树，对于每次的查找来言，树是固定的 - *若新添加的keyword含有除大小写字母、'_'和数字以外的其它字符，请联系@叶提修改这颗树。 - *实现不需要保证字典序，但是原则上还是保证字典序，方便维护和查找*/ -static const NonReservedKeyword Oracle_non_reserved_keywords[] = -{ -/* - * Oracle Non-Reserved/Reserved Keywords - * 添加新的非保留关键字请从OB Non-Reserved Keywords处开始添加，前面是兼容oracle的保留关键字及非保留关键字！！！ - * */ - - {"absent", ABSENT}, - {"access", ACCESS}, - {"accessed", ACCESSED}, - {"add", ADD}, - {"admin", ADMIN}, - {"after", AFTER}, - {"all", ALL}, - {"allocate", ALLOCATE}, - {"alter", ALTER}, - {"analyze", ANALYZE}, - {"and", AND}, - {"any", ANY}, - {"archive", ARCHIVE}, - {"archivelog", ARCHIVELOG}, - {"as", AS}, - {"asc", ASC}, - {"authorization", AUTHORIZATION}, - {"audit", AUDIT}, - {"avg", AVG}, - {"backup", BACKUP}, - {"become", BECOME}, - {"before", BEFORE}, - {"begin", BEGI}, - {"between", BETWEEN}, - {"blob", BLOB}, - {"block", BLOCK}, - {"binary_double", BINARY_DOUBLE}, - {"binary_float", BINARY_FLOAT}, - {"body", BODY}, - {"by", BY}, - {"cache", CACHE}, - {"cancel", CANCEL}, - {"cascade", CASCADE}, - {"change", CHANGE}, - {"char", CHAR}, - {"character", CHARACTER}, - {"check", CHECK}, - {"checkpoint", CHECKPOINT}, - {"clob", CLOB}, - {"close", CLOSE}, - {"cluster", CLUSTER}, - {"cipher", CIPHER}, - {"cobol", COBOL}, - {"column", COLUMN}, - {"commit", COMMIT}, - {"comment", COMMENT}, - {"compile", COMPILE}, - {"compress", COMPRESS}, - {"constraint", CONSTRAINT}, - {"constraints", CONSTRAINTS}, - {"contents", CONTENTS}, - {"continue", CONTINUE}, - {"connect", CONNECT}, - {"controlfile", CONTROLFILE}, - {"count", COUNT}, - {"create", CREATE}, - {"current", CURRENT}, - {"cursor", CURSOR}, - {"cycle", CYCLE}, - {"database", DATABASE}, - {"datafile", DATAFILE}, - {"date", DATE}, - {"dba", DBA}, - {"dba_recyclebin", DBA_RECYCLEBIN}, - {"dec", DEC}, - {"decimal", DECIMAL}, - {"declare", DECLARE}, - {"desc", DESC}, - {"default", DEFAULT}, - {"delete", DELETE}, - {"deleting", DELETING}, - {"disable", DISABLE}, - {"distinct", DISTINCT}, - {"dismount", DISMOUNT}, - {"dml", DML}, - {"ddl", DDL}, - {"dop", DOP}, - {"double", DOUBLE}, - {"drop", DROP}, - {"dump", DUMP}, - {"each", EACH}, - {"else", ELSE}, - {"enable", ENABLE}, - {"end", END}, - {"escape", ESCAPE}, - {"events", EVENTS}, - {"except", EXCEPT}, - {"exceptions", EXCEPTIONS}, - {"exclude", EXCLUDE}, - {"exclusive", EXCLUSIVE}, - {"exists", EXISTS}, - {"exec", EXEC}, - {"execute", EXECUTE}, - {"exempt", EXEMPT}, - {"explain", EXPLAIN}, - {"extent", EXTENT}, - {"externally", EXTERNALLY}, - {"fetch", FETCH}, - {"flush", FLUSH}, - {"float", FLOAT}, - {"file", FILE_KEY}, - {"for", FOR}, - {"force", FORCE}, - {"foreign", FOREIGN}, - {"fortran", FORTRAN}, - {"found", FOUND}, - {"freelist", FREELIST}, - {"freelists", FREELISTS}, - {"from", FROM}, - {"function", FUNCTION}, - {"globally", GLOBALLY}, - {"go", GO}, - {"goto", GOTO}, - {"grant", GRANT}, - {"group", GROUP}, - {"groups", GROUPS}, - {"having", HAVING}, - {"host", HOST}, - {"identified", IDENTIFIED}, - {"immediate", IMMEDIATE}, - {"increment", INCREMENT}, - {"include", INCLUDE}, - {"index", INDEX}, - {"initial", INITIAL_}, - {"initialized", INITIALIZED}, - {"insert", INSERT}, - {"inserting", INSERTING}, - {"int", INT}, - {"integer", INTEGER}, - {"intersect", INTERSECT}, - {"into", INTO}, - {"including", INCLUDING}, - {"indicator", INDICATOR}, - {"initrans", INITRANS}, - {"instance", INSTANCE}, - {"is", IS}, - {"issuer", ISSUER}, - {"key", KEY}, - {"language", LANGUAGE}, - {"layer", LAYER}, - {"level", LEVEL}, - {"link", LINK}, - {"like", LIKE}, - {"lists", LISTS}, - {"lock", LOCK}, - {"logfile", LOGFILE}, - {"long", LONG}, - {"manage", MANAGE}, - {"manual", MANUAL}, - {"max", MAX}, - {"maxdatafiles", MAXDATAFILES}, - {"maxextents", MAXEXTENTS}, - {"maxinstances", MAXINSTANCES}, - {"maxlogfiles", MAXLOGFILES}, - {"maxloghistory", MAXLOGHISTORY}, - {"maxlogmembers", MAXLOGMEMBERS}, - {"maxtrans", MAXTRANS}, - {"maxvalue", MAXVALUE}, - {"min", MIN}, - {"minextents", MINEXTENTS}, - {"minvalue", MINVALUE}, - {"minus", MINUS}, - {"mode", MODE}, - {"modify", MODIFY}, - {"module", MODULE}, - {"mount", MOUNT}, - {"new", NEW}, - {"next", NEXT}, - {"noaudit", NOAUDIT}, - {"noarchivelog", NOARCHIVELOG}, - {"nocache", NOCACHE}, - {"nocompress", NOCOMPRESS}, - {"nocycle", NOCYCLE}, - {"nomaxvalue", NOMAXVALUE}, - {"nominvalue", NOMINVALUE}, - {"none", NONE}, - {"noorder", NOORDER}, - {"noresetlogs", NORESETLOGS}, - {"nosort", NOSORT}, - {"not", NOT}, - {"notfound", NOTFOUND}, - {"number", NUMBER}, - {"numeric", NUMERIC}, - {"of", OF}, - {"off", OFF}, - {"offline", OFFLINE}, - {"old", OLD}, - {"on", ON}, - {"online", ONLINE}, - {"only", ONLY}, - {"open", OPEN}, - {"optimal", OPTIMAL}, - {"option", OPTION}, - {"or", OR}, - {"order", ORDER}, - {"own", OWN}, - {"package", PACKAGE_KEY}, - {"parallel", PARALLEL}, - {"passing", PASSING}, - {"password_lock_time", PASSWORD_LOCK_TIME}, - {"password_verify_function", PASSWORD_VERIFY_FUNCTION}, - {"pctincrease", PCTINCREASE}, - {"pctfree", PCTFREE}, - {"pctused", PCTUSED}, - {"plan", PLAN}, - {"pli", PLI}, - {"pivot", PIVOT}, - {"precision", PRECISION}, - {"primary", PRIMARY}, - {"private", PRIVATE}, - {"prior", PRIOR}, - {"privileges", PRIVILEGES}, - {"procedure", PROCEDURE}, - {"profile", PROFILE}, - {"public", PUBLIC}, - {"quota", QUOTA}, - {"raw", RAW}, - {"read", READ}, - {"real", REAL}, - {"recover", RECOVER}, - {"references", REFERENCES}, - {"referencing", REFERENCING}, - {"rename", RENAME}, - {"reject", REJECT}, - {"resetlogs", RESETLOGS}, - {"resource", RESOURCE}, - {"restricted", RESTRICTED}, - {"revoke", REVOKE}, - {"reuse", REUSE}, - {"remote_oss", REMOTE_OSS}, - {"role", ROLE}, - {"roles", ROLES}, - {"rollback", ROLLBACK}, - {"row", ROW}, - {"rowid", ROWID}, - {"rowlabel", ROWLABEL}, - {"rownum", ROWNUM}, - {"rows", ROWS}, - {"savepoint", SAVEPOINT}, - {"schema", SCHEMA}, - {"scn", SCN}, - {"section", SECTION}, - {"select", SELECT}, - {"segment", SEGMENT}, - {"sequence", SEQUENCE}, - {"session", SESSION}, - {"set", SET}, - {"sets", SETS}, - {"share", SHARE}, - {"shared", SHARED}, - {"snapshot", SNAPSHOT}, - {"size", SIZE}, - {"smallint", SMALLINT}, - {"some", SOME}, - {"sort", SORT}, - {"sql", SQL}, - {"sqlcode", SQLCODE}, - {"sqlerror", SQLERROR}, - {"sqlstate", SQLSTATE}, - {"statement_id", STATEMENT_ID}, - {"statistics", STATISTICS}, - {"start", START}, - {"stop", STOP}, - {"storage", STORAGE}, - {"successful", SUCCESSFUL}, - {"sum", SUM}, - {"switch", SWITCH}, - {"synonym", SYNONYM}, - {"sysdate", SYSDATE}, - {"system", SYSTEM}, - {"systimestamp", SYSTIMESTAMP}, - {"sets", SETS}, - {"table", TABLE}, - {"tables", TABLES}, - {"tablespace", TABLESPACE}, - {"temporary", TEMPORARY}, - {"then", THEN}, - {"thread", THREAD}, - {"time", TIME}, - {"to", TO}, - {"tracing", TRACING}, - {"transaction", TRANSACTION}, - {"trigger", TRIGGER}, - {"triggers", TRIGGERS}, - {"truncate", TRUNCATE}, - {"under", UNDER}, - {"unlimited", UNLIMITED}, - {"union", UNION}, - {"unique", UNIQUE}, - {"unpivot", UNPIVOT}, - {"until", UNTIL}, - {"uid", UID}, - {"update", UPDATE}, - {"updating", UPDATING}, - {"use", USE}, - {"user", USER}, - {"using", USING}, - {"validate", VALIDATE}, - {"values", VALUES}, - {"varchar", VARCHAR}, - {"varchar2", VARCHAR2}, - {"view", VIEW}, - {"when", WHEN}, - {"whenever", WHENEVER}, - {"where", WHERE}, - {"work", WORK}, - {"write", WRITE}, - {"with", WITH}, - {"within", WITHIN}, -/* - * OB Non-Reserved Keywords - * 新添加的非保留关键字，请按照顺序添加到下面的数组中 - */ - {"account", ACCOUNT}, - {"accessible", ACCESSIBLE}, - {"action", ACTION}, - {"active", ACTIVE}, - {"activate", ACTIVATE}, - {"adddate", ADDDATE}, - {"administer", ADMINISTER}, - {"against", AGAINST}, - {"aggregate", AGGREGATE}, - {"algorithm", ALGORITHM}, - {"allow", ALLOW}, - {"always", ALWAYS}, - {"analyse", ANALYSE}, - {"any", ANY}, - {"array", ARRAY}, - {"approx_count_distinct", APPROX_COUNT_DISTINCT}, - {"approx_count_distinct_synopsis", APPROX_COUNT_DISTINCT_SYNOPSIS}, - {"approx_count_distinct_synopsis_merge", APPROX_COUNT_DISTINCT_SYNOPSIS_MERGE}, - {"ascii", ASCII}, - {"asensitive", ASENSITIVE}, - {"asis", ASIS}, - {"at", AT}, - {"authors", AUTHORS}, - {"auto", AUTO}, - {"autoextend_size", AUTOEXTEND_SIZE}, - {"avg_row_length", AVG_ROW_LENGTH}, - {"backup_copies", BACKUP_COPIES}, - {"balance", BALANCE}, - {"base", BASE}, - {"baseline", BASELINE}, - {"baseline_id", BASELINE_ID}, - {"basic", BASIC}, - {"binlog", BINLOG}, - {"binary", BINARY}, - {"binary_double", BINARY_DOUBLE}, - {"binary_double_infinity", BINARY_DOUBLE_INFINITY}, - {"binary_double_nan", BINARY_DOUBLE_NAN}, - {"binary_float", BINARY_FLOAT}, - {"binary_float_infinity", BINARY_FLOAT_INFINITY}, - {"binary_float_nan", BINARY_FLOAT_NAN}, - {"bit", BIT}, - {"blob", BLOB}, - {"block_size", BLOCK_SIZE}, - {"bool", BOOL}, - {"boolean", BOOLEAN}, - {"bootstrap", BOOTSTRAP}, - {"both", BOTH}, - {"breadth", BREADTH}, - {"btree", BTREE}, - {"bulk", BULK}, - {"bulk_exceptions", BULK_EXCEPTIONS}, - {"bulk_rowcount", BULK_ROWCOUNT}, - {"badfile", BADFILE}, - {"byte", BYTE}, - {"calc_partition_id", CALC_PARTITION_ID}, - {"call", CALL}, - {"cascaded", CASCADED}, - {"case", CASE}, - {"cast", CAST}, - {"catalog_name", CATALOG_NAME}, - {"chain", CHAIN}, - {"changed", CHANGED}, - {"charset", CHARSET}, - {"char_cs", CHAR_CS}, - {"checksum", CHECKSUM}, - {"cipher", CIPHER}, - {"class_origin", CLASS_ORIGIN}, - {"clean", CLEAN}, - {"clear", CLEAR}, - {"client", CLIENT}, - {"clob", CLOB}, - {"cluster", CLUSTER}, - {"coalesce", COALESCE}, - {"code", CODE}, - {"collate", COLLATE}, - {"collation", COLLATION}, - {"collect", COLLECT}, - {"column_format", COLUMN_FORMAT}, - {"column_name", COLUMN_NAME}, - {"column_outer_join_symbol", COLUMN_OUTER_JOIN_SYMBOL}, - {"columns", COLUMNS}, - {"committed", COMMITTED}, - {"compact", COMPACT}, - {"completion", COMPLETION}, - {"compressed", COMPRESSED}, - {"compression", COMPRESSION}, - {"compute", COMPUTE}, - {"concurrent", CONCURRENT}, - {"conditional", CONDITIONAL}, - {"connect", CONNECT}, - {"connection", CONNECTION}, - {"consistent", CONSISTENT}, - {"constraint_catalog", CONSTRAINT_CATALOG}, - {"constraint_name", CONSTRAINT_NAME}, - {"constraint_schema", CONSTRAINT_SCHEMA}, - {"contains", CONTAINS}, - {"context", CONTEXT}, - {"content", CONTENT}, - {"contributors", CONTRIBUTORS}, - {"copy", COPY}, - {"corr", CORR}, - {"covar_pop", COVAR_POP}, - {"covar_samp", COVAR_SAMP}, - {"cpu", CPU}, - {"create_timestamp", CREATE_TIMESTAMP}, - {"cross", CROSS}, - {"cube", CUBE}, - {"cume_dist", CUME_DIST}, - {"current_schema", CURRENT_SCHEMA}, - {"current_date", CURRENT_DATE}, - {"current_timestamp", CURRENT_TIMESTAMP}, - {"current_user", CURRENT_USER}, - {"data", DATA}, - {"data_table_id", DATA_TABLE_ID}, - {"databases", DATABASES}, - {"database_id", DATABASE_ID}, - {"date_add", DATE_ADD}, - {"date_sub", DATE_SUB}, - {"datetime", DATETIME}, - {"day", DAY}, - {"day_hour", DAY_HOUR}, - {"day_microsecond", DAY_MICROSECOND}, - {"day_minute", DAY_MINUTE}, - {"day_second", DAY_SECOND}, - {"deallocate", DEALLOCATE}, - {"default_auth", DEFAULT_AUTH}, - {"defaults", DEFAULTS}, - {"definer", DEFINER}, - {"delay", DELAY}, - {"delayed", DELAYED}, - {"delay_key_write", DELAY_KEY_WRITE}, - {"depth", DEPTH}, - {"des_key_file", DES_KEY_FILE}, - {"describe", DESCRIBE}, - {"destination", DESTINATION}, - {"deterministic", DETERMINISTIC}, - {"dense_rank", DENSE_RANK}, - {"description", DESCRIPTION}, - {"diagnostics", DIAGNOSTICS}, - {"dictionary", DICTIONARY}, - {"directory", DIRECTORY}, - {"disallow", DISALLOW}, - {"discard", DISCARD}, - {"disk", DISK}, - {"distinctrow", DISTINCTROW}, - {"div", DIV}, - {"do", DO}, - {"document", DOCUMENT}, - {"dot", DOT}, - {"dual", DUAL}, - {"dumpfile", DUMPFILE}, - {"duplicate", DUPLICATE}, - {"dynamic", DYNAMIC}, - {"default_tablegroup", DEFAULT_TABLEGROUP}, - {"debug", DEBUG}, - {"e", E}, - {"effective", EFFECTIVE}, - {"elseif", ELSEIF}, - {"enable_extended_rowid", ENABLE_EXTENDED_ROWID}, - {"enclosed", ENCLOSED}, - {"encoding", ENCODING}, - {"encrypted", ENCRYPTED}, - {"encryption", ENCRYPTION}, - {"ends", ENDS}, - {"engine", ENGINE_}, - {"engines", ENGINES}, - {"entityescaping", ENTITYESCAPING}, - {"enum", ENUM}, - {"error", ERROR_P}, - {"error_index", ERROR_INDEX}, - {"error_code", ERROR_CODE}, - {"errors", ERRORS}, - {"estimate", ESTIMATE}, - {"escaped", ESCAPED}, - {"evalname", EVALNAME}, - {"event", EVENT}, - {"every", EVERY}, - {"exchange", EXCHANGE}, - {"exclude", EXCLUDE}, - {"exit", EXIT}, - {"expansion", EXPANSION}, - {"expire", EXPIRE}, - {"expire_info", EXPIRE_INFO}, - {"export", EXPORT}, - {"external", EXTERNAL}, - {"extended", EXTENDED}, - {"extended_noaddr", EXTENDED_NOADDR}, - {"extent_size", EXTENT_SIZE}, - {"extra", EXTRA}, - {"extract", EXTRACT}, - {"extractvalue", EXTRACTVALUE}, - {"fast", FAST}, - {"failed_login_attempts", FAILED_LOGIN_ATTEMPTS}, - {"faults", FAULTS}, - {"fields", FIELDS}, - {"field_delimiter", FIELD_DELIMITER}, - {"field_optionally_enclosed_by", FIELD_OPTIONALLY_ENCLOSED_BY}, - {"file_id", FILE_ID}, - {"final_count", FINAL_COUNT}, - {"first", FIRST}, - {"first_value", FIRST_VALUE}, - {"fixed", FIXED}, - {"flashback", FLASHBACK}, - {"float4", FLOAT4}, - {"float8", FLOAT8}, - {"follower", FOLLOWER}, - {"format", FORMAT}, - {"freeze", FREEZE}, - {"frequency", FREQUENCY}, - {"frozen", FROZEN}, - {"following", FOLLOWING}, - {"full", FULL}, - {"g", G}, - {"general", GENERAL}, - {"generated", GENERATED}, - {"geometry", GEOMETRY}, - {"geometrycollection", GEOMETRYCOLLECTION}, - {"get", GET}, - {"get_format", GET_FORMAT}, - {"global", GLOBAL}, - {"global_alias", GLOBAL_ALIAS}, - {"grants", GRANTS}, - {"grouping", GROUPING}, - {"grouping_id", GROUPING_ID}, - {"group_id", GROUP_ID}, - {"gts", GTS}, - {"handler", HANDLER}, - {"hash", HASH}, - {"help", HELP}, - {"hide", HIDE}, - {"hidden", HIDDEN}, - {"high", HIGH}, - {"high_priority", HIGH_PRIORITY}, - {"hour_microsecond", HOUR_MICROSECOND}, - {"hour_minute", HOUR_MINUTE}, - {"hour_second", HOUR_SECOND}, - {"host", HOST}, - {"hosts", HOSTS}, - {"hour", HOUR}, - {"hybrid_hist", HYBRID_HIST}, - {"id", ID}, - {"idc", IDC}, - {"identity", IDENTITY}, - {"indexed", INDEXED}, - {"if", IF}, - {"ifignore", IFIGNORE}, - {"ignore", IGNORE}, - {"ignore_server_ids", IGNORE_SERVER_IDS}, - {"ilogcache", ILOGCACHE}, - {"import", IMPORT}, - {"in", IN}, - {"incr", INCR}, - {"include", INCLUDE}, - {"incremental", INCREMENTAL}, - {"indent", INDENT}, - {"indexes", INDEXES}, - {"index_table_id", INDEX_TABLE_ID}, - {"info", INFO}, - {"infile", INFILE}, - {"infinite", INFINITE_VALUE}, - {"initial_size", INITIAL_SIZE}, - {"inner", INNER}, - {"inner_parse", INNER_PARSE}, - {"inout", INOUT}, - {"insensitive", INSENSITIVE}, - {"insert_method", INSERT_METHOD}, - {"install", INSTALL}, - {"int1", INT1}, - {"int2", INT2}, - {"int3", INT3}, - {"int4", INT4}, - {"int8", INT8}, - {"intersect", INTERSECT}, - {"invisible", INVISIBLE}, - {"interval", INTERVAL}, - {"invoker", INVOKER}, - {"io", IO}, - {"iops_weight", IOPS_WEIGHT}, - {"io_after_gtids", IO_AFTER_GTIDS}, - {"io_before_gtids", IO_BEFORE_GTIDS}, - {"io_thread", IO_THREAD}, - {"ipc", IPC}, - {"isnull", ISNULL}, - {"isolation", ISOLATION}, - {"isolation_level", ISOLATION_LEVEL}, - {"isopen", ISOPEN}, - {"issuer", ISSUER}, - {"iterate", ITERATE}, - {"job", JOB}, - {"join", JOIN}, - {"json", JSON}, - {"json_array", JSON_ARRAY}, - {"json_equal", JSON_EQUAL}, - {"json_exists", JSON_EXISTS}, - {"json_value", JSON_VALUE}, - {"json_query", JSON_QUERY}, - {"json_mergepatch", JSON_MERGEPATCH}, - {"json_arrayagg", JSON_ARRAYAGG}, - {"json_objectagg", JSON_OBJECTAGG}, - {"json_table", JSON_TABLE}, - {"json_object", JSON_OBJECT}, - {"k", K}, - {"key_block_size", KEY_BLOCK_SIZE}, - {"keys", KEYS}, - {"keystore", KEYSTORE}, - {"key_version", KEY_VERSION}, - {"kill", KILL}, - {"kvcache", KVCACHE}, - {"keep", KEEP}, - {"lag", LAG}, - {"last", LAST}, - {"last_value", LAST_VALUE}, - {"lax", LAX}, - {"lead", LEAD}, - {"leader", LEADER}, - {"leading", LEADING}, - {"leave", LEAVE}, - {"leaves", LEAVES}, - {"left", LEFT}, - {"less", LESS}, - {"level", LEVEL}, - {"lib", LIB}, - {"limit", LIMIT}, - {"linear", LINEAR}, - {"lines", LINES}, - {"linestring", LINESTRING}, - {"line_delimiter", LINE_DELIMITER}, - {"list", LIST}, - {"listagg", LISTAGG}, - {"lnnvl", LNNVL}, - {"load", LOAD}, - {"local", LOCAL}, - {"locality", LOCALITY}, - {"localtimestamp", LOCALTIMESTAMP}, - {"location", LOCATION}, - {"lock_", LOCK_}, - {"locked", LOCKED}, - {"locks", LOCKS}, - {"logonly_replica_num", LOGONLY_REPLICA_NUM}, - {"log", LOG}, - {"logs", LOGS}, - {"longblob", LONGBLOB}, - {"longtext", LONGTEXT}, - {"loop", LOOP}, - {"low", LOW}, - {"low_priority", LOW_PRIORITY}, - {"m", M}, - {"major", MAJOR}, - {"management", MANAGEMENT}, - {"master", MASTER}, - {"master_auto_position", MASTER_AUTO_POSITION}, - {"master_bind", MASTER_BIND}, - {"master_connect_retry", MASTER_CONNECT_RETRY}, - {"master_delay", MASTER_DELAY}, - {"master_heartbeat_period", MASTER_HEARTBEAT_PERIOD}, - {"master_host", MASTER_HOST}, - {"master_log_file", MASTER_LOG_FILE}, - {"master_log_pos", MASTER_LOG_POS}, - {"master_password", MASTER_PASSWORD}, - {"master_port", MASTER_PORT}, - {"master_retry_count", MASTER_RETRY_COUNT}, - {"master_server_id", MASTER_SERVER_ID}, - {"master_ssl", MASTER_SSL}, - {"master_ssl_ca", MASTER_SSL_CA}, - {"master_ssl_capath", MASTER_SSL_CAPATH}, - {"master_ssl_cert", MASTER_SSL_CERT}, - {"master_ssl_cipher", MASTER_SSL_CIPHER}, - {"master_ssl_crl", MASTER_SSL_CRL}, - {"master_ssl_crlpath", MASTER_SSL_CRLPATH}, - {"master_ssl_key", MASTER_SSL_KEY}, - {"master_ssl_verify_server_cert", MASTER_SSL_VERIFY_SERVER_CERT}, - {"master_user", MASTER_USER}, - {"match", MATCH}, - {"matched", MATCHED}, - {"max_connections_per_hour", MAX_CONNECTIONS_PER_HOUR}, - {"max_cpu", MAX_CPU}, - {"log_disk_size", LOG_DISK_SIZE}, - {"max_iops", MAX_IOPS}, - {"memory_size", MEMORY_SIZE}, - {"max_queries_per_hour", MAX_QUERIES_PER_HOUR}, - {"max_rows", MAX_ROWS}, - {"max_size", MAX_SIZE}, - {"max_updates_per_hour", MAX_UPDATES_PER_HOUR}, - {"max_user_connections", MAX_USER_CONNECTIONS}, - {"max_used_part_id", MAX_USED_PART_ID}, - {"median", MEDIAN}, - {"medium", MEDIUM}, - {"mediumblob", MEDIUMBLOB}, - {"mediumint", MEDIUMINT}, - {"mediumtext", MEDIUMTEXT}, - {"memory", MEMORY}, - {"memstore_percent", MEMSTORE_PERCENT}, - {"memtable", MEMTABLE}, - {"merge", MERGE}, - {"message_text", MESSAGE_TEXT}, - {"meta", META}, - {"microsecond", MICROSECOND}, - {"middleint", MIDDLEINT}, - {"migrate", MIGRATE}, - {"migration", MIGRATION}, - {"min_cpu", MIN_CPU}, - {"min_iops", MIN_IOPS}, - {"min_rows", MIN_ROWS}, - {"minor", MINOR}, - {"minute", MINUTE}, - {"minute_microsecond", MINUTE_MICROSECOND}, - {"minute_second", MINUTE_SECOND}, - {"mismatch", MISMATCH}, - {"missing", MISSING}, - {"mod", MOD}, - {"modifies", MODIFIES}, - {"month", MONTH}, - {"move", MOVE}, - {"movement", MOVEMENT}, - {"multilinestring", MULTILINESTRING}, - {"multipoint", MULTIPOINT}, - {"multipolygon", MULTIPOLYGON}, - {"multiset", MULTISET}, - {"mutex", MUTEX}, - {"mysql_errno", MYSQL_ERRNO}, - {"my_name", MY_NAME}, - {"name", NAME}, - {"names", NAMES}, - {"namespace", NAMESPACE}, - {"nan", NAN_VALUE}, - {"national", NATIONAL}, - {"natural", NATURAL}, - {"nchar", NCHAR}, - {"nchar_cs", NCHAR_CS}, - {"ndb", NDB}, - {"ndbcluster", NDBCLUSTER}, - {"nested", NESTED}, - {"no", NO}, - {"noentityescaping", NOENTITYESCAPING}, - {"noschemacheck", NOSCHEMACHECK}, - {"no_parallel", NO_PARALLEL}, - {"no_rewrite", NO_REWRITE}, - {"no_wait", NO_WAIT}, - {"no_write_to_binlog", NO_WRITE_TO_BINLOG}, - {"nologging", NOLOGGING}, - {"nodegroup", NODEGROUP}, - {"noparallel", NOPARALLEL}, - {"now", NOW}, - {"nowait", NOWAIT}, - {"nulls", NULLS}, - {"null_if", NULL_IF_EXETERNAL }, - {"nvarchar2", NVARCHAR2}, - {"ntile", NTILE}, - {"nth_value", NTH_VALUE}, - {"object", OBJECT}, - {"occur", OCCUR}, - {"ordinality", ORDINALITY}, - {"offset", OFFSET}, - {"old_key", OLD_KEY}, - {"over", OVER}, - {"old_password", OLD_PASSWORD}, - {"oltp", OLTP}, - {"one", ONE}, - {"one_shot", ONE_SHOT}, - {"options", OPTIONS}, - {"optimize", OPTIMIZE}, - {"optionally", OPTIONALLY}, - {"ora_rowscn", ORA_ROWSCN}, - {"orig_default", ORIG_DEFAULT}, - {"out", OUT}, - {"outer", OUTER}, - {"outfile", OUTFILE}, - {"outline", OUTLINE}, - {"owner", OWNER}, - {"p", P}, - {"pack_keys", PACK_KEYS}, - {"page", PAGE}, - {"parameters", PARAMETERS}, - {"param_assign_operator", PARAM_ASSIGN_OPERATOR}, - {"parser", PARSER}, - {"partial", PARTIAL}, - {"partition", PARTITION}, - {"partition_id", PARTITION_ID}, - {"partitioning", PARTITIONING}, - {"partitions", PARTITIONS}, - {"password", PASSWORD}, - {"password_grace_time", PASSWORD_GRACE_TIME}, - {"password_life_time", PASSWORD_LIFE_TIME}, - {"password_lock_time", PASSWORD_LOCK_TIME}, - {"password_verify_function", PASSWORD_VERIFY_FUNCTION}, - {"path", PATH}, - {"pattern", PATTERN}, - {"pause", PAUSE}, - {"percent", PERCENT}, - {"percentage", PERCENTAGE}, - {"percentile_cont", PERCENTILE_CONT}, - {"percentile_disc", PERCENTILE_DISC}, - {"percent_rank", PERCENT_RANK}, - {"phase", PHASE}, - {"planregress", PLANREGRESS}, - {"plugin", PLUGIN}, - {"plugin_dir", PLUGIN_DIR}, - {"plugins", PLUGINS}, - {"plus", PLUS}, - {"pivot", PIVOT}, - {"policy", POLICY}, - {"point", POINT}, - {"polygon", POLYGON}, - {"pool", POOL}, - {"port", PORT}, - {"position", POSITION}, - {"prepare", PREPARE}, - {"preserve", PRESERVE}, - {"pretty", PRETTY }, - {"pretty_color", PRETTY_COLOR }, - {"prev", PREV}, - {"primary_zone", PRIMARY_ZONE}, - {"privilege", PRIVILEGE}, - {"process", PROCESS}, - {"processlist", PROCESSLIST}, - {"profiles", PROFILES}, - {"progressive_merge_num", PROGRESSIVE_MERGE_NUM}, - {"proxy", PROXY}, - {"preceding", PRECEDING}, - {"purge", PURGE}, - {"quarter", QUARTER}, - {"query", QUERY}, - {"quick", QUICK}, - {"rank", RANK}, - {"range", RANGE}, - {"ratio_to_report", RATIO_TO_REPORT}, - {"read_consistency", READ_CONSISTENCY}, - {"read_write", READ_WRITE}, - {"reads", READS}, - {"read_only", READ_ONLY}, - {"rebuild", REBUILD}, - {"recovery_window", RECOVERY_WINDOW}, - {"recursive", RECURSIVE}, - {"recycle", RECYCLE}, - {"recyclebin", RECYCLEBIN}, - {"redaction", REDACTION}, - {"redo_buffer_size", REDO_BUFFER_SIZE}, - {"redofile", REDOFILE}, - {"redundancy", REDUNDANCY}, - {"redundant", REDUNDANT}, - {"refresh", REFRESH}, - {"regexp_like", REGEXP_LIKE}, - {"regr_slope", REGR_SLOPE}, - {"regr_intercept", REGR_INTERCEPT}, - {"regr_count", REGR_COUNT}, - {"regr_r2", REGR_R2}, - {"regr_avgx", REGR_AVGX}, - {"regr_avgy", REGR_AVGY}, - {"regr_sxx", REGR_SXX}, - {"regr_syy", REGR_SYY}, - {"regr_sxy", REGR_SXY}, - {"relay", RELAY}, - {"relay_log_file", RELAY_LOG_FILE}, - {"relay_log_pos", RELAY_LOG_POS}, - {"relay_thread", RELAY_THREAD}, - {"relaylog", RELAYLOG}, - {"release", RELEASE}, - {"reload", RELOAD}, - {"remove", REMOVE}, - {"reorganize", REORGANIZE}, - {"repair", REPAIR}, - {"repeat", REPEAT}, - {"repeatable", REPEATABLE}, - {"replace", REPLACE}, - {"replica", REPLICA}, - {"replica_num", REPLICA_NUM}, - {"replica_type", REPLICA_TYPE}, - {"duplicate_scope", DUPLICATE_SCOPE}, - {"replication", REPLICATION}, - {"report", REPORT}, - {"require", REQUIRE}, - {"reset", RESET}, - {"resignal", RESIGNAL}, - {"resource", RESOURCE}, - {"resource_pool_list", RESOURCE_POOL_LIST}, - {"respect", RESPECT}, - {"restart", RESTART}, - {"restore", RESTORE}, - {"restrict", RESTRICT}, - {"resume", RESUME}, - {"return", RETURN}, - {"returned_sqlstate", RETURNED_SQLSTATE}, - {"returns", RETURNS}, - {"reverse", REVERSE}, - {"rlike", RLIKE}, - {"right", RIGHT}, - {"rollup", ROLLUP}, - {"root", ROOT}, - {"rootservice", ROOTSERVICE}, - {"roottable", ROOTTABLE}, - {"routine", ROUTINE}, - {"rowcount", ROWCOUNT}, - {"row_count", ROW_COUNT}, - {"row_format", ROW_FORMAT}, - {"row_number", ROW_NUMBER}, - {"rtree", RTREE}, - {"run", RUN}, - {"sample", SAMPLE}, - {"search", SEARCH}, - {"scalars", SCALARS}, - {"schedule", SCHEDULE}, - {"schema", SCHEMAS}, - {"schemacheck", SCHEMACHECK}, - {"schema_name", SCHEMA_NAME}, - {"scope", SCOPE}, - {"second", SECOND}, - {"second_microsecond", SECOND_MICROSECOND}, - {"security", SECURITY}, - {"seed", SEED}, - {"sensitive", SENSITIVE}, - {"separator", SEPARATOR}, - {"serial", SERIAL}, - {"serializable", SERIALIZABLE}, - {"server", SERVER}, - {"server_ip", SERVER_IP}, - {"server_port", SERVER_PORT}, - {"server_type", SERVER_TYPE}, - {"session_alias", SESSION_ALIAS}, - {"session_user", SESSION_USER}, - {"binding", BINDING}, - {"sharding", SHARDING}, - {"set_master_cluster", SET_MASTER_CLUSTER}, - {"set_slave_cluster", SET_SLAVE_CLUSTER}, - {"set_tp", SET_TP}, - {"show", SHOW}, - {"shutdown", SHUTDOWN}, - {"shrink", SHRINK}, - {"signal", SIGNAL}, - {"signed", SIGNED}, - {"simple", SIMPLE}, - {"slave", SLAVE}, - {"size", SIZE}, - {"skip", SKIP}, - {"skip_blank_lines", SKIP_BLANK_LINES}, - {"skip_header", SKIP_HEADER}, - {"slow", SLOW}, - {"skewonly", SKEWONLY}, - {"socket", SOCKET}, - {"soname", SONAME}, - {"sounds", SOUNDS}, - {"source", SOURCE}, - {"space", SPACE}, - {"spatial", SPATIAL}, - {"specific", SPECIFIC}, - {"statements", STATEMENTS}, - {"stddev", STDDEV}, - {"stddev_pop", STDDEV_POP}, - {"stddev_samp", STDDEV_SAMP}, - {"strict", STRICT}, - {"spfile", SPFILE}, - {"split", SPLIT}, - {"sqlexception", SQLEXCEPTION}, - {"sqlwarning", SQLWARNING}, - {"sql_big_result", SQL_BIG_RESULT}, - {"sql_calc_found_row", SQL_CALC_FOUND_ROW}, - {"sql_calc_found_rows", SQL_CALC_FOUND_ROWS}, - {"sql_small_result", SQL_SMALL_RESULT}, - {"sql_after_gtids", SQL_AFTER_GTIDS}, - {"sql_after_mts_gaps", SQL_AFTER_MTS_GAPS}, - {"sql_before_gtids", SQL_BEFORE_GTIDS}, - {"sql_buffer_result", SQL_BUFFER_RESULT}, - {"sql_cache", SQL_CACHE}, - {"sql_id", SQL_ID}, - {"sql_no_cache", SQL_NO_CACHE}, - {"sql_thread", SQL_THREAD}, - {"sql_tsi_day", SQL_TSI_DAY}, - {"sql_tsi_hour", SQL_TSI_HOUR}, - {"sql_tsi_minute", SQL_TSI_MINUTE}, - {"sql_tsi_month", SQL_TSI_MONTH}, - {"sql_tsi_quarter", SQL_TSI_QUARTER}, - {"sql_tsi_second", SQL_TSI_SECOND}, - {"sql_tsi_week", SQL_TSI_WEEK}, - {"sql_tsi_year", SQL_TSI_YEAR}, - {"ssl", SSL}, - {"straight_join", STRAIGHT_JOIN}, - {"starting", STARTING}, - {"starts", STARTS}, - {"stats_auto_recalc", STATS_AUTO_RECALC}, - {"stats_persistent", STATS_PERSISTENT}, - {"stats_sample_pages", STATS_SAMPLE_PAGES}, - {"status", STATUS}, - {"storage_format_version", STORAGE_FORMAT_VERSION}, - {"stored", STORED}, - {"storing", STORING}, - {"strong", STRONG}, - {"standby", STANDBY}, - {"subclass_origin", SUBCLASS_ORIGIN}, - {"subdate", SUBDATE}, - {"subject", SUBJECT}, - {"subpartition", SUBPARTITION}, - {"subpartitions", SUBPARTITIONS}, - {"substr", SUBSTR}, - {"super", SUPER}, - {"suspend", SUSPEND}, - {"swaps", SWAPS}, - {"switches", SWITCHES}, - {"switchover", SWITCHOVER}, - {"sysbackup", SYSBACKUP}, - {"sysdba", SYSDBA}, - {"syskm", SYSKM}, - {"sysoper", SYSOPER}, - {"system_user", SYSTEM_USER}, - {"systimestamp", SYSTIMESTAMP}, - {"t", T}, - {"tablegroup", TABLEGROUP}, - {"table_checksum", TABLE_CHECKSUM}, - {"table_mode", TABLE_MODE}, - {"table_id", TABLE_ID}, - {"table_name", TABLE_NAME}, - {"tablegroups", TABLEGROUPS}, - {"tablegroup_id", TABLEGROUP_ID}, - {"tablet", TABLET}, - {"tablet_max_size", TABLET_MAX_SIZE}, - {"task", TASK}, - {"template", TEMPLATE}, - {"temptable", TEMPTABLE}, - {"tenant", TENANT}, - {"terminated", TERMINATED}, - {"text", TEXT}, - {"than", THAN}, - {"ties", TIES}, - {"timestamp", TIMESTAMP}, - {"timestampadd", TIMESTAMPADD}, - {"timestampdiff", TIMESTAMPDIFF}, - {"timezone_abbr", TIMEZONE_ABBR}, - {"timezone_hour", TIMEZONE_HOUR}, - {"timezone_minute", TIMEZONE_MINUTE}, - {"timezone_region", TIMEZONE_REGION}, - {"tinyblob", TINYBLOB}, - {"tinytext", TINYTEXT}, - {"tablet_size", TABLET_SIZE}, - {"top_k_fre_hist", TOP_K_FRE_HIST}, - {"tp_name", TP_NAME}, - {"tp_no", TP_NO}, - {"trace", TRACE}, - {"traditional", TRADITIONAL}, - {"trailing", TRAILING}, - {"translate", TRANSLATE}, - {"trigger", TRIGGER}, - {"trim", TRIM}, - {"trim_space", TRIM_SPACE}, - {"treat", TREAT}, - {"type", TYPE}, - {"typename", TYPENAME}, - {"types", TYPES}, - {"uncommitted", UNCOMMITTED}, - {"unconditional", UNCONDITIONAL}, - {"undefined", UNDEFINED}, - {"undo", UNDO}, - {"undo_buffer_size", UNDO_BUFFER_SIZE}, - {"undofile", UNDOFILE}, - {"unicode", UNICODE}, - {"uninstall", UNINSTALL}, - {"unit", UNIT}, - {"unit_group", UNIT_GROUP}, - {"unit_num", UNIT_NUM}, - {"unknown", UNKNOWN}, - {"unlock", UNLOCK}, - {"unlocked", UNLOCKED}, - {"unpivot", UNPIVOT}, - {"unusual", UNUSUAL}, - {"updatexml",UPDATEXML}, - {"upgrade", UPGRADE}, - {"usage", USAGE}, - {"use_bloom_filter", USE_BLOOM_FILTER}, - {"use_frm", USE_FRM}, - {"user_resources", USER_RESOURCES}, - {"utc_date", UTC_DATE}, - {"utc_timestamp", UTC_TIMESTAMP}, - {"unbounded", UNBOUNDED}, - {"urowid", UROWID}, - {"valid", VALID}, - {"value", VALUE}, - {"var_pop", VAR_POP}, - {"var_samp", VAR_SAMP}, - {"varcharacter", VARCHAR}, - {"varying", VARYING}, - {"variables", VARIABLES}, - {"variance", VARIANCE}, - {"verbose", VERBOSE}, - {"verify", VERIFY}, - {"version", VERSION}, - {"materialized", MATERIALIZED}, - {"virtual", VIRTUAL}, - {"visible", VISIBLE}, - {"wait", WAIT}, - {"warnings", WARNINGS}, - {"weak", WEAK}, - {"week", WEEK}, - {"weight_string", WEIGHT_STRING}, - {"wellformed", WELLFORMED}, - {"without", WITHOUT}, - {"wmsys", WMSYS}, - {"wm_concat", WM_CONCAT}, - {"wrapper", WRAPPER}, - {"while", WHILE}, - {"x509", X509}, - {"xa", XA}, - {"xml", XML}, - {"xmlattributes", XMLATTRIBUTES}, - {"xmlcast", XMLCAST}, - {"xmlelement", XMLELEMENT}, - {"xmlagg", XMLAGG}, - {"xmlparse", XMLPARSE}, - {"xmlserialize", XMLSERIALIZE}, - {"xmltype", XMLTYPE}, - {"xor", XOR}, - {"year_month", YEAR_MONTH}, - {"year", YEAR}, - {"zerofill", ZEROFILL}, - {"zone", ZONE}, - {"zone_list", ZONE_LIST}, - {"time_zone_info", TIME_ZONE_INFO}, - {"connect_by_isleaf", CONNECT_BY_ISLEAF}, - {"connect_by_iscycle", CONNECT_BY_ISCYCLE}, - {"connect_by_root", CONNECT_BY_ROOT}, - {"sys_connect_by_path", SYS_CONNECT_BY_PATH}, - {"siblings", SIBLINGS}, - {"zone_type", ZONE_TYPE}, - {"returning", RETURNING}, - {"sessiontimezone", SESSIONTIMEZONE}, - {"dbtimezone", DBTIMEZONE}, - {"rely", RELY}, - {"norely", NORELY}, - {"novalidate", NOVALIDATE}, - {"member", MEMBER}, - {"submultiset", SUBMULTISET}, - {"empty", EMPTY}, - {"empty_field_as_null", EMPTY_FIELD_AS_NULL}, - {"a", A}, - {"throttle", THROTTLE}, - {"priority", PRIORITY}, - {"rt", RT}, - {"network", NETWORK}, - {"logical_reads", LOGICAL_READS}, - {"queue_time", QUEUE_TIME} -}; - -const NonReservedKeyword *oracle_non_reserved_keyword_lookup(const char *word) -{ - return find_word(word, oracle_keywords_root, Oracle_non_reserved_keywords); -} - -int oracle_sql_reserved_keyword_lookup(const char *word) -{ - int ret = -1; - if (casesame_cstr("SET", word)) { - ret = SET; - } else if (casesame_cstr("SELECT", word)) { - ret = SELECT; - } else if (casesame_cstr("INSERT", word)) { - ret = INSERT; - } else if (casesame_cstr("UPDATE", word)) { - ret = UPDATE; - } else if (casesame_cstr("DELETE", word)) { - ret = DELETE; - //merge is not reserved keyword in oracle - // } else if (casesame_cstr("MERGE", word)) { - // ret = MERGE; - } else if (casesame_cstr("WITH", word)) { - ret = WITH; - } - return ret; -} - -//return 0 if succ, return 1 if fail -int create_oracle_trie_tree() -{ - return create_trie_tree(Oracle_non_reserved_keywords, LENGTH_OF(Oracle_non_reserved_keywords), &oracle_keywords_root); -} - -void __attribute__((constructor)) init_oracle_non_reserved_keywords_tree() -{ - int ret = 0; - if (0 != (ret = create_oracle_trie_tree())) { - (void)printf("ERROR build oracle_non_reserved_keywords tree failed=>%d", ret); - } -} - -static t_node *oracle_reserved_keywords_root = NULL; - -static const ReservedKeyword Oracle_reserved_keywords[] = { - /* - * oracle reserved keyword, for dbms_utility.cannonize and resvered keyword view - */ - { "access", ACCESS }, - { "add", ADD }, - { "all", ALL }, - { "alter", ALTER }, - { "and", AND }, - { "any", ANY }, - // { "arraylen", ARRAYLEN }, - { "as", AS }, - { "asc", ASC }, - { "audit", AUDIT }, - { "between", BETWEEN }, - // { "bfile", BFILE }, - { "blob", BLOB }, - { "by", BY }, - { "binary_double", BINARY_DOUBLE }, - { "binary_float", BINARY_FLOAT }, - { "char", CHAR }, - { "check", CHECK }, - { "clob", CLOB }, - { "cluster", CLUSTER }, - { "column", COLUMN }, - { "comment", COMMENT }, - { "compress", COMPRESS }, - { "connect", CONNECT }, - { "create", CREATE }, - { "current", CURRENT }, - { "date", DATE }, - { "decimal", DECIMAL }, - { "default", DEFAULT }, - { "delete", DELETE }, - { "desc", DESC }, - { "distinct", DISTINCT }, - { "drop", DROP }, - { "else", ELSE }, - { "exclusive", EXCLUSIVE }, - { "exists", EXISTS }, - { "file_key", FILE_KEY }, - { "float", FLOAT }, - { "for", FOR }, - { "from", FROM }, - { "grant", GRANT }, - { "group", GROUP }, - { "having", HAVING }, - { "identified", IDENTIFIED }, - { "immediate", IMMEDIATE }, - { "in", IN }, - { "increment", INCREMENT }, - { "index", INDEX }, - { "initial_", INITIAL_ }, - { "insert", INSERT }, - { "integer", INTEGER }, - { "intersect", INTERSECT }, - { "into", INTO }, - { "is", IS }, - { "level", LEVEL }, - { "like", LIKE }, - { "lock", LOCK }, - { "long", LONG }, - { "maxextents", MAXEXTENTS }, - { "minus", MINUS }, - { "mode", MODE }, - { "modify", MODIFY }, - // { "nclob", NCLOB }, - { "noaudit", NOAUDIT }, - { "nocompress", NOCOMPRESS }, - { "not", NOT }, - { "notfound", NOTFOUND }, - { "nowait", NOWAIT }, - { "null", NULLX }, - { "number", NUMBER }, - { "of", OF }, - { "offline", OFFLINE }, - { "on", ON }, - { "online", ONLINE }, - { "option", OPTION }, - { "or", OR }, - { "order", ORDER }, - { "pctfree", PCTFREE }, - { "prior", PRIOR }, - { "privileges", PRIVILEGES }, - { "public", PUBLIC }, - { "raw", RAW }, - { "rename", RENAME }, - { "resource", RESOURCE }, - { "revoke", REVOKE }, - { "row", ROW }, - { "rowid", ROWID }, - { "rowlabel", ROWLABEL }, - { "rownum", ROWNUM }, - { "rows", ROWS }, - { "start", START }, - { "select", SELECT }, - { "session", SESSION }, - { "set", SET }, - { "share", SHARE }, - { "size", SIZE }, - { "smallint", SMALLINT }, - // { "sqlbuf", SQLBUF }, - { "successful", SUCCESSFUL }, - { "synonym", SYNONYM }, - { "sysdate", SYSDATE }, - { "systimestamp", SYSTIMESTAMP }, - { "some", SOME }, - { "table", TABLE }, - { "then", THEN }, - { "to", TO }, - { "trigger", TRIGGER }, - { "uid", UID }, - { "union", UNION }, - { "unique", UNIQUE }, - { "update", UPDATE }, - { "user", USER }, - { "validate", VALIDATE }, - { "values", VALUES }, - { "varchar", VARCHAR }, - { "varchar2", VARCHAR2 }, - { "view", VIEW }, - { "whenever", WHENEVER }, - { "where", WHERE }, - { "with", WITH }, - { "isopen", ISOPEN }, - { "rowcount", ROWCOUNT }, - { "bulk_rowcount", BULK_ROWCOUNT }, - { "error_index", ERROR_INDEX }, - { "bulk_exceptions", BULK_EXCEPTIONS }, - { "case", CASE }, - { "current_date", CURRENT_DATE }, - { "current_timestamp", CURRENT_TIMESTAMP }, - { "dual", DUAL }, - { "localtimestamp", LOCALTIMESTAMP } -}; - -const ReservedKeyword *oracle_reserved_keyword_lookup(const char *word) -{ - return find_word(word, oracle_reserved_keywords_root, Oracle_reserved_keywords); -} -int create_oracle_reserved_trie_tree() -{ - return create_trie_tree(Oracle_reserved_keywords, LENGTH_OF(Oracle_reserved_keywords), &oracle_reserved_keywords_root); -} - -void __attribute__((constructor)) init_oracle_reserved_keywords_tree() -{ - int ret = 0; - if (0 != (ret = create_oracle_reserved_trie_tree())) { - (void)printf("ERROR build oracle_non_reserved_keywords tree failed=>%d", ret); - } -} - -struct FuncPair{ - char *func_name_; - int yytokentype_; -}; -//兼容Oracle 将"FUN_NAME"(xx) 和 func_name(xx) 的行为 -//添加新的窗口函数需要在此处以及window_function_name_compare函数中添加对应逻辑 -//使用二分查找， 所以请按照字典序添加 -static const struct FuncPair FUNCLIST[WINDOW_FUNCTION_NUM]= -{ - {"APPROX_COUNT_DISTINCT", APPROX_COUNT_DISTINCT}, - {"APPROX_COUNT_DISTINCT_SYNOPSIS", APPROX_COUNT_DISTINCT_SYNOPSIS}, - {"APPROX_COUNT_DISTINCT_SYNOPSIS_MERGE", APPROX_COUNT_DISTINCT_SYNOPSIS_MERGE}, - {"AVG", AVG},{"CORR", CORR},{"COUNT", COUNT},{"COVAR_POP", COVAR_POP}, - {"COVAR_SAMP", COVAR_SAMP},{"CUME_DIST", CUME_DIST},{"DENSE_RANK", DENSE_RANK}, - {"FIRST_VALUE", FIRST_VALUE},{"LAG", LAG},{"LAST_VALUE", LAST_VALUE}, - {"LEAD", LEAD},{"LISTAGG", LISTAGG},{"MAX", MAX},{"MEDIAN", MEDIAN}, - {"MIN", MIN},{"NTH_VALUE", NTH_VALUE},{"NTILE", NTILE},{"OVER", OVER}, - {"PERCENT_RANK", PERCENT_RANK},{"RANK", RANK},{"RATIO_TO_REPORT", RATIO_TO_REPORT}, - {"REGR_AVGX", REGR_AVGX},{"REGR_AVGY", REGR_AVGY},{"REGR_COUNT", REGR_COUNT}, - {"REGR_INTERCEPT", REGR_INTERCEPT},{"REGR_R2", REGR_R2},{"REGR_SLOPE", REGR_SLOPE}, - {"REGR_SXX", REGR_SXX},{"REGR_SXY", REGR_SXY},{"REGR_SYY", REGR_SYY}, - {"ROW_NUMBER", ROW_NUMBER},{"STDDEV", STDDEV},{"STDDEV_POP", STDDEV_POP}, - {"STDDEV_SAMP", STDDEV_SAMP},{"SUM", SUM},{"VARIANCE", VARIANCE}, - {"VAR_POP", VAR_POP},{"VAR_SAMP", VAR_SAMP} -}; - -int binary_search(const struct FuncPair *window_func, int64_t begin, int64_t end, const char *value) -{ - int result = -1; - int need_break = 0; - while(!need_break && begin <= end) { - int mid = begin + (end - begin) / 2; - int cmp = strcmp(window_func[mid].func_name_, value); - if (cmp > 0) { - end = mid - 1; - } else if (cmp < 0) { - begin = mid + 1; - } else { - result = window_func[mid].yytokentype_; - need_break = 1; - } - } - return result; -} - -int window_function_name_compare(const char *dup_value, int *window_fun_idx) -{ - int compared = 0; - if (NULL == dup_value) { - } else { - *window_fun_idx = binary_search(FUNCLIST, 0, WINDOW_FUNCTION_NUM - 1, dup_value); - if (*window_fun_idx > 0) { - compared = 1; - } - } - - return compared; -} diff --git a/src/sql/parser/sql_parser_base.c b/src/sql/parser/sql_parser_base.c index ea9015701..c36063325 100644 --- a/src/sql/parser/sql_parser_base.c +++ b/src/sql/parser/sql_parser_base.c @@ -16,6 +16,8 @@ #define yyconst const typedef void* yyscan_t; typedef struct yy_buffer_state *YY_BUFFER_STATE; +#define IS_ORACLE_MODE(mode) (0 != (mode & SMO_ORACLE)) +#define IS_ORACLE_COMPATIBLE (IS_ORACLE_MODE(p->sql_mode_)) extern int obsql_mysql_yylex_init_extra(YY_EXTRA_TYPE yy_user_defined,yyscan_t* ptr_yy_globals ); extern int obsql_mysql_yyparse(ParseResult *result); extern int obsql_mysql_multi_fast_parse(ParseResult *p); @@ -25,6 +27,35 @@ extern int obsql_mysql_yylex_destroy (yyscan_t yyscanner ); extern YY_BUFFER_STATE obsql_mysql_yy_scan_bytes (yyconst char *bytes,int len ,yyscan_t yyscanner ); extern void obsql_mysql_yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner ); extern void obsql_mysql_yy_delete_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner ); +#ifdef OB_BUILD_ORACLE_PARSER +extern int obsql_oracle_latin1_yylex_init_extra(YY_EXTRA_TYPE yy_user_defined,yyscan_t* ptr_yy_globals ); +extern int obsql_oracle_latin1_yyparse(ParseResult *result); +extern int obsql_oracle_latin1_multi_fast_parse(ParseResult *p); +extern int obsql_oracle_latin1_multi_values_parse(ParseResult *p); +extern int obsql_oracle_latin1_fast_parse(ParseResult *p); +extern int obsql_oracle_latin1_yylex_destroy (yyscan_t yyscanner ); +extern YY_BUFFER_STATE obsql_oracle_latin1_yy_scan_bytes (yyconst char *bytes,int len ,yyscan_t yyscanner ); +extern void obsql_oracle_latin1_yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner ); +extern void obsql_oracle_latin1_yy_delete_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner ); +extern int obsql_oracle_utf8_yylex_init_extra(YY_EXTRA_TYPE yy_user_defined,yyscan_t* ptr_yy_globals ); +extern int obsql_oracle_utf8_yyparse(ParseResult *result); +extern int obsql_oracle_utf8_multi_fast_parse(ParseResult *p); +extern int obsql_oracle_utf8_multi_values_parse(ParseResult *p); +extern int obsql_oracle_utf8_fast_parse(ParseResult *p); +extern int obsql_oracle_utf8_yylex_destroy (yyscan_t yyscanner ); +extern YY_BUFFER_STATE obsql_oracle_utf8_yy_scan_bytes (yyconst char *bytes,int len ,yyscan_t yyscanner ); +extern void obsql_oracle_utf8_yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner ); +extern void obsql_oracle_utf8_yy_delete_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner ); +extern int obsql_oracle_gbk_yylex_init_extra(YY_EXTRA_TYPE yy_user_defined,yyscan_t* ptr_yy_globals ); +extern int obsql_oracle_gbk_yyparse(ParseResult *result); +extern int obsql_oracle_gbk_multi_fast_parse(ParseResult *p); +extern int obsql_oracle_gbk_multi_values_parse(ParseResult *p); +extern int obsql_oracle_gbk_fast_parse(ParseResult *p); +extern int obsql_oracle_gbk_yylex_destroy (yyscan_t yyscanner ); +extern YY_BUFFER_STATE obsql_oracle_gbk_yy_scan_bytes (yyconst char *bytes,int len ,yyscan_t yyscanner ); +extern void obsql_oracle_gbk_yy_switch_to_buffer (YY_BUFFER_STATE new_buffer ,yyscan_t yyscanner ); +extern void obsql_oracle_gbk_yy_delete_buffer (YY_BUFFER_STATE b ,yyscan_t yyscanner ); +#endif int parse_init(ParseResult *p) { int ret = 0; // can not include C++ file "ob_define.h" @@ -38,7 +69,45 @@ int parse_init(ParseResult *p) } if (OB_LIKELY( 0 == ret)) { +#ifdef OB_BUILD_ORACLE_PARSER + if (IS_ORACLE_COMPATIBLE) { + switch (p->connection_collation_) { + case 28/*CS_TYPE_GBK_CHINESE_CI*/: + case 87/*CS_TYPE_GBK_BIN*/: + case 216/*CS_TYPE_GB18030_2022_BIN*/: + case 217/*CS_TYPE_GB18030_2022_PINYIN_CI*/: + case 218/*CS_TYPE_GB18030_2022_PINYIN_CS*/: + case 219/*CS_TYPE_GB18030_2022_RADICAL_CI*/: + case 220/*CS_TYPE_GB18030_2022_RADICAL_CS*/: + case 221/*CS_TYPE_GB18030_2022_STROKE_CI*/: + case 222/*CS_TYPE_GB18030_2022_STROKE_CS*/: + case 248/*CS_TYPE_GB18030_CHINESE_CI*/: + case 249/*CS_TYPE_GB18030_BIN*/: + ret = obsql_oracle_gbk_yylex_init_extra(p, &(p->yyscan_info_)); + break; + case 45/*CS_TYPE_UTF8MB4_GENERAL_CI*/: + case 46/*CS_TYPE_UTF8MB4_BIN*/: + case 63/*CS_TYPE_BINARY*/: + case 224/*CS_TYPE_UTF8MB4_UNICODE_CI*/: + ret = obsql_oracle_utf8_yylex_init_extra(p, &(p->yyscan_info_)); + break; + case 8/*CS_TYPE_LATIN1_SWEDISH_CI*/: + case 47/*CS_TYPE_LATIN1_BIN*/: + ret = obsql_oracle_latin1_yylex_init_extra(p, &(p->yyscan_info_)); + break; + default: { + ret = -1; + (void)snprintf(p->error_msg_, MAX_ERROR_MSG, "get not support connection collation: %u", + p->connection_collation_); + break; + } + } + } else { +#endif ret = obsql_mysql_yylex_init_extra(p, &(p->yyscan_info_)); +#ifdef OB_BUILD_ORACLE_PARSER + } +#endif } return ret; } @@ -47,7 +116,45 @@ int parse_terminate(ParseResult *p) { int ret = 0; if (OB_LIKELY(NULL != p->yyscan_info_)) { +#ifdef OB_BUILD_ORACLE_PARSER + if (IS_ORACLE_COMPATIBLE) { + switch (p->connection_collation_) { + case 28/*CS_TYPE_GBK_CHINESE_CI*/: + case 87/*CS_TYPE_GBK_BIN*/: + case 216/*CS_TYPE_GB18030_2022_BIN*/: + case 217/*CS_TYPE_GB18030_2022_PINYIN_CI*/: + case 218/*CS_TYPE_GB18030_2022_PINYIN_CS*/: + case 219/*CS_TYPE_GB18030_2022_RADICAL_CI*/: + case 220/*CS_TYPE_GB18030_2022_RADICAL_CS*/: + case 221/*CS_TYPE_GB18030_2022_STROKE_CI*/: + case 222/*CS_TYPE_GB18030_2022_STROKE_CS*/: + case 248/*CS_TYPE_GB18030_CHINESE_CI*/: + case 249/*CS_TYPE_GB18030_BIN*/: + ret = obsql_oracle_gbk_yylex_destroy(p->yyscan_info_); + break; + case 45/*CS_TYPE_UTF8MB4_GENERAL_CI*/: + case 46/*CS_TYPE_UTF8MB4_BIN*/: + case 63/*CS_TYPE_BINARY*/: + case 224/*CS_TYPE_UTF8MB4_UNICODE_CI*/: + ret = obsql_oracle_utf8_yylex_destroy(p->yyscan_info_); + break; + case 8/*CS_TYPE_LATIN1_SWEDISH_CI*/: + case 47/*CS_TYPE_LATIN1_BIN*/: + ret = obsql_oracle_latin1_yylex_destroy(p->yyscan_info_); + break; + default: { + ret = -1; + (void)snprintf(p->error_msg_, MAX_ERROR_MSG, "get not support connection collation: %u", + p->connection_collation_); + break; + } + } + } else { +#endif ret = obsql_mysql_yylex_destroy(p->yyscan_info_); +#ifdef OB_BUILD_ORACLE_PARSER + } +#endif } return ret; } @@ -105,6 +212,115 @@ int parse_sql(ParseResult *p, const char *buf, size_t input_len) ret = OB_PARSER_ERR_NO_MEMORY; #endif } else { +#ifdef OB_BUILD_ORACLE_PARSER + if (IS_ORACLE_COMPATIBLE) { + switch (p->connection_collation_) { + case 28/*CS_TYPE_GBK_CHINESE_CI*/: + case 87/*CS_TYPE_GBK_BIN*/: + case 216/*CS_TYPE_GB18030_2022_BIN*/: + case 217/*CS_TYPE_GB18030_2022_PINYIN_CI*/: + case 218/*CS_TYPE_GB18030_2022_PINYIN_CS*/: + case 219/*CS_TYPE_GB18030_2022_RADICAL_CI*/: + case 220/*CS_TYPE_GB18030_2022_RADICAL_CS*/: + case 221/*CS_TYPE_GB18030_2022_STROKE_CI*/: + case 222/*CS_TYPE_GB18030_2022_STROKE_CS*/: + case 248/*CS_TYPE_GB18030_CHINESE_CI*/: + case 249/*CS_TYPE_GB18030_BIN*/: { + YY_BUFFER_STATE bp = obsql_oracle_gbk_yy_scan_bytes(buf, len, p->yyscan_info_); + obsql_oracle_gbk_yy_switch_to_buffer(bp, p->yyscan_info_); + int tmp_ret = -1; + if (p->is_fp_) { + tmp_ret = obsql_oracle_gbk_fast_parse(p); + } else if (p->is_multi_query_) { + tmp_ret = obsql_oracle_gbk_multi_fast_parse(p); + } else if (p->is_multi_values_parser_) { + tmp_ret = obsql_oracle_gbk_multi_values_parse(p); + } else { + tmp_ret = obsql_oracle_gbk_yyparse(p); + } + if (0 == tmp_ret) { + ret = OB_PARSER_SUCCESS; + } else if (2 == tmp_ret) { + ret = OB_PARSER_ERR_NO_MEMORY; + } else { + if (0 != p->extra_errno_) { + ret = p->extra_errno_; + } else { + ret = OB_PARSER_ERR_PARSE_SQL; + } + } + obsql_oracle_gbk_yy_delete_buffer(bp, p->yyscan_info_); + break; + } + case 45/*CS_TYPE_UTF8MB4_GENERAL_CI*/: + case 46/*CS_TYPE_UTF8MB4_BIN*/: + case 63/*CS_TYPE_BINARY*/: + case 224/*CS_TYPE_UTF8MB4_UNICODE_CI*/:{ + YY_BUFFER_STATE bp = obsql_oracle_utf8_yy_scan_bytes(buf, len, p->yyscan_info_); + obsql_oracle_utf8_yy_switch_to_buffer(bp, p->yyscan_info_); + int tmp_ret = -1; + if (p->is_fp_) { + tmp_ret = obsql_oracle_utf8_fast_parse(p); + } else if (p->is_multi_query_) { + tmp_ret = obsql_oracle_utf8_multi_fast_parse(p); + } else if (p->is_multi_values_parser_) { + tmp_ret = obsql_oracle_utf8_multi_values_parse(p); + } else { + tmp_ret = obsql_oracle_utf8_yyparse(p); + } + if (0 == tmp_ret) { + ret = OB_PARSER_SUCCESS; + } else if (2 == tmp_ret) { + ret = OB_PARSER_ERR_NO_MEMORY; + } else { + if (0 != p->extra_errno_) { + ret = p->extra_errno_; + } else { + ret = OB_PARSER_ERR_PARSE_SQL; + } + } + obsql_oracle_utf8_yy_delete_buffer(bp, p->yyscan_info_); + break; + } + case 8/*CS_TYPE_LATIN1_SWEDISH_CI*/: + case 47/*CS_TYPE_LATIN1_BIN*/:{ + YY_BUFFER_STATE bp = obsql_oracle_latin1_yy_scan_bytes(buf, len, p->yyscan_info_); + obsql_oracle_latin1_yy_switch_to_buffer(bp, p->yyscan_info_); + int tmp_ret = -1; + if (p->is_fp_) { + tmp_ret = obsql_oracle_latin1_fast_parse(p); + } else if (p->is_multi_query_) { + tmp_ret = obsql_oracle_latin1_multi_fast_parse(p); + } else if (p->is_multi_values_parser_) { + tmp_ret = obsql_oracle_latin1_multi_values_parse(p); + } else { + tmp_ret = obsql_oracle_latin1_yyparse(p); + } + if (0 == tmp_ret) { + ret = OB_PARSER_SUCCESS; + } else if (2 == tmp_ret) { + ret = OB_PARSER_ERR_NO_MEMORY; + } else { + if (0 != p->extra_errno_) { + ret = p->extra_errno_; + } else { + ret = OB_PARSER_ERR_PARSE_SQL; + } + } + obsql_oracle_latin1_yy_delete_buffer(bp, p->yyscan_info_); + break; + } + default: { + ret = OB_PARSER_ERR_UNEXPECTED; + (void)snprintf(p->error_msg_, MAX_ERROR_MSG, "get not support conn collation: %u", + p->connection_collation_); + break; + } + } +#endif +#ifdef OB_BUILD_ORACLE_PARSER + } else { +#endif YY_BUFFER_STATE bp = obsql_mysql_yy_scan_bytes(buf, len, p->yyscan_info_); obsql_mysql_yy_switch_to_buffer(bp, p->yyscan_info_); int tmp_ret = -1; @@ -129,6 +345,9 @@ int parse_sql(ParseResult *p, const char *buf, size_t input_len) } } obsql_mysql_yy_delete_buffer(bp, p->yyscan_info_); +#ifdef OB_BUILD_ORACLE_PARSER + } +#endif } } } diff --git a/src/sql/parser/sql_parser_base.h b/src/sql/parser/sql_parser_base.h index 9e203f7d5..62881775b 100644 --- a/src/sql/parser/sql_parser_base.h +++ b/src/sql/parser/sql_parser_base.h @@ -240,6 +240,50 @@ do { setup_token_pos_info(node, word_start - 1, word_end - word_start + 1); \ } while (0) +//oralce下生成非保留关键字结点请使用该宏,区别于mysql的是做了大写的转换 +#define get_oracle_non_reserved_node(node, malloc_pool, expr_start, expr_end) \ + do { \ + malloc_terminal_node(node, malloc_pool, T_IDENT); \ + if (OB_UNLIKELY(NULL == node || NULL == result || NULL == result->input_sql_)) {\ + yyerror(NULL, result, "invalid argument, node:%p, result:%p or input_sql is NULL", node, result);\ + YYABORT_UNEXPECTED; \ + } else if (OB_UNLIKELY(expr_start < 0 || expr_end < 0 || expr_start > expr_end)) { \ + yyerror(NULL, result, "invalid argument, expr_start:%d, expr_end:%d", (int32_t)expr_start, (int32_t)expr_end);\ + YYABORT_UNEXPECTED; \ + } else { \ + int start = expr_start; \ + char * upper_value = NULL; \ + char * raw_str = NULL; \ + node->str_value_ = NULL; \ + node->str_len_ = 0; \ + node->raw_text_ = NULL; \ + node->text_len_ = 0; \ + while (start <= expr_end && ISSPACE(result->input_sql_[start - 1])) {\ + start++; \ + } \ + if ('"' == result->input_sql_[start - 1]) { \ + start++; \ + expr_end--; \ + } \ + if (start >= expr_start \ + && (OB_UNLIKELY((NULL == (upper_value = copy_expr_string(result, start, expr_end)))))) { \ + yyerror(NULL, result, "No more space for copying expression string"); \ + YYABORT_NO_MEMORY; \ + } else { \ + if (start >= expr_start \ + && (OB_UNLIKELY((NULL == (raw_str = copy_expr_string(result, start, expr_end)))))) { \ + yyerror(NULL, result, "No more space for copying expression string"); \ + YYABORT_NO_MEMORY; \ + } else { \ + node->raw_text_ = raw_str; \ + node->text_len_ = expr_end - start + 1; \ + node->str_value_ = str_toupper(upper_value, expr_end - start + 1); \ + node->str_len_ = expr_end - start + 1; \ + setup_token_pos_info(node, expr_start - 1, expr_end - expr_start + 1); \ + } \ + } \ + } \ + } while(0) #define make_name_node(node, malloc_pool, name) \ do { \ @@ -863,6 +907,32 @@ for (int32_t _i = 0; _i < _yyleng; ++_i) { } \ } while (0); +#define PARSE_INT_STR_ORACLE(param_node, malloc_pool, errno) \ + do { \ + if ('-' == param_node->str_value_[0]) { \ + char *copied_str = parse_strndup(param_node->str_value_, param_node->str_len_, malloc_pool); \ + if (OB_ISNULL(copied_str)) { \ + ((ParseResult *)yyextra)->extra_errno_ = OB_PARSER_ERR_NO_MEMORY;\ + yyerror(NULL, yyextra, "No more space for mallocing"); \ + return ERROR; \ + } else { \ + int pos = 1; \ + for (; pos < param_node->str_len_ && ISSPACE(copied_str[pos]); pos++) ; \ + copied_str[--pos] = '-'; \ + param_node->value_ = ob_strntoll(copied_str + pos, param_node->str_len_ - pos, 10, NULL, &errno); \ + if (ERANGE == errno) { \ + param_node->type_ = T_NUMBER; \ + token_ret = DECIMAL_VAL; \ + } \ + } \ + } else { \ + param_node->value_ = ob_strntoll(param_node->str_value_, param_node->str_len_, 10, NULL, &errno); \ + if (ERANGE == errno) { \ + param_node->type_ = T_NUMBER; \ + token_ret = DECIMAL_VAL; \ + } \ + } \ + } while (0); #define RM_MULTI_STMT_END_P(p) \ do \ diff --git a/src/sql/plan_cache/ob_lib_cache_register.h b/src/sql/plan_cache/ob_lib_cache_register.h index 46ec949e3..f4b026065 100644 --- a/src/sql/plan_cache/ob_lib_cache_register.h +++ b/src/sql/plan_cache/ob_lib_cache_register.h @@ -19,6 +19,9 @@ LIB_CACHE_OBJ_DEF(NS_TRGR, "TRGR", pl::ObPLObjectKey, pl::ObPLObjectSet, pl::ObP LIB_CACHE_OBJ_DEF(NS_PKG, "PKG", pl::ObPLObjectKey, pl::ObPLObjectSet, pl::ObPLPackage, ObNewModIds::OB_SQL_PHY_PL_OBJ) // package cache LIB_CACHE_OBJ_DEF(NS_TABLEAPI, "TABLEAPI", table::ObTableApiCacheKey, table::ObTableApiCacheNode, table::ObTableApiCacheObj, "OB_TABLEAPI_OBJ") // tableapi cache LIB_CACHE_OBJ_DEF(NS_CALLSTMT, "CALLSTMT", pl::ObPLObjectKey, pl::ObPLObjectSet, ObCallProcedureInfo, ObNewModIds::OB_SQL_PHY_PL_OBJ) // call stmt cache +#ifdef OB_BUILD_SPM +LIB_CACHE_OBJ_DEF(NS_SPM, "SPM", ObBaselineKey, ObSpmSet, ObPlanBaselineItem, "OB_SQL_SPM_OBJ") // baseline cache +#endif /*OB_BUILD_SPM*/ #endif /*LIB_CACHE_OBJ_DEF*/ #ifndef OCEANBASE_SQL_PLAN_CACHE_OB_LIB_CACHE_REGISTER_ diff --git a/src/sql/plan_cache/ob_pcv_set.cpp b/src/sql/plan_cache/ob_pcv_set.cpp index e7683d22b..43e25a089 100644 --- a/src/sql/plan_cache/ob_pcv_set.cpp +++ b/src/sql/plan_cache/ob_pcv_set.cpp @@ -32,6 +32,9 @@ int ObPCVSet::init(ObILibCacheCtx &ctx, const ObILibCacheObject *obj) { int ret = OB_SUCCESS; ObSQLSessionInfo* sess; +#ifdef OB_BUILD_SPM + bool is_spm_on = false; +#endif ObPlanCacheCtx &pc_ctx = static_cast(ctx); const ObPlanCacheObject *cache_obj = static_cast(obj); if (is_inited_) { @@ -43,6 +46,12 @@ int ObPCVSet::init(ObILibCacheCtx &ctx, const ObILibCacheObject *obj) } else if (NULL == (sess = pc_ctx.sql_ctx_.session_info_)) { ret = OB_ERR_UNEXPECTED; SQL_PC_LOG(WARN, "session info is null", K(ret)); +#ifdef OB_BUILD_SPM + } else if (OB_FAIL(sess->get_use_plan_baseline(is_spm_on))) { + LOG_WARN("fail to get spm config"); + } else if (FALSE_IT(is_spm_closed_ = (!is_spm_on))) { + // do nothing +#endif } else if (NULL == (pc_alloc_ = lib_cache_->get_pc_allocator())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid plan cache allocator", K_(pc_alloc), K(ret)); @@ -110,6 +119,9 @@ int ObPCVSet::inner_get_cache_obj(ObILibCacheCtx &ctx, { UNUSED(key); int ret = OB_SUCCESS; +#ifdef OB_BUILD_SPM + bool is_spm_on = false; +#endif ObPlanCacheObject *plan = NULL; ObPlanCacheCtx &pc_ctx = static_cast(ctx); if (PC_PS_MODE == pc_ctx.mode_ || PC_PL_MODE == pc_ctx.mode_) { @@ -150,6 +162,13 @@ int ObPCVSet::inner_get_cache_obj(ObILibCacheCtx &ctx, ret = OB_INVALID_ARGUMENT; LOG_WARN("unexpected null schema guard", K(ret), K(pc_ctx.sql_ctx_.schema_guard_), K(pc_ctx.sql_ctx_.session_info_)); +#ifdef OB_BUILD_SPM + } else if (OB_FAIL(pc_ctx.sql_ctx_.session_info_->get_use_plan_baseline(is_spm_on))) { + LOG_WARN("failed to get spm status", K(ret)); + } else if (is_spm_closed_ != (!is_spm_on)) { + // spm param is altered + ret = OB_OLD_SCHEMA_VERSION; +#endif } else { ObSEArray schema_array; //plan cache匹配临时表应该始终使用用户创建的session才能保证语义的正确性 @@ -521,6 +540,21 @@ int ObPCVSet::check_contains_table(uint64_t db_id, common::ObString tab_name, bo return ret; } +#ifdef OB_BUILD_SPM +int ObPCVSet::get_evolving_evolution_task(EvolutionPlanList &evo_task_list) +{ + int ret = OB_SUCCESS; + DLIST_FOREACH(pcv, pcv_list_) { + if (OB_ISNULL(pcv)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(pcv), K(ret)); + } else if (OB_FAIL(pcv->get_evolving_evolution_task(evo_task_list))) { + LOG_WARN("fail to get evolving evolution task", K(ret)); + } + } + return ret; +} +#endif } } diff --git a/src/sql/plan_cache/ob_pcv_set.h b/src/sql/plan_cache/ob_pcv_set.h index 9e7daefcc..40e2332dc 100644 --- a/src/sql/plan_cache/ob_pcv_set.h +++ b/src/sql/plan_cache/ob_pcv_set.h @@ -68,7 +68,12 @@ public: normal_parse_const_cnt_(0), min_cluster_version_(0), plan_num_(0), +#ifndef OB_BUILD_SPM need_check_gen_tbl_col_(false) +#else + need_check_gen_tbl_col_(false), + is_spm_closed_(false) +#endif { } virtual ~ObPCVSet() @@ -96,6 +101,10 @@ public: const ObString &get_sql() { return sql_; } int deep_copy_sql(const common::ObString &sql); int check_contains_table(uint64_t db_id, common::ObString tab_name, bool &contains); +#ifdef OB_BUILD_SPM + bool get_spm_closed() const { return is_spm_closed_; } + int get_evolving_evolution_task(EvolutionPlanList &evo_task_list); +#endif TO_STRING_KV(K_(is_inited)); @@ -128,6 +137,9 @@ private: int64_t plan_num_; common::ObSEArray sql_ids_; bool need_check_gen_tbl_col_; +#ifdef OB_BUILD_SPM + bool is_spm_closed_; +#endif common::ObFixedArray col_field_arr_; }; diff --git a/src/sql/plan_cache/ob_plan_cache.cpp b/src/sql/plan_cache/ob_plan_cache.cpp index 1eda6035b..64ef2a585 100644 --- a/src/sql/plan_cache/ob_plan_cache.cpp +++ b/src/sql/plan_cache/ob_plan_cache.cpp @@ -37,6 +37,11 @@ #include "pl/ob_pl.h" #include "pl/ob_pl_package.h" #include "observer/ob_req_time_service.h" +#ifdef OB_BUILD_SPM +#include "sql/spm/ob_spm_define.h" +#include "sql/spm/ob_spm_controller.h" +#include "sql/spm/ob_spm_evolution_plan.h" +#endif #include "pl/pl_cache/ob_pl_cache_mgr.h" using namespace oceanbase::common; @@ -153,6 +158,37 @@ struct ObGetKVEntryBySQLIDOp : public ObKVEntryTraverseOp common::ObString sql_id_; }; +#ifdef OB_BUILD_SPM +struct ObGetPlanBaselineBySQLIDOp : public ObKVEntryTraverseOp +{ + explicit ObGetPlanBaselineBySQLIDOp(uint64_t db_id, + common::ObString sql_id, + LCKeyValueArray *key_val_list, + const CacheRefHandleID ref_handle) + : ObKVEntryTraverseOp(key_val_list, ref_handle), + db_id_(db_id), + sql_id_(sql_id) + { + } + virtual int check_entry_match(LibCacheKVEntry &entry, bool &is_match) + { + int ret = OB_SUCCESS; + is_match = false; + if (ObLibCacheNameSpace::NS_SPM == entry.first->namespace_) { + ObBaselineKey *key = static_cast(entry.first); + if (db_id_ != common::OB_INVALID_ID && db_id_ != key->db_id_) { + // skip entry that has non-matched db_id + } else if (sql_id_ == key->sql_id_) { + is_match = true; + } + } + return ret; + } + + uint64_t db_id_; + common::ObString sql_id_; +}; +#endif struct ObGetPcvSetByTabNameOp : public ObKVEntryTraverseOp { @@ -188,6 +224,36 @@ struct ObGetPcvSetByTabNameOp : public ObKVEntryTraverseOp common::ObString tab_name_; }; +#ifdef OB_BUILD_SPM +struct ObGetEvolutionTaskPcvSetOp : public ObKVEntryTraverseOp +{ + explicit ObGetEvolutionTaskPcvSetOp(EvolutionPlanList *evo_task_list, + LCKeyValueArray *key_val_list, + const CacheRefHandleID ref_handle) + : ObKVEntryTraverseOp(key_val_list, ref_handle), + evo_task_list_(evo_task_list) + { + } + + virtual int check_entry_match(LibCacheKVEntry &entry, bool &is_match) + { + int ret = common::OB_SUCCESS; + is_match = false; + if (entry.first->namespace_ == ObLibCacheNameSpace::NS_CRSR) { + ObPCVSet *node = static_cast(entry.second); + int64_t origin_count = evo_task_list_->count(); + if (OB_FAIL(node->get_evolving_evolution_task(*evo_task_list_))) { + LOG_WARN("failed to get evolving evolution task", K(ret)); + } else { + is_match = evo_task_list_->count() > origin_count; + } + } + return ret; + } + + EvolutionPlanList *evo_task_list_; +}; +#endif struct ObGetTableIdOp { @@ -1295,6 +1361,20 @@ int ObPlanCache::cache_evict_plan_by_sql_id(uint64_t db_id, common::ObString sql return ret; } +#ifdef OB_BUILD_SPM +int ObPlanCache::cache_evict_baseline_by_sql_id(uint64_t db_id, common::ObString sql_id) +{ + int ret = OB_SUCCESS; + SQL_PC_LOG(TRACE, "cache evict plan baseline by sql id start"); + LCKeyValueArray to_evict_keys; + ObGetPlanBaselineBySQLIDOp get_ids_op(db_id, sql_id, &to_evict_keys, PLAN_BASELINE_HANDLE); + if (OB_FAIL(foreach_cache_evict(get_ids_op))) { + SQL_PC_LOG(WARN, "failed to foreach cache evict", K(ret)); + } + SQL_PC_LOG(TRACE, "cache evict plan baseline by sql id end"); + return ret; +} +#endif int ObPlanCache::evict_plan_by_table_name(uint64_t database_id, ObString tab_name) { @@ -1465,6 +1545,69 @@ int ObPlanCache::cache_evict_by_glitch_node() // return ret; // } +#ifdef OB_BUILD_SPM +int ObPlanCache::load_plan_baseline(const obrpc::ObLoadPlanBaselineArg &arg, uint64_t &load_count) +{ + int ret = OB_SUCCESS; + common::ObSEArray plan_ids; + ObGlobalReqTimeService::check_req_timeinfo(); + ObGetPlanIdBySqlIdOp plan_id_op(&plan_ids, arg.sql_id_, arg.with_plan_hash_, arg.plan_hash_value_); + load_count = 0; + if (OB_FAIL(co_mgr_.foreach_cache_obj(plan_id_op))) { + LOG_WARN("fail to traverse id2stat_map", K(ret)); + } else { + ObPhysicalPlan *plan = NULL; + LOG_INFO("load plan baseline by sql ids", K(arg), K(plan_ids)); + for (int64_t i = 0; i < plan_ids.count(); i++) { + uint64_t plan_id= plan_ids.at(i); + ObCacheObjGuard guard(LOAD_BASELINE_HANDLE); + int tmp_ret = ref_plan(plan_id, guard); //plan引用计数加1 + plan = static_cast(guard.cache_obj_); + if (OB_HASH_NOT_EXIST == tmp_ret) { + //do nothing; + } else if (OB_SUCCESS != tmp_ret || NULL == plan) { + LOG_WARN("get plan failed", K(tmp_ret), KP(plan)); + } else { + LOG_INFO("load plan baseline by sql id", K(arg)); + if (OB_FAIL(ObSpmController::load_baseline(arg, plan))) { + LOG_WARN("failed to load baseline", K(ret)); + } else { + ++load_count; + } + } + } + } + return ret; +} + +int ObPlanCache::check_baseline_finish() +{ + int ret = OB_SUCCESS; + LCKeyValueArray hold_keys; + EvolutionPlanList evo_task_list; + ObGetEvolutionTaskPcvSetOp get_evo_op(&evo_task_list, &hold_keys, CHECK_EVOLUTION_PLAN_HANDLE); + ObGlobalReqTimeService::check_req_timeinfo(); + if (OB_FAIL(cache_key_node_map_.foreach_refactored(get_evo_op))) { + LOG_WARN("traversing cache_key_node_map failed"); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < evo_task_list.count(); ++i) { + ObEvolutionPlan *evo_plan = evo_task_list.at(i); + if (OB_NOT_NULL(evo_plan) && evo_plan->get_is_evolving_flag()) { + evo_plan->check_task_need_finish(); + } + } + } + //decrement reference count anyway + int64_t N = hold_keys.count(); + for (int64_t i = 0; i < N; i++) { + if (NULL != hold_keys.at(i).node_) { + hold_keys.at(i).node_->dec_ref_count(get_evo_op.get_ref_handle()); + } + } + return ret; +} + +#endif // 计算plan_cache需要淘汰的pcv_set个数 // ret = true表示执行正常，否则失败 diff --git a/src/sql/plan_cache/ob_plan_cache.h b/src/sql/plan_cache/ob_plan_cache.h index bafa577b3..913d13375 100644 --- a/src/sql/plan_cache/ob_plan_cache.h +++ b/src/sql/plan_cache/ob_plan_cache.h @@ -52,6 +52,9 @@ class ObLibCacheAtomicOp; class ObEvolutionPlan; typedef common::hash::ObHashMap PlanCacheMap; +#ifdef OB_BUILD_SPM +typedef common::ObSEArray EvolutionPlanList; +#endif struct ObKVEntryTraverseOp { @@ -356,6 +359,13 @@ public: int cache_evict_by_ns(ObLibCacheNameSpace ns); template int foreach_cache_evict(CallBack &cb); +#ifdef OB_BUILD_SPM + int cache_evict_baseline_by_sql_id(uint64_t db_id, common::ObString sql_id); + // load plan baseline from plan cache + // int load_plan_baseline(); + int load_plan_baseline(const obrpc::ObLoadPlanBaselineArg &arg, uint64_t &load_count); + int check_baseline_finish(); +#endif void destroy(); common::ObAddr &get_host() { return host_; } void set_host(common::ObAddr &addr) { host_ = addr; } diff --git a/src/sql/plan_cache/ob_plan_cache_value.cpp b/src/sql/plan_cache/ob_plan_cache_value.cpp index 66892add9..06e3b15e4 100644 --- a/src/sql/plan_cache/ob_plan_cache_value.cpp +++ b/src/sql/plan_cache/ob_plan_cache_value.cpp @@ -169,6 +169,9 @@ ObPlanCacheValue::ObPlanCacheValue() sessid_(OB_INVALID_ID), sess_create_time_(0), contain_sys_name_table_(false), +#ifdef OB_BUILD_SPM + is_spm_closed_(false), +#endif need_param_(true), is_nested_sql_(false), is_batch_execute_(false), @@ -256,6 +259,9 @@ int ObPlanCacheValue::init(ObPCVSet *pcv_set, const ObILibCacheObject *cache_obj sys_schema_version_ = plan->get_sys_schema_version(); tenant_schema_version_ = plan->get_tenant_schema_version(); sql_traits_ = pc_ctx.sql_traits_; +#ifdef OB_BUILD_SPM + is_spm_closed_ = pcv_set->get_spm_closed(); +#endif stmt_type_ = plan->get_stmt_type(); need_param_ = plan->need_param(); is_nested_sql_ = ObSQLUtils::is_nested_sql(&pc_ctx.exec_ctx_); @@ -2344,6 +2350,23 @@ int ObPlanCacheValue::check_contains_table(uint64_t db_id, common::ObString tab_ return ret; } +#ifdef OB_BUILD_SPM +int ObPlanCacheValue::get_evolving_evolution_task(EvolutionPlanList &evo_task_list) +{ + int ret = OB_SUCCESS; + DLIST_FOREACH(plan_set, plan_sets_) { + if (OB_ISNULL(plan_set)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(plan_set), K(ret)); + } else if (!plan_set->is_sql_planset()) { + // do nothing + } else if (OB_FAIL(static_cast(plan_set)->get_evolving_evolution_task(evo_task_list))) { + LOG_WARN("fail to get evolving evolution task", K(ret)); + } + } + return ret; +} +#endif }//end of namespace sql }//end of namespace oceanbase diff --git a/src/sql/plan_cache/ob_plan_cache_value.h b/src/sql/plan_cache/ob_plan_cache_value.h index c1ae67f59..29bb40ad2 100644 --- a/src/sql/plan_cache/ob_plan_cache_value.h +++ b/src/sql/plan_cache/ob_plan_cache_value.h @@ -278,6 +278,9 @@ public: int lift_tenant_schema_version(int64_t new_schema_version); int check_contains_table(uint64_t db_id, common::ObString tab_name, bool &contains); +#ifdef OB_BUILD_SPM + int get_evolving_evolution_task(EvolutionPlanList &evo_task_list); +#endif private: //used for add plan //check table version, view table version, merged version @@ -406,6 +409,9 @@ private: uint64_t sess_create_time_; // wether this pcv's plans contains sys table (oracle mode) bool contain_sys_name_table_; +#ifdef OB_BUILD_SPM + bool is_spm_closed_; +#endif bool need_param_; //at present, if a SQL is in nested sql, it is forced to use DAS plan diff --git a/src/sql/plan_cache/ob_plan_set.cpp b/src/sql/plan_cache/ob_plan_set.cpp index fa0ed5e80..6596cfbfa 100644 --- a/src/sql/plan_cache/ob_plan_set.cpp +++ b/src/sql/plan_cache/ob_plan_set.cpp @@ -1233,6 +1233,9 @@ int ObSqlPlanSet::add_plan(ObPhysicalPlan &plan, if (OB_FAIL(add_physical_plan(OB_PHY_PLAN_LOCAL, pc_ctx, plan))) { SQL_PC_LOG(TRACE, "fail to add local plan", K(ret)); } else if (OB_SUCC(ret) +#ifdef OB_BUILD_SPM + && is_spm_closed_ +#endif && FALSE_IT(direct_local_plan_ = &plan)) { // do nothing } else { @@ -1318,10 +1321,19 @@ int ObSqlPlanSet::init_new_set(const ObPlanCacheCtx &pc_ctx, outline_param_idx_ = outline_param_idx; need_try_plan_ = 0; has_duplicate_table_ = false; +#ifdef OB_BUILD_SPM + bool is_spm_on = false; +#endif const ObSQLSessionInfo *session_info = sql_ctx.session_info_; if (OB_ISNULL(session_info)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid null plan cache or session info", K(ret), K(session_info)); +#ifdef OB_BUILD_SPM + } else if (OB_FAIL(session_info->get_use_plan_baseline(is_spm_on))) { + LOG_WARN("failed to get spm configs", K(ret)); + } else if (FALSE_IT(is_spm_closed_ = (!is_spm_on))) { + // do nothing +#endif } else if (OB_ISNULL(pc_malloc_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("pc_allocator has not been initialized.", K(ret)); @@ -1330,6 +1342,12 @@ int ObSqlPlanSet::init_new_set(const ObPlanCacheCtx &pc_ctx, } else if (OB_FAIL(table_locations_.prepare_allocate_and_keep_count(sql_ctx.partition_infos_.count(), *plan_cache_value_->get_pcv_set()->get_allocator()))) { LOG_WARN("fail to init table location count", K(ret)); +#ifdef OB_BUILD_SPM + } else if (OB_FAIL(local_evolution_plan_.init(this))) { + SQL_PC_LOG(WARN, "failed to init local evolution plan", K(ret)); + } else if (OB_FAIL(dist_evolution_plan_.init(this))) { + SQL_PC_LOG(WARN, "failed to init dist evolution plan", K(ret)); +#endif } else if (OB_FAIL(dist_plans_.init(this))) { SQL_PC_LOG(WARN, "failed to init dist plans", K(ret)); } else { @@ -1567,6 +1585,7 @@ int ObSqlPlanSet::add_physical_plan(const ObPhyPlanType plan_type, if (plan_type != OB_PHY_PLAN_LOCAL && plan_type != OB_PHY_PLAN_DISTRIBUTED) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid plan type", K(ret), K(plan_type)); +#ifndef OB_BUILD_SPM } else if (OB_PHY_PLAN_LOCAL == plan_type) { if (NULL != local_plan_) { ret = OB_SQL_PC_PLAN_DUPLICATE; @@ -1576,6 +1595,48 @@ int ObSqlPlanSet::add_physical_plan(const ObPhyPlanType plan_type, } else if (OB_FAIL(dist_plans_.add_plan(plan, pc_ctx))) { LOG_WARN("failed to add dist plan", K(ret), K(plan)); } +#else + } else if (!pc_ctx.need_evolution_ || is_spm_closed_) { + // Addition of other non-evolving plans is prohibited in plan evolution + if (OB_PHY_PLAN_LOCAL == plan_type) { + if (local_evolution_plan_.get_is_evolving_flag()) { + ret = OB_SQL_PC_PLAN_DUPLICATE; + LOG_WARN("addition of other non-evolving plans is prohibited in local plan evolution"); + } else if (NULL != local_plan_) { + ret = OB_SQL_PC_PLAN_DUPLICATE; + } else { + local_plan_ = &plan; + } + } else { + if (OB_FAIL(dist_plans_.add_plan(plan, pc_ctx))) { + LOG_WARN("failed to add dist plan", K(ret), K(plan)); + } + } + } else { + // Clear local_plan_ and dist_plans_ before adding evolution plans + ObSpmCacheCtx& spm_ctx = pc_ctx.sql_ctx_.spm_ctx_; + if (ObSpmCacheCtx::STAT_ADD_BASELINE_PLAN == spm_ctx.spm_stat_) { + if (OB_PHY_PLAN_LOCAL == spm_ctx.evolution_plan_type_) { + OZ (local_evolution_plan_.add_plan(pc_ctx, &plan)); + } else { + OZ (dist_evolution_plan_.add_plan(pc_ctx, &plan)); + } + } else { + if (OB_PHY_PLAN_LOCAL == plan_type) { + if (NULL != local_plan_) { + remove_cache_obj_entry(local_plan_->get_plan_id()); + local_plan_ = NULL; + } + OZ (local_evolution_plan_.add_plan(pc_ctx, &plan)); + OX (spm_ctx.evolution_plan_type_ = OB_PHY_PLAN_LOCAL); + } else { + OZ (dist_plans_.remove_plan_stat()); + OZ (dist_evolution_plan_.add_plan(pc_ctx, &plan)); + OX (spm_ctx.evolution_plan_type_ = OB_PHY_PLAN_DISTRIBUTED); + } + } + } +#endif return ret; } @@ -1586,6 +1647,7 @@ int ObSqlPlanSet::get_physical_plan(const ObPhyPlanType plan_type, int ret = OB_SUCCESS; plan = NULL; bool get_next = false; +#ifndef OB_BUILD_SPM if (plan_type != OB_PHY_PLAN_LOCAL && plan_type != OB_PHY_PLAN_DISTRIBUTED) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid plan type", K(ret), K(plan_type)); @@ -1594,9 +1656,63 @@ int ObSqlPlanSet::get_physical_plan(const ObPhyPlanType plan_type, } else if (OB_FAIL(dist_plans_.get_plan(pc_ctx, plan))) { LOG_DEBUG("failed to get dist plan", K(ret)); } +#else + if (plan_type != OB_PHY_PLAN_LOCAL && plan_type != OB_PHY_PLAN_DISTRIBUTED) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid plan type", K(ret), K(plan_type)); + } else if (OB_PHY_PLAN_LOCAL == plan_type) { + if (OB_FAIL(try_get_local_evolution_plan(pc_ctx, plan, get_next))) { + LOG_WARN("failed to try get local evloution plan", K(ret)); + } else if (get_next) { + plan = local_plan_; + } + if (OB_SUCC(ret) && NULL == plan) { + ret = OB_SQL_PC_NOT_EXIST; + } + } else if (OB_PHY_PLAN_DISTRIBUTED == plan_type) { + if (OB_FAIL(try_get_dist_evolution_plan(pc_ctx, plan, get_next))) { + LOG_WARN("failed to try get local evloution plan", K(ret)); + } else if (get_next && OB_FAIL(dist_plans_.get_plan(pc_ctx, plan))) { + LOG_TRACE("failed to get dist plan", K(ret)); + } + if (OB_SUCC(ret) && NULL == plan) { + ret = OB_SQL_PC_NOT_EXIST; + } + } +#endif return ret; } +#ifdef OB_BUILD_SPM +int ObSqlPlanSet::add_evolution_plan_for_spm(ObPhysicalPlan *plan, ObPlanCacheCtx &ctx) +{ + int ret = OB_SUCCESS; + ObPlanCache *pc = NULL; + if (NULL == plan_cache_value_ + || NULL == plan_cache_value_->get_pcv_set() + || NULL == plan_cache_value_->get_pcv_set()->get_plan_cache()) { + ret = OB_NOT_INIT; + } else if (OB_ISNULL(plan)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("plan is null", K(ret)); + } else if (OB_PHY_PLAN_LOCAL != plan->get_plan_type() + && OB_PHY_PLAN_DISTRIBUTED != plan->get_plan_type()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("not supported type", K(ret), K(plan->get_plan_type())); + } else if (FALSE_IT(pc = plan_cache_value_->get_pcv_set()->get_plan_cache())) { + } else if (OB_PHY_PLAN_LOCAL == plan->get_plan_type()) { + if (NULL != local_plan_) { + ret = OB_SQL_PC_PLAN_DUPLICATE; + LOG_WARN("local plan duplicate", K(ret)); + } else { + local_plan_ = plan; + } + } else if (OB_FAIL(dist_plans_.add_evolution_plan(*plan, ctx))) { + LOG_WARN("failed to add dist plan", K(ret), K(plan)); + } + return ret; +} +#endif int ObSqlPlanSet::get_plan_normal(ObPlanCacheCtx &pc_ctx, ObPhysicalPlan *&plan) @@ -1695,6 +1811,66 @@ int ObSqlPlanSet::get_plan_normal(ObPlanCacheCtx &pc_ctx, return ret; } +#ifdef OB_BUILD_SPM +int ObSqlPlanSet::try_get_local_evolution_plan(ObPlanCacheCtx &pc_ctx, + ObPhysicalPlan *&plan, + bool &get_next) +{ + int ret = OB_SUCCESS; + plan = NULL; + get_next = false; + if ((!is_spm_closed_ && local_evolution_plan_.get_is_evolving_flag()) + || pc_ctx.sql_ctx_.spm_ctx_.force_get_evolution_plan()) { + if (OB_FAIL(local_evolution_plan_.get_plan(pc_ctx, plan))) { + if (OB_SQL_PC_NOT_EXIST == ret) { + ret = OB_SUCCESS; + } + SQL_PC_LOG(TRACE, "get evolution plan failed", K(ret)); + } + } + if (OB_SUCC(ret) && NULL == plan) { + get_next = true; + } + return ret; +} + +int ObSqlPlanSet::try_get_dist_evolution_plan(ObPlanCacheCtx &pc_ctx, + ObPhysicalPlan *&plan, + bool &get_next) +{ + int ret = OB_SUCCESS; + plan = NULL; + get_next = false; + if ((!is_spm_closed_ && dist_evolution_plan_.get_is_evolving_flag()) + || pc_ctx.sql_ctx_.spm_ctx_.force_get_evolution_plan()) { + if (OB_FAIL(dist_evolution_plan_.get_plan(pc_ctx, plan))) { + if (OB_SQL_PC_NOT_EXIST == ret) { + ret = OB_SUCCESS; + } + SQL_PC_LOG(TRACE, "get evolution plan failed", K(ret)); + } + } + if (OB_SUCC(ret) && NULL == plan) { + get_next = true; + } + return ret; +} + +int ObSqlPlanSet::try_get_evolution_plan(ObPlanCacheCtx &pc_ctx, + ObPhysicalPlan *&plan, + bool &get_next) +{ + int ret = OB_SUCCESS; + plan = NULL; + get_next = false; + if (OB_FAIL(try_get_local_evolution_plan(pc_ctx, plan, get_next))) { + LOG_WARN("failed to try get local evolution plan", K(ret)); + } else if (get_next && OB_FAIL(try_get_dist_evolution_plan(pc_ctx, plan, get_next))) { + LOG_WARN("failed to try get dist evolution plan", K(ret)); + } + return ret; +} +#endif int ObSqlPlanSet::try_get_local_plan(ObPlanCacheCtx &pc_ctx, ObPhysicalPlan *&plan, @@ -1784,6 +1960,11 @@ int ObSqlPlanSet::get_plan_special(ObPlanCacheCtx &pc_ctx, bool get_next = true; ObPhyPlanType real_type = OB_PHY_PLAN_UNINITIALIZED; ObSEArray candi_table_locs; +#ifdef OB_BUILD_SPM + if (OB_FAIL(try_get_evolution_plan(pc_ctx, plan, get_next))) { + SQL_PC_LOG(TRACE, "get evolution plan failed", K(ret)); + } +#endif // try local plan if (OB_SUCC(ret) && get_next) { if (OB_FAIL(try_get_local_plan(pc_ctx, plan, get_next))) { @@ -1811,6 +1992,10 @@ int ObSqlPlanSet::get_plan_special(ObPlanCacheCtx &pc_ctx, int64_t ObSqlPlanSet::get_mem_size() { int64_t plan_set_mem = 0; +#ifdef OB_BUILD_SPM + plan_set_mem += local_evolution_plan_.get_mem_size(); + plan_set_mem += dist_evolution_plan_.get_mem_size(); +#endif if (NULL != local_plan_) { plan_set_mem += local_plan_->get_mem_size(); } @@ -1835,6 +2020,10 @@ void ObSqlPlanSet::reset() //do nothing LOG_WARN_RET(OB_ERR_UNEXPECTED, "plan_cache_value or pc allocator is NULL"); } +#ifdef OB_BUILD_SPM + local_evolution_plan_.reset(); + dist_evolution_plan_.reset(); +#endif local_plan_ = NULL; array_binding_plan_ = NULL; remote_plan_ = NULL; @@ -2030,6 +2219,10 @@ int ObSqlPlanSet::is_partition_in_same_server(const ObIArray &c void ObSqlPlanSet::remove_all_plan() { +#ifdef OB_BUILD_SPM + IGNORE_RETURN local_evolution_plan_.remove_all_plan(); + IGNORE_RETURN dist_evolution_plan_.remove_all_plan(); +#endif IGNORE_RETURN dist_plans_.remove_all_plan(); } @@ -2195,6 +2388,20 @@ bool ObSqlPlanSet::is_sql_planset() return true; } +#ifdef OB_BUILD_SPM +int ObSqlPlanSet::get_evolving_evolution_task(EvolutionPlanList &evo_task_list) +{ + int ret = OB_SUCCESS; + if (local_evolution_plan_.get_is_evolving_flag() && + OB_FAIL(evo_task_list.push_back(&local_evolution_plan_))) { + LOG_WARN("failed to push back evolution plan", K(ret)); + } else if (dist_evolution_plan_.get_is_evolving_flag() && + OB_FAIL(evo_task_list.push_back(&dist_evolution_plan_))) { + LOG_WARN("failed to push back evolution plan", K(ret)); + } + return ret; +} +#endif } } diff --git a/src/sql/plan_cache/ob_plan_set.h b/src/sql/plan_cache/ob_plan_set.h index d6c0a510c..75828d914 100644 --- a/src/sql/plan_cache/ob_plan_set.h +++ b/src/sql/plan_cache/ob_plan_set.h @@ -23,6 +23,9 @@ #include "sql/plan_cache/ob_dist_plans.h" #include "share/schema/ob_schema_struct.h" #include "lib/hash/ob_hashset.h" +#ifdef OB_BUILD_SPM +#include "sql/spm/ob_spm_evolution_plan.h" +#endif namespace test { @@ -275,7 +278,12 @@ public: has_duplicate_table_(false), //has_array_binding_(false), is_contain_virtual_table_(false), +#ifdef OB_BUILD_SPM + enable_inner_part_parallel_exec_(false), + is_spm_closed_(false) +#else enable_inner_part_parallel_exec_(false) +#endif { } @@ -306,6 +314,10 @@ public: inline bool has_duplicate_table() const { return has_duplicate_table_; } //inline bool has_array_binding() const { return has_array_binding_; } inline bool enable_inner_part_parallel() const { return enable_inner_part_parallel_exec_; } +#ifdef OB_BUILD_SPM + int add_evolution_plan_for_spm(ObPhysicalPlan *plan, ObPlanCacheCtx &ctx); + int get_evolving_evolution_task(EvolutionPlanList &evo_task_list); +#endif private: enum { @@ -335,6 +347,17 @@ private: int get_plan_special(ObPlanCacheCtx &pc_ctx, ObPhysicalPlan *&plan); +#ifdef OB_BUILD_SPM + int try_get_local_evolution_plan(ObPlanCacheCtx &pc_ctx, + ObPhysicalPlan *&plan, + bool &get_next); + int try_get_dist_evolution_plan(ObPlanCacheCtx &pc_ctx, + ObPhysicalPlan *&plan, + bool &get_next); + int try_get_evolution_plan(ObPlanCacheCtx &pc_ctx, + ObPhysicalPlan *&plan, + bool &get_next); +#endif int try_get_local_plan(ObPlanCacheCtx &pc_ctx, ObPhysicalPlan *&plan, bool &get_next); @@ -375,6 +398,10 @@ private: //used for array binding, only local plan ObPhysicalPlan *array_binding_plan_; ObPhysicalPlan *local_plan_; +#ifdef OB_BUILD_SPM + ObEvolutionPlan local_evolution_plan_; + ObEvolutionPlan dist_evolution_plan_; +#endif ObPhysicalPlan *remote_plan_; // for directly get plan ObPhysicalPlan *direct_local_plan_; @@ -391,6 +418,9 @@ private: bool is_contain_virtual_table_; // px并行度是否大于1 bool enable_inner_part_parallel_exec_; +#ifdef OB_BUILD_SPM + bool is_spm_closed_; +#endif }; inline ObPlanSetType ObPlanSet::get_plan_set_type_by_cache_obj_type(ObLibCacheNameSpace ns) diff --git a/src/sql/plan_cache/ob_sql_parameterization.cpp b/src/sql/plan_cache/ob_sql_parameterization.cpp index 85e9b0aaf..651ab6b8e 100644 --- a/src/sql/plan_cache/ob_sql_parameterization.cpp +++ b/src/sql/plan_cache/ob_sql_parameterization.cpp @@ -1018,6 +1018,9 @@ int ObSqlParameterization::parameterize_syntax_tree(common::ObIAllocator &alloca SQL_PC_LOG(ERROR, "got session is NULL", K(ret)); } else if (is_prepare_mode(mode) || is_transform_outline +#ifdef OB_BUILD_SPM + || pc_ctx.sql_ctx_.spm_ctx_.is_retry_for_spm_ +#endif ) { // if so, faster parser is needed // otherwise, fast parser has been done before diff --git a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp index 428915068..9fbadca34 100644 --- a/src/sql/resolver/cmd/ob_alter_system_resolver.cpp +++ b/src/sql/resolver/cmd/ob_alter_system_resolver.cpp @@ -19,6 +19,9 @@ #include "share/ob_locality_parser.h" #include "share/ob_time_utility2.h" #include "share/ob_encryption_util.h" +#ifdef OB_BUILD_TDE_SECURITY +#include "share/ob_encrypt_kms.h" +#endif #include "observer/ob_server_struct.h" #include "observer/omt/ob_tenant_config_mgr.h" #include "share/ob_zone_table_operation.h" @@ -1961,6 +1964,20 @@ int ObSetConfigResolver::resolve(const ParseNode &parse_tree) if (OB_FAIL(item.tenant_name_.assign(tenant_name))) { LOG_WARN("assign tenant name failed", K(tenant_name), K(ret)); break; +#ifdef OB_BUILD_TDE_SECURITY + } else if (0 == config_name.case_compare(TDE_METHOD) + || 0 == config_name.case_compare(EXTERNAL_KMS_INFO)) { + if (OB_ISNULL(schema_checker_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("schema checker is null", K(ret)); + } else if (OB_FAIL(schema_checker_->get_tenant_id(tenant_name, tenant_node_id))) { + LOG_WARN("fail to get tenant id", K(ret)); + } else if (OB_FAIL(check_param_valid(tenant_node_id, + config_name, + ObString(item.value_.size(), item.value_.ptr())))) { + LOG_WARN("fail to check param valid", K(ret)); + } +#endif } } } else { @@ -2042,14 +2059,32 @@ int ObSetConfigResolver::check_param_valid(int64_t tenant_id , int ret = OB_SUCCESS; if (tenant_id == OB_SYS_TENANT_ID) { if (0 == name.case_compare(SSL_EXTERNAL_KMS_INFO)) { +#ifndef OB_BUILD_TDE_SECURITY ret = OB_NOT_SUPPORTED; +#else + share::ObSSLClient client; + if (OB_FAIL(client.init(value.ptr(), value.length()))) { + OB_LOG(WARN, "ssl client init", K(ret), K(value)); + } else if (OB_FAIL(client.check_param_valid())) { + OB_LOG(WARN, "ssl client param is not valid", K(ret)); + } +#endif } else if (0 == name.case_compare("ssl_client_authentication") && (0 == value.case_compare("1") || 0 == value.case_compare("ON") || 0 == value.case_compare("TRUE"))) { ObString ssl_config(GCONF.ssl_external_kms_info.str()); if (!ssl_config.empty()) { +#ifndef OB_BUILD_TDE_SECURITY ret = OB_NOT_SUPPORTED; +#else + share::ObSSLClient client; + if (OB_FAIL(client.init(ssl_config.ptr(), ssl_config.length()))) { + OB_LOG(WARN, "kms client init", K(ret), K(ssl_config)); + } else if (OB_FAIL(client.check_param_valid())) { + OB_LOG(WARN, "kms client param is not valid", K(ret)); + } +#endif } else { if (EASY_OK != easy_ssl_ob_config_check(OB_SSL_CA_FILE, OB_SSL_CERT_FILE, OB_SSL_KEY_FILE, NULL, NULL, true, false)) { @@ -2075,6 +2110,69 @@ int ObSetConfigResolver::check_param_valid(int64_t tenant_id , } } } +#ifdef OB_BUILD_TDE_SECURITY + if (OB_SUCC(ret)) { + if (0 == name.case_compare("tde_method")) { + ObString tde_method; + if (!ObTdeMethodUtil::is_valid(value)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not supported other method", K(value), K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter invalid tde_method"); + } else if (OB_FAIL(share::ObEncryptionUtil::get_tde_method(tenant_id, tde_method))) { + LOG_WARN("fail to check tenant is method internal", K(ret)); + } else if (0 != tde_method.case_compare("none") && 0 != value.case_compare(tde_method)) { + // tde_method修改规则 + // 当主密钥已经存在于租户内, 不允许修改为其他类型. + // 主备库场景放开此限制, 检查主密钥的类型是否和要修改的类型一致. + ObSchemaGetterGuard schema_guard; + const share::schema::ObKeystoreSchema *keystore_schema = NULL; + if (OB_ISNULL(GCTX.schema_service_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("schema service is empty"); + } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard( + tenant_id, schema_guard))) { + LOG_WARN("get_schema_guard failed"); + } else if (OB_FAIL(schema_guard.get_keystore_schema(tenant_id, keystore_schema))) { + LOG_WARN("fail get keystore schema", K(ret)); + } else if (OB_ISNULL(keystore_schema)) { + ret = OB_OBJECT_NAME_NOT_EXIST; + LOG_WARN("fail to get keystore schema", K(ret)); + } else if (0 != keystore_schema->get_master_key_id()) { + if (!GCTX.is_standby_cluster()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("alter tde method is not support", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter tde method with master key exists"); + } else if (0 == keystore_schema->get_master_key().case_compare("kms") + && 0 == value.case_compare("bkmi")) { + /*do nothing*/ + } else if (0 == keystore_schema->get_master_key().case_compare(value)) { + /*do nothing*/ + } else if (0 == value.case_compare("internal")) { + /*do nothing*/ + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("alter tde method is not support", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter tde method with master key exists"); + } + } + } + } else if (0 == name.case_compare(EXTERNAL_KMS_INFO)) { + ObString tde_method; + ObKmsClient *client = NULL; + ObArenaAllocator allocator(ObModIds::OB_SQL_COMPILE); + if (OB_FAIL(share::ObEncryptionUtil::get_tde_method(tenant_id, tde_method))) { + LOG_WARN("fail to get method internal", K(ret)); + } else if (OB_FAIL(ObKmsClientUtil::get_kms_client(allocator, tde_method, client))) { + LOG_WARN("fail to get kms client", K(tde_method), K(ret)); + } else if (OB_ISNULL(client)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("kms_client is null", K(tde_method), K(ret)); + } else if (OB_FAIL(client->init(value.ptr(), value.length()))) { + LOG_WARN("the json str is not valid", K(ret)); + } + } + } +#endif return ret; } @@ -2350,30 +2448,105 @@ int ObMigrateUnitResolver::resolve(const ParseNode &parse_tree) int ObAddArbitrationServiceResolver::resolve(const ParseNode &parse_tree) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ARBITRATION UNUSEDx(parse_tree); ret = OB_NOT_SUPPORTED; LOG_WARN("not supported in CE version", KR(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "add arbitration service in CE version"); +#else + if (OB_UNLIKELY(T_ADD_ARBITRATION_SERVICE != parse_tree.type_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("type is not T_ADD_ARBITRATION_SERVICE", "type", get_type_name(parse_tree.type_)); + } else if (OB_UNLIKELY(NULL == parse_tree.children_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("children should not be null"); + } else { + ObAddArbitrationServiceStmt *stmt = create_stmt(); + if (NULL == stmt) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("create ObAddArbitrationServiceStmt failed"); + } else { + stmt_ = stmt; + ObString dest(""); + if (OB_FAIL(Util::resolve_string(parse_tree.children_[0], dest))) { + LOG_WARN("resolve string failed", K(ret)); + } else if (OB_FAIL(stmt->get_rpc_arg().init(dest))) { + LOG_WARN("fail to init arg", K(ret), K(dest)); + } + } + } +#endif return ret; } int ObRemoveArbitrationServiceResolver::resolve(const ParseNode &parse_tree) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ARBITRATION UNUSEDx(parse_tree); ret = OB_NOT_SUPPORTED; LOG_WARN("not supported in CE version", KR(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "remove arbitration service in CE version"); +#else + if (OB_UNLIKELY(T_REMOVE_ARBITRATION_SERVICE != parse_tree.type_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("type is not T_REMOVE_ARBITRATION_SERVICE", "type", get_type_name(parse_tree.type_)); + } else if (OB_UNLIKELY(NULL == parse_tree.children_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("children should not be null"); + } else { + ObRemoveArbitrationServiceStmt *stmt = create_stmt(); + if (NULL == stmt) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("create ObRemoveArbitrationServiceStmt failed"); + } else { + stmt_ = stmt; + ObString dest(""); + if (OB_FAIL(Util::resolve_string(parse_tree.children_[0], dest))) { + LOG_WARN("resolve string failed", K(ret)); + } else if (OB_FAIL(stmt->get_rpc_arg().init(dest))) { + LOG_WARN("fail to init arg", K(ret), K(dest)); + } + } + } +#endif return ret; } int ObReplaceArbitrationServiceResolver::resolve(const ParseNode &parse_tree) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ARBITRATION UNUSEDx(parse_tree); ret = OB_NOT_SUPPORTED; LOG_WARN("not supported in CE version", KR(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "replace arbitration service in CE version"); +#else + if (OB_UNLIKELY(T_REPLACE_ARBITRATION_SERVICE != parse_tree.type_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("type is not T_REPLACE_ARBITRATION_SERVICE", "type", get_type_name(parse_tree.type_)); + } else if (OB_UNLIKELY(NULL == parse_tree.children_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("children should not be null"); + } else { + ObReplaceArbitrationServiceStmt *stmt = create_stmt(); + if (NULL == stmt) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("create ObReplaceArbitrationServiceStmt failed"); + } else { + stmt_ = stmt; + ObString arbitration_service(""); + ObString previous_arbitration_service(""); + if (OB_FAIL(Util::resolve_string(parse_tree.children_[0], previous_arbitration_service))) { + LOG_WARN("resolve string failed", K(ret)); + } else if (OB_FAIL(Util::resolve_string(parse_tree.children_[1], arbitration_service))) { + LOG_WARN("resolve string for previous arb service failed", K(ret)); + } else if (OB_FAIL(stmt->get_rpc_arg().init(arbitration_service, previous_arbitration_service))) { + LOG_WARN("fail to init arg", KR(ret), K(arbitration_service), K(previous_arbitration_service)); + } + } + } +#endif return ret; } @@ -2528,6 +2701,10 @@ int ObPhysicalRestoreTenantResolver::resolve(const ParseNode &parse_tree) } else if (OB_NOT_NULL(description_node) && OB_FAIL(Util::resolve_string(description_node, stmt->get_rpc_arg().description_))) { LOG_WARN("fail to resolve description", K(ret)); +#ifdef OB_BUILD_TDE_SECURITY + } else if (OB_FAIL(resolve_kms_encrypt_info(stmt->get_rpc_arg().restore_option_))) { + LOG_WARN("fail to resolve encrypt info", K(ret)); +#endif } else if (OB_FAIL(resolve_decryption_passwd(stmt->get_rpc_arg()))) { LOG_WARN("failed to resolve decrytion passwd", K(ret)); } else if (OB_FAIL(resolve_restore_source_array(stmt->get_rpc_arg()))) { @@ -2574,7 +2751,7 @@ int ObPhysicalRestoreTenantResolver::resolve(const ParseNode &parse_tree) stmt->set_is_preview(false); } else { if (T_TABLE_LIST == node->type_) { - // TODO table list restore not support, fix this 4.3 + // TODO(chongrong.th) table list restore not support, fix this in 4.3 ret = OB_NOT_SUPPORTED; LOG_USER_ERROR(OB_NOT_SUPPORTED, "table list restore is"); // store database_name/table_name with case sensitive. @@ -2616,6 +2793,54 @@ int ObPhysicalRestoreTenantResolver::resolve(const ParseNode &parse_tree) return ret; } +#ifdef OB_BUILD_TDE_SECURITY +int ObPhysicalRestoreTenantResolver::resolve_kms_encrypt_info(common::ObString store_option) +{ + int ret = OB_SUCCESS; + const char *encrypt_option_str = "kms_encrypt=true"; + ObString kms_var("kms_encrypt_info"); + int64_t encrypt_opt_str_len = strlen(encrypt_option_str); + bool is_kms_encrypt = false; + for (int i = 0; i <= store_option.length() - encrypt_opt_str_len; ++i) { + if (0 == STRNCASECMP(store_option.ptr() + i, encrypt_option_str, encrypt_opt_str_len)) { + is_kms_encrypt = true; + break; + } + } + if (is_kms_encrypt) { + ObObj value; + if (OB_ISNULL(session_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("session is null" , K(ret)); + } else if (OB_FAIL(session_info_->get_user_variable_value(kms_var, value))) { + LOG_WARN("fail to get user variable", K(ret)); + } else { + ObPhysicalRestoreTenantStmt *stmt = static_cast(stmt_); + stmt->get_rpc_arg().kms_info_ = value.get_varchar(); //浅拷贝即可 + bool is_valid = false; + if (OB_FAIL(check_kms_info_valid(stmt->get_rpc_arg().kms_info_, is_valid))) { + LOG_WARN("fail to check kms info valid", K(ret)); + } else if (!is_valid) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("kms info is not valid", K(ret)); + } + } + } + + return ret; +} + +int ObPhysicalRestoreTenantResolver::check_kms_info_valid(const ObString &kms_info, bool &is_valid) +{ + int ret = OB_SUCCESS; + if (kms_info.length() > 4000 || kms_info.length() < 0) { + is_valid = false; + } else { + is_valid = true; + } + return ret; +} +#endif int ObPhysicalRestoreTenantResolver::resolve_decryption_passwd( ObPhysicalRestoreTenantArg &arg) @@ -3406,6 +3631,60 @@ int ObAlterSystemSetResolver::check_param_valid(int64_t tenant_id , const ObString &name, const ObString &value) { int ret = OB_SUCCESS; +#ifdef OB_BUILD_TDE_SECURITY + if (0 == name.case_compare("tde_method")) { + ObString tde_method; + if (!ObTdeMethodUtil::is_valid(value)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("not supported other method", K(value), K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter invalid tde_method"); + } else if (OB_FAIL(share::ObEncryptionUtil::get_tde_method(tenant_id, tde_method))) { + LOG_WARN("fail to check tenant is method internal", K(ret)); + } else if (0 != tde_method.case_compare("none") && 0 != value.case_compare(tde_method)) { + // tde_method修改规则 + // 当主密钥已经存在于租户内, 不允许修改为其他类型. + // 主备库场景放开此限制, 检查主密钥的类型是否和要修改的类型一致. + const share::schema::ObKeystoreSchema *keystore_schema = NULL; + if (OB_FAIL(schema_checker_->get_keystore_schema(tenant_id, keystore_schema))) { + LOG_WARN("fail get keystore schema", K(ret)); + } else if (OB_ISNULL(keystore_schema)) { + ret = OB_OBJECT_NAME_NOT_EXIST; + LOG_WARN("fail to get keystore schema", K(ret)); + } else if (0 != keystore_schema->get_master_key_id()) { + if (!GCTX.is_standby_cluster()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("alter tde method is not support", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter tde method with master key exists"); + } else if (0 == keystore_schema->get_master_key().case_compare("kms") + && 0 == value.case_compare("bkmi")) { + /*do nothing*/ + } else if (0 == keystore_schema->get_master_key().case_compare(value)) { + /*do nothing*/ + } else if (ObTdeMethodUtil::is_internal(value)) { + /*do nothing*/ + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("alter tde method is not support", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter tde method with master key exists"); + } + } + } + } else if (0 == name.case_compare(EXTERNAL_KMS_INFO)) { + ObString tde_method; + ObKmsClient *client = NULL; + ObArenaAllocator allocator(ObModIds::OB_SQL_COMPILE); + if (OB_FAIL(share::ObEncryptionUtil::get_tde_method(tenant_id, tde_method))) { + LOG_WARN("fail to get method internal", K(ret)); + } else if (OB_FAIL(ObKmsClientUtil::get_kms_client(allocator, tde_method, client))) { + LOG_WARN("fail to get kms client", K(tde_method), K(ret)); + } else if (OB_ISNULL(client)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("kms_client is null", K(tde_method), K(ret)); + } else if (OB_FAIL(client->init(value.ptr(), value.length()))) { + LOG_WARN("the json str is not valid", K(ret)); + } + } +#endif return ret; } @@ -4136,7 +4415,75 @@ int ObDeletePolicyResolver::resolve(const ParseNode &parse_tree) int ObBackupKeyResolver::resolve(const ParseNode &parse_tree) { +#ifndef OB_BUILD_TDE_SECURITY int ret = OB_ERR_PARSE_SQL; +#else + int ret = OB_SUCCESS; + if (OB_UNLIKELY(T_BACKUP_KEY != parse_tree.type_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("type is not T_BACKUP_KEY", "type", get_type_name(parse_tree.type_)); + } else if (OB_UNLIKELY(NULL == parse_tree.children_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("children should not be null", K(ret)); + } else if (OB_UNLIKELY(4 != parse_tree.num_child_ && 3 != parse_tree.num_child_ )) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("children num not match", K(ret), "num_child", parse_tree.num_child_); + } else if (OB_ISNULL(session_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("session info should not be null", K(ret)); + } else { + uint64_t tenant_id = session_info_->get_login_tenant_id(); + const int64_t with_tenant = parse_tree.children_[0]->value_; + common::ObSArray backup_tenant_ids; + ObBackupPathString backup_dest; + ObString encrypt_key; + if (0 == with_tenant) { + ParseNode *dest_node = parse_tree.children_[1]; + ParseNode *encrypt_key_node = parse_tree.children_[2]; + if (nullptr != dest_node && OB_FAIL(backup_dest.assign(dest_node->str_value_))) { + LOG_WARN("failed to assign backup_dest", K(ret)); + } else if (nullptr != encrypt_key_node) { + encrypt_key.assign_ptr(encrypt_key_node->str_value_, + static_cast(encrypt_key_node->str_len_)); + } + } else if (1 == with_tenant && OB_SYS_TENANT_ID != tenant_id) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("user tenant cannot specify tenant names", K(ret), K(tenant_id)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "user tenant cannot specify tenant names"); + } else if (1 == with_tenant/*tenant level*/ && OB_SYS_TENANT_ID == tenant_id) { + ParseNode *t_node = parse_tree.children_[1]; + ParseNode *dest_node = parse_tree.children_[2]; + ParseNode *encrypt_key_node = parse_tree.children_[3]; + if (nullptr == t_node) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("No tenant name specified", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "No tenant name specified"); + } else if (OB_FAIL(Util::get_tenant_ids(*t_node, backup_tenant_ids))) { + LOG_WARN("failed to get tenant ids"); + } else if (backup_tenant_ids.count() != 1) { + ret = common::OB_INVALID_ARGUMENT; + LOG_USER_ERROR(OB_INVALID_ARGUMENT, "tenant list, count of tenants must be 1"); + } else if (FALSE_IT(tenant_id = backup_tenant_ids.at(0))) { + } else if (nullptr != dest_node && OB_FAIL(backup_dest.assign(dest_node->str_value_))) { + LOG_WARN("failed to assign backup_dest", K(ret)); + } else if (nullptr != encrypt_key_node) { + encrypt_key.assign_ptr(encrypt_key_node->str_value_, + static_cast(encrypt_key_node->str_len_)); + } + } + + ObBackupKeyStmt *stmt = create_stmt(); + if (OB_FAIL(ret)) { + } else if (nullptr == stmt) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_ERROR("create ObBackupKeyStmt failed"); + } else if (OB_FAIL(stmt->set_param(tenant_id, backup_dest, encrypt_key))) { + LOG_WARN("Failed to set param", K(ret), K(tenant_id)); + } else { + stmt_ = stmt; + } + } +#endif return ret; } diff --git a/src/sql/resolver/cmd/ob_alter_system_resolver.h b/src/sql/resolver/cmd/ob_alter_system_resolver.h index e57162212..412f9cf89 100644 --- a/src/sql/resolver/cmd/ob_alter_system_resolver.h +++ b/src/sql/resolver/cmd/ob_alter_system_resolver.h @@ -172,6 +172,10 @@ class ObPhysicalRestoreTenantResolver : public ObSystemCmdResolver virtual ~ObPhysicalRestoreTenantResolver() {} virtual int resolve(const ParseNode &parse_tree); private: +#ifdef OB_BUILD_TDE_SECURITY + int resolve_kms_encrypt_info(common::ObString store_option); + int check_kms_info_valid(const common::ObString &kms_info, bool &is_valid); +#endif int resolve_decryption_passwd(obrpc::ObPhysicalRestoreTenantArg &arg); int resolve_restore_source_array(obrpc::ObPhysicalRestoreTenantArg &arg); }; diff --git a/src/sql/resolver/cmd/ob_alter_system_stmt.h b/src/sql/resolver/cmd/ob_alter_system_stmt.h index ab7f92aa1..1eca5b452 100644 --- a/src/sql/resolver/cmd/ob_alter_system_stmt.h +++ b/src/sql/resolver/cmd/ob_alter_system_stmt.h @@ -389,7 +389,14 @@ class ObAddArbitrationServiceStmt : public ObSystemCmdStmt public: ObAddArbitrationServiceStmt() : ObSystemCmdStmt(stmt::T_ADD_ARBITRATION_SERVICE) {} virtual ~ObAddArbitrationServiceStmt() {} +#ifndef OB_BUILD_ARBITRATION TO_STRING_KV(N_STMT_TYPE, ((int)stmt_type_)); +#else + obrpc::ObAdminAddArbitrationServiceArg &get_rpc_arg() { return rpc_arg_; } + TO_STRING_KV(N_STMT_TYPE, ((int)stmt_type_), K_(rpc_arg)); +private: + obrpc::ObAdminAddArbitrationServiceArg rpc_arg_; +#endif }; class ObRemoveArbitrationServiceStmt : public ObSystemCmdStmt @@ -397,7 +404,14 @@ class ObRemoveArbitrationServiceStmt : public ObSystemCmdStmt public: ObRemoveArbitrationServiceStmt() : ObSystemCmdStmt(stmt::T_REMOVE_ARBITRATION_SERVICE) {} virtual ~ObRemoveArbitrationServiceStmt() {} +#ifndef OB_BUILD_ARBITRATION TO_STRING_KV(N_STMT_TYPE, ((int)stmt_type_)); +#else + obrpc::ObAdminRemoveArbitrationServiceArg &get_rpc_arg() { return rpc_arg_; } + TO_STRING_KV(N_STMT_TYPE, ((int)stmt_type_), K_(rpc_arg)); +private: + obrpc::ObAdminRemoveArbitrationServiceArg rpc_arg_; +#endif }; class ObReplaceArbitrationServiceStmt : public ObSystemCmdStmt @@ -405,7 +419,14 @@ class ObReplaceArbitrationServiceStmt : public ObSystemCmdStmt public: ObReplaceArbitrationServiceStmt() : ObSystemCmdStmt(stmt::T_REPLACE_ARBITRATION_SERVICE) {} virtual ~ObReplaceArbitrationServiceStmt() {} +#ifndef OB_BUILD_ARBITRATION TO_STRING_KV(N_STMT_TYPE, ((int)stmt_type_)); +#else + obrpc::ObAdminReplaceArbitrationServiceArg &get_rpc_arg() { return rpc_arg_; } + TO_STRING_KV(N_STMT_TYPE, ((int)stmt_type_), K_(rpc_arg)); +private: + obrpc::ObAdminReplaceArbitrationServiceArg rpc_arg_; +#endif }; class ObClearLocationCacheStmt : public ObSystemCmdStmt diff --git a/src/sql/resolver/cmd/ob_show_resolver.cpp b/src/sql/resolver/cmd/ob_show_resolver.cpp index 2a3ca4ff2..12230410e 100644 --- a/src/sql/resolver/cmd/ob_show_resolver.cpp +++ b/src/sql/resolver/cmd/ob_show_resolver.cpp @@ -1420,7 +1420,7 @@ int ObShowResolver::resolve(const ParseNode &parse_tree) break; } case T_SHOW_RESTORE_PREVIEW: { - // TODO: fix restore preview later. + // TODO(chongrong.th): fix restore preview in 4.1 ret = OB_NOT_SUPPORTED; LOG_USER_ERROR(OB_NOT_SUPPORTED, "show restore preview is"); break; diff --git a/src/sql/resolver/cmd/ob_variable_set_resolver.cpp b/src/sql/resolver/cmd/ob_variable_set_resolver.cpp index 04b54395f..04a924408 100644 --- a/src/sql/resolver/cmd/ob_variable_set_resolver.cpp +++ b/src/sql/resolver/cmd/ob_variable_set_resolver.cpp @@ -155,14 +155,7 @@ int ObVariableSetResolver::resolve(const ParseNode &parse_tree) MEMCPY(&value_node, set_node->children_[1], sizeof(ParseNode)); } if (OB_SUCC(ret)) { - if (0 == var_node.variable_name_.case_compare("ob_compatibility_mode") && - 0 == strncasecmp(value_node.str_value_, "oracle", - std::min(static_cast(value_node.str_len_), 6))) { - ret = OB_NOT_SUPPORTED; - LOG_USER_ERROR(OB_NOT_SUPPORTED, "Not support oracle mode"); - } - if (OB_SUCC(ret) && - OB_FAIL(ObResolverUtils::resolve_const_expr(params_, value_node, var_node.value_expr_, NULL))) { + if (OB_FAIL(ObResolverUtils::resolve_const_expr(params_, value_node, var_node.value_expr_, NULL))) { LOG_WARN("resolve variable value failed", K(ret)); } } diff --git a/src/sql/resolver/ddl/ob_alter_table_resolver.cpp b/src/sql/resolver/ddl/ob_alter_table_resolver.cpp index 8ee3763dd..24ed18420 100644 --- a/src/sql/resolver/ddl/ob_alter_table_resolver.cpp +++ b/src/sql/resolver/ddl/ob_alter_table_resolver.cpp @@ -24,6 +24,10 @@ #include "sql/resolver/dml/ob_delete_resolver.h" #include "share/ob_index_builder_util.h" #include "sql/engine/expr/ob_expr_lob_utils.h" +#ifdef OB_BUILD_ORACLE_XML +#include "lib/xml/ob_xml_parser.h" +#include "lib/xml/ob_xml_util.h" +#endif namespace oceanbase { @@ -4332,8 +4336,60 @@ int ObAlterTableResolver::add_udt_hidden_column(ObAlterTableStmt *alter_table_st LOG_WARN("fail to calc xmltype default value expr", K(ret)); } else { // calc result is 1. string type (not lob) or 2. xmltype binary (need to remove lob header) +#ifdef OB_BUILD_ORACLE_XML + ObString res_string; + ObObj xml_default; + xml_default.set_null(); + if (orig_default_value.is_character_type()) { + // convert to xml binary + ObXmlDocument *xml_doc = NULL; + ObMulModeMemCtx* mem_ctx = nullptr; + lib::ObMallocHookAttrGuard malloc_guard(lib::ObMemAttr(MTL_ID(), "XMLCodeGen")); + if (OB_FAIL(ObXmlUtil::create_mulmode_tree_context(allocator_, mem_ctx))) { + LOG_WARN("fail to create tree memory context", K(ret)); + } else if (OB_FAIL(ObXmlParserUtils::parse_document_text(mem_ctx, + orig_default_value.get_string(), + xml_doc))) { + LOG_WARN("parse xml plain text as document failed.", K(ret), K(orig_default_value.get_string())); + ret = OB_ERR_XML_PARSE; + LOG_USER_ERROR(OB_ERR_XML_PARSE); + } else if (OB_ISNULL(xml_doc)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("get null xml document.", K(ret)); + } else { + ObIMulModeBase *xml_root = static_cast(xml_doc); + ObString res_string; + if (OB_FAIL(xml_root->get_raw_binary(res_string, allocator_))) { + LOG_WARN("failed to get xml binary", K(ret), K(orig_default_value.get_string())); + } else { + xml_default.set_lob_value(ObLongTextType, res_string.ptr(), res_string.length()); + xml_default.set_collation_type(CS_TYPE_BINARY); + } + } + } else if (orig_default_value.is_xml_sql_type()) { + // move lob header + res_string = orig_default_value.get_string(); + if (OB_FAIL(ObTextStringHelper::read_real_string_data(allocator_, + ObLongTextType, + CS_TYPE_BINARY, + true, res_string))) { + LOG_WARN("failed to get xml binary data", K(ret), K(orig_default_value.get_string())); + } else { + xml_default.set_lob_value(ObLongTextType, res_string.ptr(), res_string.length()); + xml_default.set_collation_type(CS_TYPE_BINARY); + } + } else { + ret = OB_NOT_SUPPORTED; + LOG_WARN("error xml default type", K(ret), K(orig_default_value)); + } + if (OB_FAIL(ret)) { + } else if (OB_FAIL(hidden_blob.set_orig_default_value(xml_default))) { + LOG_WARN("fail to set orig default value", K(xml_default), K(ret)); + } +#else ret = OB_NOT_SUPPORTED; LOG_WARN("xml type not supported in opensource version", K(ret), K(orig_default_value)); +#endif } if (OB_FAIL(ret)) { diff --git a/src/sql/resolver/ddl/ob_create_routine_resolver.cpp b/src/sql/resolver/ddl/ob_create_routine_resolver.cpp index e95607cb1..ea6a90f2f 100644 --- a/src/sql/resolver/ddl/ob_create_routine_resolver.cpp +++ b/src/sql/resolver/ddl/ob_create_routine_resolver.cpp @@ -19,6 +19,9 @@ #include "pl/ob_pl_package.h" #include "pl/ob_pl_resolver.h" #include "share/schema/ob_trigger_info.h" +#ifdef OB_BUILD_ORACLE_PL +#include "pl/ob_pl_udt_object_manager.h" +#endif namespace oceanbase { @@ -286,6 +289,38 @@ int ObCreateRoutineResolver::set_routine_param(const ObIArray &a } else if (ObObjAccessIdx::is_pkg_type(access_idxs)) { CK (access_idxs.count() >= 1 && access_idxs.count() <= 3); if (OB_FAIL(ret)) { +#ifdef OB_BUILD_ORACLE_PL + } else if (1 == access_idxs.count()) { // standard package type + ObPLPackageGuard package_guard(params_.session_info_->get_effective_tenant_id()); + const ObUserDefinedType *user_type = NULL; + const ObUserDefinedSubType *sub_type = NULL; + CK (OB_NOT_NULL(params_.schema_checker_)); + OZ (ObResolverUtils::get_user_type(params_.allocator_, + params_.session_info_, + params_.sql_proxy_, + params_.schema_checker_->get_schema_guard(), + package_guard, + access_idxs.at(0).var_index_, + user_type)); + CK (OB_NOT_NULL(sub_type = static_cast(user_type))); + if (OB_FAIL(ret)) { + } else if (sub_type->get_base_type()->is_obj_type()) { + ObDataType data_type = *(sub_type->get_base_type()->get_data_type()); + if (ObNumberTC == data_type.get_type_class() && + 38 == data_type.get_precision() && + 0 == data_type.get_scale()) { + data_type.set_precision(-1); + data_type.set_scale(-85); + } + routine_param.set_param_type(data_type); + } else { + OX (routine_param.set_param_type(ObExtendType)); + OX (routine_param.set_pkg_type()); + OX (routine_param.set_type_name(access_idxs.at(access_idxs.count()-1).var_name_)); + OX (routine_param.set_type_subname("STANDARD")); + OX (routine_param.set_type_owner(OB_SYS_DATABASE_ID)); + } +#endif } else { OX (routine_param.set_param_type(ObExtendType)); OX (routine_param.set_pkg_type()); @@ -605,6 +640,14 @@ int ObCreateRoutineResolver::resolve_param_list(const ParseNode *param_list, ObR if (OB_SUCC(ret) && 1 == param_node->int32_values_[1]) { routine_param.set_nocopy_param(); } +#ifdef OB_BUILD_ORACLE_PL + if (OB_SUCC(ret)) { + // 借用text_len_ , 后续修改 + if (pl::ObPLUDTObjectManager::is_self_param(param_name)) { + routine_param.set_is_self_param(); + } + } +#endif // 设置default value expr str if (OB_SUCC(ret) && 3 == param_node->num_child_ // oracle mode has default node diff --git a/src/sql/resolver/ddl/ob_trigger_resolver.cpp b/src/sql/resolver/ddl/ob_trigger_resolver.cpp index dbe835396..191bf4bfb 100644 --- a/src/sql/resolver/ddl/ob_trigger_resolver.cpp +++ b/src/sql/resolver/ddl/ob_trigger_resolver.cpp @@ -842,6 +842,41 @@ int ObTriggerResolver::resolve_alter_clause(const ParseNode &alter_clause, } else { tg_info.set_disable(); } +#ifdef OB_BUILD_ORACLE_PL + } else if (TRIGGER_ALTER_COMPILE == alter_clause.int16_values_[0]) { + is_set_status = false; + is_alter_compile = true; + if (1 == alter_clause.int16_values_[2]) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("alter trigger with reuse_setting not supported yet!", K(ret)); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter trigger with reuse setting"); + } else { + share::schema::ObErrorInfo error_info; + ObSEArray dep_infos; + if (OB_FAIL(analyze_trigger(*schema_checker_->get_schema_guard(), + session_info_, + params_.sql_proxy_, + *allocator_, + tg_info, + db_name, + dep_infos, + true))) { + LOG_USER_WARN(OB_ERR_TRIGGER_COMPILE_ERROR, "TRIGGER", + db_name.length(), db_name.ptr(), + tg_info.get_trigger_name().length(), tg_info.get_trigger_name().ptr()); + ret = OB_SUCCESS; + error_info.handle_error_info(&tg_info); + } else { + OZ (error_info.delete_error(&tg_info)); + } + } + } else if (TRIGGER_ALTER_RENAME == alter_clause.int16_values_[0]) { + is_set_status = false; + is_alter_compile = false; + CK (1 == alter_clause.num_child_); + CK (OB_NOT_NULL(alter_clause.children_[0])) + OZ (resolve_rename_trigger(*alter_clause.children_[0], *schema_checker_->get_schema_guard(), tg_info, *allocator_)); +#endif } return ret; } @@ -1152,6 +1187,47 @@ int ObTriggerResolver::analyze_trigger(ObSchemaGetterGuard &schema_guard, return ret; } +#ifdef OB_BUILD_ORACLE_PL +int ObTriggerResolver::resolve_rename_trigger(const ParseNode &rename_clause, + ObSchemaGetterGuard &schema_guard, + ObTriggerInfo &trigger_info, + common::ObIAllocator &alloc) +{ + int ret = OB_SUCCESS; + if (2 != rename_clause.num_child_) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("num child error", K(rename_clause.num_child_)); + } else if (OB_NOT_NULL(rename_clause.children_[0])) { + ret = OB_ERR_CMD_NOT_PROPERLY_ENDED; + LOG_WARN("database name must null", K(ret)); + } else { + ObString mock_db_name; + ObString new_trg_name; + if (OB_FAIL(resolve_schema_name(rename_clause, mock_db_name, new_trg_name))) { + LOG_WARN("resolve schema name failed", K(new_trg_name), K(ret)); + } else if (0 == trigger_info.get_trigger_name().case_compare(new_trg_name)) { + ret = OB_OBJ_ALREADY_EXIST; + LOG_WARN("name is already exist", K(trigger_info.get_trigger_name()), K(new_trg_name), K(ret)); + } else { + const ObTriggerInfo *other_trg_info = NULL; + if (OB_FAIL(schema_guard.get_trigger_info(trigger_info.get_tenant_id(), + trigger_info.get_database_id(), + new_trg_name, + other_trg_info))) { + LOG_WARN("get trigger info failed", K(trigger_info.get_database_id()), K(new_trg_name), K(ret)); + } else if (OB_NOT_NULL(other_trg_info)) { + ret = OB_OBJ_ALREADY_EXIST; + LOG_WARN("name is already exist", KPC(other_trg_info), K(new_trg_name), K(ret)); + } else if (OB_FAIL(trigger_info.set_trigger_name(new_trg_name))) { + LOG_WARN("set trigger name failed", K(trigger_info), K(new_trg_name), K(ret)); + } else if (OB_FAIL(ObTriggerInfo::replace_trigger_name_in_body(trigger_info, alloc, schema_guard))) { + LOG_WARN("rebuild trigger body failed", K(trigger_info), K(ret)); + } + } + } + return ret; +} +#endif const ObString ObTriggerResolver::REF_OLD = "OLD"; const ObString ObTriggerResolver::REF_NEW = "NEW"; diff --git a/src/sql/resolver/ddl/ob_trigger_resolver.h b/src/sql/resolver/ddl/ob_trigger_resolver.h index a6b8b08d6..17854e155 100644 --- a/src/sql/resolver/ddl/ob_trigger_resolver.h +++ b/src/sql/resolver/ddl/ob_trigger_resolver.h @@ -95,6 +95,12 @@ private: int resolve_base_object(obrpc::ObCreateTriggerArg &trigger_arg, bool search_public_schema); int resolve_order_clause(const ParseNode *parse_node, obrpc::ObCreateTriggerArg &trigger_arg); +#ifdef OB_BUILD_ORACLE_PL + int resolve_rename_trigger(const ParseNode &rename_clause, + ObSchemaGetterGuard &schema_guard, + share::schema::ObTriggerInfo &trigger_info, + common::ObIAllocator &alloc); +#endif private: static const common::ObString REF_OLD; diff --git a/src/sql/resolver/dml/ob_dml_resolver.cpp b/src/sql/resolver/dml/ob_dml_resolver.cpp index cc4760da6..5059f6c1d 100755 --- a/src/sql/resolver/dml/ob_dml_resolver.cpp +++ b/src/sql/resolver/dml/ob_dml_resolver.cpp @@ -2012,6 +2012,34 @@ int ObDMLResolver::resolve_into_variables(const ParseNode *node, ret = OB_NOT_SUPPORTED; LOG_WARN("dynamic sql returning bulk collect is not supported!", K(ret)); LOG_USER_ERROR(OB_NOT_SUPPORTED, "dynamic sql returning bulk collect"); +#ifdef OB_BUILD_ORACLE_PL + } else if (1 == node->value_) { //Bulk Into的变量一定是NestedTable，这里做下CHECK + if (!user_vars.empty()) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("Bulk Collection Into not support User Variables", K(ret), K(user_vars.count())); + LOG_USER_ERROR(OB_NOT_SUPPORTED, "Bulk Collection to User Variables"); + } else { + for (uint64_t i = 0; OB_SUCC(ret) && i < pl_vars.count(); ++i) { + CK (OB_NOT_NULL(pl_vars.at(i))); + if (OB_SUCC(ret) + && !pl_vars.at(i)->is_obj_access_expr() && !pl_vars.at(i)->is_const_raw_expr()) { + ret = OB_ERR_MIX_SINGLE_MULTI; + LOG_WARN("Bulk Collection Into is not Nested Table", K(ret)); + } + if (OB_SUCC(ret) && pl_vars.at(i)->is_obj_access_expr()) { + pl::ObPLDataType pl_type; + const ObUserDefinedType *user_type = NULL; + ObObjAccessRawExpr* access_expr = static_cast(pl_vars.at(i)); + OZ (access_expr->get_final_type(pl_type)); + if (OB_SUCC(ret) && !pl_type.is_collection_type()) { + ret = OB_ERR_MIX_SINGLE_MULTI; + LOG_WARN("Bulk Collection Into is not collection", K(ret), K(pl_type)); + } + } + } + OX (session_info_->get_cur_exec_ctx()->get_sql_ctx()->is_bulk_ = true); + } +#endif } } if (OB_SUCC(ret) && !pl_vars.empty() && !user_vars.empty()) { diff --git a/src/sql/resolver/dml/ob_sql_hint.cpp b/src/sql/resolver/dml/ob_sql_hint.cpp index 94ed10ba9..3ea280ea4 100644 --- a/src/sql/resolver/dml/ob_sql_hint.cpp +++ b/src/sql/resolver/dml/ob_sql_hint.cpp @@ -1407,19 +1407,32 @@ bool ObStmtHint::has_disable_hint(ObItemType hint_type) const void ObLogPlanHint::reset() { is_outline_data_ = false; +#ifdef OB_BUILD_SPM + is_spm_evolution_ = false; +#endif join_order_.reset(); table_hints_.reuse(); join_hints_.reuse(); normal_hints_.reuse(); } +#ifndef OB_BUILD_SPM int ObLogPlanHint::init_log_plan_hint(ObSqlSchemaGuard &schema_guard, const ObDMLStmt &stmt, const ObQueryHint &query_hint) +#else +int ObLogPlanHint::init_log_plan_hint(ObSqlSchemaGuard &schema_guard, + const ObDMLStmt &stmt, + const ObQueryHint &query_hint, + const bool is_spm_evolution) +#endif { int ret = OB_SUCCESS; reset(); is_outline_data_ = query_hint.has_outline_data(); +#ifdef OB_BUILD_SPM + is_spm_evolution_ = is_spm_evolution; +#endif const ObStmtHint &stmt_hint = stmt.get_stmt_hint(); if (OB_FAIL(join_order_.init_leading_info(stmt, query_hint, stmt_hint.get_normal_hint(T_LEADING)))) { LOG_WARN("failed to get leading hint info", K(ret)); @@ -1948,6 +1961,12 @@ int ObLogPlanHint::check_valid_set_left_branch(const ObSelectStmt *select_stmt, int ObLogPlanHint::check_status() const { int ret = OB_SUCCESS; +#ifdef OB_BUILD_SPM + if (is_spm_evolution_) { + ret = OB_OUTLINE_NOT_REPRODUCIBLE; + LOG_WARN("failed to generate plan use outline data for spm evolution", K(ret)); + } +#endif return ret; } diff --git a/src/sql/resolver/dml/ob_sql_hint.h b/src/sql/resolver/dml/ob_sql_hint.h index bb92a954f..2a417df52 100644 --- a/src/sql/resolver/dml/ob_sql_hint.h +++ b/src/sql/resolver/dml/ob_sql_hint.h @@ -382,9 +382,16 @@ struct ObLogPlanHint ObLogPlanHint() { reset(); } void reset(); int init_normal_hints(const ObIArray &normal_hints); +#ifndef OB_BUILD_SPM int init_log_plan_hint(ObSqlSchemaGuard &schema_guard, const ObDMLStmt &stmt, const ObQueryHint &query_hint); +#else + int init_log_plan_hint(ObSqlSchemaGuard &schema_guard, + const ObDMLStmt &stmt, + const ObQueryHint &query_hint, + const bool is_spm_evolution); +#endif int init_other_opt_hints(ObSqlSchemaGuard &schema_guard, const ObDMLStmt &stmt, const ObQueryHint &query_hint, @@ -467,6 +474,9 @@ struct ObLogPlanHint K_(normal_hints)); bool is_outline_data_; +#ifdef OB_BUILD_SPM + bool is_spm_evolution_; +#endif LogLeadingHint join_order_; common::ObSEArray table_hints_; common::ObSEArray join_hints_; diff --git a/src/sql/resolver/expr/ob_raw_expr.cpp b/src/sql/resolver/expr/ob_raw_expr.cpp index 0fded5a2e..75fbd0195 100644 --- a/src/sql/resolver/expr/ob_raw_expr.cpp +++ b/src/sql/resolver/expr/ob_raw_expr.cpp @@ -2703,6 +2703,27 @@ int ObObjAccessRawExpr::add_access_indexs(const ObIArray &ac case pl::ObObjAccessIdx::IS_PROPERTY: { if (OB_FAIL(expr_access.push_back(access_idx))) { LOG_WARN("store access index failed", K(ret)); +#ifdef OB_BUILD_ORACLE_PL + } else if (access_idx.is_property()) { + if (0 == access_idx.var_name_.case_compare("count")) { + set_property(pl::ObAssocArrayType::COUNT_PROPERTY); + } else if (0 == access_idx.var_name_.case_compare("first")) { + set_property(pl::ObAssocArrayType::FIRST_PROPERTY); + } else if (0 == access_idx.var_name_.case_compare("last")) { + set_property(pl::ObAssocArrayType::LAST_PROPERTY); + } else if (0 == access_idx.var_name_.case_compare("limit")) { + set_property(pl::ObAssocArrayType::LIMIT_PROPERTY); + } else if (0 == access_idx.var_name_.case_compare("prior")) { + set_property(pl::ObAssocArrayType::PRIOR_PROPERTY); + } else if (0 == access_idx.var_name_.case_compare("next")) { + set_property(pl::ObAssocArrayType::NEXT_PROPERTY); + } else if (0 == access_idx.var_name_.case_compare("exists")) { + set_property(pl::ObAssocArrayType::EXISTS_PROPERTY); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("Invalid property type", K(i), K(access_idx), K(ret)); + } +#endif } else { /*do nothing*/ } } break; diff --git a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp index 5ea9b87ee..2597bca67 100644 --- a/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp +++ b/src/sql/resolver/expr/ob_raw_expr_resolver_impl.cpp @@ -28,6 +28,9 @@ #include "pl/ob_pl_resolver.h" #include "common/ob_smart_call.h" #include "share/schema/ob_udt_info.h" +#ifdef OB_BUILD_ORACLE_PL +#include "pl/ob_pl_udt_object_manager.h" +#endif #include "sql/resolver/ob_stmt.h" #include "sql/resolver/dml/ob_del_upd_stmt.h" #include "deps/oblib/src/lib/json_type/ob_json_path.h" @@ -2517,10 +2520,25 @@ int ObRawExprResolverImpl::process_datatype_or_questionmark(const ParseNode &nod const ObObjParam ¶m = ctx_.param_list_->at(val.get_unknown()); c_expr->set_is_literal_bool(param.is_boolean()); if (param.is_ext()) { +#ifdef OB_BUILD_ORACLE_PL + if (OB_NOT_NULL(ctx_.session_info_->get_pl_implicit_cursor()) + && ctx_.session_info_->get_pl_implicit_cursor()->get_in_forall() + && param.is_ext_sql_array()) { + ObSqlArrayObj *param_array = reinterpret_cast(param.get_ext()); + CK (OB_NOT_NULL(param_array)); + OX (c_expr->set_meta_type(param_array->element_.get_meta_type())); + OX (c_expr->set_expr_obj_meta(param_array->element_.get_meta_type())); + OX (c_expr->set_accuracy(param_array->element_.get_accuracy())); + OX (c_expr->set_param(param)); + } else { +#endif c_expr->set_meta_type(param.get_meta()); c_expr->set_expr_obj_meta(param.get_param_meta()); c_expr->set_udt_id(param.get_udt_id()); c_expr->set_param(param); +#ifdef OB_BUILD_ORACLE_PL + } +#endif } else { c_expr->set_meta_type(ObSQLUtils::is_oracle_empty_string(param) ? param.get_param_meta() : param.get_meta()); @@ -2555,8 +2573,21 @@ int ObRawExprResolverImpl::process_datatype_or_questionmark(const ParseNode &nod const ObObjParam ¶m = ctx_.param_list_->at(val.get_unknown()); c_expr->set_is_literal_bool(param.is_boolean()); if (param.is_ext()) { +#ifndef OB_BUILD_ORACLE_PL ret = OB_NOT_SUPPORTED; LOG_WARN("not support array binding", K(ret)); +#else + pl::ObPLNestedTable *param_array = reinterpret_cast(param.get_ext()); + if (OB_ISNULL(param_array)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("param array is invalid", KPC(param_array)); + } else { + c_expr->set_meta_type(param_array->get_element_type().get_meta_type()); + c_expr->set_expr_obj_meta(param_array->get_element_type().get_meta_type()); + c_expr->set_accuracy(param_array->get_element_type().get_accuracy()); + c_expr->set_param(param); + } +#endif } else { //questionmark won't set meta_type again if (param.get_param_meta().get_type() != param.get_type()) { diff --git a/src/sql/resolver/ob_resolver.cpp b/src/sql/resolver/ob_resolver.cpp index 6af078904..e040fe2aa 100644 --- a/src/sql/resolver/ob_resolver.cpp +++ b/src/sql/resolver/ob_resolver.cpp @@ -131,6 +131,18 @@ #include "pl/ob_pl_package.h" #include "sql/resolver/ddl/ob_create_context_resolver.h" #include "sql/resolver/ddl/ob_drop_context_resolver.h" +#ifdef OB_BUILD_TDE_SECURITY +#include "sql/resolver/ddl/ob_create_tablespace_resolver.h" +#include "sql/resolver/ddl/ob_alter_tablespace_resolver.h" +#include "sql/resolver/ddl/ob_drop_tablespace_resolver.h" +#include "sql/resolver/ddl/ob_create_keystore_resolver.h" +#include "sql/resolver/ddl/ob_alter_keystore_resolver.h" +#endif +#ifdef OB_BUILD_ORACLE_PL +#include "sql/resolver/ddl/ob_create_udt_resolver.h" +#include "sql/resolver/ddl/ob_drop_udt_resolver.h" +#include "sql/resolver/ddl/ob_audit_resolver.h" +#endif namespace oceanbase { using namespace common; @@ -288,6 +300,12 @@ int ObResolver::resolve(IsPrepared if_prepared, const ParseNode &parse_tree, ObS REGISTER_STMT_RESOLVER(CreateTable); break; } +#ifdef OB_BUILD_AUDIT_SECURITY + case T_AUDIT: { + REGISTER_STMT_RESOLVER(Audit); + break; + } +#endif case T_CREATE_FUNC: { REGISTER_STMT_RESOLVER(CreateFunc); break; @@ -818,6 +836,20 @@ int ObResolver::resolve(IsPrepared if_prepared, const ParseNode &parse_tree, ObS REGISTER_STMT_RESOLVER(AnonymousBlock); break; } +#ifdef OB_BUILD_ORACLE_PL + case T_SP_CREATE_TYPE: { + REGISTER_STMT_RESOLVER(CreateUDT); + break; + } + case T_SP_CREATE_TYPE_BODY: { + REGISTER_STMT_RESOLVER(CreateUDTBody); + break; + } + case T_SP_DROP_TYPE: { + REGISTER_STMT_RESOLVER(DropUDT); + break; + } +#endif case T_PACKAGE_CREATE: { REGISTER_STMT_RESOLVER(CreatePackage); break; @@ -957,6 +989,31 @@ int ObResolver::resolve(IsPrepared if_prepared, const ParseNode &parse_tree, ObS REGISTER_STMT_RESOLVER(SetRole); break; }*/ +#ifdef OB_BUILD_TDE_SECURITY + case T_CREATE_KEYSTORE: { + REGISTER_STMT_RESOLVER(CreateKeystore); + break; + } + case T_ALTER_KEYSTORE_OPEN: + case T_ALTER_KEYSTORE_CLOSE: + case T_ALTER_KEYSTORE_SET_KEY: + case T_ALTER_KEYSTORE_PASSWORD: { + REGISTER_STMT_RESOLVER(AlterKeystore); + break; + } + case T_CREATE_TABLESPACE: { + REGISTER_STMT_RESOLVER(CreateTablespace); + break; + } + case T_ALTER_TABLESPACE: { + REGISTER_STMT_RESOLVER(AlterTablespace); + break; + } + case T_DROP_TABLESPACE: { + REGISTER_STMT_RESOLVER(DropTablespace); + break; + } +#endif case T_CREATE_PROFILE: { REGISTER_STMT_RESOLVER(UserProfile); break; diff --git a/src/sql/resolver/ob_resolver_utils.cpp b/src/sql/resolver/ob_resolver_utils.cpp index 9f465ee87..218884b9f 100644 --- a/src/sql/resolver/ob_resolver_utils.cpp +++ b/src/sql/resolver/ob_resolver_utils.cpp @@ -999,6 +999,16 @@ int ObResolverUtils::check_match(const pl::ObPLResolveCtx &resolve_ctx, resolve_ctx.allocator_, resolve_ctx.sql_proxy_, dst_pl_type)); +#ifdef OB_BUILD_ORACLE_PL + if (OB_SUCC(ret) && dst_pl_type.is_subtype()) { + const ObUserDefinedType *user_type = NULL; + const ObUserDefinedSubType *sub_type = NULL; + OZ (resolve_ctx.get_user_type( + dst_pl_type.get_user_type_id(), user_type, &(resolve_ctx.allocator_))); + CK (OB_NOT_NULL(sub_type = static_cast(sub_type))); + OX (dst_pl_type = *(sub_type->get_base_type())); + } +#endif } else { dst_pl_type = routine_param->get_pl_data_type(); } diff --git a/src/sql/rewrite/ob_transform_pre_process.cpp b/src/sql/rewrite/ob_transform_pre_process.cpp index 62da9d7fb..fae5ea2b7 100644 --- a/src/sql/rewrite/ob_transform_pre_process.cpp +++ b/src/sql/rewrite/ob_transform_pre_process.cpp @@ -133,6 +133,17 @@ int ObTransformPreProcess::transform_one_stmt(common::ObIArray LOG_TRACE("succeed to replace function", K(is_happened)); } } +#ifdef OB_BUILD_LABEL_SECURITY + if (OB_SUCC(ret)) { + if (OB_FAIL(transform_for_label_se_table(stmt, is_happened))) { + LOG_WARN("failed to transform for label security table", K(ret)); + } else { + trans_happened |= is_happened; + OPT_TRACE("transform for label security table:", is_happened); + LOG_TRACE("succeed to transform for label security table", K(is_happened), K(ret)); + } + } +#endif if (OB_SUCC(ret)) { if (OB_FAIL(transform_for_rls_table(stmt, is_happened))) { LOG_WARN("failed to transform for rls table", K(ret)); @@ -3195,6 +3206,199 @@ int ObTransformPreProcess::collect_all_tableitem(ObDMLStmt *stmt, return ret; } +#ifdef OB_BUILD_LABEL_SECURITY +/** + * 假如t1的安全列是 c_label，对应的安全策略是 policy_name, 会在下面三种语句添加filter + * 1) select * from t1 where .. + * 2) update t1 set c1 = 1 where .. + * 3) delete from t1 where .. + * + * filter: 0 == ols_label_value_cmp_le(ols_session_label(policy_name), c_label) + * + */ +int ObTransformPreProcess::transform_for_label_se_table(ObDMLStmt *stmt, bool &trans_happened) +{ + int ret = OB_SUCCESS; + trans_happened = false; + if (OB_ISNULL(stmt) || OB_ISNULL(ctx_) || OB_ISNULL(ctx_->schema_checker_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("null unexpected", K(ret), K(stmt), K(ctx_), K(ctx_->schema_checker_)); + } else if (OB_ISNULL(ctx_->session_info_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("session info is NULL", K(ret)); + } else if (!ctx_->session_info_->is_inner()) { //内部session不用条件进行过滤 + common::ObArray table_item_list; + int64_t num_from_items = stmt->get_from_item_size(); + //1, collect all table item + for(int64_t i = 0; OB_SUCC(ret) && i < num_from_items; ++i) { + const FromItem &from_item = stmt->get_from_item(i); + if (from_item.is_joined_) { + JoinedTable *joined_table_item = stmt->get_joined_table(from_item.table_id_); + if (OB_FAIL(collect_all_tableitem(stmt, joined_table_item, table_item_list))) { + LOG_WARN("failed to collect table item", K(ret)); + } + } else { + TableItem *table_item = NULL; + table_item = stmt->get_table_item_by_id(from_item.table_id_); + if (OB_FAIL(collect_all_tableitem(stmt, table_item, table_item_list))) { + LOG_WARN("failed to collect table item", K(ret)); + LOG_WARN("failed to collect table item", K(ret), K(*stmt)); + } + } + } + + //2, for each label security table item, add label column filter + for (int64_t i = 0; OB_SUCC(ret) && i < table_item_list.count(); ++i) { + TableItem *table_item = table_item_list.at(i); + if (OB_ISNULL(table_item)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table item is null", K(ret)); + } else if (table_item->is_basic_table() || table_item->is_link_table()) { + uint64_t table_ref_id = table_item->ref_id_; + const ObTableSchema *table_schema = NULL; + if (OB_FAIL(ctx_->schema_checker_->get_table_schema(ctx_->session_info_->get_effective_tenant_id(), table_ref_id, table_schema, table_item->is_link_table()))) { + LOG_WARN("failed to get table schema", K(table_ref_id), K(ret)); + } else if (OB_ISNULL(table_schema)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("table should not be null", K(table_ref_id)); + } else if (table_schema->has_label_se_column()) { + const ObIArray &label_se_column_ids = table_schema->get_label_se_column_ids(); + for (int64_t j = 0; OB_SUCC(ret) && j < label_se_column_ids.count(); ++j) { + const ObColumnSchemaV2 *column_schema = NULL; + ObString policy_name; + if (OB_ISNULL(column_schema = table_schema->get_column_schema(label_se_column_ids.at(j)))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("column schema not exist", K(ret), K(label_se_column_ids.at(j))); + } else if (OB_FAIL(ctx_->schema_checker_->get_label_se_policy_name_by_column_name( + ctx_->session_info_->get_effective_tenant_id(), + column_schema->get_column_name_str(), + policy_name))) { + LOG_WARN("fail to get policy name", K(ret)); + } else if (OB_FAIL(add_filter_for_label_se_table(*stmt, *table_item, policy_name, *column_schema))) { + LOG_WARN("add filter for temporary table failed", K(ret)); + } + } + if (OB_SUCC(ret)) { + trans_happened = true; + } + } + } + } + } + return ret; +} + +int ObTransformPreProcess::add_filter_for_label_se_table(ObDMLStmt &stmt, + const TableItem &table_item, + const ObString &policy_name, + const ObColumnSchemaV2 &column_schema) +{ + int ret = OB_SUCCESS; + ObRawExprFactory *expr_factory = NULL; + ObSQLSessionInfo *session_info = NULL; + + if (OB_ISNULL(ctx_) + || OB_ISNULL(session_info = ctx_->session_info_) + || OB_ISNULL(expr_factory = ctx_->expr_factory_)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("some parameter is NULL", K(ret), K(ctx_), K(expr_factory), K(session_info)); + } + + ObConstRawExpr *policy_name_expr = NULL; + if (OB_SUCC(ret)) { + if (OB_FAIL(ObRawExprUtils::build_const_string_expr(*expr_factory, + ObVarcharType, + policy_name, + ObCharset::get_default_collation(ObCharset::get_default_charset()), + policy_name_expr))) { + LOG_WARN("fail to build expr", K(ret)); + } + } + + ObSysFunRawExpr *session_label_expr = NULL; + if (OB_SUCC(ret)) { + if (OB_FAIL(expr_factory->create_raw_expr(T_FUN_LABEL_SE_SESSION_LABEL, session_label_expr))) { + LOG_WARN("create expr failed", K(ret)); + } else { + ObString func_name = ObString::make_string(N_OLS_SESSION_LABEL); + session_label_expr->set_func_name(func_name); + } + if (OB_SUCC(ret)) { + if (OB_FAIL(session_label_expr->add_param_expr(policy_name_expr))) { + LOG_WARN("fail to add param expr", K(ret)); + } + } + } + + ObColumnRefRawExpr *col_expr = NULL; + if (OB_SUCC(ret)) { + ColumnItem *col_item = NULL; + if (NULL != (col_item = stmt.get_column_item_by_id(table_item.table_id_, column_schema.get_column_id()))) { + col_expr = col_item->expr_; + col_expr->set_explicited_reference(); + } else { + if (OB_FAIL(ObRawExprUtils::build_column_expr(*expr_factory, column_schema, col_expr))) { + LOG_WARN("fail to build column expr", K(ret)); + } else { + col_expr->set_table_id(table_item.table_id_); + col_expr->set_table_name(table_item.table_name_); + col_expr->set_explicited_reference(); + ColumnItem column_item; + column_item.expr_ = col_expr; + column_item.table_id_ = col_expr->get_table_id(); + column_item.column_id_ = col_expr->get_column_id(); + column_item.column_name_ = col_expr->get_column_name(); + if (OB_FAIL(stmt.add_column_item(column_item))) { + LOG_WARN("add column item to stmt failed", K(ret)); + } + } + } + } + ObSysFunRawExpr *label_cmp_expr = NULL; + if (OB_SUCC(ret)) { + ObString func_name = ObString::make_string(N_OLS_LABEL_VALUE_CMP_LE); + + if (OB_FAIL(expr_factory->create_raw_expr(T_FUN_LABEL_SE_LABEL_VALUE_CMP_LE, label_cmp_expr))) { + LOG_WARN("create expr failed", K(ret)); + } else { + label_cmp_expr->set_func_name(func_name); + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(label_cmp_expr->add_param_expr(col_expr))) { + LOG_WARN("fail to add param expr", K(ret)); + } else if (OB_FAIL(label_cmp_expr->add_param_expr(session_label_expr))) { + LOG_WARN("fail to add parm expr", K(ret)); + } + } + } + ObConstRawExpr *zero_int_expr = NULL; + if (OB_SUCC(ret)) { + if (OB_FAIL(ObRawExprUtils::build_const_int_expr(*expr_factory, ObIntType, 0, zero_int_expr))) { + LOG_WARN("fail to build expr", K(ret)); + } + } + ObRawExpr *equal_expr = NULL; + if (OB_SUCC(ret)) { + if (OB_FAIL(ObRawExprUtils::create_equal_expr(*expr_factory, + session_info, + label_cmp_expr, + zero_int_expr, + equal_expr))) { + LOG_WARN("fail to create equal expr", K(ret)); + } + } + + if (OB_SUCC(ret)) { + if (OB_FAIL(stmt.get_condition_exprs().push_back(equal_expr))) { + LOG_WARN("failed to push back new filter", K(ret)); + } else { + LOG_TRACE("add new filter succeed", K(stmt.get_condition_exprs()), K(*equal_expr)); + } + } + return ret; +} +#endif /** * ObTransformPreProcess::transform_for_rls_table diff --git a/src/sql/rewrite/ob_transform_pre_process.h b/src/sql/rewrite/ob_transform_pre_process.h index 4e1ee3538..16be75bef 100644 --- a/src/sql/rewrite/ob_transform_pre_process.h +++ b/src/sql/rewrite/ob_transform_pre_process.h @@ -284,6 +284,13 @@ struct DistinctObjMeta int add_filter_for_temporary_table(ObDMLStmt &stmt, const TableItem &table_item, bool is_trans_scope_temp_table); +#ifdef OB_BUILD_LABEL_SECURITY + int transform_for_label_se_table(ObDMLStmt *stmt, bool &trans_happened); + int add_filter_for_label_se_table(ObDMLStmt &stmt, + const TableItem &table_item, + const common::ObString &policy_name, + const share::schema::ObColumnSchemaV2 &column_schema); +#endif int collect_all_tableitem(ObDMLStmt *stmt, TableItem *table_item, common::ObArray &table_item_list); diff --git a/src/sql/rewrite/ob_transformer_impl.cpp b/src/sql/rewrite/ob_transformer_impl.cpp index 868cb798d..7f1c90334 100644 --- a/src/sql/rewrite/ob_transformer_impl.cpp +++ b/src/sql/rewrite/ob_transformer_impl.cpp @@ -764,7 +764,16 @@ int ObTransformerImpl::add_trans_happended_hints(ObQueryCtx &query_ctx, { int ret = OB_SUCCESS; ObQueryHint &query_hint = query_ctx.get_query_hint_for_update(); +#ifndef OB_BUILD_SPM if (OB_FAIL(query_hint.outline_trans_hints_.assign(trans_ctx.outline_trans_hints_))) { +#else + if (OB_UNLIKELY(query_ctx.is_spm_evolution_ + && query_hint.has_outline_data() + && query_hint.trans_list_.count() != trans_ctx.outline_trans_hints_.count())) { + ret = OB_OUTLINE_NOT_REPRODUCIBLE; + LOG_WARN("failed to do transform for spm evolution plan", K(ret)); + } else if (OB_FAIL(query_hint.outline_trans_hints_.assign(trans_ctx.outline_trans_hints_))) { +#endif LOG_WARN("failed to assign trans hints", K(ret)); } else if (OB_FAIL(query_hint.used_trans_hints_.assign(trans_ctx.used_trans_hints_))) { LOG_WARN("failed to assign trans hints", K(ret)); diff --git a/src/sql/session/ob_basic_session_info.cpp b/src/sql/session/ob_basic_session_info.cpp index f9015b4b7..7d3777ee9 100644 --- a/src/sql/session/ob_basic_session_info.cpp +++ b/src/sql/session/ob_basic_session_info.cpp @@ -6440,8 +6440,7 @@ observer::ObSMConnection *ObBasicSessionInfo::get_sm_connection() } else { conn = &sess->conn_; } - } - else { + } else { LOG_ERROR_RET(OB_ERR_UNEXPECTED, "invalid sock_desc type", K(sock_desc.type_)); } return conn; diff --git a/src/sql/session/ob_sql_session_info.cpp b/src/sql/session/ob_sql_session_info.cpp index fe7c5a10e..25ea00636 100644 --- a/src/sql/session/ob_sql_session_info.cpp +++ b/src/sql/session/ob_sql_session_info.cpp @@ -46,6 +46,10 @@ #include "sql/engine/px/ob_px_target_mgr.h" #include "lib/utility/utility.h" #include "lib/utility/ob_proto_trans_util.h" +#ifdef OB_BUILD_ORACLE_PL +#include "pl/debug/ob_pl_debugger_manager.h" +#include "pl/sys_package/ob_pl_utl_file.h" +#endif #include "lib/allocator/ob_mod_define.h" #include "lib/string/ob_hex_utils_base.h" #include "share/stat/ob_opt_stat_manager.h" @@ -146,6 +150,12 @@ ObSQLSessionInfo::ObSQLSessionInfo(const uint64_t tenant_id) : curr_session_context_size_(0), pl_context_(NULL), pl_can_retry_(true), +#ifdef OB_BUILD_ORACLE_PL + pl_debugger_(NULL), +#endif +#ifdef OB_BUILD_SPM + select_plan_type_(ObSpmCacheCtx::INVALID_TYPE), +#endif pl_attach_session_id_(0), pl_query_sender_(NULL), pl_ps_protocol_(false), @@ -293,6 +303,9 @@ void ObSQLSessionInfo::reset(bool skip_sys_var) curr_session_context_size_ = 0; pl_context_ = NULL; pl_can_retry_ = true; +#ifdef OB_BUILD_ORACLE_PL + pl_debugger_ = NULL; +#endif pl_attach_session_id_ = 0; pl_query_sender_ = NULL; pl_ps_protocol_ = false; @@ -359,6 +372,30 @@ void ObSQLSessionInfo::clean_status() bool ObSQLSessionInfo::is_encrypt_tenant() { bool ret = false; +#ifdef OB_BUILD_TDE_SECURITY + uint64_t cur_time = ObClockGenerator::getClock(); + int64_t tenant_id = get_effective_tenant_id(); + if (cur_time - encrypt_info_.last_modify_time_ > 10 * 1000 * 1000L) { + if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id)) { + LOG_WARN("Invalid tenant id to init fast freeze checker", K(tenant_id)); + } else { + encrypt_info_.last_modify_time_ = cur_time; + omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id)); + if (tenant_config.is_valid()) { + ObString method_str(tenant_config->tde_method.get_value()); + if (ObTdeMethodUtil::is_kms(method_str)) { + encrypt_info_.is_encrypt_ = true; + ret = true; + } else { + encrypt_info_.is_encrypt_ = false; + ret = false; + } + } + } + } else { + ret = encrypt_info_.is_encrypt_; + } +#endif return ret; } @@ -514,7 +551,22 @@ void ObSQLSessionInfo::destroy(bool skip_sys_var) piece_cache_ = NULL; } +#ifdef OB_BUILD_ORACLE_PL + if (OB_SUCC(ret)) { + const int64_t session_id = get_sessid(); + // utl file should close all fd when user session exits, + // so we should check session type here to avoid fd closing + // unexpectedly when inner session exists + if (is_user_session() && OB_FAIL(ObPLUtlFile::close_all(session_id))) { + LOG_WARN("failed to close all fd in utl file", K(ret), K(session_id)); + } + } +#endif +#ifdef OB_BUILD_ORACLE_PL + // pl debug 功能, pl debug不支持分布式调试，但调用也不会有副作用 + reset_pl_debugger_resource(); +#endif // 非分布式需要的话，分布式也需要，用于清理package的全局变量值 reset_all_package_state(); reset(skip_sys_var); @@ -849,7 +901,7 @@ ObMySQLRequestManager* ObSQLSessionInfo::get_request_manager() return request_manager_; } -sql::ObFLTSpanMgr* ObSQLSessionInfo::get_flt_span_manager() +ObFLTSpanMgr* ObSQLSessionInfo::get_flt_span_manager() { int ret = OB_SUCCESS; if (NULL == flt_span_mgr_) { @@ -1429,7 +1481,20 @@ int ObSQLSessionInfo::close_dbms_cursor(int64_t cursor_id) int ObSQLSessionInfo::make_cursor(pl::ObPLCursorInfo *&cursor) { int ret = OB_SUCCESS; +#ifndef OB_BUILD_ORACLE_PL UNUSED(cursor); +#else + const pl::ObRefCursorType pl_type; + ObObj param; + param.set_ext(reinterpret_cast(cursor)); + int64_t param_size = 0; + ObSchemaGetterGuard dummy_schema_guard; + OZ (init_cursor_cache()); + OZ (pl_type.init_obj(dummy_schema_guard, get_cursor_allocator(), param, param_size)); + OX (cursor = reinterpret_cast(param.get_ext())); + OZ (add_cursor(cursor)); + LOG_DEBUG("cursor alloc, session cursor", K(cursor)); +#endif return ret; } @@ -1809,10 +1874,77 @@ void ObSQLSessionInfo::set_has_pl_implicit_savepoint(bool v) bool ObSQLSessionInfo::is_pl_debug_on() { bool is_on = false; +#ifdef OB_BUILD_ORACLE_PL + is_on = pl_debugger_ != NULL && pl_debugger_->is_debug_on(); +#endif return is_on; } +#ifdef OB_BUILD_ORACLE_PL +int ObSQLSessionInfo::initialize_pl_debugger() +{ + int ret = OB_SUCCESS; + ObPDBManager *instance = ObPDBManager::get_instance(); + if (OB_NOT_NULL(instance)) { + OZ (instance->alloc(pl_debugger_, this)); + CK (OB_NOT_NULL(pl_debugger_)); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("current observer has not debugger instance!", K(ret)); + } + return ret; +} + +int ObSQLSessionInfo::get_pl_debugger( + uint32_t id, pl::debugger::ObPLDebugger *& pl_debugger) +{ + int ret = OB_SUCCESS; + ObPDBManager *instance = ObPDBManager::get_instance(); + if (OB_NOT_NULL(instance)) { + OZ (instance->get(id, pl_debugger)); + CK (OB_NOT_NULL(pl_debugger)); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("current observer has not debugger instance!", K(ret)); + } + return ret; +} + +int ObSQLSessionInfo::release_pl_debugger(pl::debugger::ObPLDebugger *pl_debugger) +{ + int ret = OB_SUCCESS; + ObPDBManager *instance = ObPDBManager::get_instance(); + if (OB_ISNULL(pl_debugger)) { + } else if (OB_NOT_NULL(instance)) { + instance->release(pl_debugger); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_ERROR("current observer has not debugger instance!", K(ret)); + } + return ret; +} + +int ObSQLSessionInfo::free_pl_debugger() +{ + int ret = OB_SUCCESS; + ObPDBManager *instance = ObPDBManager::get_instance(); + if (OB_ISNULL(pl_debugger_)) { + } else if (OB_NOT_NULL(instance)) { + instance->free(pl_debugger_); + pl_debugger_ = NULL; + LOG_INFO("free current session debugger", K(ret)); + } else { + LOG_ERROR("current observer has not debugger instance!"); + } + return ret; +} + +void ObSQLSessionInfo::reset_pl_debugger_resource() +{ + free_pl_debugger(); +} +#endif void ObSQLSessionInfo::reset_all_package_changed_info() { @@ -2401,6 +2533,9 @@ int ObSQLSessionInfo::save_sql_session(StmtSavedValue &saved_value) OX (saved_value.read_uncommited_ = read_uncommited_); OX (saved_value.is_ignore_stmt_ = is_ignore_stmt_); OX (inner_flag_ = true); +#ifdef OB_BUILD_SPM + OX (saved_value.select_plan_type_ = select_plan_type_); +#endif return ret; } @@ -2412,6 +2547,9 @@ int ObSQLSessionInfo::restore_sql_session(StmtSavedValue &saved_value) OX (read_uncommited_ = saved_value.read_uncommited_); OX (is_ignore_stmt_ = saved_value.is_ignore_stmt_); OX (audit_record_.assign(saved_value.audit_record_)); +#ifdef OB_BUILD_SPM + OX (select_plan_type_ = saved_value.select_plan_type_); +#endif return ret; } diff --git a/src/sql/session/ob_sql_session_info.h b/src/sql/session/ob_sql_session_info.h index 07cdb0e47..91db7998a 100644 --- a/src/sql/session/ob_sql_session_info.h +++ b/src/sql/session/ob_sql_session_info.h @@ -62,6 +62,12 @@ struct ObPLExecRecursionCtx; struct ObPLSqlCodeInfo; class ObPLContext; class ObDbmsCursorInfo; +#ifdef OB_BUILD_ORACLE_PL +namespace debugger +{ +class ObPLDebugger; +} +#endif } // namespace pl namespace obmysql @@ -525,12 +531,18 @@ public: session_type_ = INVALID_TYPE; inner_flag_ = false; is_ignore_stmt_ = false; + #ifdef OB_BUILD_SPM + select_plan_type_ = ObSpmCacheCtx::INVALID_TYPE; + #endif } public: ObAuditRecordData audit_record_; SessionType session_type_; bool inner_flag_; bool is_ignore_stmt_; + #ifdef OB_BUILD_SPM + ObSpmCacheCtx::SpmSelectPlanType select_plan_type_; + #endif }; class CursorCache { @@ -832,6 +844,9 @@ public: pl_context_ = pl_stack_ctx; } +#ifdef OB_BUILD_ORACLE_PL + inline pl::debugger::ObPLDebugger *get_pl_debugger() const { return pl_debugger_; } +#endif bool is_pl_debug_on(); inline void set_pl_attached_id(uint32_t id) { pl_attach_session_id_ = id; } @@ -864,6 +879,12 @@ public: int set_package_variable(ObExecContext &ctx, const common::ObString &key, const common::ObObj &value, bool from_proxy = false); +#ifdef OB_BUILD_ORACLE_PL + int initialize_pl_debugger(); + int free_pl_debugger(); + int get_pl_debugger(uint32_t id, pl::debugger::ObPLDebugger *& pl_debugger); + int release_pl_debugger(pl::debugger::ObPLDebugger *pl_debugger); +#endif inline bool get_pl_can_retry() { return pl_can_retry_; } inline void set_pl_can_retry(bool can_retry) { pl_can_retry_ = can_retry; } @@ -911,6 +932,14 @@ public: inner_flag_ = false; session_type_ = USER_SESSION; } +#ifdef OB_BUILD_SPM + inline void set_spm_select_plan_type(ObSpmCacheCtx::SpmSelectPlanType type) + { + select_plan_type_ = type; + } + inline ObSpmCacheCtx::SpmSelectPlanType get_spm_select_plan_type() const { return select_plan_type_; } + inline void reset_spm_select_plan_type() { select_plan_type_ = ObSpmCacheCtx::INVALID_TYPE; } +#endif void set_session_type_with_flag(); void set_session_type(SessionType session_type) { session_type_ = session_type; } inline SessionType get_session_type() const { return session_type_; } @@ -1294,6 +1323,12 @@ private: // if false == pl_can_retry_, we can only retry query in PL blocks locally bool pl_can_retry_; //标记当前执行的PL是否可以整体重试 +#ifdef OB_BUILD_ORACLE_PL + pl::debugger::ObPLDebugger *pl_debugger_; // 如果开启了debug, 该字段不为null +#endif +#ifdef OB_BUILD_SPM + ObSpmCacheCtx::SpmSelectPlanType select_plan_type_; +#endif uint32_t pl_attach_session_id_; // 如果当前session执行过dbms_debug.attach_session, 记录目标session的ID observer::ObQueryDriver *pl_query_sender_; // send query result in mysql pl diff --git a/src/sql/session/ob_sql_session_mgr.cpp b/src/sql/session/ob_sql_session_mgr.cpp index c336384b2..aee3a6d99 100644 --- a/src/sql/session/ob_sql_session_mgr.cpp +++ b/src/sql/session/ob_sql_session_mgr.cpp @@ -458,6 +458,20 @@ int ObSQLSessionMgr::free_session(const ObFreeSessionCtx &ctx) ObSQLSessionInfo *sess_info = NULL; sessinfo_map_.get(Key(sessid), sess_info); if (NULL != sess_info) { +#ifdef OB_BUILD_AUDIT_SECURITY + if (!sess_info->get_is_deserialized()) { + ObString empty_comment_text; + sess_info->update_alive_time_stat(); + int64_t cur_timeout_backup = THIS_WORKER.get_timeout_ts(); + THIS_WORKER.set_timeout_ts(ObTimeUtility::current_time() + OB_MAX_USER_SPECIFIED_TIMEOUT); + ObSecurityAuditUtils::handle_security_audit(*sess_info, + stmt::T_LOGOFF, + ObString::make_string("DISCONNECT"), + empty_comment_text, + ret); + THIS_WORKER.set_timeout_ts(cur_timeout_backup); + } +#endif if (OB_UNLIKELY(OB_SUCCESS != sess_info->on_user_disconnect())) { LOG_WARN("user disconnect failed", K(ret), K(sess_info->get_user_id())); } diff --git a/src/sql/spm/ob_plan_baseline.h b/src/sql/spm/ob_plan_baseline.h deleted file mode 100644 index 5698bc111..000000000 --- a/src/sql/spm/ob_plan_baseline.h +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#ifndef OCEANBASE_SQL_SPM_OB_PLAN_BASELINE_ -#define OCEANBASE_SQL_SPM_OB_PLAN_BASELINE_ - -#include "sql/spm/ob_spm_define.h" -#include "lib/container/ob_2d_array.h" - -namespace oceanbase -{ -namespace sql -{ - - -class ObPlanBaselineSet -{ -public: - ObPlanBaselineSet(); - virtual ~ObPlanBaselineSet(); -private: - // PlanBaselineItemArray pbi_array_; -}; - -} //namespace sql end -} //namespace oceanbase end - -#endif diff --git a/src/sql/spm/ob_plan_baseline_cache.h b/src/sql/spm/ob_plan_baseline_cache.h deleted file mode 100644 index e994e4081..000000000 --- a/src/sql/spm/ob_plan_baseline_cache.h +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright (c) 2021 OceanBase - * OceanBase CE is licensed under Mulan PubL v2. - * You can use this software according to the terms and conditions of the Mulan PubL v2. - * You may obtain a copy of Mulan PubL v2 at: - * http://license.coscl.org.cn/MulanPubL-2.0 - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PubL v2 for more details. - */ - -#ifndef OB_SQL_SPM_OB_PLAN_BASELINE_H_ -#define OB_SQL_SPM_OB_PLAN_BASELINE_H_ - -namespace oceanbase -{ -namespace sql -{ - -} // namespace sql end -} // namespace oceanbase end -#endif \ No newline at end of file diff --git a/src/sql/spm/ob_plan_baseline_mgr.h b/src/sql/spm/ob_plan_baseline_mgr.h index 00c0c51af..d8d9257fe 100644 --- a/src/sql/spm/ob_plan_baseline_mgr.h +++ b/src/sql/spm/ob_plan_baseline_mgr.h @@ -13,7 +13,6 @@ #ifndef OB_SQL_SPM_OB_PLAN_BASELINE_MGR_H_ #define OB_SQL_SPM_OB_PLAN_BASELINE_MGR_H_ #include "sql/spm/ob_spm_define.h" -#include "sql/spm/ob_plan_baseline_cache.h" #include "sql/spm/ob_plan_baseline_sql_service.h" #include "lib/task/ob_timer.h" #include "lib/list/ob_list.h" diff --git a/src/storage/backup/ob_backup_data_store.cpp b/src/storage/backup/ob_backup_data_store.cpp index 83a2fb94e..8aafee66d 100755 --- a/src/storage/backup/ob_backup_data_store.cpp +++ b/src/storage/backup/ob_backup_data_store.cpp @@ -15,6 +15,9 @@ #include "share/backup/ob_backup_io_adapter.h" #include "lib/restore/ob_storage.h" #include "lib/oblog/ob_log_module.h" +#ifdef OB_BUILD_TDE_SECURITY +#include "share/ob_master_key_getter.h" +#endif using namespace oceanbase; using namespace common; @@ -700,12 +703,39 @@ int ObBackupDataStore::read_backup_set_info(ObExternBackupSetInfoDesc &backup_se int ObBackupDataStore::write_root_key_info(const uint64_t tenant_id) { int ret = OB_SUCCESS; +#ifdef OB_BUILD_TDE_SECURITY + share::ObBackupPath path; + ObString empty_str; + // TODO(sean.yyj): support use user specified encrypt key to encrypt rook key backup + if (!is_init()) { + ret = OB_NOT_INIT; + LOG_WARN("backup data extern mgr not init", K(ret)); + } else if (OB_FAIL(ObBackupPathUtil::get_backup_root_key_path(backup_set_dest_, path))) { + LOG_WARN("fail to get path", K(ret)); + } else if (OB_FAIL(ObMasterKeyUtil::backup_root_key(tenant_id, path.get_obstr(), + backup_set_dest_.get_storage_info(), empty_str))) { + LOG_WARN("fail to backup root key", K(ret)); + } +#endif return ret; } int ObBackupDataStore::read_root_key_info(const uint64_t tenant_id) { int ret = OB_SUCCESS; +#ifdef OB_BUILD_TDE_SECURITY + share::ObBackupPath path; + ObString empty_str; + if (!is_init()) { + ret = OB_NOT_INIT; + LOG_WARN("backup data extern mgr not init", K(ret)); + } else if (OB_FAIL(ObBackupPathUtil::get_backup_root_key_path(backup_set_dest_, path))) { + LOG_WARN("fail to get path", K(ret)); + } else if (OB_FAIL(ObMasterKeyUtil::restore_root_key(tenant_id, path.get_obstr(), + backup_set_dest_.get_storage_info(), empty_str))) { + LOG_WARN("fail to backup root key", K(ret)); + } +#endif return ret; } diff --git a/src/storage/blocksstable/ob_macro_block_reader.cpp b/src/storage/blocksstable/ob_macro_block_reader.cpp index 4ee3ad21d..91027f8fe 100644 --- a/src/storage/blocksstable/ob_macro_block_reader.cpp +++ b/src/storage/blocksstable/ob_macro_block_reader.cpp @@ -56,6 +56,24 @@ ObMacroBlockReader::~ObMacroBlockReader() } } +#ifdef OB_BUILD_TDE_SECURITY +int ObMacroBlockReader::init_encrypter_if_needed() +{ + int ret = OB_SUCCESS; + void *buf = nullptr; + ObMemAttr attr(MTL_ID(), ObModIds::OB_CS_SSTABLE_READER); + + if (nullptr != encryption_) { + } else if (OB_ISNULL(buf = ob_malloc(sizeof(ObMicroBlockEncryption), attr))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + STORAGE_LOG(WARN, "Failed to alloc memory for encrypter", K(ret)); + } else { + encryption_ = new (buf) ObMicroBlockEncryption(); + } + + return ret; +} +#endif int ObMacroBlockReader::decompress_data( @@ -317,7 +335,27 @@ int ObMacroBlockReader::decrypt_and_decompress_data( } else { const char *data_buf = input + header_size; int64_t data_buf_size = size - header_size; +#ifndef OB_BUILD_TDE_SECURITY is_compressed = header.is_compressed_data(); +#else + if (OB_UNLIKELY(share::ObEncryptionUtil::need_encrypt(deserialize_meta.encrypt_id_))) { + LOG_DEBUG("Macro data need decrypt", K(deserialize_meta.encrypt_id_), K(data_buf_size)); + const char *decrypt_buf = NULL; + int64_t decrypt_size = 0; + if (OB_FAIL(ObMacroBlockReader::decrypt_buf( + deserialize_meta, data_buf, data_buf_size, decrypt_buf, decrypt_size))) { + STORAGE_LOG(WARN, "fail to decrypt buf", K(ret)); + } else { + data_buf = decrypt_buf; + data_buf_size = decrypt_size; + is_compressed = (header.data_length_ != decrypt_size); + is_encrypted = true; + } + } else { + is_compressed = header.is_compressed_data(); + is_encrypted = false; + } +#endif if (OB_SUCC(ret) && !is_compressed) { uncomp_size = header_size + data_buf_size; @@ -362,6 +400,36 @@ int ObMacroBlockReader::decrypt_and_decompress_data( return ret; } +#ifdef OB_BUILD_TDE_SECURITY +int ObMacroBlockReader::decrypt_buf( + const ObMicroBlockDesMeta &deserialize_meta, + const char *buf, + const int64_t size, + const char *&decrypt_buf, + int64_t &decrypt_size) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(buf)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("Invalid argument of input data", K(ret), KP(buf)); + } else if (OB_UNLIKELY(!share::ObEncryptionUtil::need_encrypt(deserialize_meta.encrypt_id_))) { + decrypt_buf = buf; + decrypt_size = size; + } else if (OB_FAIL(init_encrypter_if_needed())) { + LOG_WARN("Failed to init encrypter", K(ret)); + } else if (OB_FAIL(encryption_->init( + deserialize_meta.encrypt_id_, + MTL_ID(), + deserialize_meta.master_key_id_, + deserialize_meta.encrypt_key_, + share::OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH))) { + LOG_WARN("Fail to init micro block encryption", K(ret)); + } else if (OB_FAIL(encryption_->decrypt(buf, size, decrypt_buf, decrypt_size))) { + LOG_WARN("Fail to decrypt data", K(ret)); + } + return ret; +} +#endif ObSSTableDataBlockReader::ObSSTableDataBlockReader() : data_(NULL), size_(0), common_header_(), macro_header_(), linked_header_(), diff --git a/src/storage/blocksstable/ob_macro_block_reader.h b/src/storage/blocksstable/ob_macro_block_reader.h index 3b0219593..c13a076be 100644 --- a/src/storage/blocksstable/ob_macro_block_reader.h +++ b/src/storage/blocksstable/ob_macro_block_reader.h @@ -88,9 +88,20 @@ public: bool &is_compressed, const bool need_deep_copy = false, ObIAllocator *ext_allocator = nullptr); +#ifdef OB_BUILD_TDE_SECURITY + int decrypt_buf( + const ObMicroBlockDesMeta &deserialize_meta, + const char *buf, + const int64_t size, + const char *&decrypt_buf, + int64_t &decrypt_size); +#endif private: int alloc_buf(const int64_t req_size, char *&buf, int64_t &buf_size); int alloc_buf(ObIAllocator &allocator, const int64_t buf_size, char *&buf); +#ifdef OB_BUILD_TDE_SECURITY + int init_encrypter_if_needed(); +#endif private: common::ObCompressor *compressor_; diff --git a/src/storage/blocksstable/ob_macro_block_writer.cpp b/src/storage/blocksstable/ob_macro_block_writer.cpp index f2d33608e..a3e4fffa0 100644 --- a/src/storage/blocksstable/ob_macro_block_writer.cpp +++ b/src/storage/blocksstable/ob_macro_block_writer.cpp @@ -59,6 +59,15 @@ int ObMicroBlockBufferHelper::open( STORAGE_LOG(WARN, "invalid input argument.", K(ret), K(data_store_desc), K(read_info)); } else if (OB_FAIL(compressor_.init(data_store_desc.micro_block_size_, data_store_desc.compressor_type_))) { STORAGE_LOG(WARN, "Fail to init micro block compressor, ", K(ret), K(data_store_desc)); +#ifdef OB_BUILD_TDE_SECURITY + } else if (OB_FAIL(encryption_.init( + data_store_desc.encrypt_id_, + MTL_ID(), + data_store_desc.master_key_id_, + data_store_desc.encrypt_key_, + OB_MAX_TABLESPACE_ENCRYPT_KEY_LENGTH))) { + STORAGE_LOG(WARN, "fail to init micro block encryption", K(ret), K(data_store_desc)); +#endif } else if (OB_FAIL(check_datum_row_.init(allocator, read_info.get_request_count()))) { STORAGE_LOG(WARN, "Failed to init datum row", K(ret), K(read_info)); } else if (OB_FAIL(check_reader_helper_.init(allocator))) { @@ -75,6 +84,9 @@ void ObMicroBlockBufferHelper::reset() data_store_desc_ = nullptr; micro_block_merge_verify_level_ = 0; compressor_.reset(); +#ifdef OB_BUILD_TDE_SECURITY + encryption_.reset(); +#endif check_reader_helper_.reset(); check_datum_row_.reset(); } @@ -96,10 +108,18 @@ int ObMicroBlockBufferHelper::compress_encrypt_micro_block(ObMicroBlockDesc &mic && OB_FAIL(check_micro_block(compress_buf, compress_buf_size, block_buffer, block_size, micro_block_desc))) { STORAGE_LOG(WARN, "failed to check micro block", K(ret)); +#ifndef OB_BUILD_TDE_SECURITY } else { ObMicroBlockHeader *header = const_cast(micro_block_desc.header_); micro_block_desc.buf_ = compress_buf; micro_block_desc.buf_size_ = compress_buf_size; +#else + } else if (OB_FAIL(encryption_.encrypt(compress_buf, compress_buf_size, micro_block_desc.buf_, micro_block_desc.buf_size_))) { + STORAGE_LOG(WARN, "fail to encrypt micro block", K(ret)); + } else { + // fill header after compress/encrypt + ObMicroBlockHeader *header = const_cast(micro_block_desc.header_); +#endif header->data_length_ = block_size; header->data_zlength_ = micro_block_desc.buf_size_; header->data_checksum_ = ob_crc64_sse42(0, micro_block_desc.buf_, micro_block_desc.buf_size_); diff --git a/src/storage/blocksstable/ob_micro_block_cache.cpp b/src/storage/blocksstable/ob_micro_block_cache.cpp index 80a488f4b..226f6ecee 100755 --- a/src/storage/blocksstable/ob_micro_block_cache.cpp +++ b/src/storage/blocksstable/ob_micro_block_cache.cpp @@ -351,6 +351,10 @@ int ObIMicroBlockIOCallback::process_block( ObIMicroBlockCache::BaseBlockCache *kvcache = nullptr; ObKVCachePair *kvpair = nullptr; ObKVCacheInstHandle inst_handle; +#ifdef OB_BUILD_TDE_SECURITY + const char *decrypt_buf = NULL; + int64_t decrypt_len = 0; +#endif const int64_t block_size = header.header_size_ + header.data_length_; int64_t extra_size = 0; bool need_decoder = false; @@ -375,12 +379,20 @@ int ObIMicroBlockIOCallback::process_block( LOG_WARN("Fail to serialize header", K(ret), K(header)); } else if (FALSE_IT(payload_buf = payload_buf + pos)) { } else if (FALSE_IT(payload_size = payload_size - pos)) { +#ifndef OB_BUILD_TDE_SECURITY } else if (OB_FAIL(reader->decompress_data_with_prealloc_buf( block_des_meta_.compressor_type_, payload_buf, payload_size, block_buf + pos, block_size - pos))) { +#else + } else if (OB_FAIL(reader->decrypt_buf( + block_des_meta_, payload_buf, payload_size, decrypt_buf, decrypt_len))) { + LOG_WARN("Fail to decrypt data", K(ret)); + } else if (OB_FAIL(reader->decompress_data_with_prealloc_buf(block_des_meta_.compressor_type_, decrypt_buf, + decrypt_len, block_buf + pos, block_size - pos))) { +#endif LOG_WARN("Fail to decompress data with preallocated buffer", K(ret)); } else if (need_write_extra_buf_ && OB_FAIL(cache_->write_extra_buf(block_buf, block_size, extra_size, block_buf + block_size, micro_data))) { diff --git a/src/storage/blocksstable/ob_micro_block_encryption.h b/src/storage/blocksstable/ob_micro_block_encryption.h index 292f476f9..d37a89747 100644 --- a/src/storage/blocksstable/ob_micro_block_encryption.h +++ b/src/storage/blocksstable/ob_micro_block_encryption.h @@ -21,8 +21,37 @@ namespace blocksstable { class ObMicroBlockEncryption { public: +#ifndef OB_BUILD_TDE_SECURITY ObMicroBlockEncryption() {} virtual ~ObMicroBlockEncryption() {} +#else + ObMicroBlockEncryption(); + virtual ~ObMicroBlockEncryption(); + void reset(); + int init(const int64_t encrypt_id, const int64_t tenant_id, + const int64_t master_key_id = 0, const char *encrypt_key = NULL, const int64_t encrypt_key_len = 0); + int encrypt(const char *in, const int64_t in_size, const char *&out, int64_t &out_size); + int decrypt(const char *in, const int64_t in_size, const char *&out, int64_t &out_size); +private: + //此密钥用于mysql模式下表加解密 + int generate_key(); + //此密钥用于oracle模式下加解密 + int generate_key(char *master_key, const int64_t master_key_len, char *encrypt_key, const int64_t encrypt_key_len); + bool need_encrypt(); + bool need_refresh(const int64_t encrypt_id, const int64_t tenant_id, const int64_t master_key_id, + const char *encrypt_key, const int64_t encrypt_key_len); +private: + bool is_inited_; + uint64_t tenant_id_; + ObSelfBufferWriter encrypt_buf_; + ObSelfBufferWriter decrypt_buf_; + int64_t encrypt_id_; + int64_t raw_key_len_; + char raw_key_[common::OB_MAX_ENCRYPTION_KEY_NAME_LENGTH]; // 真正用于数据加解密的密钥 + char encrypt_key_[common::OB_MAX_ENCRYPTION_KEY_NAME_LENGTH]; // 密钥的密文 + int64_t encrypt_key_len_; // 密钥的密文的长度 + int64_t master_key_id_; // 主密钥版本 +#endif DISALLOW_COPY_AND_ASSIGN(ObMicroBlockEncryption); }; diff --git a/src/storage/high_availability/ob_ls_restore.h b/src/storage/high_availability/ob_ls_restore.h index d5f0f6d97..8f553d400 100644 --- a/src/storage/high_availability/ob_ls_restore.h +++ b/src/storage/high_availability/ob_ls_restore.h @@ -120,6 +120,7 @@ private: backup::ObBackupMetaIndexStoreWrapper meta_index_store_; backup::ObBackupMetaIndexStoreWrapper second_meta_index_store_; backup::ObBackupIndexKVCache *kv_cache_; + //TODO(muwei.ym) put bandwidth_throttle, svr_rpc_proxy, and storage_rpc into ctx 4.3 common::ObInOutBandwidthThrottle *bandwidth_throttle_; obrpc::ObStorageRpcProxy *svr_rpc_proxy_; storage::ObStorageRpc *storage_rpc_; diff --git a/src/storage/high_availability/ob_storage_ha_macro_block_writer.cpp b/src/storage/high_availability/ob_storage_ha_macro_block_writer.cpp index 2e7bb8ba6..d2d3fe113 100644 --- a/src/storage/high_availability/ob_storage_ha_macro_block_writer.cpp +++ b/src/storage/high_availability/ob_storage_ha_macro_block_writer.cpp @@ -148,6 +148,7 @@ int ObStorageHAMacroBlockWriter::process(blocksstable::ObMacroBlocksWriteCtx &co } else if (!write_handle.is_empty() && OB_FAIL(write_handle.wait(io_timeout_ms))) { STORAGE_LOG(WARN, "failed to wait write handle", K(ret)); } else if (header.is_reuse_macro_block_) { + //TODO(muwei.ym) reuse macro block in 4.3 ret = OB_NOT_SUPPORTED; LOG_WARN("header is reuse macro block", K(ret)); } else { diff --git a/src/storage/high_availability/ob_storage_ha_service.h b/src/storage/high_availability/ob_storage_ha_service.h index eaef87c6e..5b712e894 100644 --- a/src/storage/high_availability/ob_storage_ha_service.h +++ b/src/storage/high_availability/ob_storage_ha_service.h @@ -46,7 +46,7 @@ private: private: - // TODO: change SCHEDULER_WAIT_TIME_MS to 5 min when rs use rpc to wake up the ha service. + // TODO(chongrong.th): change SCHEDULER_WAIT_TIME_MS to 5 min when rs use rpc to wake up the ha service in 4.3 static const int64_t SCHEDULER_WAIT_TIME_MS = 1000L; // 1s bool is_inited_; common::ObThreadCond thread_cond_; diff --git a/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp b/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp index 6b054c2b7..baac3af4f 100755 --- a/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp +++ b/src/storage/high_availability/ob_storage_ha_tablet_builder.cpp @@ -845,6 +845,7 @@ int ObStorageHATabletsBuilder::build_copy_tablet_sstable_info_arg_( //ddl if (OB_SUCC(ret)) { + //TODO(muwei.ym) now do not reuse ddl sstable, will reuse it in 4.3 if (OB_FAIL(get_need_copy_ddl_sstable_range_(tablet, ddl_sstable_array, arg.ddl_sstable_scn_range_))) { LOG_WARN("failed to get need copy ddl sstable range", K(ret)); } @@ -1066,7 +1067,7 @@ int ObStorageHATabletsBuilder::hold_local_reuse_sstable_( } else { while (OB_SUCC(ret)) { if (tablet->get_tablet_meta().has_next_tablet_) { - //TODO: In this condition can work without L replica. + //TODO(muwei.ym) In this condition can work without L replica. 4.3 //With L replica inner tablet should keep multi version tablet if (OB_FAIL(remove_uncomplete_tablet_(tablet_id))) { LOG_WARN("failed to remove uncomplete tablet", K(ret), K(tablet_id)); @@ -1105,7 +1106,7 @@ int ObStorageHATabletsBuilder::hold_local_complete_tablet_sstable_( { int ret = OB_SUCCESS; ObTabletMemberWrapper table_store_wrapper; - + //TODO(muwwei.ym) here do not reuse andy ddl sstables and minor sstables if (!is_inited_) { ret = OB_NOT_INIT; LOG_WARN("storage ha tablets builder do not init", K(ret)); @@ -1116,6 +1117,7 @@ int ObStorageHATabletsBuilder::hold_local_complete_tablet_sstable_( LOG_INFO("ls inner tablet do not reuse any sstable", K(ret), KPC(tablet)); } else if (OB_FAIL(tablet->fetch_table_store(table_store_wrapper))) { LOG_WARN("fail to fetch table store", K(ret)); + //TODO(muwei.ym) ls inner tablet now do not reuse any sstable, will reuse in 4.3 } else { const ObSSTableArray &major_sstable = table_store_wrapper.get_member()->get_major_sstables(); for (int64_t i = 0; OB_SUCC(ret) && i < major_sstable.count(); ++i) { @@ -1257,6 +1259,7 @@ int ObStorageHATabletsBuilder::create_remote_logical_sstable_( return ret; } +//TODO(muwei.ym) put this param in tablet_table_store 4.3 int ObStorageHATabletsBuilder::build_remote_logical_sstable_param_( const SCN start_scn, const SCN end_scn, diff --git a/src/storage/high_availability/ob_tablet_group_restore.h b/src/storage/high_availability/ob_tablet_group_restore.h index 80b9167d1..e8b5325d5 100644 --- a/src/storage/high_availability/ob_tablet_group_restore.h +++ b/src/storage/high_availability/ob_tablet_group_restore.h @@ -373,6 +373,7 @@ protected: protected: bool is_inited_; ObTabletRestoreCtx tablet_restore_ctx_; + //TODO(muwei.ym) put this into dag net ctx 4.3 common::ObInOutBandwidthThrottle *bandwidth_throttle_; obrpc::ObStorageRpcProxy *svr_rpc_proxy_; storage::ObStorageRpc *storage_rpc_; diff --git a/src/storage/ls/ob_ls.cpp b/src/storage/ls/ob_ls.cpp index 6f808c75b..96f62d140 100755 --- a/src/storage/ls/ob_ls.cpp +++ b/src/storage/ls/ob_ls.cpp @@ -18,10 +18,16 @@ #include "logservice/ob_log_service.h" #include "logservice/archiveservice/ob_archive_service.h" #include "logservice/data_dictionary/ob_data_dict_service.h" +#ifdef OB_BUILD_ARBITRATION +#include "logservice/arbserver/ob_arb_srv_garbage_collect_service.h" +#endif #include "observer/net/ob_ingress_bw_alloc_service.h" #include "observer/ob_srv_network_frame.h" #include "observer/report/ob_i_meta_report.h" #include "rootserver/freeze/ob_major_freeze_service.h" +#ifdef OB_BUILD_ARBITRATION +#include "rootserver/ob_arbitration_service.h" +#endif #include "rootserver/backup/ob_backup_task_scheduler.h" #include "rootserver/backup/ob_backup_service.h" #include "rootserver/backup/ob_archive_scheduler_service.h" @@ -280,6 +286,12 @@ int ObLS::init(const share::ObLSID &ls_id, REGISTER_TO_LOGSERVICE(logservice::RESTORE_SERVICE_LOG_BASE_TYPE, MTL(rootserver::ObRestoreService *)); } +#ifdef OB_BUILD_ARBITRATION + if (OB_SUCC(ret) && !is_user_tenant(tenant_id) && ls_id.is_sys_ls()) { + REGISTER_TO_LOGSERVICE(logservice::ARBITRATION_SERVICE_LOG_BASE_TYPE, MTL(rootserver::ObArbitrationService *)); + LOG_INFO("arbitration service regist to logservice success"); + } +#endif if (OB_SUCC(ret) && is_sys_tenant(tenant_id) && ls_id.is_sys_ls()) { rootserver::ObIngressBWAllocService *ingress_service = GCTX.net_frame_->get_ingress_service(); REGISTER_TO_LOGSERVICE(logservice::NET_ENDPOINT_INGRESS_LOG_BASE_TYPE, ingress_service); @@ -813,6 +825,13 @@ void ObLS::destroy() rootserver::ObHeartbeatService * heartbeat_service = MTL(rootserver::ObHeartbeatService*); UNREGISTER_FROM_LOGSERVICE(logservice::HEARTBEAT_SERVICE_LOG_BASE_TYPE, heartbeat_service); } +#ifdef OB_BUILD_ARBITRATION + if (!is_user_tenant(MTL_ID()) && ls_meta_.ls_id_.is_sys_ls()) { + rootserver::ObArbitrationService * arbitration_service = MTL(rootserver::ObArbitrationService*); + UNREGISTER_FROM_LOGSERVICE(logservice::ARBITRATION_SERVICE_LOG_BASE_TYPE, arbitration_service); + } + +#endif if (is_sys_tenant(MTL_ID()) && ls_meta_.ls_id_.is_sys_ls()) { rootserver::ObIngressBWAllocService *ingress_service = GCTX.net_frame_->get_ingress_service(); UNREGISTER_FROM_LOGSERVICE(logservice::NET_ENDPOINT_INGRESS_LOG_BASE_TYPE, ingress_service); @@ -2135,6 +2154,11 @@ int ObLS::diagnose(DiagnoseInfo &info) const STORAGE_LOG(WARN, "diagnose palf failed", K(ret), K(ls_id)); } else if (OB_FAIL(restore_handler_.diagnose(info.restore_diagnose_info_))) { STORAGE_LOG(WARN, "diagnose restore_handler failed", K(ret), K(ls_id), K(info)); +#ifdef OB_BUILD_ARBITRATION + } else if (common::ObRole::LEADER == info.palf_diagnose_info_.palf_role_ && + OB_FAIL(log_service->diagnose_arb_srv(ls_id, info.arb_srv_diagnose_info_))) { + STORAGE_LOG(WARN, "diagnose_arb_srv failed", K(ret), K(ls_id)); +#endif } else if (info.is_role_sync()) { // 角色同步时不需要诊断role change service info.rc_diagnose_info_.state_ = logservice::TakeOverState::TAKE_OVER_FINISH; diff --git a/src/storage/ls/ob_ls.h b/src/storage/ls/ob_ls.h index 722195187..679bd54a0 100755 --- a/src/storage/ls/ob_ls.h +++ b/src/storage/ls/ob_ls.h @@ -46,6 +46,9 @@ #include "logservice/rcservice/ob_role_change_handler.h" #include "logservice/restoreservice/ob_log_restore_handler.h" // ObLogRestoreHandler #include "logservice/ob_log_handler.h" +#ifdef OB_BUILD_ARBITRATION +#include "logservice/ob_arbitration_service.h" +#endif #include "logservice/restoreservice/ob_log_restore_handler.h" // ObLogRestoreHandler #include "storage/ls/ob_ls_meta_package.h" #include "storage/ls/ob_ls_get_mod.h" @@ -129,6 +132,9 @@ struct DiagnoseInfo logservice::GCDiagnoseInfo gc_diagnose_info_; checkpoint::CheckpointDiagnoseInfo checkpoint_diagnose_info_; logservice::RestoreDiagnoseInfo restore_diagnose_info_; +#ifdef OB_BUILD_ARBITRATION + logservice::LogArbSrvDiagnoseInfo arb_srv_diagnose_info_; +#endif TO_STRING_KV(K(ls_id_), K(log_handler_diagnose_info_), K(palf_diagnose_info_), @@ -138,6 +144,9 @@ struct DiagnoseInfo K(gc_diagnose_info_), K(checkpoint_diagnose_info_), K(restore_diagnose_info_) +#ifdef OB_BUILD_ARBITRATION + ,K(arb_srv_diagnose_info_) +#endif ); void reset() { ls_id_ = -1; @@ -149,6 +158,9 @@ struct DiagnoseInfo gc_diagnose_info_.reset(); checkpoint_diagnose_info_.reset(); restore_diagnose_info_.reset(); +#ifdef OB_BUILD_ARBITRATION + arb_srv_diagnose_info_.reset(); +#endif } }; @@ -645,6 +657,10 @@ public: DELEGATE_WITH_RET(log_handler_, disable_vote, int); DELEGATE_WITH_RET(log_handler_, remove_member, int); DELEGATE_WITH_RET(log_handler_, remove_learner, int); +#ifdef OB_BUILD_ARBITRATION + DELEGATE_WITH_RET(log_handler_, add_arbitration_member, int); + DELEGATE_WITH_RET(log_handler_, remove_arbitration_member, int); +#endif DELEGATE_WITH_RET(log_handler_, is_in_sync, int); DELEGATE_WITH_RET(log_handler_, get_end_scn, int); DELEGATE_WITH_RET(log_handler_, disable_sync, int); diff --git a/src/storage/ls/ob_ls_meta.cpp b/src/storage/ls/ob_ls_meta.cpp index c6ea5f127..f9a5f218c 100644 --- a/src/storage/ls/ob_ls_meta.cpp +++ b/src/storage/ls/ob_ls_meta.cpp @@ -470,7 +470,7 @@ int ObLSMeta::update_ls_meta( guard.click(); tmp.all_id_meta_.update_all_id_meta(src_ls_meta.all_id_meta_); if (tmp.clog_checkpoint_scn_ < clog_checkpoint_scn_) { - // TODO: now do not allow clog checkpoint ts rollback, may support it in 4.3 + // TODO(muwei.ym): now do not allow clog checkpoint ts rollback, may support it in 4.3 ret = OB_ERR_UNEXPECTED; LOG_WARN("do not allow clog checkpoint ts rollback", K(ret), K(src_ls_meta), KPC(this)); } else if (OB_FAIL(write_slog_(tmp))) { diff --git a/src/storage/memtable/ob_memtable.cpp b/src/storage/memtable/ob_memtable.cpp index 701faf089..e766ed19b 100755 --- a/src/storage/memtable/ob_memtable.cpp +++ b/src/storage/memtable/ob_memtable.cpp @@ -336,6 +336,14 @@ int ObMemtable::set( TRANS_LOG(WARN, "invalid param", K(param), K(columns.count()), K(row.row_val_.count_)); ret = OB_INVALID_ARGUMENT; +#ifdef OB_BUILD_TDE_SECURITY + //TODO: table_id is just used as encrypt_index, we may rename it in the future. + // If the function(set) no longer passes in this parameter(table_id), + // we need to construct ObTxEncryptMeta in advance, and pass tx_encrypt_meta(ObTxEncryptMeta*) + // instead of encrypt_meta(ObEncryptMeta*) into this function(set) + } else if (need_for_save(encrypt_meta) && OB_FAIL(save_encrypt_meta(param.table_id_, encrypt_meta))) { + TRANS_LOG(WARN, "store encrypt meta to memtable failed", KPC(encrypt_meta), KR(ret)); +#endif } if (OB_FAIL(ret)) { @@ -371,6 +379,14 @@ int ObMemtable::set( || param.get_schema_rowkey_count() > columns.count()) { ret = OB_INVALID_ARGUMENT; TRANS_LOG(ERROR, "invalid param", K(ret), K(param)); +#ifdef OB_BUILD_TDE_SECURITY + //TODO: table_id is just used as encrypt_index, we may rename it in the future. + // If the function(set) no longer passes in this parameter(table_id), + // we need to construct ObTxEncryptMeta in advance, and pass tx_encrypt_meta(ObTxEncryptMeta*) + // instead of encrypt_meta(ObEncryptMeta*) into this function(set) + } else if (need_for_save(encrypt_meta) && OB_FAIL(save_encrypt_meta(param.table_id_, encrypt_meta))) { + TRANS_LOG(WARN, "store encrypt meta to memtable failed", KPC(encrypt_meta), KR(ret)); +#endif } if (OB_FAIL(ret)){ @@ -2385,6 +2401,59 @@ bool ObMemtable::is_partition_memtable_empty(const uint64_t table_id) const return query_engine_.is_partition_memtable_empty(table_id); } +#ifdef OB_BUILD_TDE_SECURITY +int ObMemtable::save_encrypt_meta(const uint64_t table_id, const share::ObEncryptMeta *encrypt_meta) +{ + int ret = OB_SUCCESS; + SpinWLockGuard guard(encrypt_meta_lock_); + if (OB_NOT_NULL(encrypt_meta)) { + if (OB_ISNULL(encrypt_meta_) && + (OB_ISNULL(encrypt_meta_ = (ObTxEncryptMeta *)local_allocator_.alloc(sizeof(ObTxEncryptMeta))) || + OB_ISNULL(new(encrypt_meta_) ObTxEncryptMeta()))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + TRANS_LOG(WARN, "alloc failed", KP(encrypt_meta), K(ret)); + } else { + ret = encrypt_meta_->store_encrypt_meta(table_id, *encrypt_meta); + } + } else { + //maybe the table is removed from encrypted tablespace + local_allocator_.free((void *)encrypt_meta_); + encrypt_meta_ = nullptr; + } + return ret; +} + +int ObMemtable::get_encrypt_meta(transaction::ObTxEncryptMeta *&encrypt_meta) +{ + int ret = OB_SUCCESS; + SpinRLockGuard guard(encrypt_meta_lock_); + if (NULL != encrypt_meta_) { + if (NULL == encrypt_meta && NULL == (encrypt_meta = op_alloc(ObTxEncryptMeta))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + } else { + ret = encrypt_meta->assign(*encrypt_meta_); + } + } else { + if (NULL != encrypt_meta) { + encrypt_meta->reset(); + } + } + return ret; +} + +bool ObMemtable::need_for_save(const share::ObEncryptMeta *encrypt_meta) +{ + bool need_save = true; + SpinRLockGuard guard(encrypt_meta_lock_); + if (encrypt_meta == NULL && encrypt_meta_ == NULL) { + need_save = false; + } else if (encrypt_meta != NULL && encrypt_meta_ != NULL && + encrypt_meta_->is_memtable_equal(*encrypt_meta)) { + need_save = false; + } + return need_save; +} +#endif int RowHeaderGetter::get() { diff --git a/src/storage/memtable/ob_memtable.h b/src/storage/memtable/ob_memtable.h index e75f2a589..f09dde9f5 100644 --- a/src/storage/memtable/ob_memtable.h +++ b/src/storage/memtable/ob_memtable.h @@ -463,6 +463,12 @@ public: void set_allow_freeze(const bool allow_freeze); inline bool allow_freeze() const { return ATOMIC_LOAD(&allow_freeze_); } +#ifdef OB_BUILD_TDE_SECURITY + /*clog encryption related*/ + int save_encrypt_meta(const uint64_t table_id, const share::ObEncryptMeta *encrypt_meta); + int get_encrypt_meta(transaction::ObTxEncryptMeta *&encrypt_meta); + bool need_for_save(const share::ObEncryptMeta *encrypt_meta); +#endif // Print stat data in log. // For memtable debug. diff --git a/src/storage/memtable/ob_memtable_mutator.cpp b/src/storage/memtable/ob_memtable_mutator.cpp index 0481a468a..32038e02d 100644 --- a/src/storage/memtable/ob_memtable_mutator.cpp +++ b/src/storage/memtable/ob_memtable_mutator.cpp @@ -398,6 +398,64 @@ int ObMemtableMutatorRow::copy(uint64_t &table_id, return ret; } +#ifdef OB_BUILD_TDE_SECURITY +int ObMemtableMutatorRow::handle_encrypt_row_(char *buf, const int64_t buf_len, + const int64_t start_pos, int64_t &end_pos, + const share::ObEncryptMeta &meta) +{ + int ret = OB_SUCCESS; + int64_t encrypted_len = 0; + const int64_t expected_encrypted_len = ObEncryptionUtil::encrypted_length(end_pos - start_pos); + char *encrypted_buf = nullptr; + int64_t encrypted_buf_size = 0; + ObEncryptRowBuf row_buf; + #ifdef ERRSIM + ret = OB_E(EventTable::EN_ENCRYPT_ALLOCATE_ROW_BUF_FAILED) OB_SUCCESS; + if (OB_FAIL(ret)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + TRANS_LOG(WARN, "ERRSIM, alloc encrypt buf failed", K(ret)); + return ret; + } + #endif + if (OB_ISNULL(buf) || buf_len < 0 || start_pos < 0 || start_pos > end_pos) { + TRANS_LOG(WARN, "invalid argument", KP(buf), K(buf_len), K(start_pos), K(end_pos)); + ret = OB_INVALID_ARGUMENT; + } else if (expected_encrypted_len >= ObEncryptRowBuf::TMP_ENCRYPT_BUF_LEN) { + if (OB_FAIL(row_buf.alloc(expected_encrypted_len))) { + TRANS_LOG(WARN, "alloc encrypt buf failed", K(ret), K(table_id_), K(expected_encrypted_len)); + } else { + encrypted_buf = row_buf.ptr_; + encrypted_buf_size = expected_encrypted_len; + } + } else { + encrypted_buf = row_buf.buf_; + encrypted_buf_size = ObEncryptRowBuf::TMP_ENCRYPT_BUF_LEN; + } + if (OB_SUCC(ret)) { + if (NULL == encrypted_buf) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "encrypted_buf init fail", "ret", ret); + } else if (OB_FAIL(ObEncryptionUtil::encrypt_data(meta, + buf + start_pos, + end_pos - start_pos, + encrypted_buf, + encrypted_buf_size, + encrypted_len))) { + TRANS_LOG(WARN, "encyrpt row data failed", K(ret), K(table_id_), KP(buf), K(buf_len), K(start_pos), K(end_pos)); + } else if (start_pos + encrypted_len > buf_len) { + TRANS_LOG(WARN, "buffer not enough for encrypted row", K(table_id_), KP(buf), K(buf_len), K(start_pos), K(end_pos)); + ret = OB_BUF_NOT_ENOUGH; + } else { + MEMCPY(buf + start_pos, encrypted_buf, encrypted_len); + end_pos = start_pos + encrypted_len; + } + } + if (EXECUTE_COUNT_PER_SEC(1)) { + TRANS_LOG(INFO, "encrypt data for table", K(table_id_), K(ret)); + } + return ret; +} +#endif int ObMemtableMutatorRow::serialize(char *buf, int64_t &buf_len, int64_t &pos, const transaction::ObTxEncryptMeta *encrypt_meta, @@ -412,6 +470,32 @@ int ObMemtableMutatorRow::serialize(char *buf, int64_t &buf_len, int64_t &pos, if (OB_ISNULL(buf) || pos < 0 || pos > buf_len) { ret = OB_INVALID_ARGUMENT; +#ifdef OB_BUILD_TDE_SECURITY + } else if (encrypt_meta != NULL && encrypt_meta->is_valid()) { + need_encrypt = true; + table_id_ = encrypt_meta->table_id_; + if (OB_FAIL(encrypt_info.store_and_get_old(table_id_, *encrypt_meta, &old_meta))) { + if (ret == OB_ENTRY_EXIST) { + //if hash exist, we give priority to the old_meta + //in order to avoid inconsistent encrypt_meta for the same table_id + use_old = true; + ret = OB_SUCCESS; + } else { + TRANS_LOG(WARN, "store clog encrypt info failed", K(ret)); + } + } else { + int64_t meta_size = encoded_length_vi64(table_id_) + encrypt_meta->meta_.get_serialize_size(); + if (buf_len - meta_size < OB_MAX_COUNT_NEED_BYTES) { + //need delete this encrypt meta + if (OB_FAIL(encrypt_info.remove(table_id_))) { + TRANS_LOG(WARN, "remove clog encrypt info failed", K(ret)); + } + ret = OB_SIZE_OVERFLOW; + } else { + buf_len -= meta_size; + } + } +#endif } if (OB_SUCC(ret)) { @@ -433,6 +517,14 @@ int ObMemtableMutatorRow::serialize(char *buf, int64_t &buf_len, int64_t &pos, if (OB_BUF_NOT_ENOUGH != ret || buf_len > common::OB_MAX_LOG_ALLOWED_SIZE) { TRANS_LOG(INFO, "serialize row fail", K(ret), KP(buf), K(buf_len), K(pos)); } +#ifdef OB_BUILD_TDE_SECURITY + } else if (need_encrypt) { + if (use_old) { + ret = handle_encrypt_row_(buf, buf_len, data_pos, new_pos, old_meta->meta_); + } else { + ret = handle_encrypt_row_(buf, buf_len, data_pos, new_pos, encrypt_meta->meta_); + } +#endif } if (FAILEDx(encode_vi64(buf, buf_len, new_pos, column_cnt_))) { TRANS_LOG(WARN, "failed to serialize column cnt", K(column_cnt_)); @@ -446,6 +538,51 @@ int ObMemtableMutatorRow::serialize(char *buf, int64_t &buf_len, int64_t &pos, return ret; } +#ifdef OB_BUILD_TDE_SECURITY +int ObMemtableMutatorRow::handle_decrypt_row_(const char *buf, const int64_t start_pos, + const int64_t end_pos, ObEncryptRowBuf &row_buf, const char* &out_buf, int64_t &out_len, + const share::ObEncryptMeta &meta) +{ + int ret = OB_SUCCESS; + const int64_t expected_decrypted_len = end_pos - start_pos; + int64_t buf_size = 0; + char *tmp_buf = nullptr; + #ifdef ERRSIM + ret = OB_E(EventTable::EN_DECRYPT_ALLOCATE_ROW_BUF_FAILED) OB_SUCCESS; + if (OB_FAIL(ret)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + TRANS_LOG(WARN, "ERRSIM, alloc decrypt buf failed", K(ret)); + return ret; + } + #endif + if (OB_ISNULL(buf) || start_pos < 0 || start_pos > end_pos) { + TRANS_LOG(WARN, "handle_decrypt_row_, invalid argument", KP(buf), K(start_pos), K(end_pos)); + ret = OB_INVALID_ARGUMENT; + } else if (expected_decrypted_len >= ObEncryptRowBuf::TMP_ENCRYPT_BUF_LEN) { + if (OB_FAIL(row_buf.alloc(expected_decrypted_len))) { + TRANS_LOG(WARN, "alloc decrypt buf failed", K(ret), K(table_id_), K(expected_decrypted_len)); + } else { + tmp_buf = row_buf.ptr_; + buf_size = expected_decrypted_len; + } + } else { + tmp_buf = row_buf.buf_; + buf_size = ObEncryptRowBuf::TMP_ENCRYPT_BUF_LEN; + } + if (OB_SUCC(ret)) { + if (OB_FAIL(ObEncryptionUtil::decrypt_data(meta, + buf + start_pos, end_pos - start_pos, tmp_buf, buf_size, out_len))) { + TRANS_LOG(WARN, "decrypt row data failed", K(ret), K(table_id_)); + } else { + out_buf = tmp_buf; + } + } + if (EXECUTE_COUNT_PER_SEC(1)) { + TRANS_LOG(INFO, "decrypt data for table", K(table_id_), K(ret)); + } + return ret; +} +#endif int ObMemtableMutatorRow::deserialize(const char *buf, const int64_t buf_len, int64_t &pos, ObEncryptRowBuf &row_buf, @@ -472,10 +609,47 @@ int ObMemtableMutatorRow::deserialize(const char *buf, const int64_t buf_len, in TRANS_LOG(WARN, "deserialize table id failed", K(ret), K(buf_len), K(new_pos)); } else { int64_t data_pos = new_pos; +#ifdef OB_BUILD_TDE_SECURITY + //TODO: table_id_ is just used as encrypt_index, we may rename it in the future + bool need_decrypt = table_id_ > 0 ? true : false; + if (need_decrypt) { + transaction::ObTxEncryptMeta *encrypt_meta = NULL; + if (OB_FAIL(encrypt_info.get_encrypt_info(table_id_, encrypt_meta))) { + TRANS_LOG(ERROR, "failed to get encrypt info", K(ret), K(table_id_)); + } else { + ret = handle_decrypt_row_(buf, data_pos, pos + encrypted_len, + row_buf, decrypted_buf, decrypted_len, encrypt_meta->meta_); + if (OB_SUCC(ret)) { + row_size_ = decrypted_len + (data_pos - pos); + new_pos = 0; + if (need_extract_encrypt_meta) { + encrypt_stat_map.set_map(CLOG_CONTAIN_ENCRYPTED_ROW); + ObAesOpMode final_mode = static_cast(final_encrypt_meta.encrypt_algorithm_); + ObAesOpMode cur_mode = static_cast(encrypt_meta->meta_.encrypt_algorithm_); + if (ObAesEncryption::compare_aes_mod_safety(final_mode, cur_mode)) { + if (OB_FAIL(final_encrypt_meta.assign(encrypt_meta->meta_))) { + TRANS_LOG(WARN, "failed to assign encrypt_meta", K(ret), K(table_id_)); + } + } + } + } + } + } else { + //no encryption + decrypted_buf = buf + data_pos; + decrypted_len = encrypted_len - (data_pos - pos); + row_size_ = encrypted_len; + new_pos = 0; + if (need_extract_encrypt_meta) { + encrypt_stat_map.set_map(CLOG_CONTAIN_NON_ENCRYPTED_ROW); + } + } +#else decrypted_buf = buf + data_pos; decrypted_len = encrypted_len - (data_pos - pos); row_size_ = encrypted_len; new_pos = 0; +#endif if (OB_SUCC(ret)) { if (NULL == decrypted_buf) { ret = OB_ERR_UNEXPECTED; @@ -1080,6 +1254,11 @@ int ObMutatorWriter::serialize(const uint8_t row_flag, int64_t &res_len, } else if (OB_FAIL(meta_.fill_header(buf_.get_data() + meta_size, buf_.get_position() - meta_size))) { } else if (OB_FAIL(meta_.serialize(buf_.get_data(), meta_size, meta_pos))) { +#ifdef OB_BUILD_TDE_SECURITY + } else if (((row_flag & ObTransRowFlag::ENCRYPT) > 0) && + OB_FAIL(encrypt_info.serialize(buf_.get_data(), buf_.get_capacity(), end_pos))) { + TRANS_LOG(WARN, "serialize clog encrypt info failed", K(ret), K(buf_.get_capacity()), K(end_pos)); +#endif } else { buf_.get_position() = end_pos; res_len = buf_.get_position(); @@ -1097,6 +1276,55 @@ int64_t ObMutatorWriter::get_serialize_size() const return SIZE; } +#ifdef OB_BUILD_TDE_SECURITY +int ObMutatorWriter::encrypt_big_row_data( + const char *in_buf, const int64_t in_buf_len, int64_t &in_buf_pos, + char *out_buf, const int64_t out_buf_len, int64_t &out_buf_pos, + const int64_t table_id, const transaction::ObCLogEncryptInfo &encrypt_info, + bool &need_encrypt) +{ + int ret = OB_SUCCESS; + transaction::ObTxEncryptMeta *encrypt_meta = NULL; + need_encrypt = false; + if (!encrypt_info.is_valid() || OB_ISNULL(in_buf) || in_buf_len < 0 || in_buf_pos > in_buf_len + || OB_ISNULL(out_buf) || out_buf_len < 0 || out_buf_pos > out_buf_len) { + TRANS_LOG(WARN, "invalid argument", KP(in_buf), K(in_buf_len), K(in_buf_pos), K(table_id)); + ret = OB_INVALID_ARGUMENT; + } else if (OB_FAIL(encrypt_info.get_encrypt_info(table_id, encrypt_meta))) { + TRANS_LOG(ERROR, "failed to get encrypt info", K(ret), K(table_id)); + } else if (need_encrypt) { + if (OB_FAIL(serialization::encode_vi64(out_buf, out_buf_len, out_buf_pos, table_id))) { + TRANS_LOG(WARN, "failed to encode table id", K(ret)); + } else { + int64_t encrypted_len = 0; + int64_t data_len = ObEncryptionUtil::decrypted_length(out_buf_len - out_buf_pos); + data_len = min(data_len, in_buf_len - in_buf_pos); + if (OB_UNLIKELY(data_len < 0)) { + TRANS_LOG(WARN, "buf to short to hold encrypted data", + K(out_buf_len), K(out_buf_pos), K(in_buf_len), K(in_buf_pos)); + ret = OB_BUF_NOT_ENOUGH; + } else if (OB_FAIL(ObEncryptionUtil::encrypt_data(encrypt_meta->meta_, + in_buf + in_buf_pos, data_len, + out_buf + out_buf_pos, out_buf_len - out_buf_pos, + encrypted_len))) { + TRANS_LOG(WARN, "failed to encrypt big row data", K(ret)); + } else { + in_buf_pos += data_len; + out_buf_pos += encrypted_len; + } + } + if (EXECUTE_COUNT_PER_SEC(1)) { + TRANS_LOG(INFO, "encrypt big row data", K(table_id), K(ret)); + } + } else { + int64_t data_len = min(in_buf_len - in_buf_pos, out_buf_len - out_buf_pos); + MEMCPY(out_buf + out_buf_pos, in_buf + in_buf_pos, data_len); + in_buf_pos += data_len; + out_buf_pos += data_len; + } + return ret; +} +#endif //////////////////////////////////////////////////////////////////////////////////////////////////// ObMemtableMutatorIterator::ObMemtableMutatorIterator() @@ -1138,6 +1366,11 @@ int ObMemtableMutatorIterator::deserialize(const char *buf, const int64_t data_l } else if (!buf_.set_data(const_cast(buf + pos), meta_.get_total_size())) { TRANS_LOG(WARN, "set_data fail", KP(buf), K(pos), K(meta_.get_total_size())); } else if (FALSE_IT(end_pos += meta_.get_total_size())) { +#ifdef OB_BUILD_TDE_SECURITY + } else if (((meta_.get_flags() & ObTransRowFlag::ENCRYPT) > 0) && + (OB_FAIL(encrypt_info.deserialize(buf, data_len, end_pos)))) { + TRANS_LOG(WARN, "decode clog encrypt info fail", K(ret), KP(buf), K(data_len), K(end_pos)); +#endif } else { pos = end_pos; buf_.get_limit() = meta_.get_total_size(); diff --git a/src/storage/memtable/ob_memtable_mutator.h b/src/storage/memtable/ob_memtable_mutator.h index 03220a4f9..51be074b6 100644 --- a/src/storage/memtable/ob_memtable_mutator.h +++ b/src/storage/memtable/ob_memtable_mutator.h @@ -217,6 +217,16 @@ public: int32_t flag_; // currently, unused uint8_t rowid_version_; int64_t column_cnt_; +#ifdef OB_BUILD_TDE_SECURITY +private: + int handle_encrypt_row_(char *buf, const int64_t buf_len, + const int64_t start_pos, int64_t &end_pos, + const share::ObEncryptMeta &meta); + int handle_decrypt_row_(const char *buf, const int64_t start_pos, + const int64_t end_pos, ObEncryptRowBuf &row_buf, const char* &out_buf, int64_t &out_len, + const share::ObEncryptMeta &meta); + static const int64_t OB_MAX_COUNT_NEED_BYTES = 10; //max int64_t size need bytes +#endif }; class ObMutatorTableLock : public ObMutator @@ -327,6 +337,13 @@ public: transaction::ObCLogEncryptInfo &encrypt_info); ObMemtableMutatorMeta& get_meta() { return meta_; } int64_t get_serialize_size() const; +#ifdef OB_BUILD_TDE_SECURITY + static int encrypt_big_row_data( + const char *in_buf, const int64_t in_buf_len, int64_t &in_buf_pos, + char *out_buf, const int64_t out_buf_len, int64_t &out_buf_pos, + const int64_t table_id, const transaction::ObCLogEncryptInfo &encrypt_info, + bool &need_encrypt); +#endif private: ObMemtableMutatorMeta meta_; common::ObDataBuffer buf_; diff --git a/src/storage/memtable/ob_redo_log_generator.cpp b/src/storage/memtable/ob_redo_log_generator.cpp index 984ce4278..bdc2e12da 100644 --- a/src/storage/memtable/ob_redo_log_generator.cpp +++ b/src/storage/memtable/ob_redo_log_generator.cpp @@ -159,6 +159,11 @@ int ObRedoLogGenerator::fill_redo_log(char *buf, if (OB_LIKELY(OB_ERR_TOO_BIG_ROWSIZE != ret)) { int64_t res_len = 0; uint8_t row_flag = ObTransRowFlag::NORMAL_ROW; +#ifdef OB_BUILD_TDE_SECURITY + if (encrypt_info.has_encrypt_meta()) { + row_flag |= ObTransRowFlag::ENCRYPT; + } +#endif if (OB_SUCCESS != (tmp_ret = mmw.serialize(row_flag, res_len, encrypt_info))) { if (OB_ENTRY_NOT_EXIST != tmp_ret) { TRANS_LOG(ERROR, "mmw.serialize fail", K(ret), K(tmp_ret)); @@ -303,6 +308,10 @@ int ObRedoLogGenerator::fill_row_redo(ObITransCallbackIterator &cursor, if (OB_ISNULL(memtable)) { TRANS_LOG(ERROR, "memtable is null", K(riter)); ret = OB_ERR_UNEXPECTED; +#ifdef OB_BUILD_TDE_SECURITY + } else if (OB_FAIL(memtable->get_encrypt_meta(clog_encrypt_meta_))) { + TRANS_LOG(ERROR, "get encrypt meta failed", K(memtable), K(ret)); +#endif } else if (OB_FAIL(mmw.append_row_kv(mem_ctx_->get_max_table_version(), redo, clog_encrypt_meta_, diff --git a/src/storage/ob_locality_manager.cpp b/src/storage/ob_locality_manager.cpp index cbd9021e1..2fcc263dd 100644 --- a/src/storage/ob_locality_manager.cpp +++ b/src/storage/ob_locality_manager.cpp @@ -16,6 +16,9 @@ #include "share/schema/ob_schema_getter_guard.h" #include "rpc/obmysql/obsm_struct.h" // easy_connection_str #include "share/schema/ob_multi_version_schema_service.h" +#ifdef OB_BUILD_ARBITRATION +#include "share/arbitration_service/ob_arbitration_service_info.h" +#endif namespace oceanbase { @@ -36,6 +39,9 @@ void ObLocalityManager::reset() { is_inited_ = false; self_.reset(); +#ifdef OB_BUILD_ARBITRATION + arb_service_addr_.reset(); +#endif sql_proxy_ = NULL; locality_info_.reset(); server_locality_cache_.reset(); @@ -240,6 +246,53 @@ int ObLocalityManager::load_region() return ret; } +#ifdef OB_BUILD_ARBITRATION +int ObLocalityManager::load_arb_service_info() +{ + int ret = OB_SUCCESS; + const ObString arbitration_service_key("default"); + const bool lock_line = false; + ObArbitrationServiceInfo arb_service_info; + ObAddr arb_service_addr; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + STORAGE_LOG(WARN, "ObLocalityManager not init", K(ret)); + } else if (OB_FAIL(arbitration_service_table_operator_.get( + *sql_proxy_, + arbitration_service_key, + lock_line, + arb_service_info))) { + STORAGE_LOG(WARN, "fail to get arbitration service info", KR(ret), K(arbitration_service_key), + K(lock_line), K(arb_service_info)); + if (OB_ARBITRATION_SERVICE_NOT_EXIST == ret) { + SpinWLockGuard guard(rwlock_); + arb_service_addr_.reset(); + } + } else if (OB_FAIL(arb_service_addr.parse_from_string(arb_service_info.get_arbitration_service_string()))) { + STORAGE_LOG(WARN, "parse_from_string failed", K(ret)); + } else if (arb_service_addr == arb_service_addr_) { + // no need update + } else { + SpinWLockGuard guard(rwlock_); + arb_service_addr_ = arb_service_addr; + } + STORAGE_LOG(INFO, "load_arb_service_info finshed", K(ret), K_(arb_service_addr)); + return ret; +} + +int ObLocalityManager::get_arb_service_addr(common::ObAddr &arb_service_addr) const +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + STORAGE_LOG(ERROR, "locality manager not inited, cannot start."); + ret = OB_NOT_INIT; + } else { + SpinRLockGuard guard(rwlock_); + arb_service_addr = arb_service_addr_; + } + return ret; +} +#endif int ObLocalityManager::get_local_region(ObRegion ®ion) const { @@ -707,6 +760,10 @@ int ObLocalityManager::ObRefreshLocalityTask::process() STORAGE_LOG(WARN, "locality manager is null", K(ret)); } else if (OB_FAIL(locality_mgr_->load_region())) { STORAGE_LOG(WARN, "process refresh locality task fail", K(ret)); +#ifdef OB_BUILD_ARBITRATION + } else if (OB_FAIL(locality_mgr_->load_arb_service_info())) { + STORAGE_LOG(WARN, "load_arb_service_info fail", K(ret)); +#endif } return ret; } diff --git a/src/storage/ob_locality_manager.h b/src/storage/ob_locality_manager.h index 6689c2aef..8a61620db 100644 --- a/src/storage/ob_locality_manager.h +++ b/src/storage/ob_locality_manager.h @@ -14,6 +14,9 @@ #define OCEANBASE_STORAGE_OB_LOCALITY_MANAGER_H_ #include "lib/mysqlclient/ob_mysql_proxy.h" +#ifdef OB_BUILD_ARBITRATION +#include "share/arbitration_service/ob_arbitration_service_table_operator.h" +#endif #include "share/ob_locality_info.h" #include "share/ob_locality_table_operator.h" #include "common/ob_zone_type.h" @@ -57,6 +60,10 @@ public: virtual int get_server_region(const common::ObAddr &server, common::ObRegion ®ion) const; virtual int get_server_idc(const common::ObAddr &server, common::ObIDC &idc) const; int get_server_cluster_id(const common::ObAddr &server, int64_t &cluster_id) const; +#ifdef OB_BUILD_ARBITRATION + int load_arb_service_info(); + int get_arb_service_addr(common::ObAddr &arb_service_addr) const; +#endif int record_server_region(const common::ObAddr &server, const common::ObRegion ®ion); int record_server_idc(const common::ObAddr &server, const common::ObIDC &idc); int record_server_cluster_id(const common::ObAddr &server, const int64_t &cluster_id); @@ -107,6 +114,10 @@ private: share::ObLocalityTableOperator locality_operator_; common::ObDedupQueue refresh_locality_task_queue_; ReloadLocalityTask reload_locality_task_; +#ifdef OB_BUILD_ARBITRATION + common::ObAddr arb_service_addr_; + share::ObArbitrationServiceTableOperator arbitration_service_table_operator_; +#endif char *ssl_invited_nodes_buf_;//common::OB_MAX_CONFIG_VALUE_LEN, use new bool is_loaded_; static const int64_t FAIL_TO_LOAD_LOCALITY_CACHE_TIMEOUT = 60L * 1000L * 1000L; diff --git a/src/storage/tablet/ob_tablet.cpp b/src/storage/tablet/ob_tablet.cpp index bf5f19947..11d6e33ef 100755 --- a/src/storage/tablet/ob_tablet.cpp +++ b/src/storage/tablet/ob_tablet.cpp @@ -2796,6 +2796,11 @@ int ObTablet::update_row( LOG_WARN("fail to protect table", K(ret)); } else if (OB_FAIL(prepare_memtable(relative_table, store_ctx, write_memtable))) { LOG_WARN("prepare write memtable fail", K(ret), K(relative_table)); +#ifdef OB_BUILD_TDE_SECURITY + // XXX we do not turn on clog encryption now + } else if (false && NULL != encrypt_meta_arr && !encrypt_meta_arr->empty() && + FALSE_IT(get_encrypt_meta(relative_table.get_table_id(), encrypt_meta_arr, encrypt_meta))) { +#endif } else { ObArenaAllocator allocator(common::ObMemAttr(MTL_ID(), ObModIds::OB_STORE_ROW_EXISTER)); ObTableIterParam param; @@ -2846,6 +2851,11 @@ int ObTablet::insert_row_without_rowkey_check( LOG_WARN("fail to protect table", K(ret)); } else if (OB_FAIL(prepare_memtable(relative_table, store_ctx, write_memtable))) { LOG_WARN("prepare write memtable fail", K(ret), K(relative_table)); +#ifdef OB_BUILD_TDE_SECURITY + // XXX we do not turn on clog encryption now + } else if (false && NULL != encrypt_meta_arr && !encrypt_meta_arr->empty() && + FALSE_IT(get_encrypt_meta(relative_table.get_table_id(), encrypt_meta_arr, encrypt_meta))) { +#endif } else { ObArenaAllocator allocator(common::ObMemAttr(MTL_ID(), ObModIds::OB_STORE_ROW_EXISTER)); ObTableIterParam param; diff --git a/src/storage/tablet/ob_tablet.h b/src/storage/tablet/ob_tablet.h index fd043e177..f962806ba 100755 --- a/src/storage/tablet/ob_tablet.h +++ b/src/storage/tablet/ob_tablet.h @@ -689,6 +689,12 @@ private: ObTableIterParam ¶m, ObTableAccessContext &context); +#ifdef OB_BUILD_TDE_SECURITY + void get_encrypt_meta( + const uint64_t table_id, + const common::ObIArray *encrypt_meta_arr, + const transaction::ObSerializeEncryptMeta *&encrypt_meta); +#endif // memtable operation int pull_memtables(ObArenaAllocator &allocator, ObITable **&ddl_kvs_addr, int64_t &ddl_kv_count); @@ -845,6 +851,20 @@ inline int64_t ObTablet::get_lock_wait_timeout( (abs_lock_timeout > stmt_timeout ? stmt_timeout : abs_lock_timeout)); } +#ifdef OB_BUILD_TDE_SECURITY +inline void ObTablet::get_encrypt_meta( + const uint64_t table_id, + const common::ObIArray *encrypt_meta_arr, + const transaction::ObSerializeEncryptMeta *&encrypt_meta) +{ + for (int64_t i = 0; i < encrypt_meta_arr->count(); ++i) { + if (encrypt_meta_arr->at(i).real_table_id() == table_id) { + encrypt_meta = &(encrypt_meta_arr->at(i).meta_); + break; + } + } +} +#endif } // namespace storage } // namespace oceanbase diff --git a/src/storage/tablet/ob_tablet_table_store.cpp b/src/storage/tablet/ob_tablet_table_store.cpp index fcce01049..36cb7aad5 100755 --- a/src/storage/tablet/ob_tablet_table_store.cpp +++ b/src/storage/tablet/ob_tablet_table_store.cpp @@ -2024,6 +2024,7 @@ int ObTabletTableStore::build_ha_ddl_tables_( const ObBatchUpdateTableStoreParam ¶m, const ObTabletTableStore &old_store) { + //TODO(muwei.ym) need reuse local minor sstable and cut sstable log ts 4.3 int ret = OB_SUCCESS; ObArray ddl_tables; ObITable *new_table = nullptr; diff --git a/src/storage/tx/ob_clog_encrypt_info_os.cpp b/src/storage/tx/ob_clog_encrypt_info_os.cpp index e518da4c4..504a0cc73 100644 --- a/src/storage/tx/ob_clog_encrypt_info_os.cpp +++ b/src/storage/tx/ob_clog_encrypt_info_os.cpp @@ -14,6 +14,7 @@ #include "lib/allocator/ob_malloc.h" #include "lib/utility/serialization.h" #include "lib/utility/ob_tracepoint.h" +#ifndef OB_BUILD_TDE_SECURITY namespace oceanbase { @@ -134,3 +135,4 @@ OB_SERIALIZE_MEMBER(ObEncryptMetaCache, table_id_, local_index_id_, meta_); }//transaction }//oceanbase +#endif diff --git a/src/storage/tx/ob_clog_encrypter.h b/src/storage/tx/ob_clog_encrypter.h index 127efa660..1f4c90ea7 100644 --- a/src/storage/tx/ob_clog_encrypter.h +++ b/src/storage/tx/ob_clog_encrypter.h @@ -20,6 +20,20 @@ namespace oceanbase namespace transaction { +#ifdef OB_BUILD_TDE_SECURITY +class ObTransMutator; + +class ObCLogEncrypter +{ +public: + // static int decrypt_log_mutator(const common::ObPartitionKey &pkey, ObTransMutator &mutator, + // const ObCLogEncryptInfo &encrypt_info, ObTransMutator &new_mutator, + // const bool need_extract_encrypt_meta, share::ObEncryptMeta &final_encrypt_meta, + // share::ObCLogEncryptStatMap &encrypt_stat_map); + // static int archive_encrypt_log_mutator(const common::ObPartitionKey &pkey, ObTransMutator &mutator, + // const ObCLogEncryptInfo &encrypt_info, ObTransMutator &new_mutator); +}; +#endif } } diff --git a/src/storage/tx/ob_dblink_client.cpp b/src/storage/tx/ob_dblink_client.cpp index 891877041..c2079e750 100644 --- a/src/storage/tx/ob_dblink_client.cpp +++ b/src/storage/tx/ob_dblink_client.cpp @@ -13,6 +13,9 @@ #include "lib/mysqlclient/ob_mysql_proxy.h" #include "observer/ob_server_struct.h" #include "lib/mysqlclient/ob_mysql_connection.h" +#ifdef OB_BUILD_ORACLE_PL +#include "pl/sys_package/ob_dbms_xa.h" +#endif namespace oceanbase { diff --git a/src/storage/tx/ob_tx_replay_executor.cpp b/src/storage/tx/ob_tx_replay_executor.cpp index 83a282216..6f3252cbd 100755 --- a/src/storage/tx/ob_tx_replay_executor.cpp +++ b/src/storage/tx/ob_tx_replay_executor.cpp @@ -548,6 +548,10 @@ int ObTxReplayExecutor::replay_redo_in_memtable_(ObTxRedoLog &redo) pos, encrypt_info)) || redo.get_mutator_size() != pos) { TRANS_LOG(WARN, "[Replay Tx] deserialize fail or pos does not match data_len", K(ret)); +#ifdef OB_BUILD_TDE_SECURITY + } else if (OB_FAIL(encrypt_info.decrypt_table_key())) { + TRANS_LOG(WARN, "[Replay Tx] failed to decrypt table key", K(ret)); +#endif } else { meta_flag = mmi_ptr_->get_meta().get_flags(); ObEncryptRowBuf row_buf; diff --git a/src/storage/tx/ob_xa_query.cpp b/src/storage/tx/ob_xa_query.cpp index 7a1485943..d2f92c6f0 100644 --- a/src/storage/tx/ob_xa_query.cpp +++ b/src/storage/tx/ob_xa_query.cpp @@ -12,6 +12,9 @@ #include "share/ob_define.h" #include "lib/mysqlclient/ob_mysql_proxy.h" #include "observer/ob_server_struct.h" +#ifdef OB_BUILD_ORACLE_PL +#include "pl/sys_package/ob_dbms_xa.h" +#endif #include "sql/ob_sql_utils.h" namespace oceanbase @@ -69,8 +72,30 @@ int ObXAQueryObImpl::xa_start(const ObXATransID &xid, const int64_t flags) ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "unexpected connection", K(ret), K(xid)); } else { +#ifdef OB_BUILD_DBLINK + ObSqlString sql; + int xa_result = ObDbmsXA::XA_OK; + if (OB_FAIL(sql.assign_fmt(RM_XA_START_SQL, + xid.get_format_id(), + xid.get_gtrid_str().length(), xid.get_gtrid_str().ptr(), + xid.get_bqual_str().length(), xid.get_bqual_str().ptr()))) { + TRANS_LOG(WARN, "fail to generate query sql", K(ret), K(xid)); + } else if (OB_FAIL(execute_query_(sql, xa_result))) { + TRANS_LOG(WARN, "fail to execute query", K(ret), K(xid), K(sql)); + } else { + if (ObDbmsXA::XA_OK == xa_result) { + // return success + } else { + // TODO, accurate error number + ret = OB_TRANS_XA_RMFAIL; + TRANS_LOG(WARN, "an error is returned from xa start", K(ret), K(xa_result), + K(xid), K(sql)); + } + } +#else ret = OB_NOT_SUPPORTED; TRANS_LOG(WARN, "fail to xa_start", K(ret)); +#endif } return ret; } @@ -97,8 +122,30 @@ int ObXAQueryObImpl::xa_end(const ObXATransID &xid, const int64_t flags) ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "unexpected connection", K(ret), K(xid)); } else { +#ifdef OB_BUILD_DBLINK + ObSqlString sql; + int xa_result = ObDbmsXA::XA_OK; + if (OB_FAIL(sql.assign_fmt(RM_XA_END_SQL, + xid.get_format_id(), + xid.get_gtrid_str().length(), xid.get_gtrid_str().ptr(), + xid.get_bqual_str().length(), xid.get_bqual_str().ptr()))) { + TRANS_LOG(WARN, "fail to generate query sql", K(ret), K(xid)); + } else if (OB_FAIL(execute_query_(sql, xa_result))) { + TRANS_LOG(WARN, "fail to execute query", K(ret), K(xid), K(sql)); + } else { + if (ObDbmsXA::XA_OK == xa_result) { + // return success + } else { + // TODO, accurate error number + ret = OB_TRANS_XA_RMFAIL; + TRANS_LOG(WARN, "an error is returned from xa end", K(ret), K(xa_result), + K(xid), K(sql)); + } + } +#else ret = OB_NOT_SUPPORTED; TRANS_LOG(WARN, "fail to xa_end", K(ret)); +#endif } return ret; } @@ -125,8 +172,33 @@ int ObXAQueryObImpl::xa_prepare(const ObXATransID &xid) ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "unexpected connection", K(ret), K(xid)); } else { +#ifdef OB_BUILD_DBLINK + ObSqlString sql; + int xa_result = ObDbmsXA::XA_OK; + if (OB_FAIL(sql.assign_fmt(RM_XA_PREPARE_SQL, + xid.get_format_id(), + xid.get_gtrid_str().length(), xid.get_gtrid_str().ptr(), + xid.get_bqual_str().length(), xid.get_bqual_str().ptr()))) { + TRANS_LOG(WARN, "fail to generate query sql", K(ret), K(xid)); + } else if (OB_FAIL(execute_query_(sql, xa_result))) { + TRANS_LOG(WARN, "fail to execute query", K(ret), K(xid), K(sql)); + } else { + if (ObDbmsXA::XA_OK == xa_result) { + // return success + } else if (ObDbmsXA::XA_RDONLY == xa_result) { + // return rdonly + ret = OB_TRANS_XA_RDONLY; + } else { + // TODO, accurate error number + ret = OB_TRANS_XA_RMFAIL; + TRANS_LOG(WARN, "an error is returned from xa prepare", K(ret), K(xa_result), + K(xid), K(sql)); + } + } +#else ret = OB_NOT_SUPPORTED; TRANS_LOG(WARN, "fail to xa_prepare", K(ret)); +#endif } return ret; } @@ -153,8 +225,31 @@ int ObXAQueryObImpl::xa_commit(const ObXATransID &xid, const int64_t flags) ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "unexpected connection", K(ret), K(xid)); } else { +#ifdef OB_BUILD_DBLINK + ObSqlString sql; + int xa_result = ObDbmsXA::XA_OK; + if (OB_FAIL(sql.assign_fmt(RM_XA_COMMIT_SQL, + xid.get_format_id(), + xid.get_gtrid_str().length(), xid.get_gtrid_str().ptr(), + xid.get_bqual_str().length(), xid.get_bqual_str().ptr(), + flags))) { + TRANS_LOG(WARN, "fail to generate query sql", K(ret), K(xid)); + } else if (OB_FAIL(execute_query_(sql, xa_result))) { + TRANS_LOG(WARN, "fail to execute query", K(ret), K(xid), K(sql)); + } else { + if (ObDbmsXA::XA_OK == xa_result) { + // return success + } else { + // TODO, accurate error number + ret = OB_TRANS_XA_RMFAIL; + TRANS_LOG(WARN, "an error is returned from xa commit", K(ret), K(xa_result), + K(xid), K(sql)); + } + } +#else ret = OB_NOT_SUPPORTED; TRANS_LOG(WARN, "fail to xa_commit", K(ret)); +#endif } return ret; } @@ -181,8 +276,30 @@ int ObXAQueryObImpl::xa_rollback(const ObXATransID &xid) ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "unexpected connection", K(ret), K(xid)); } else { +#ifdef OB_BUILD_DBLINK + ObSqlString sql; + int xa_result = ObDbmsXA::XA_OK; + if (OB_FAIL(sql.assign_fmt(RM_XA_ROLLBACK_SQL, + xid.get_format_id(), + xid.get_gtrid_str().length(), xid.get_gtrid_str().ptr(), + xid.get_bqual_str().length(), xid.get_bqual_str().ptr()))) { + TRANS_LOG(WARN, "fail to generate query sql", K(ret), K(xid)); + } else if (OB_FAIL(execute_query_(sql, xa_result))) { + TRANS_LOG(WARN, "fail to execute query", K(ret), K(xid), K(sql)); + } else { + if (ObDbmsXA::XA_OK == xa_result) { + // return success + } else { + // TODO, accurate error number + ret = OB_TRANS_XA_RMFAIL; + TRANS_LOG(WARN, "an error is returned from xa rollback", K(ret), K(xa_result), + K(xid), K(sql)); + } + } +#else ret = OB_NOT_SUPPORTED; TRANS_LOG(WARN, "fail to xa_rollback", K(ret)); +#endif } return ret; } @@ -190,8 +307,27 @@ int ObXAQueryObImpl::xa_rollback(const ObXATransID &xid) int ObXAQueryObImpl::execute_query_(const ObSqlString &sql, int &xa_result) { int ret = OB_SUCCESS; +#ifdef OB_BUILD_DBLINK + int tmp_xa_result = ObDbmsXA::XA_OK; + const uint64_t tenant_id = OB_INVALID_TENANT_ID; + SMART_VAR(ObMySQLProxy::MySQLResult, res) { + ObMySQLResult *result; + if (OB_FAIL(conn_->execute_read(tenant_id, sql.ptr(), res))) { + TRANS_LOG(WARN, "fail to execute xa query sql", K(ret), K(sql)); + } else if (OB_ISNULL(result = res.get_result())) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN,"fail to execute sql", K(ret), K(tenant_id), K(sql)); + } else if (OB_FAIL(result->next())) { + TRANS_LOG(WARN,"fail to iterate next result", K(ret), K(sql)); + } else { + EXTRACT_INT_FIELD_MYSQL(*result, "RESULT", tmp_xa_result, int64_t); + xa_result = tmp_xa_result; + } + } +#else ret = OB_NOT_SUPPORTED; TRANS_LOG(WARN, "fail to execute_query_", K(ret)); +#endif return ret; } @@ -239,8 +375,32 @@ int ObXAQueryOraImpl::xa_start(const ObXATransID &xid, const int64_t flags) } else if (OB_FAIL(convert_flag_(flags, ObXAReqType::XA_START, oci_flag))) { TRANS_LOG(WARN, "fail to convert xa flag to oci flag", K(ret), K(xid), K(flags), K(oci_flag)); } else { +#ifdef OB_BUILD_DBLINK + common::ObOciConnection *oci_conn = dynamic_cast(conn_); + if (NULL == oci_conn) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "unexpected connection", K(ret), KP_(conn)); + } else { + // if success, ora errcode is 0 + int ora_errcode = 0; + const int64_t timeout_seconds = 60; + if (OB_FAIL(oci_conn->trans_start(xid.get_gtrid_str(), xid.get_bqual_str(), + xid.get_format_id(), timeout_seconds, oci_flag, ora_errcode))) { + TRANS_LOG(WARN, "fail to do trans start", K(ret), K(xid), K(ora_errcode), + K(timeout_seconds), K(flags), K(oci_flag)); + } else if (0 != ora_errcode) { + ret = OB_TRANS_XA_RMFAIL; + TRANS_LOG(WARN, "an error is returned from trans start", K(ret), K(xid), K(ora_errcode), + K(timeout_seconds), K(flags), K(oci_flag)); + } else { + TRANS_LOG(INFO, "oci trans start", K(ret), K(xid), K(ora_errcode), K(flags)); + // do nothing + } + } +#else ret = OB_NOT_SUPPORTED; TRANS_LOG(WARN, "fail to xa_start", K(ret)); +#endif } return ret; } @@ -259,8 +419,30 @@ int ObXAQueryOraImpl::xa_end(const ObXATransID &xid, const int64_t flags) ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "unexpected connection", K(ret), K(xid)); } else { +#ifdef OB_BUILD_DBLINK + common::ObOciConnection *oci_conn = dynamic_cast(conn_); + if (NULL == oci_conn) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "unexpected connection", K(ret), KP_(conn)); + } else { + // if success, ora errcode is 0 + int ora_errcode = 0; + if (OB_FAIL(oci_conn->trans_detach(xid.get_gtrid_str(), xid.get_bqual_str(), + xid.get_format_id(), ora_errcode))) { + TRANS_LOG(WARN, "fail to do trans detach", K(ret), K(xid), K(ora_errcode), K(flags)); + } else if (0 != ora_errcode) { + ret = OB_TRANS_XA_RMFAIL; + TRANS_LOG(WARN, "an error is returned from trans detach", K(ret), K(xid), K(ora_errcode), + K(flags)); + } else { + TRANS_LOG(INFO, "oci trans detach", K(ret), K(xid), K(ora_errcode), K(flags)); + // do nothing + } + } +#else ret = OB_NOT_SUPPORTED; TRANS_LOG(WARN, "fail to xa_end", K(ret)); +#endif } return ret; } @@ -279,8 +461,37 @@ int ObXAQueryOraImpl::xa_prepare(const ObXATransID &xid) ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "unexpected connection", K(ret), K(xid)); } else { +#ifdef OB_BUILD_DBLINK + common::ObOciConnection *oci_conn = dynamic_cast(conn_); + if (NULL == oci_conn) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "unexpected connection", K(ret), KP_(conn)); + } else { + // if success, ora errcode is 0 + int ora_errcode = 0; + if (OB_FAIL(oci_conn->trans_prepare(xid.get_gtrid_str(), xid.get_bqual_str(), + xid.get_format_id(), ora_errcode))) { + TRANS_LOG(WARN, "fail to do trans prepare", K(ret), K(xid), K(ora_errcode)); + } else if (0 != ora_errcode) { + const int ORA_RDONLY = 24767; + if (ORA_RDONLY == ora_errcode) { + ret = OB_TRANS_XA_RDONLY; + TRANS_LOG(INFO, "read-only code is returned from trans prepare", K(ret), + K(xid), K(ora_errcode)); + } else { + ret = OB_TRANS_XA_RMFAIL; + TRANS_LOG(WARN, "an error is returned from trans prepare", K(ret), K(xid), + K(ora_errcode)); + } + } else { + TRANS_LOG(INFO, "oci trans prepare", K(ret), K(xid), K(ora_errcode)); + // do nothing + } + } +#else ret = OB_NOT_SUPPORTED; TRANS_LOG(WARN, "fail to xa_prepare", K(ret)); +#endif } return ret; } @@ -302,8 +513,31 @@ int ObXAQueryOraImpl::xa_commit(const ObXATransID &xid, const int64_t flags) } else if (OB_FAIL(convert_flag_(flags, ObXAReqType::XA_COMMIT, oci_flag))) { TRANS_LOG(WARN, "fail to convert xa flag to oci flag", K(ret), K(xid), K(flags), K(oci_flag)); } else { +#ifdef OB_BUILD_DBLINK + common::ObOciConnection *oci_conn = dynamic_cast(conn_); + if (NULL == oci_conn) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "unexpected connection", K(ret), KP_(conn)); + } else { + // if success, ora errcode is 0 + int ora_errcode = 0; + if (OB_FAIL(oci_conn->trans_commit(xid.get_gtrid_str(), xid.get_bqual_str(), + xid.get_format_id(), oci_flag, ora_errcode))) { + TRANS_LOG(WARN, "fail to do trans commit", K(ret), K(xid), K(ora_errcode), + K(flags), K(oci_flag)); + } else if (0 != ora_errcode) { + ret = OB_TRANS_XA_RMFAIL; + TRANS_LOG(WARN, "an error is returned from trans commit", K(ret), K(xid), K(ora_errcode), + K(flags), K(oci_flag)); + } else { + TRANS_LOG(INFO, "oci trans commit", K(ret), K(xid), K(ora_errcode), K(flags), K(oci_flag)); + // do nothing + } + } +#else ret = OB_NOT_SUPPORTED; TRANS_LOG(WARN, "fail to xa_commit", K(ret)); +#endif } return ret; } @@ -321,8 +555,30 @@ int ObXAQueryOraImpl::xa_rollback(const ObXATransID &xid) ret = OB_ERR_UNEXPECTED; TRANS_LOG(WARN, "unexpected connection", K(ret), K(xid)); } else { +#ifdef OB_BUILD_DBLINK + common::ObOciConnection *oci_conn = dynamic_cast(conn_); + if (NULL == oci_conn) { + ret = OB_ERR_UNEXPECTED; + TRANS_LOG(WARN, "unexpected connection", K(ret), KP_(conn)); + } else { + // if success, ora errcode is 0 + int ora_errcode = 0; + if (OB_FAIL(oci_conn->trans_rollback(xid.get_gtrid_str(), xid.get_bqual_str(), + xid.get_format_id(), ora_errcode))) { + TRANS_LOG(WARN, "fail to do trans rollback", K(ret), K(xid), K(ora_errcode)); + } else if (0 != ora_errcode) { + ret = OB_TRANS_XA_RMFAIL; + TRANS_LOG(WARN, "an error is returned from trans rollback", K(ret), K(xid), + K(ora_errcode)); + } else { + TRANS_LOG(INFO, "oci trans rollback", K(ret), K(xid), K(ora_errcode)); + // do nothing + } + } +#else ret = OB_NOT_SUPPORTED; TRANS_LOG(WARN, "fail to xa_rollback", K(ret)); +#endif } return ret; } diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index fbdc80d44..77264b805 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1,3 +1,7 @@ add_subdirectory(ob_error) add_subdirectory(ob_admin) add_subdirectory(upgrade) +if(OB_BUILD_CLOSE_MODULES) + add_subdirectory(agentserver EXCLUDE_FROM_ALL) + add_subdirectory(ob_bench EXCLUDE_FROM_ALL) +endif() diff --git a/tools/ob_admin/CMakeLists.txt b/tools/ob_admin/CMakeLists.txt index 68dba9b01..4d43d94e2 100644 --- a/tools/ob_admin/CMakeLists.txt +++ b/tools/ob_admin/CMakeLists.txt @@ -1,3 +1,9 @@ +set(ob_admin_tde_files "") +if(OB_BUILD_TDE_SECURITY) + set(ob_admin_tde_files "${CMAKE_SOURCE_DIR}/close_modules/tde_security/tools/ob_admin/dump_key/ob_admin_dump_key_executor.cpp") +endif() + +if (OB_BUILD_OPENSOURCE) add_executable(ob_admin # clog_tool/cmd_args_parser.h # clog_tool/ob_admin_clog_v2_executor.cpp @@ -55,8 +61,72 @@ add_executable(ob_admin log_tool/parser/ob_admin_parser_log_entry.cpp log_tool/parser/ob_admin_parser_group_entry.cpp log_tool/ob_admin_log_tool_executor.cpp + ${ob_admin_tde_files} ) +else() +add_executable(ob_admin + # clog_tool/cmd_args_parser.h + # clog_tool/ob_admin_clog_v2_executor.cpp + # clog_tool/ob_admin_clog_v2_executor.h + # clog_tool/ob_func_utils.cpp + # clog_tool/ob_func_utils.h + # clog_tool/ob_ilog_entry_parser.cpp + # clog_tool/ob_ilog_entry_parser.h + # clog_tool/ob_log_entry_filter.cpp + # clog_tool/ob_log_entry_filter.h + # clog_tool/ob_log_entry_parser.cpp + # clog_tool/ob_log_entry_parser.h + + # dumpsst/ob_admin_cmp_micro_executor.cpp + # dumpsst/ob_admin_cmp_micro_executor.h + dumpsst/ob_admin_dumpsst_executor.cpp + dumpsst/ob_admin_dumpsst_executor.h + dumpsst/ob_admin_dumpsst_print_helper.cpp + dumpsst/ob_admin_dumpsst_print_helper.h + io_bench/ob_admin_io_executor.cpp + io_bench/ob_admin_io_executor.h + main.cpp + ob_admin_executor.cpp + ob_admin_executor.h + server_tool/ob_admin_routine.cpp + server_tool/ob_admin_routine.h + server_tool/ob_admin_server_executor.cpp + server_tool/ob_admin_server_executor.h + # server_tool/ob_admin_utils.h + slog_tool/ob_admin_slog_executor.cpp + slog_tool/ob_admin_slog_executor.h + + dump_enum_value/ob_admin_dump_enum_value_executor.cpp + dump_enum_value/ob_admin_dump_enum_value_executor.h + dump_ckpt/ob_admin_dump_ckpt_executor.cpp + dump_ckpt/ob_admin_dump_ckpt_executor.h + + # archive_tool/ob_fake_archive_log_file_store.h + # archive_tool/ob_fake_archive_log_file_store.cpp + # archive_tool/ob_admin_log_archive_executor.h + # archive_tool/ob_admin_log_archive_executor.cpp + # archive_tool/ob_archive_entry_parser.h + # archive_tool/ob_archive_entry_parser.cpp + # archive_tool/ob_archive_fake_entry_iterator.h + # archive_tool/ob_archive_fake_entry_iterator.cpp + # archive_tool/ob_archive_fake_file_store.h + # archive_tool/ob_archive_fake_file_store.cpp + + backup_tool/ob_admin_dump_backup_data_executor.h + backup_tool/ob_admin_dump_backup_data_executor.cpp + + #trx_tool/ob_admin_trx_executor.h + #trx_tool/ob_admin_trx_executor.cpp + + log_tool/dump/ob_admin_dump_block.cpp + log_tool/parser/ob_admin_parser_log_entry.cpp + log_tool/parser/ob_admin_parser_group_entry.cpp + log_tool/ob_admin_log_tool_executor.cpp + ${ob_admin_tde_files} + ) + +endif() target_include_directories(ob_admin PRIVATE) target_link_libraries(ob_admin @@ -66,6 +136,7 @@ target_link_libraries(ob_admin ob_sql_static ob_storage_static ob_share_static + ${ob_close_modules_static_name} -Wl,--end-group -static-libgcc -static-libstdc++) diff --git a/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp b/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp index a26ee2a6b..aa3bef664 100644 --- a/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp +++ b/tools/ob_admin/backup_tool/ob_admin_dump_backup_data_executor.cpp @@ -16,6 +16,9 @@ #include "storage/blocksstable/ob_data_buffer.h" #include "../dumpsst/ob_admin_dumpsst_print_helper.h" #include "storage/blocksstable/ob_logic_macro_id.h" +#ifdef OB_BUILD_TDE_SECURITY +#include "share/ob_master_key_getter.h" +#endif #include #include @@ -520,6 +523,11 @@ int ObAdminDumpBackupDataExecutor::execute(int argc, char *argv[]) lib::set_tenant_memory_limit(500, 96 * 1024 * 1024 * 1024LL); int64_t data_type = 0; ObBackupDestType::TYPE path_type; +#ifdef OB_BUILD_TDE_SECURITY + // The root_key is set to ensure the successful parsing of backup_dest, because there is encryption and decryption of access_key + share::ObMasterKeyGetter::instance().init(NULL); + ObMasterKeyGetter::instance().set_root_key(OB_SYS_TENANT_ID, obrpc::RootKeyType::DEFAULT, ObString()); +#endif if (OB_FAIL(parse_cmd_(argc, argv))) { STORAGE_LOG(WARN, "failed to parse cmd", K(ret), K(argc), K(argv)); } else if (is_quiet_) { diff --git a/tools/ob_admin/dumpsst/ob_admin_dumpsst_executor.cpp b/tools/ob_admin/dumpsst/ob_admin_dumpsst_executor.cpp index d821fc28c..eda2ed371 100644 --- a/tools/ob_admin/dumpsst/ob_admin_dumpsst_executor.cpp +++ b/tools/ob_admin/dumpsst/ob_admin_dumpsst_executor.cpp @@ -14,6 +14,9 @@ #include "ob_admin_dumpsst_executor.h" #include "observer/ob_server_struct.h" #include "share/ob_io_device_helper.h" +#ifdef OB_BUILD_TDE_SECURITY +#include "share/ob_master_key_getter.h" +#endif #include "share/ob_tenant_mem_limit_getter.h" #include "share/rc/ob_tenant_base.h" #include "storage/ob_file_system_router.h" @@ -310,6 +313,11 @@ int ObAdminDumpsstExecutor::dump_macro_block(const ObDumpMacroBlockContext ¯ if (OB_UNLIKELY(!macro_block_context.is_valid())) { ret = OB_INVALID_ARGUMENT; STORAGE_LOG(ERROR, "invalid macro block id", K(macro_block_context), K(ret)); +#ifdef OB_BUILD_TDE_SECURITY + } else if (OB_FAIL(init_master_key_getter())) { + STORAGE_LOG(ERROR, "failed to init master key getter", K(ret)); + STORAGE_LOG(ERROR, "failed to init macro reader", K(ret)); +#endif } else if (OB_FAIL(ObBlockManager::read_block(read_info, macro_handle))) { STORAGE_LOG(ERROR, "Fail to read macro block, ", K(ret), K(read_info)); } else if (OB_FAIL(common_header.deserialize(macro_handle.get_buffer(), macro_handle.get_data_size(), pos))) { @@ -331,6 +339,30 @@ int ObAdminDumpsstExecutor::dump_macro_block(const ObDumpMacroBlockContext ¯ return ret; } +#ifdef OB_BUILD_TDE_SECURITY +int ObAdminDumpsstExecutor::init_master_key_getter() +{ + int ret = OB_SUCCESS; + if (OB_FAIL(share::ObMasterKeyGetter::instance().init(NULL, nullptr))) { + STORAGE_LOG(WARN, "fail to init master key", K(ret)); + } + if (OB_INIT_TWICE == ret) { + ret = OB_SUCCESS; + } + // if (OB_SUCC(ret) && OB_NOT_NULL(key_hex_str_)) { + // int64_t len = STRLEN(key_hex_str_); + // char master_key[share::OB_MAX_MASTER_KEY_LENGTH + 1]; + // if (OB_FAIL(common::hex_to_cstr(key_hex_str_, len, + // master_key, share::OB_MAX_MASTER_KEY_LENGTH + 1))) { + // STORAGE_LOG(WARN, "fail to hex to cstr", K(ret)); + // } else if (share::ObMasterKeyGetter::instance().set_master_key( + // master_key_id_, master_key, STRLEN(master_key))) { + // STORAGE_LOG(WARN, "fail to set master key", K(ret)); + // } + // } + return ret; +} +#endif void ObAdminDumpsstExecutor::print_usage() { diff --git a/tools/ob_admin/dumpsst/ob_admin_dumpsst_executor.h b/tools/ob_admin/dumpsst/ob_admin_dumpsst_executor.h index 8a071c0cd..6356d7796 100644 --- a/tools/ob_admin/dumpsst/ob_admin_dumpsst_executor.h +++ b/tools/ob_admin/dumpsst/ob_admin_dumpsst_executor.h @@ -68,6 +68,9 @@ private: int dump_shared_macro_block(const char* buf, const int64_t size); void dump_sstable(); void dump_sstable_meta(); +#ifdef OB_BUILD_TDE_SECURITY + int init_master_key_getter(); +#endif bool is_quiet_; ObAdminDumpsstCmd cmd_; diff --git a/tools/ob_admin/main.cpp b/tools/ob_admin/main.cpp index 54a8e68c7..19616a11c 100644 --- a/tools/ob_admin/main.cpp +++ b/tools/ob_admin/main.cpp @@ -19,6 +19,9 @@ #include "io_bench/ob_admin_io_executor.h" #include "server_tool/ob_admin_server_executor.h" #include "backup_tool/ob_admin_dump_backup_data_executor.h" +#ifdef OB_BUILD_TDE_SECURITY +#include "tools/ob_admin/dump_key/ob_admin_dump_key_executor.h" +#endif #include "dump_enum_value/ob_admin_dump_enum_value_executor.h" #include "log_tool/ob_admin_log_tool_executor.h" #include "slog_tool/ob_admin_slog_executor.h" @@ -34,6 +37,9 @@ void print_usage() " ob_admin dump_ckpt ## dump slog checkpoint, only support for 4.x\n" " ob_admin dumpsst\n" " ob_admin dump_enum_value\n" +#ifdef OB_BUILD_TDE_SECURITY + " ob_admin dump_key\n" +#endif " ob_admin log_tool ## './ob_admin log_tool' for more detail\n" " ob_admin -h127.0.0.1 -p2883 xxx\n" " ob_admin -h127.0.0.1 -p2883 (-sintl/-ssm -mbkmi/-mlocal) [command]\n" @@ -61,6 +67,10 @@ int main(int argc, char *argv[]) } else { if (0 == strcmp("io_bench", argv[1])) { executor = new ObAdminIOExecutor(); +#ifdef OB_BUILD_TDE_SECURITY + } else if (0 == strcmp("dump_key", argv[1])) { + executor = new ObAdminDumpKeyExecutor(); +#endif } else if (0 == strcmp("dump_enum_value", argv[1])) { executor = new ObAdminDumpEnumValueExecutor(); } else if (0 == strcmp("dumpsst", argv[1])) { diff --git a/tools/ob_admin/server_tool/ob_admin_routine.cpp b/tools/ob_admin/server_tool/ob_admin_routine.cpp index 40474375d..a159d2ae1 100644 --- a/tools/ob_admin/server_tool/ob_admin_routine.cpp +++ b/tools/ob_admin/server_tool/ob_admin_routine.cpp @@ -786,3 +786,41 @@ DEF_COMMAND(TRANS, update_lock, 1, "tenant_id ls_id obj_type obj_id lock_mode ow COMMON_LOG(INFO, "update_lock", K(arg)); return ret; } + +#ifdef OB_BUILD_ARBITRATION +// force_clear_arb_cluster_info +// @params [in] cluster_id, which cluster to modify +// @params [in] svr_ip, the arbitration server IP +// @params [in] svr_port, the arbitration server IP +// ATTENTION: +// Please make sure let log stream's leader to execute this command +// For permanant offline, orig_paxos_number should equals to new_paxos_number +DEF_COMMAND(TRANS, force_clear_arb_cluster_info, 1, "cluster_id # force_clear_arb_cluster_info") +{ + int ret = OB_SUCCESS; + string arg_str; + int64_t cluster_id_to_clean = OB_INVALID_TENANT_ID; + + if (cmd_ == action_name_) { + ret = OB_INVALID_ARGUMENT; + ADMIN_WARN("should provide cluster_id arb_svr_ip arb_svr_port"); + } else { + arg_str = cmd_.substr(action_name_.length() + 1); + } + + if (OB_FAIL(ret)) { + } else if (1 != sscanf(arg_str.c_str(), "%ld", &cluster_id_to_clean)) { + ret = OB_INVALID_ARGUMENT; + COMMON_LOG(WARN, "invalid arg", K(ret), K(arg_str.c_str()), K(cmd_.c_str()), K(cluster_id_to_clean)); + } else if (false == is_valid_cluster_id(cluster_id_to_clean)) { + COMMON_LOG(WARN, "invalid cluster_id", K(ret), K(cluster_id_to_clean)); + } else { + ObForceClearArbClusterInfoArg arg(cluster_id_to_clean); + if (OB_FAIL(client_->force_clear_arb_cluster_info(arg))) { + COMMON_LOG(ERROR, "send req fail", K(ret)); + } + } + COMMON_LOG(INFO, "force_clear_arb_cluster_info", K(cluster_id_to_clean)); + return ret; +} +#endif \ No newline at end of file diff --git a/tools/ob_admin/server_tool/ob_admin_server_executor.cpp b/tools/ob_admin/server_tool/ob_admin_server_executor.cpp index 82948995b..3015365ab 100644 --- a/tools/ob_admin/server_tool/ob_admin_server_executor.cpp +++ b/tools/ob_admin/server_tool/ob_admin_server_executor.cpp @@ -19,6 +19,9 @@ #include #include #include "observer/ob_srv_network_frame.h" +#ifdef OB_BUILD_TDE_SECURITY +#include "share/ob_encrypt_kms.h" +#endif #include "lib/utility/ob_print_utils.h" #include "ob_admin_routine.h" using namespace std; @@ -209,6 +212,59 @@ static int ob_admin_server_read_bkmi_cfg(const char* buf, int64_t &sz) return ret; } +#ifdef OB_BUILD_TDE_SECURITY +int ObAdminServerExecutor::load_ssl_config() +{ + int ret = OB_SUCCESS; + const bool enable_ssl_client_authentication = (SSL_MODE_NONE == ssl_mode_) ? false : true; + if (enable_ssl_client_authentication) { + bool use_bkmi = (0 == ssl_cfg_mode_) ? false : true; + bool is_sm = (SSL_MODE_SM == ssl_mode_) ? true : false; + int64_t ssl_key_expired_time = 0; + const char *ca_cert = NULL; + const char *public_cert = NULL; + const char *private_key = NULL; + if (!use_bkmi) { + ca_cert = OB_CLIENT_SSL_CA_FILE; + public_cert = OB_CLIENT_SSL_CERT_FILE; + private_key = OB_CLIENT_SSL_KEY_FILE; + } else { + share::ObSSLClient ssl_client; + char ssl_kms_info[OB_MAX_CONFIG_VALUE_LEN]; + memset(ssl_kms_info, 0, sizeof(ssl_kms_info)); + int64_t sz = 0; + if (OB_FAIL(ob_admin_server_read_bkmi_cfg(ssl_kms_info, sz))) { + COMMON_LOG(ERROR, "read from bkmi config file failed", K(ret)); + } else { + ObString ssl_config(ssl_kms_info); + if (OB_FAIL(ssl_client.init(ssl_config.ptr(), ssl_config.length()))) { + COMMON_LOG(ERROR, "ssl_client_init failed", K(ret), K(ssl_config)); + } else if (OB_FAIL(ssl_client.check_param_valid())) { + COMMON_LOG(ERROR, "kms client param is not valid", K(ret)); + } else { + ca_cert = ssl_client.get_root_ca().ptr(); + public_cert = ssl_client.public_cert_.content_.ptr(); + private_key = ssl_client.private_key_.content_.ptr(); + } + } + } + + if (EASY_OK != easy_ssl_ob_config_check(ca_cert, public_cert, private_key, NULL, NULL, !use_bkmi, is_sm)) { + COMMON_LOG(ERROR, "Local file mode: key and cert not match"); + ret = OB_INVALID_CONFIG; + } else if (OB_FAIL(observer::ObSrvNetworkFrame::extract_expired_time(OB_CLIENT_SSL_CERT_FILE, + ssl_key_expired_time))) { + COMMON_LOG(ERROR, "extract_expired_time failed", KR(ret)); + } else if (OB_FAIL(client_.load_ssl_config(use_bkmi, is_sm, ca_cert, public_cert, private_key, NULL, NULL))) { + COMMON_LOG(ERROR, "ObNetClient load_ssl_config failed", K(ret), K(is_sm), K(ssl_key_expired_time)); + } + } else { + COMMON_LOG(INFO, "no need to open ssl"); + } + + return ret; +} +#endif int ObAdminServerExecutor::execute(int argc, char *argv[]) { @@ -218,6 +274,10 @@ int ObAdminServerExecutor::execute(int argc, char *argv[]) ret = OB_NOT_SUPPORTED; } else if (OB_FAIL(client_.init())) { COMMON_LOG(WARN, "client init failed", K(ret)); +#ifdef OB_BUILD_TDE_SECURITY + } else if (OB_FAIL(load_ssl_config())) { + COMMON_LOG(WARN, "client load_ssl_config failed", K(ret)); +#endif } else if (OB_FAIL(client_.get_proxy(srv_proxy_))) { COMMON_LOG(WARN, "get_proxy failed", K(ret)); } else { diff --git a/unittest/logservice/CMakeLists.txt b/unittest/logservice/CMakeLists.txt index ea8d28371..4b647715b 100644 --- a/unittest/logservice/CMakeLists.txt +++ b/unittest/logservice/CMakeLists.txt @@ -40,3 +40,7 @@ ob_unittest(test_palf_throttling) ob_unittest(test_net_standby_restore_source) ob_unittest(test_log_external_storage_handler) ob_unittest(test_log_external_storage_io_task) +if(OB_BUILD_CLOSE_MODULES) + ob_unittest(test_arb_gc_utils) + ob_unittest(test_ob_arbitration_service) +endif() diff --git a/unittest/observer/rpc/CMakeLists.txt b/unittest/observer/rpc/CMakeLists.txt index 8ff2e8913..04fbd83d3 100644 --- a/unittest/observer/rpc/CMakeLists.txt +++ b/unittest/observer/rpc/CMakeLists.txt @@ -13,6 +13,7 @@ function(rpc_server_test case) oceanbase_static ob_sql_static ob_storage_static + ${ob_close_modules_static_name} -Wl,--end-group -static-libgcc -static-libstdc++) diff --git a/unittest/share/CMakeLists.txt b/unittest/share/CMakeLists.txt index f44f7495b..51c848e34 100644 --- a/unittest/share/CMakeLists.txt +++ b/unittest/share/CMakeLists.txt @@ -1,5 +1,9 @@ #ob_unittest(test_unit_getter) storage_unittest(test_scanner) +if(OB_BUILD_CLOSE_MODULES) + storage_unittest(test_encryption_util) + storage_unittest(test_master_key_getter) +endif() #ob_unittest(test_kms_client) #ob_unittest(test_zone_info) #ob_unittest(test_zone_merge_info) @@ -71,6 +75,14 @@ ob_unittest(test_json_bin) ob_unittest(test_json_path) ob_unittest(test_json_tree) +if(OB_BUILD_CLOSE_MODULES) + ob_unittest(test_xml_bin) + ob_unittest(test_xml_parser) + ob_unittest(test_xml_tree) + ob_unittest(test_xml_tree_base) + ob_unittest(test_xpath) + ob_unittest(test_xpath_filter) +endif() add_subdirectory(allocator) add_subdirectory(auto_increment) diff --git a/unittest/share/backup/test_backup_struct.cpp b/unittest/share/backup/test_backup_struct.cpp index 5aad69464..2aa506821 100644 --- a/unittest/share/backup/test_backup_struct.cpp +++ b/unittest/share/backup/test_backup_struct.cpp @@ -16,6 +16,9 @@ #include "lib/container/ob_array_iterator.h" #define private public #include "share/backup/ob_backup_path.h" +#ifdef OB_BUILD_TDE_SECURITY +#include "share/ob_master_key_getter.h" +#endif using namespace oceanbase; using namespace common; @@ -74,6 +77,74 @@ TEST(ObBackupDest, nfs) ASSERT_EQ(0, strcmp(dest.root_path_, "file:///backup_dir")); } +#ifdef OB_BUILD_TDE_SECURITY +TEST(ObBackupDest, oss) +{ + const char *backup_test = "oss://backup_dir/?host=xxx.com&access_id=111&access_key=222&delete_mode=tagging"; + ObBackupDest dest; + ObBackupDest dest1; + char backup_dest_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 }; + char backup_path_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 }; + ASSERT_EQ(OB_SUCCESS, dest.set(backup_test)); + ASSERT_EQ(OB_SUCCESS, dest1.set(backup_test)); + LOG_INFO("dump backup dest", K(dest), K(dest.get_root_path()), K(*(dest.get_storage_info()))); + ASSERT_EQ(0, strcmp(dest.root_path_, "oss://backup_dir")); + ASSERT_TRUE(dest.storage_info_->device_type_ == 0); + + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().init(NULL)); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().set_root_key(OB_SYS_TENANT_ID, + obrpc::RootKeyType::DEFAULT, ObString())); + ASSERT_EQ(OB_SUCCESS, dest.get_backup_dest_str(backup_dest_str, sizeof(backup_dest_str))); + ASSERT_EQ(0, strcmp(backup_dest_str, "oss://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F&delete_mode=tagging")); + ASSERT_EQ(OB_SUCCESS, dest.get_backup_path_str(backup_path_str, sizeof(backup_path_str))); + ASSERT_EQ(0, strcmp(backup_path_str, "oss://backup_dir?host=xxx.com")); + ASSERT_TRUE(dest.is_root_path_equal(dest1)); + bool is_equal = false; + ASSERT_EQ(OB_SUCCESS, dest.is_backup_path_equal(dest1, is_equal)); + ASSERT_TRUE(is_equal); + ASSERT_TRUE(dest == dest1); + dest1.reset(); + ASSERT_EQ(OB_SUCCESS, dest1.set(dest.get_root_path().ptr(), dest.get_storage_info())); + ASSERT_TRUE(dest == dest1); + ObMasterKeyGetter::instance().stop(); + ObMasterKeyGetter::instance().wait(); + ObMasterKeyGetter::instance().reset(); +} + +TEST(ObBackupDest, oss_encrypt) +{ + const char *backup_test = "oss://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F"; + ObBackupDest dest; + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().init(NULL)); + EXPECT_EQ(OB_SUCCESS, ObMasterKeyGetter::instance().set_root_key(OB_SYS_TENANT_ID, + obrpc::RootKeyType::DEFAULT, ObString())); + ASSERT_EQ(OB_SUCCESS, dest.set(backup_test)); + LOG_INFO("dump backup dest", K(dest.get_root_path()), K(*(dest.get_storage_info()))); + ASSERT_EQ(0, strcmp(dest.root_path_, "oss://backup_dir")); + ASSERT_TRUE(dest.storage_info_->device_type_ == 0); + const char *path = "oss://backup_dir/"; + const char *endpoint = "host=xxx.com"; + const char *authorization = "access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F"; + const char *extension = ""; + ObBackupDest dest1; + ASSERT_EQ(OB_SUCCESS, dest1.set(path, endpoint, authorization, extension)); + ASSERT_TRUE(dest == dest1); + + char backup_dest_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 }; + char backup_path_str[OB_MAX_BACKUP_DEST_LENGTH] = { 0 }; + ASSERT_EQ(OB_SUCCESS, dest.get_backup_dest_str(backup_dest_str, sizeof(backup_dest_str))); + ASSERT_EQ(0, strcmp(backup_dest_str, "oss://backup_dir?host=xxx.com&access_id=111&encrypt_key=9B6FDE7E1E54CD292CDE5494CEB86B6F")); + ASSERT_EQ(OB_SUCCESS, dest.get_backup_path_str(backup_path_str, sizeof(backup_path_str))); + ASSERT_EQ(0, strcmp(backup_path_str, "oss://backup_dir?host=xxx.com")); + + dest1.reset(); + ASSERT_EQ(OB_SUCCESS, dest1.set(path, endpoint, authorization, extension)); + ASSERT_TRUE(dest == dest1); + ObMasterKeyGetter::instance().stop(); + ObMasterKeyGetter::instance().wait(); + ObMasterKeyGetter::instance().reset(); +} +#endif // int main(int argc, char **argv) { diff --git a/unittest/share/deadlock/CMakeLists.txt b/unittest/share/deadlock/CMakeLists.txt index 1820b83a2..2caf0341d 100644 --- a/unittest/share/deadlock/CMakeLists.txt +++ b/unittest/share/deadlock/CMakeLists.txt @@ -1,3 +1,6 @@ +if(OB_BUILD_CLOSE_MODULES) + storage_unittest(test_deadlock) +endif() storage_unittest(test_key_wrapper) storage_unittest(test_deadlock_utility) storage_unittest(test_lcl_message_drop) diff --git a/unittest/storage/CMakeLists.txt b/unittest/storage/CMakeLists.txt index 5b9f3a113..13f73c6a7 100644 --- a/unittest/storage/CMakeLists.txt +++ b/unittest/storage/CMakeLists.txt @@ -89,6 +89,12 @@ storage_dml_unittest(test_major_rows_merger) storage_dml_unittest(test_tablet tablet/test_tablet.cpp) storage_unittest(test_compaction_iter compaction/test_compaction_iter.cpp) +if(OB_BUILD_CLOSE_MODULES) +storage_dml_unittest(test_compaction_policy) +endif() #storage_dml_unittest(test_table_scan_pure_index_table) +if(OB_BUILD_CLOSE_MODULES) + storage_unittest(test_lob_seq_id) +endif() storage_unittest(test_sstable_log_ts_range_cut test_sstable_log_ts_range_cut.cpp) diff --git a/unittest/storage/blocksstable/CMakeLists.txt b/unittest/storage/blocksstable/CMakeLists.txt index 7625db010..aaad89d20 100644 --- a/unittest/storage/blocksstable/CMakeLists.txt +++ b/unittest/storage/blocksstable/CMakeLists.txt @@ -13,6 +13,9 @@ storage_unittest(test_row_reader) storage_unittest(test_micro_block_reader) storage_unittest(test_micro_block_writer) #storage_unittest(test_bloom_filter_data) +if(OB_BUILD_TDE_SECURITY) +#storage_unittest(test_micro_block_encryption) +endif() storage_unittest(test_ref_cnt) storage_unittest(test_macro_block_id) #storage_unittest(test_lob_data_reader_writer)