From 0759f42e449b96765c0f90d6303085ce301b4006 Mon Sep 17 00:00:00 2001 From: 0xacc Date: Thu, 15 Aug 2024 07:50:55 +0000 Subject: [PATCH] [CP] [FEAT MERGE] 424 PL stable features --- .gitignore | 1 + .../src/lib/mysqlclient/ob_isql_connection.h | 1 + .../src/lib/mysqlclient/ob_mysql_proxy.cpp | 1 + .../src/lib/mysqlclient/ob_mysql_proxy.h | 5 +- src/objit/CMakeLists.txt | 1 - src/objit/include/objit/ob_llvm_helper.h | 2 +- src/objit/src/core/jit_context.cpp | 36 +- src/objit/src/core/jit_context.h | 17 +- src/objit/src/core/ob_jit_allocator.cpp | 26 +- src/objit/src/core/ob_orc_jit.cpp | 257 +- src/objit/src/core/ob_orc_jit.h | 180 +- src/objit/src/ob_llvm_helper.cpp | 46 +- src/observer/mysql/ob_query_driver.cpp | 25 +- src/observer/mysql/ob_query_retry_ctrl.cpp | 21 +- src/observer/mysql/obmp_stmt_execute.cpp | 7 +- src/observer/mysql/obmp_stmt_prexecute.cpp | 10 +- src/observer/ob_inner_sql_connection.cpp | 5 + src/observer/ob_inner_sql_connection.h | 1 + src/pl/CMakeLists.txt | 33 +- src/pl/diagnosis/ob_pl_sql_audit_guard.cpp | 119 + src/pl/diagnosis/ob_pl_sql_audit_guard.h | 132 + src/pl/ob_pl.cpp | 7 +- src/pl/ob_pl_code_generator.cpp | 26 +- src/pl/ob_pl_package_manager.cpp | 497 ++- src/pl/ob_pl_package_manager.h | 52 +- src/pl/ob_pl_type.cpp | 13 +- src/pl/sys_package/ob_dbms_upgrade.cpp | 42 +- src/rootserver/ob_upgrade_executor.cpp | 26 +- .../sys_package/__dbms_upgrade_body_mysql.sql | 4 +- .../sys_package/__dbms_upgrade_mysql.sql | 5 +- src/sql/engine/basic/ob_ra_row_store.cpp | 9 + src/sql/ob_spi.cpp | 2779 ++++++----------- src/sql/ob_spi.h | 161 +- src/sql/session/ob_basic_session_info.cpp | 12 + src/sql/session/ob_basic_session_info.h | 14 +- 35 files changed, 2324 insertions(+), 2249 deletions(-) create mode 100644 src/pl/diagnosis/ob_pl_sql_audit_guard.cpp create mode 100644 src/pl/diagnosis/ob_pl_sql_audit_guard.h diff --git a/.gitignore b/.gitignore index 4ef465080..06b531bae 100644 --- a/.gitignore +++ b/.gitignore @@ -171,6 +171,7 @@ src/pl/parser/_gen_parser.error src/pl/parser/_gen_pl_parser.output src/pl/parser/pl_parser_mysql_mode.output src/pl/parser/pl_parser_oracle_mode.output +src/share/inner_table/sys_package/system_package.cpp ############# close_modules ############# close_modules/oracle_pl/pl/parser/*.output diff --git a/deps/oblib/src/lib/mysqlclient/ob_isql_connection.h b/deps/oblib/src/lib/mysqlclient/ob_isql_connection.h index e764c624b..42fc7c92e 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_isql_connection.h +++ b/deps/oblib/src/lib/mysqlclient/ob_isql_connection.h @@ -206,6 +206,7 @@ public: virtual void set_is_load_data_exec(bool v) { UNUSED(v); } virtual void set_force_remote_exec(bool v) { UNUSED(v); } virtual void set_use_external_session(bool v) { UNUSED(v); } + virtual void set_ob_enable_pl_cache(bool v) { UNUSED(v); } virtual int64_t get_cluster_id() const { return common::OB_INVALID_ID; } void set_session_init_status(bool status) { is_inited_ = status;} virtual void set_user_timeout(int64_t user_timeout) { UNUSED(user_timeout); } diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp index 3e83a15ea..3152d5e72 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.cpp @@ -205,6 +205,7 @@ int ObCommonSqlProxy::write(const uint64_t tenant_id, const ObString sql, conn->set_is_load_data_exec(param->is_load_data_exec_); conn->set_use_external_session(param->use_external_session_); conn->set_group_id(param->consumer_group_id_); + conn->set_ob_enable_pl_cache(param->enable_pl_cache_); if (param->is_load_data_exec_) { is_user_sql = true; } diff --git a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h index 24ffa4682..ec19ac524 100644 --- a/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h +++ b/deps/oblib/src/lib/mysqlclient/ob_mysql_proxy.h @@ -91,8 +91,8 @@ struct ObSessionParam final { public: ObSessionParam() - : sql_mode_(nullptr), tz_info_wrap_(nullptr), ddl_info_(), is_load_data_exec_(false), use_external_session_(false), consumer_group_id_(0), nls_formats_{} - {} + : sql_mode_(nullptr), tz_info_wrap_(nullptr), ddl_info_(), is_load_data_exec_(false), + use_external_session_(false), consumer_group_id_(0), nls_formats_{}, enable_pl_cache_(true) {} ~ObSessionParam() = default; public: int64_t *sql_mode_; @@ -102,6 +102,7 @@ public: bool use_external_session_; // need init remote inner sql conn with sess getting from sess mgr int64_t consumer_group_id_; common::ObString nls_formats_[common::ObNLSFormatEnum::NLS_MAX]; + bool enable_pl_cache_; }; // thread safe sql proxy diff --git a/src/objit/CMakeLists.txt b/src/objit/CMakeLists.txt index d215a7cb8..da695e6c2 100644 --- a/src/objit/CMakeLists.txt +++ b/src/objit/CMakeLists.txt @@ -20,7 +20,6 @@ message(STATUS "This is SOURCE dir " ${PROJECT_SOURCE_DIR}) include_directories(${LLVM_INCLUDE_DIRS}) add_definitions(${LLVM_DEFINITIONS}) -add_definitions(-Wno-deprecated) add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0 -g -O2 -frtti) # Find the libraries that correspond to the LLVM components diff --git a/src/objit/include/objit/ob_llvm_helper.h b/src/objit/include/objit/ob_llvm_helper.h index 7ba69952b..9165b99a2 100644 --- a/src/objit/include/objit/ob_llvm_helper.h +++ b/src/objit/include/objit/ob_llvm_helper.h @@ -365,7 +365,7 @@ public: void dump_module(); void dump_debuginfo(); int verify_module(); - uint64_t get_function_address(const common::ObString &name); + int get_function_address(const common::ObString &name, uint64_t &addr); static void add_symbol(const common::ObString &name, void *value); ObDIRawData get_debug_info() const; diff --git a/src/objit/src/core/jit_context.cpp b/src/objit/src/core/jit_context.cpp index dc61ba5e3..32adf61e0 100644 --- a/src/objit/src/core/jit_context.cpp +++ b/src/objit/src/core/jit_context.cpp @@ -23,22 +23,20 @@ namespace core using namespace llvm; -int JitContext::InitializeModule(ObOrcJit &jit) +int JitContext::InitializeModule(const ObDataLayout &DL) { int ret = OB_SUCCESS; - TheJIT = &jit; - if (nullptr == (TheModule = std::make_unique("PL/SQL", TheJIT->getContext()))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memory for LLVM Module", K(ret)); - } else if (nullptr == (Builder = std::make_unique>(TheJIT->getContext()))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("failed to alloc memory for LLVM Builder", K(ret)); + if (OB_FAIL(ob_jit_make_unique(TheContext))) { + LOG_WARN("failed to make jit context", K(ret)); + } else if (OB_FAIL(ob_jit_make_unique(TheModule, "PL/SQL", *TheContext))) { + LOG_WARN("failed to make jit module", K(ret)); + } else if (OB_FAIL(ob_jit_make_unique(Builder, *TheContext))) { + LOG_WARN("failed to make ir builder", K(ret)); + } else if (OB_FAIL(ob_jit_make_unique(TheFPM, TheModule.get()))) { + LOG_WARN("failed to make FPM", K(ret)); } else { - TheContext = &TheJIT->getContext(); - TheModule->setDataLayout(TheJIT->getDataLayout()); - Builder = std::make_unique>(*TheContext); - TheFPM = std::make_unique(TheModule.get()); + TheModule->setDataLayout(DL); TheFPM->add(createInstructionCombiningPass()); TheFPM->add(createReassociatePass()); TheFPM->add(createGVNPass()); @@ -49,12 +47,20 @@ int JitContext::InitializeModule(ObOrcJit &jit) return ret; } -void JitContext::compile() +int JitContext::compile(ObOrcJit &jit) { - if (!Compile && nullptr != TheJIT) { - TheJIT->addModule(std::move(TheModule)); + int ret = OB_SUCCESS; + + if (Compile) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("already compiled", K(ret), K(lbt())); + } else if (OB_FAIL(jit.addModule(std::move(TheModule), std::move(TheContext)))) { + LOG_WARN("failed to add module to jit engine", K(ret)); + } else { Compile = true; } + + return ret; } int JitContext::optimize() diff --git a/src/objit/src/core/jit_context.h b/src/objit/src/core/jit_context.h index ac54c61df..3a8be94b0 100644 --- a/src/objit/src/core/jit_context.h +++ b/src/objit/src/core/jit_context.h @@ -31,26 +31,27 @@ struct JitContext { public: explicit JitContext() - : Compile(false), TheContext(nullptr), TheJIT(nullptr) - { - } + : Compile(false), + TheContext(nullptr), + Builder(nullptr), + TheModule(nullptr), + TheFPM(nullptr) + { } - int InitializeModule(ObOrcJit &jit); - void compile(); + int InitializeModule(const ObDataLayout &DL); + int compile(ObOrcJit &jit); int optimize(); ObLLVMContext& get_context() { return *TheContext; } IRBuilder<>& get_builder() { return *Builder; } Module& get_module() { return *TheModule; } - ObOrcJit* get_jit() { return TheJIT; } public: bool Compile; - ObLLVMContext *TheContext; + std::unique_ptr TheContext; std::unique_ptr> Builder; std::unique_ptr TheModule; - ObOrcJit *TheJIT; std::unique_ptr TheFPM; }; diff --git a/src/objit/src/core/ob_jit_allocator.cpp b/src/objit/src/core/ob_jit_allocator.cpp index 9888f8bef..bf6c89987 100644 --- a/src/objit/src/core/ob_jit_allocator.cpp +++ b/src/objit/src/core/ob_jit_allocator.cpp @@ -82,15 +82,8 @@ public: // set memory protection state. static int protect_mapped_memory(const ObJitMemoryBlock &block, int64_t p_flags); - - static bool aarch64_addr_safe(void *addr, int64_t size); }; -bool ObJitMemory::aarch64_addr_safe(void *addr, int64_t size) -{ - return reinterpret_cast(addr) >> 32 == (reinterpret_cast(addr)+size) >> 32; -} - ObJitMemoryBlock ObJitMemory::allocate_mapped_memory(int64_t num_bytes, int64_t p_flags) { @@ -107,25 +100,11 @@ ObJitMemoryBlock ObJitMemory::allocate_mapped_memory(int64_t num_bytes, do { addr = ::mmap(reinterpret_cast(start), page_size*num_pages, p_flags, mm_flags, fd, 0); - if (MAP_FAILED == addr -#if defined(__aarch64__) - || !aarch64_addr_safe(addr, page_size*num_pages) -#endif - ) { + if (MAP_FAILED == addr) { if (REACH_TIME_INTERVAL(10000000)) { //间隔10s打印日志 LOG_ERROR_RET(common::OB_ALLOCATE_MEMORY_FAILED, "allocate jit memory failed", K(addr), K(num_bytes), K(page_size), K(num_pages)); } ::usleep(100000); //100ms -#if defined(__aarch64__) - if (MAP_FAILED != addr) { - if (0 != ::munmap(addr, page_size*num_pages)) { - LOG_WARN_RET(OB_ERR_SYS, "jit block munmap failed", K(addr), K(page_size*num_pages)); - } - start = reinterpret_cast(addr) + UINT32_MAX - page_size*num_pages; //先向上移4G，再移动此次分配的大小，以保证此次分配的地址高16位一致 - LOG_INFO("aarch64 memory allocated not safe, try again", K(addr), K(start), K(page_size), K(num_pages)); - addr = MAP_FAILED; - } -#endif } else { LOG_DEBUG("allocate mapped memory success!", K(addr), K(start), @@ -370,6 +349,7 @@ void ObJitAllocator::reserve(const JitMemType mem_type, int64_t sz, int64_t alig } } +// Returns true if an error occurred, false otherwise. bool ObJitAllocator::finalize() { int ret = OB_SUCCESS; @@ -384,7 +364,7 @@ bool ObJitAllocator::finalize() LOG_WARN("fail to finalize code memory", K(ret)); } - return OB_SUCC(ret); + return OB_FAIL(ret); } void ObJitAllocator::free() { diff --git a/src/objit/src/core/ob_orc_jit.cpp b/src/objit/src/core/ob_orc_jit.cpp index 5d5eb4c80..78416eb1a 100644 --- a/src/objit/src/core/ob_orc_jit.cpp +++ b/src/objit/src/core/ob_orc_jit.cpp @@ -35,7 +35,6 @@ #include "llvm/IR/DataLayout.h" #include "llvm/IR/Mangler.h" #include "llvm/IR/Verifier.h" -#include "llvm/Support/DynamicLibrary.h" #include "llvm/Support/SourceMgr.h" #include "llvm/Support/raw_ostream.h" #include "llvm/AsmParser/Parser.h" @@ -56,94 +55,139 @@ namespace jit namespace core { -#ifndef ORC2 +DenseMap ObJitGlobalSymbolGenerator::symbol_table; + ObOrcJit::ObOrcJit(common::ObIAllocator &Allocator) : DebugBuf(nullptr), DebugLen(0), JITAllocator(), NotifyLoaded(Allocator, DebugBuf, DebugLen, SoObject), - TheContext(), - ObResolver(createLegacyLookupResolver( - ObES, - [this](StringRef Name) { return findMangledSymbol(std::string(Name)); }, - [](Error Err) { cantFail(std::move(Err), "lookupFlags failed"); })), ObTM(EngineBuilder().selectTarget()), ObDL(ObTM->createDataLayout()), - ObObjectLayer(AcknowledgeORCv1Deprecation, - ObES, - [this](ObVModuleKey) { - return ObObjLayerT::Resources{ - std::make_shared(JITAllocator), ObResolver}; }, - NotifyLoaded), - ObCompileLayer(AcknowledgeORCv1Deprecation, ObObjectLayer, SimpleCompiler(*ObTM)) + ObEngineBuilder(), + ObJitEngine() { } -ObVModuleKey ObOrcJit::addModule(std::unique_ptr M) +int ObOrcJit::init() { - auto Key = ObES.allocateVModule(); - cantFail(ObCompileLayer.addModule(Key, std::move(M))); - ObModuleKeys.push_back(Key); - return Key; + int ret = OB_SUCCESS; + + ObEngineBuilder.setObjectLinkingLayerCreator( + [this](ExecutionSession &ES, const Triple &TT) { + auto ObjLinkingLayer = + std::make_unique( + ES, + [&]() { + return std::make_unique(JITAllocator); + }); + +#ifndef NDEBUG + ObjLinkingLayer->registerJITEventListener( + *JITEventListener::createGDBRegistrationListener()); +#endif // NDEBUG + ObjLinkingLayer->registerJITEventListener(NotifyLoaded); + return ObjLinkingLayer; + }); + + auto tm_builder_wrapper = JITTargetMachineBuilder::detectHost(); + + if (!tm_builder_wrapper) { + Error err = tm_builder_wrapper.takeError(); + std::string msg = toString(std::move(err)); + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get target machine", K(msg.c_str())); + } else { + ObEngineBuilder.setJITTargetMachineBuilder(*tm_builder_wrapper); + } + + return ret; } -ObJITSymbol ObOrcJit::lookup(std::string Name) +int ObOrcJit::addModule(std::unique_ptr M, std::unique_ptr TheContext) { - return findMangledSymbol(mangle(Name)); + int ret = OB_SUCCESS; + + if (OB_FAIL(create_jit_engine())) { + LOG_WARN("failed to create jit engine", K(ret)); + } else if (OB_ISNULL(ObJitEngine)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected NULL jit engine", K(ret), K(lbt())); + } else { + Error err = ObJitEngine->addIRModule(ThreadSafeModule{std::move(M), std::move(TheContext)}); + + if (err) { + std::string msg = toString(std::move(err)); + + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to add module to jit engine", + K(ret), K(msg.c_str())); + } + } + + return ret; } -uint64_t ObOrcJit::get_function_address(const std::string Name) +int ObOrcJit::lookup(const std::string &name, ObJITSymbol &symbol) { - return static_cast(cantFail(lookup(Name).getAddress())); + int ret = OB_SUCCESS; + + if (OB_ISNULL(ObJitEngine)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected NULL jit engine", K(ret), K(lbt())); + } else { + auto value = ObJitEngine->lookup(name); + + if (!value) { + Error err = value.takeError(); + + if (err.isA()) { + ret = OB_ENTRY_NOT_EXIST; + } else { + ret = OB_ERR_UNEXPECTED; + } + + std::string msg = toString(std::move(err)); + LOG_WARN("failed to lookup symbol in jit engine", + K(ret), + "name", name.c_str(), + "msg", msg.c_str()); + } else { + symbol = *value; + } + } + + return ret; } -#else -static ExitOnError ExitOnErr; - -ObOrcJit::ObOrcJit(ObIAllocator &Allocator, JITTargetMachineBuilder JTMB, ObDataLayout ObDL) - : DebugBuf(nullptr), - DebugLen(0), - JITAllocator(), - NotifyLoaded(Allocator, DebugBuf, DebugLen), - ObObjectLayer(ObES, - [this]() { return std::make_unique(JITAllocator); }), - ObCompileLayer(ObES, - ObObjectLayer, - std::make_unique(std::move(JTMB))), - ObDL(std::move(ObDL)), - Mangle(ObES, this->ObDL), - Ctx(std::make_unique()), - MainJD(ObES.createBareJITDylib("
")) +int ObOrcJit::get_function_address(const std::string &name, uint64_t &addr) { - /* - MainJD.define(absoluteSymbols({ - { Mangle("eh_personality"), pointerToJITTargetAddress(&ObPLEH::eh_personality) } - })); - */ - MainJD.addGenerator( - cantFail(DynamicLibrarySearchGenerator::GetForCurrentProcess( - ObDL.getGlobalPrefix()))); + int ret = OB_SUCCESS; + + ObJITSymbol sym = nullptr; + + if (OB_FAIL(lookup(name, sym))) { + LOG_WARN("failed to lookup symbol addr", K(name.c_str())); + } else { + auto value = sym.getAddress(); + + if (!value) { + Error err = value.takeError(); + std::string msg = toString(std::move(err)); + + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to get symbol address", + K(ret), + "name", name.c_str(), + "msg", msg.c_str()); + } else { + addr = static_cast(*value); + } + } + + return ret; } -Error ObOrcJit::addModule(std::unique_ptr M) -{ - return ObCompileLayer.add(MainJD, ThreadSafeModule(std::move(M), Ctx)); -} - -Expected ObOrcJit::lookup(StringRef Name) -{ - return ObES.lookup({&MainJD}, Mangle(Name.str())); -} - -uint64_t ObOrcJit::get_function_address(const std::string Name) -{ - std::cerr << "get_function_address : " << Name << std::endl; - auto Sym = ExitOnErr(lookup(Name)); - std::cerr << "get_function_address finish : " << Name << std::endl; - return static_cast(Sym.getAddress()); -} -#endif - -void ObNotifyLoaded::operator()( +void ObNotifyLoaded::notifyObjectLoaded( ObVModuleKey Key, const object::ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &Info) @@ -169,13 +213,29 @@ void ObNotifyLoaded::operator()( // } } -void ObOrcJit::add_compiled_object(size_t length, const char *ptr) +int ObOrcJit::add_compiled_object(size_t length, const char *ptr) { - ObVModuleKey Key = ObES.allocateVModule(); + int ret = OB_SUCCESS; - cantFail(ObObjectLayer.addObject( - Key, MemoryBuffer::getMemBuffer(StringRef(ptr, length), "", false))); - ObModuleKeys.push_back(Key); + if (OB_FAIL(create_jit_engine())) { + LOG_WARN("failed to create jit engine", K(ret)); + } else if (OB_ISNULL(ObJitEngine)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected NULL jit engine", K(ret), K(lbt())); + } else { + Error err =ObJitEngine->addObjectFile( + MemoryBuffer::getMemBuffer(StringRef(ptr, length), "", false)); + + if (err) { + std::string msg = toString(std::move(err)); + + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to add compile result to jit engine", + K(ret), K(msg.c_str()), K(length), K(ptr)); + } + } + + return ret; } int ObOrcJit::set_optimize_level(ObPLOptLevel level) @@ -187,14 +247,53 @@ int ObOrcJit::set_optimize_level(ObPLOptLevel level) LOG_WARN("unexpected PLSQL_OPTIMIZE_LEVEL", K(ret), K(level), K(lbt())); } - if (OB_SUCC(ret) && OB_ISNULL(ObTM)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected NULL TM", K(ret), K(ObTM.get()), K(lbt())); + if (OB_SUCC(ret) && level == ObPLOptLevel::O0) { + auto &tm_builder = ObEngineBuilder.getJITTargetMachineBuilder(); + if (!tm_builder.hasValue()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected NULL JITTargetMachineBuilder", K(ret), K(lbt())); + } else { + auto &builder = *tm_builder; + builder.setCodeGenOptLevel(CodeGenOpt::Level::None); + builder.getOptions().EnableFastISel = true; + } } - if (OB_SUCC(ret) && level == ObPLOptLevel::O0) { - ObTM->setOptLevel(CodeGenOpt::Level::None); - ObTM->setFastISel(true); + return ret; +} + +int ObOrcJit::create_jit_engine() +{ + int ret = OB_SUCCESS; + + if (OB_NOT_NULL(ObJitEngine)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected NOT NULL jit engine", K(ret), K(lbt())); + } else { + std::unique_ptr symbol_generator = nullptr; + + auto engine_wrapper = ObEngineBuilder.create(); + + if (!engine_wrapper) { + Error err = engine_wrapper.takeError(); + std::string msg = toString(std::move(err)); + + ret = OB_ERR_UNEXPECTED; + LOG_WARN("failed to build LLVM JIT engine", K(msg.c_str())); + } else { + ObJitEngine = std::move(*engine_wrapper); + } + + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_ISNULL(ObJitEngine)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected NULL jit engine", K(ret)); + } else if (OB_FAIL(ob_jit_make_unique(symbol_generator))) { + LOG_WARN("failed to make ObJitGlobalSymbolGenerator unique_ptr", K(ret)); + } else { + ObJitEngine->getMainJITDylib().addGenerator(std::move(symbol_generator)); + } } return ret; diff --git a/src/objit/src/core/ob_orc_jit.h b/src/objit/src/core/ob_orc_jit.h index 836121958..d5a5f50f9 100644 --- a/src/objit/src/core/ob_orc_jit.h +++ b/src/objit/src/core/ob_orc_jit.h @@ -20,11 +20,11 @@ #include "llvm/ExecutionEngine/ObjectCache.h" #include "llvm/IR/DataLayout.h" #include "llvm/DebugInfo/DWARF/DWARFContext.h" +#include "llvm/ExecutionEngine/Orc/LLJIT.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/ExecutionEngine/JITEventListener.h" #include "llvm/IR/Mangler.h" -#include #include #include "core/ob_jit_memory_manager.h" @@ -40,6 +40,33 @@ enum class ObPLOptLevel : int O2 = 2, O3 = 3 }; +template +static inline int ob_jit_make_unique(std::unique_ptr &ptr, Args&&... args) { + int ret = OB_SUCCESS; + + std::unique_ptr result = nullptr; + + try { + result = std::make_unique(std::forward(args)...); + } catch (const std::bad_alloc &e) { + ret = OB_ALLOCATE_MEMORY_FAILED; + SERVER_LOG(WARN, "failed to allocate memory", K(ret), K(e.what())); + } catch (...) { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "unexpected exception in std::make_unique", K(ret), K(lbt())); + } + + if (OB_FAIL(ret)) { + // do nothing + } else if (OB_ISNULL(result)) { + ret = OB_ERR_UNEXPECTED; + SERVER_LOG(WARN, "unexpected NULL ptr of std::make_unque", K(ret), K(lbt())); + } else { + ptr = std::move(result); + } + + return ret; +} namespace core { using namespace llvm; @@ -51,8 +78,10 @@ typedef ::llvm::TargetMachine ObTargetMachine; typedef ::llvm::DataLayout ObDataLayout; typedef ::llvm::orc::VModuleKey ObVModuleKey; typedef ::llvm::JITSymbol ObJITSymbol; +typedef ::llvm::orc::JITDylib::DefinitionGenerator ObJitDefinitionGenerator; +typedef ::llvm::JITEventListener ObJitEventListener; -class ObNotifyLoaded +class ObNotifyLoaded: public ObJitEventListener { public: explicit ObNotifyLoaded( @@ -60,9 +89,9 @@ public: : Allocator(Allocator), DebugBuf(DebugBuf), DebugLen(DebugLen), SoObject(SoObject) {} virtual ~ObNotifyLoaded() {} - void operator()(ObVModuleKey Key, + void notifyObjectLoaded(ObVModuleKey Key, const object::ObjectFile &Obj, - const RuntimeDyld::LoadedObjectInfo &Info); + const RuntimeDyld::LoadedObjectInfo &Info) override; private: common::ObIAllocator &Allocator; char* &DebugBuf; @@ -70,128 +99,95 @@ private: ObString &SoObject; }; +class ObJitGlobalSymbolGenerator: public ObJitDefinitionGenerator { +public: + Error tryToGenerate(orc::LookupKind K, + orc::JITDylib &JD, + orc::JITDylibLookupFlags JDLookupFlags, + const orc::SymbolLookupSet &LookupSet) override + { + for (const auto &sym : LookupSet) { + auto res = symbol_table.find(*sym.first); + + if (res != symbol_table.end()) { + Error err = JD.define(orc::absoluteSymbols( + {{sym.first, JITEvaluatedSymbol(res->second, {})}})); + + if (err) { + StringRef name = *sym.first; + std::string msg = toString(std::move(err)); + + SERVER_LOG_RET(WARN, OB_ERR_UNEXPECTED, + "failed to define SPI interface symbol", + "name", ObString(name.size(), name.data()), + "msg", msg.c_str(), + K(lbt())); + + return err; + } + } + } + + return Error::success(); + } + + static void add_symbol(StringRef name, void *addr) { + symbol_table[name] = pointerToJITTargetAddress(addr); + } + +private: + static DenseMap symbol_table; +}; -#ifndef ORC2 class ObOrcJit { public: - using ObObjLayerT = llvm::orc::LegacyRTDyldObjectLinkingLayer; - using ObCompileLayerT = llvm::orc::LegacyIRCompileLayer; + using ObLLJITBuilder = llvm::orc::LLJITBuilder; + using ObJitEngineT = llvm::orc::LLJIT; explicit ObOrcJit(common::ObIAllocator &Allocator); virtual ~ObOrcJit() {}; - ObVModuleKey addModule(std::unique_ptr M); - ObJITSymbol lookup(const std::string Name); - uint64_t get_function_address(const std::string Name); + int addModule(std::unique_ptr M, std::unique_ptr TheContext); + int get_function_address(const std::string &name, uint64_t &addr); - ObLLVMContext &getContext() { return TheContext; } const ObDataLayout &getDataLayout() const { return ObDL; } char* get_debug_info_data() { return DebugBuf; } int64_t get_debug_info_size() { return DebugLen; } - void add_compiled_object(size_t length, const char *ptr); + int add_compiled_object(size_t length, const char *ptr); const ObString& get_compiled_object() const { return SoObject; } int set_optimize_level(ObPLOptLevel level); -private: - std::string mangle(const std::string &Name) - { - std::string MangledName; { - raw_string_ostream MangledNameStream(MangledName); - Mangler::getNameWithPrefix(MangledNameStream, Name, ObDL); - } - return MangledName; - } + int init(); - ObJITSymbol findMangledSymbol(const std::string &Name) - { - const bool ExportedSymbolsOnly = true; - for (auto H : make_range(ObModuleKeys.rbegin(), ObModuleKeys.rend())) { - if (auto Sym = ObCompileLayer.findSymbolIn(H, Name, ExportedSymbolsOnly)) { - return Sym; - } - } - if (auto SymAddr = RTDyldMemoryManager::getSymbolAddressInProcess(Name)) { - return ObJITSymbol(SymAddr, JITSymbolFlags::Exported); - } - return nullptr; - } + const ObDataLayout &get_DL() const { return ObDL; } private: + int lookup(const std::string &name, ObJITSymbol &symbol); + + int create_jit_engine(); + + static ObJitGlobalSymbolGenerator symbol_generator; + char *DebugBuf; int64_t DebugLen; ObJitAllocator JITAllocator; ObNotifyLoaded NotifyLoaded; - ObLLVMContext TheContext; - ObExecutionSession ObES; - std::shared_ptr ObResolver; std::unique_ptr ObTM; const ObDataLayout ObDL; - ObObjLayerT ObObjectLayer; - ObCompileLayerT ObCompileLayer; - std::vector ObModuleKeys; ObString SoObject; + + ObLLJITBuilder ObEngineBuilder; + std::unique_ptr ObJitEngine; }; -#else -class ObOrcJit -{ -public: - explicit ObOrcJit( - common::ObIAllocator &Allocator, llvm::orc::JITTargetMachineBuilder JTMB, ObDataLayout ObDL); - virtual ~ObOrcJit() {}; - - Error addModule(std::unique_ptr M); - Expected lookup(StringRef Name); - uint64_t get_function_address(const std::string Name); - - static ObOrcJit* create(ObIAllocator &allocator) - { - auto JTMB = llvm::orc::JITTargetMachineBuilder::detectHost(); - - if (!JTMB) - return nullptr; - - auto ObDL = JTMB->getDefaultDataLayoutForTarget(); - if (!ObDL) - return nullptr; - - return OB_NEWx(ObOrcJit, (&allocator), allocator, std::move(*JTMB), std::move(*ObDL)); - } - - ObLLVMContext &getContext() { return *Ctx.getContext(); } - - const ObDataLayout &getDataLayout() const { return ObDL; } - - char* get_debug_info_data() { return DebugBuf; } - int64_t get_debug_info_size() { return DebugLen; } - -private: - char *DebugBuf; - int64_t DebugLen; - - ObJitAllocator JITAllocator; - ObNotifyLoaded NotifyLoaded; - - llvm::orc::ObExecutionSession ObES; - llvm::orc::RTDyldObjectLinkingLayer ObObjectLayer; - llvm::orc::IRCompileLayer ObCompileLayer; - - llvm::ObDataLayout ObDL; - llvm::orc::MangleAndInterner Mangle; - llvm::orc::ThreadSafeContext Ctx; - - llvm::orc::JITDylib &MainJD; -}; -#endif - } // core } // jit } // oceanbase diff --git a/src/objit/src/ob_llvm_helper.cpp b/src/objit/src/ob_llvm_helper.cpp index 739eeb99c..8b9088e27 100644 --- a/src/objit/src/ob_llvm_helper.cpp +++ b/src/objit/src/ob_llvm_helper.cpp @@ -20,7 +20,6 @@ #include "llvm/IR/GlobalVariable.h" #include "llvm/ExecutionEngine/ExecutionEngine.h" #include "llvm/Support/TargetSelect.h" -#include "llvm/Support/DynamicLibrary.h" #include "share/ob_define.h" #include "objit/ob_llvm_helper.h" @@ -514,18 +513,24 @@ int ObLLVMHelper::init() } else if (nullptr == (jc_ = OB_NEWx(core::JitContext, (&allocator_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to alloc memory for jit context", K(ret)); -#ifndef ORC2 } else if (nullptr == (jit_ = OB_NEWx(core::ObOrcJit, (&allocator_), allocator_))) { -#else - } else if (nullptr == (jit_ = core::ObOrcJit::create(allocator_))) { -#endif jc_->~JitContext(); allocator_.free(jc_); jc_ = nullptr; ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("failed to alloc memory for jit", K(ret)); - } else if (OB_FAIL(jc_->InitializeModule(*jit_))) { + } else if (OB_FAIL(jit_->init())) { + jit_->~ObOrcJit(); + allocator_.free(jit_); + jit_ = nullptr; + + jc_->~JitContext(); + allocator_.free(jc_); + jc_ = nullptr; + + LOG_WARN("failed to init jit engine", K(ret)); + } else if (OB_FAIL(jc_->InitializeModule(jit_->get_DL()))) { jit_->~ObOrcJit(); allocator_.free(jit_); jit_ = nullptr; @@ -560,9 +565,9 @@ int ObLLVMHelper::initialize() llvm::InitializeNativeTargetAsmPrinter(); llvm::InitializeNativeTargetAsmParser(); - llvm::sys::DynamicLibrary::LoadLibraryPermanently(nullptr); - -#if !defined(__aarch64__) +/*Do not juse use !defined(__aarch64__) here*/ +//#if !defined(__aarch64__) +#if defined(__x86_64__) // initialize LLVM X86 unfold table llvm::lookupUnfoldTable(0); #endif @@ -590,6 +595,7 @@ int ObLLVMHelper::init_llvm() { ObLLVMFunctionType ft; ObLLVMBasicBlock block; ObLLVMValue magic; + uint64_t addr; OZ (helper.get_llvm_type(ObIntType, int64_type)); OZ (arg_types.push_back(int64_type)); @@ -601,7 +607,7 @@ int ObLLVMHelper::init_llvm() { OZ (helper.create_ret(magic)); OZ (helper.compile_module(jit::ObPLOptLevel::O2)); - OX (helper.get_function_address(init_func_name)); + OZ (helper.get_function_address(init_func_name, addr)); return ret; } @@ -620,7 +626,7 @@ int ObLLVMHelper::compile_module(jit::ObPLOptLevel optimization) dump_module(); } OB_LLVM_MALLOC_GUARD(GET_PL_MOD_STRING(pl::OB_PL_JIT)); - jc_->compile(); + jc_->compile(*jit_); } return ret; @@ -670,15 +676,23 @@ int ObLLVMHelper::verify_module() return ret; } -uint64_t ObLLVMHelper::get_function_address(const ObString &name) +int ObLLVMHelper::get_function_address(const ObString &name, uint64_t &addr) { - OB_LLVM_MALLOC_GUARD(GET_PL_MOD_STRING(pl::OB_PL_JIT)); - return jc_->TheJIT->get_function_address(std::string(name.ptr(), name.length())); + int ret = OB_SUCCESS; + + CK (OB_NOT_NULL(jit_)); + + if (OB_SUCC(ret)) { + OB_LLVM_MALLOC_GUARD(GET_PL_MOD_STRING(pl::OB_PL_JIT)); + OZ (jit_->get_function_address(std::string(name.ptr(), name.length()), addr)); + } + + return ret; } void ObLLVMHelper::add_symbol(const ObString &name, void *value) { - llvm::sys::DynamicLibrary::AddSymbol(make_string_ref(name), value); + core::ObJitGlobalSymbolGenerator::add_symbol(make_string_ref(name), value); } ObDIRawData ObLLVMHelper::get_debug_info() const @@ -2248,7 +2262,7 @@ int ObLLVMHelper::add_compiled_object(size_t length, const char *ptr) CK (OB_NOT_NULL(jit_)); CK (OB_NOT_NULL(ptr)); CK (OB_LIKELY(length > 0)); - OX (jit_->add_compiled_object(length, ptr)); + OZ (jit_->add_compiled_object(length, ptr)); return ret; } diff --git a/src/observer/mysql/ob_query_driver.cpp b/src/observer/mysql/ob_query_driver.cpp index 6b02b2174..e6458867d 100644 --- a/src/observer/mysql/ob_query_driver.cpp +++ b/src/observer/mysql/ob_query_driver.cpp @@ -177,21 +177,26 @@ int ObQueryDriver::response_query_result(ObResultSet &result, can_retry = true; bool is_first_row = true; const ObNewRow *result_row = NULL; - bool has_top_limit = result.get_has_top_limit(); bool is_cac_found_rows = result.is_calc_found_rows(); - int64_t limit_count = OB_INVALID_COUNT == fetch_limit ? INT64_MAX : fetch_limit; int64_t row_num = 0; ObSqlCtx *sql_ctx = result.get_exec_context().get_sql_ctx(); - if (!has_top_limit && OB_INVALID_COUNT == fetch_limit) { - limit_count = INT64_MAX; - if (!lib::is_oracle_mode()) { - if (OB_FAIL(session_.get_sql_select_limit(limit_count))) { - LOG_WARN("fail tp get sql select limit", K(ret)); - } - } - } bool is_packed = result.get_physical_plan() ? result.get_physical_plan()->is_packed() : false; MYSQL_PROTOCOL_TYPE protocol_type = is_ps_protocol ? MYSQL_PROTOCOL_TYPE::BINARY : MYSQL_PROTOCOL_TYPE::TEXT; + + int64_t limit_count = INT64_MAX; + if (OB_FAIL(ret)) { + } else if (OB_INVALID_COUNT != fetch_limit) { + limit_count = fetch_limit; + } else if (lib::is_mysql_mode()) { + if (!result.get_has_top_limit() && OB_FAIL(session_.get_sql_select_limit(limit_count))) { + LOG_WARN("failed to get sytem variable sql_select_limit", K(ret)); + } + } else { // lib::is_oracle_mode() + if (OB_FAIL(session_.get_oracle_sql_select_limit(limit_count))) { + LOG_WARN("failed to get sytem variable _oracle_sql_select_limit", K(ret)); + } + } + const common::ColumnsFieldIArray *fields = NULL; if (OB_SUCC(ret)) { fields = result.get_field_columns(); diff --git a/src/observer/mysql/ob_query_retry_ctrl.cpp b/src/observer/mysql/ob_query_retry_ctrl.cpp index 3536765b1..adb30d783 100644 --- a/src/observer/mysql/ob_query_retry_ctrl.cpp +++ b/src/observer/mysql/ob_query_retry_ctrl.cpp @@ -483,11 +483,24 @@ public: { // sql which in pl will local retry first. see ObInnerSQLConnection::process_retry. // sql which not in pl use the same strategy to avoid never getting the lock. - if (v.force_local_retry_ || (v.local_retry_times_ <= 1 && !v.result_.is_pl_stmt(v.result_.get_stmt_type()))) { - v.retry_type_ = RETRY_TYPE_LOCAL; + if (v.is_from_pl_) { + if (v.local_retry_times_ <= 1 || + !v.session_.get_pl_can_retry() || + ObSQLUtils::is_in_autonomous_block(v.session_.get_cur_exec_ctx())) { + v.no_more_test_ = true; + v.retry_type_ = RETRY_TYPE_LOCAL; + } else { + v.no_more_test_ = true; + v.retry_type_ = RETRY_TYPE_NONE; + v.client_ret_ = v.err_; + } } else { - const ObMultiStmtItem &multi_stmr_item = v.ctx_.multi_stmt_item_; - try_packet_retry(v); + if (v.force_local_retry_ || (v.local_retry_times_ <= 1 && !v.result_.is_pl_stmt(v.result_.get_stmt_type()))) { + v.retry_type_ = RETRY_TYPE_LOCAL; + } else { + const ObMultiStmtItem &multi_stmr_item = v.ctx_.multi_stmt_item_; + try_packet_retry(v); + } } } }; diff --git a/src/observer/mysql/obmp_stmt_execute.cpp b/src/observer/mysql/obmp_stmt_execute.cpp index 4c611f71c..82951dd7c 100644 --- a/src/observer/mysql/obmp_stmt_execute.cpp +++ b/src/observer/mysql/obmp_stmt_execute.cpp @@ -1177,7 +1177,12 @@ int ObMPStmtExecute::execute_response(ObSQLSessionInfo &session, if (OB_SUCC(ret)) { ObPLExecCtx pl_ctx(cursor->get_allocator(), &result.get_exec_context(), NULL/*params*/, NULL/*result*/, &ret, NULL/*func*/, true); - if (OB_FAIL(ObSPIService::dbms_dynamic_open(&pl_ctx, *cursor))) { + int64_t orc_max_ret_rows = INT64_MAX; + if (lib::is_oracle_mode() + && OB_FAIL(session.get_oracle_sql_select_limit(orc_max_ret_rows))) { + LOG_WARN("failed to get sytem variable _oracle_sql_select_limit", K(ret)); + } else if (OB_FAIL(ObSPIService::dbms_dynamic_open( + &pl_ctx, *cursor, false, orc_max_ret_rows))) { LOG_WARN("open cursor fail. ", K(ret), K(stmt_id_)); if (!THIS_WORKER.need_retry()) { int cli_ret = OB_SUCCESS; diff --git a/src/observer/mysql/obmp_stmt_prexecute.cpp b/src/observer/mysql/obmp_stmt_prexecute.cpp index ec9acdc3e..7f862fff6 100644 --- a/src/observer/mysql/obmp_stmt_prexecute.cpp +++ b/src/observer/mysql/obmp_stmt_prexecute.cpp @@ -478,12 +478,16 @@ int ObMPStmtPrexecute::execute_response(ObSQLSessionInfo &session, ObPLExecCtx pl_ctx(cursor->get_allocator(), &result.get_exec_context(), NULL/*params*/, NULL/*result*/, &ret, NULL/*func*/, true); get_ctx().cur_sql_ = sql_; - if ( + int64_t orc_max_ret_rows = INT64_MAX; + if (lib::is_oracle_mode() + && OB_FAIL(session.get_oracle_sql_select_limit(orc_max_ret_rows))) { + LOG_WARN("failed to get sytem variable _oracle_sql_select_limit", K(ret)); + } else if ( #ifdef ERRSIM OB_FAIL(common::EventTable::COM_STMT_PREXECUTE_PS_CURSOR_OPEN_ERROR) || #endif - OB_FAIL(ObSPIService::dbms_dynamic_open(&pl_ctx, *cursor)) - ) { + OB_FAIL(ObSPIService::dbms_dynamic_open( + &pl_ctx, *cursor, false, orc_max_ret_rows))) { LOG_WARN("cursor open faild.", K(cursor->get_id())); // select do not support arraybinding if (!THIS_WORKER.need_retry()) { diff --git a/src/observer/ob_inner_sql_connection.cpp b/src/observer/ob_inner_sql_connection.cpp index 939f83c24..35e48fb2d 100644 --- a/src/observer/ob_inner_sql_connection.cpp +++ b/src/observer/ob_inner_sql_connection.cpp @@ -326,6 +326,11 @@ void ObInnerSQLConnection::set_is_load_data_exec(bool v) get_session().set_load_data_exec_session(v); } +void ObInnerSQLConnection::set_ob_enable_pl_cache(bool v) +{ + get_session().set_local_ob_enable_pl_cache(v); +} + int ObInnerSQLConnection::init_session_info( sql::ObSQLSessionInfo *session, const bool is_extern_session, diff --git a/src/observer/ob_inner_sql_connection.h b/src/observer/ob_inner_sql_connection.h index aad53a145..7fcf81f2f 100644 --- a/src/observer/ob_inner_sql_connection.h +++ b/src/observer/ob_inner_sql_connection.h @@ -189,6 +189,7 @@ public: virtual void set_is_load_data_exec(bool v); virtual void set_force_remote_exec(bool v) { force_remote_execute_ = v; } virtual void set_use_external_session(bool v) { use_external_session_ = v; } + virtual void set_ob_enable_pl_cache(bool v) override; bool is_nested_conn(); virtual void set_user_timeout(int64_t timeout) { user_timeout_ = timeout; } virtual int64_t get_user_timeout() const { return user_timeout_; } diff --git a/src/pl/CMakeLists.txt b/src/pl/CMakeLists.txt index 2d3d82441..88701e981 100644 --- a/src/pl/CMakeLists.txt +++ b/src/pl/CMakeLists.txt @@ -41,10 +41,40 @@ ob_set_subtarget(ob_pl pl_cache pl_cache/ob_pl_cache.cpp pl_cache/ob_pl_cache_mgr.cpp pl_cache/ob_pl_cache_object.cpp + diagnosis/ob_pl_sql_audit_guard.cpp ) +set(SYS_PACK_SQL_SOURCE_DIR ${PROJECT_SOURCE_DIR}/src/share/inner_table/sys_package) +set(gen_sys_pack_file ${SYS_PACK_SQL_SOURCE_DIR}/system_package.cpp) +message(STATUS "generating ${gen_sys_pack_file} ...") +file (GLOB embed_sys_pack_files "${SYS_PACK_SQL_SOURCE_DIR}/*.sql") +list (LENGTH embed_sys_pack_files embed_sys_pack_cnt) +file (WRITE ${gen_sys_pack_file} + "// This file is generated by CMake, do not edit it!\n\n" + "#include \n" + "#include \n\n" + "namespace oceanbase {\n" + "namespace pl {\n\n" + "extern const int64_t syspack_source_count = ${embed_sys_pack_cnt};\n" + "extern const std::pair syspack_source_contents[] = {\n") +foreach(embed_file ${embed_sys_pack_files}) + get_filename_component(file_name ${embed_file} NAME) + file(SIZE ${embed_file} embed_file_size) + file(READ ${embed_file} filedata) + file(APPEND ${gen_sys_pack_file} + "{\n" + "\"${file_name}\", // size = ${embed_file_size} bytes\n" + "R\"sys_pack_del\(${filedata}\)sys_pack_del\"\n" + "},\n") +endforeach() +file (APPEND ${gen_sys_pack_file} + "};\n\n" + "} // namespace pl\n" + "} // namespace oceanbase\n") +message(STATUS "generate ${gen_sys_pack_file} done") + ob_set_subtarget(ob_pl sys_package - sys_package/ob_dbms_external_table.cpp + ${gen_sys_pack_file} sys_package/ob_dbms_stats.cpp sys_package/ob_dbms_scheduler_mysql.cpp sys_package/ob_dbms_application.cpp @@ -59,6 +89,7 @@ ob_set_subtarget(ob_pl sys_package sys_package/ob_pl_dbms_resource_manager.cpp sys_package/ob_pl_dbms_trusted_certificate_manager.cpp sys_package/ob_dbms_limit_calculator_mysql.cpp + sys_package/ob_dbms_external_table.cpp ) ob_set_subtarget(ob_pl dblink diff --git a/src/pl/diagnosis/ob_pl_sql_audit_guard.cpp b/src/pl/diagnosis/ob_pl_sql_audit_guard.cpp new file mode 100644 index 000000000..0aa8eab5a --- /dev/null +++ b/src/pl/diagnosis/ob_pl_sql_audit_guard.cpp @@ -0,0 +1,119 @@ + +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#define USING_LOG_PREFIX PL + +#include "ob_pl_sql_audit_guard.h" +#include "sql/ob_spi.h" + +namespace oceanbase +{ +namespace pl +{ + +ObPLSqlAuditGuard::ObPLSqlAuditGuard( + sql::ObExecContext &exec_ctx, + ObSQLSessionInfo &session_info, + ObSPIResultSet &spi_result, + ObPLSqlAuditRecord &record, + int &ret, + ObString ps_sql, + observer::ObQueryRetryCtrl &retry_ctrl, + ObPLSPITraceIdGuard &traceid_guard, + stmt::StmtType stmt_type) + : exec_ctx_(exec_ctx), + session_info_(session_info), + spi_result_(spi_result), + record_(record), + ret_(ret), + ps_sql_(ps_sql), + retry_ctrl_(retry_ctrl), + traceid_guard_(traceid_guard), + stmt_type_(stmt_type) +{ + enable_perf_event_ = lib::is_diagnose_info_enabled(); + enable_sql_audit_ = GCONF.enable_sql_audit && session_info_.get_local_ob_enable_sql_audit(); + // enable_sql_stat_ = session_info.is_sqlstat_enabled(); + max_wait_guard_ = new (memory1) ObMaxWaitGuard(enable_perf_event_ ? &max_wait_desc_ : NULL); + total_wait_guard_ = new (memory2) ObTotalWaitGuard(enable_perf_event_ ? &total_wait_desc_ : NULL); + if (enable_perf_event_) { + record_.exec_record_.record_start(); + } + // if (enable_sql_stat_ && OB_NOT_NULL(exec_ctx_.get_sql_ctx())) { + // record_.sqlstat_record_.record_sqlstat_start_value(); + // record_.sqlstat_record_.set_is_in_retry(session_info_.get_is_in_retry()); + // session_info_.sql_sess_record_sql_stat_start_value(record_.sqlstat_record_); + // } + // 监控项统计开始 + record_.time_record_.set_send_timestamp(ObTimeUtility::current_time()); +} + +ObPLSqlAuditGuard::~ObPLSqlAuditGuard() +{ + int &ret = ret_; + record_.time_record_.set_exec_end_timestamp(ObTimeUtility::current_time()); + if (enable_perf_event_) { + record_.exec_record_.record_end(); + } + // if (enable_sql_stat_ && OB_NOT_NULL(exec_ctx_.get_sql_ctx()) && OB_NOT_NULL(spi_result_.get_result_set())) { + // record_.sqlstat_record_.record_sqlstat_end_value(); + // record_.sqlstat_record_.move_to_sqlstat_cache( + // session_info_, exec_ctx_.get_sql_ctx()->cur_sql_, spi_result_.get_result_set()->get_physical_plan()); + // } + max_wait_guard_->~ObMaxWaitGuard(); + total_wait_guard_->~ObTotalWaitGuard(); + + LOG_TRACE("Start PL/Sql Audit Record"/*, KPC(this)*/ ); + + if (OB_NOT_NULL(spi_result_.get_result_set())) { + if (spi_result_.get_result_set()->is_inited()) { + if (ObStmt::is_execute_stmt(stmt_type_)) { + ps_sql_ = session_info_.get_current_query_string(); + } + int64_t try_cnt = session_info_.get_raw_audit_record().try_cnt_; + ObExecRecord record_bak = session_info_.get_raw_audit_record().exec_record_; + session_info_.get_raw_audit_record().try_cnt_ = retry_ctrl_.get_retry_times(); + session_info_.get_raw_audit_record().pl_trace_id_.set(traceid_guard_.origin_trace_id_); + observer::ObInnerSQLConnection::process_record(*(spi_result_.get_result_set()), + spi_result_.get_sql_ctx(), + session_info_, + record_.time_record_, + ret_, + session_info_.get_current_execution_id(), + OB_INVALID_ID, //FIXME@hr351303 + max_wait_desc_, + total_wait_desc_, + record_.exec_record_, + record_.exec_timestamp_, + true, + ps_sql_, + true, + spi_result_.get_exec_params_str_ptr()); + session_info_.get_raw_audit_record().exec_record_ = record_bak; + session_info_.get_raw_audit_record().try_cnt_ = try_cnt; + session_info_.get_raw_audit_record().pl_trace_id_.reset(); + } else { + LOG_DEBUG("result set is not inited, do not process record", K(ret_), K(ps_sql_)); + } + } else { + if (OB_SUCCESS == ret_) { + ret_ = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error, result_set is null", K(ret_), K(ps_sql_)); + } else { + LOG_WARN("result_set is null", K(ret_), K(ps_sql_)); + } + } +} + +} // namespace pl +} // namespace oceanbase diff --git a/src/pl/diagnosis/ob_pl_sql_audit_guard.h b/src/pl/diagnosis/ob_pl_sql_audit_guard.h new file mode 100644 index 000000000..6edfb7db0 --- /dev/null +++ b/src/pl/diagnosis/ob_pl_sql_audit_guard.h @@ -0,0 +1,132 @@ +/** + * Copyright (c) 2021 OceanBase + * OceanBase CE is licensed under Mulan PubL v2. + * You can use this software according to the terms and conditions of the Mulan PubL v2. + * You may obtain a copy of Mulan PubL v2 at: + * http://license.coscl.org.cn/MulanPubL-2.0 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PubL v2 for more details. + */ + +#ifndef OCEANBASE_SRC_PL_SQL_AUDIT_GUARD_OB_PL_H_ + +#define OCEANBASE_SRC_PL_SQL_AUDIT_GUARD_OB_PL_H_ + +#include "sql/monitor/ob_exec_stat.h" +#include "observer/ob_inner_sql_connection.h" +#include "sql/resolver/ob_stmt_type.h" + +namespace oceanbase +{ +namespace observer +{ +class ObQueryRetryCtrl; +} +namespace sql +{ +struct ObPLSPITraceIdGuard; +} +namespace pl +{ + +class ObPLTimeRecord : public observer::ObITimeRecord +{ +public: + ObPLTimeRecord() + : send_timestamp_(0), + receive_timestamp_(0), + enqueue_timestamp_(0), + run_timestamp_(0), + process_timestamp_(0), + single_process_timestamp_(0), + exec_start_timestamp_(0), + exec_end_timestamp_(0) {} + virtual ~ObPLTimeRecord() {} + + void set_send_timestamp(int64_t send_timestamp) { send_timestamp_ = send_timestamp; } + void set_receive_timestamp(int64_t receive_timestamp) { receive_timestamp_ = receive_timestamp; } + void set_enqueue_timestamp(int64_t enqueue_timestamp) { enqueue_timestamp_ = enqueue_timestamp; } + void set_run_timestamp(int64_t run_timestamp) { run_timestamp_ = run_timestamp; } + void set_process_timestamp(int64_t process_timestamp) { process_timestamp_ = process_timestamp; } + void set_single_process_timestamp(int64_t single_process_timestamp) { single_process_timestamp_ = single_process_timestamp; } + void set_exec_start_timestamp(int64_t exec_start_timestamp) { exec_start_timestamp_ = exec_start_timestamp; } + void set_exec_end_timestamp(int64_t exec_end_timestamp) { exec_end_timestamp_ = exec_end_timestamp; } + + int64_t get_send_timestamp() const { return send_timestamp_; } + int64_t get_receive_timestamp() const { return get_send_timestamp(); } + int64_t get_enqueue_timestamp() const { return get_send_timestamp(); } + int64_t get_run_timestamp() const { return get_send_timestamp(); } + int64_t get_process_timestamp() const { return get_send_timestamp(); } + int64_t get_single_process_timestamp() const { return get_send_timestamp(); } + int64_t get_exec_start_timestamp() const { return get_send_timestamp(); } + int64_t get_exec_end_timestamp() const { return exec_end_timestamp_; } + +public: + int64_t send_timestamp_; + int64_t receive_timestamp_; + int64_t enqueue_timestamp_; + int64_t run_timestamp_; + int64_t process_timestamp_; + int64_t single_process_timestamp_; + int64_t exec_start_timestamp_; + int64_t exec_end_timestamp_; + +}; + +class ObPLSqlAuditRecord +{ +public: + ObPLSqlAuditRecord(sql::ExecType exec_type_) { + exec_timestamp_.exec_type_ = exec_type_; + } +public: + sql::ObExecRecord exec_record_; + // ObExecutingSqlStatRecord sqlstat_record_; + sql::ObExecTimestamp exec_timestamp_; + ObPLTimeRecord time_record_; +}; + +class ObPLSqlAuditGuard +{ +public: + ObPLSqlAuditGuard(sql::ObExecContext &exec_ctx, + sql::ObSQLSessionInfo &session_info, + sql::ObSPIResultSet &spi_result, + ObPLSqlAuditRecord &record, + int &ret, + ObString ps_sql, + observer::ObQueryRetryCtrl &retry_ctrl, + sql::ObPLSPITraceIdGuard &traceid_guard, + sql::stmt::StmtType stmt_type); + + ~ObPLSqlAuditGuard(); + +private: + bool enable_perf_event_; + bool enable_sql_audit_; + bool enable_sql_stat_; + sql::ObExecContext &exec_ctx_; + sql::ObSQLSessionInfo &session_info_; + + ObWaitEventDesc max_wait_desc_; + ObWaitEventStat total_wait_desc_; + ObMaxWaitGuard *max_wait_guard_; + ObTotalWaitGuard *total_wait_guard_; + char memory1[sizeof(ObMaxWaitGuard)]; + char memory2[sizeof(ObTotalWaitGuard)]; + sql::ObSPIResultSet &spi_result_; + ObPLSqlAuditRecord &record_; + + int &ret_; + ObString ps_sql_; + observer::ObQueryRetryCtrl &retry_ctrl_; + sql::ObPLSPITraceIdGuard &traceid_guard_; + sql::stmt::StmtType stmt_type_; +}; + +} +} + +#endif /*OCEANBASE_SRC_PL_SQL_AUDIT_GUARD_OB_PL_H_*/ \ No newline at end of file diff --git a/src/pl/ob_pl.cpp b/src/pl/ob_pl.cpp index e1bf0d0c4..4663e6378 100644 --- a/src/pl/ob_pl.cpp +++ b/src/pl/ob_pl.cpp @@ -4762,8 +4762,13 @@ int ObPL::check_session_alive(const ObBasicSessionInfo &session) { int ObPLFunction::gen_action_from_precompiled(const ObString &name, size_t length, const char *ptr) { int ret = OB_SUCCESS; + + uint64_t addr = 0; + OZ (helper_.add_compiled_object(length, ptr)); - OX (set_action(helper_.get_function_address(name))); + OZ (helper_.get_function_address(name, addr)); + OX (set_action(addr)); + return ret; } diff --git a/src/pl/ob_pl_code_generator.cpp b/src/pl/ob_pl_code_generator.cpp index cb654e5b0..a9866f44e 100644 --- a/src/pl/ob_pl_code_generator.cpp +++ b/src/pl/ob_pl_code_generator.cpp @@ -8094,15 +8094,29 @@ int ObPLCodeGenerator::final_expression(ObPLCompileUnit &pl_func) int ret = OB_SUCCESS; for (int64_t i = 0; OB_SUCC(ret) && i < ast_.get_obj_access_exprs().count(); ++i) { ObRawExpr *expr = ast_.get_obj_access_expr(i); + ObObjAccessRawExpr *obj_access_expr = nullptr; + uint64_t addr = 0; + if (OB_ISNULL(expr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("obj_access_expr is null"); } else if (!expr->is_obj_access_expr()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("not a obj access", K(*expr), K(ret)); + } else if (OB_ISNULL(obj_access_expr = static_cast(expr))) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected NULL obj_access_expr", K(ret), K(*expr)); + } else if (OB_FAIL(helper_.get_function_address(obj_access_expr->get_func_name(), addr))) { + if (OB_ENTRY_NOT_EXIST == ret) { + LOG_INFO("failed to find obj_access_expr symbol in JIT engine, will ignore this error", + K(ret), K(obj_access_expr->get_func_name())); + ret = OB_SUCCESS; + obj_access_expr->set_get_attr_func_addr(0); + } else { + LOG_WARN("failed to compile obj_access_expr", K(ret), K(obj_access_expr->get_func_name()), K(addr)); + } } else { - ObObjAccessRawExpr* obj_access_expr = static_cast(expr); - obj_access_expr->set_get_attr_func_addr(helper_.get_function_address(obj_access_expr->get_func_name())); + obj_access_expr->set_get_attr_func_addr(addr); } } @@ -8618,6 +8632,8 @@ int ObPLCodeGenerator::generate_normal(ObPLFunction &pl_func) } if (OB_SUCC(ret)) { + uint64_t addr = 0; + if (OB_FAIL(final_expression(pl_func))) { LOG_WARN("generate obj access expr failed", K(ret)); } else if (OB_FAIL(pl_func.set_variables(get_ast().get_symbol_table()))) { @@ -8626,11 +8642,13 @@ int ObPLCodeGenerator::generate_normal(ObPLFunction &pl_func) LOG_WARN("failed to set ref objects", K(get_ast().get_dependency_table()), K(ret)); } else if (OB_FAIL(pl_func.set_types(get_ast().get_user_type_table()))) { LOG_WARN("failed to set types", K(ret)); + } else if (OB_FAIL(helper_.get_function_address(get_ast().get_name(), addr))) { + LOG_WARN("failed to compile pl routine", K(ret), K(get_ast().get_name()), K(addr)); } else { pl_func.add_members(get_ast().get_flag()); pl_func.set_pipelined(get_ast().get_pipelined()); - pl_func.set_action(helper_.get_function_address(get_ast().get_name())); - pl_func.set_can_cached(get_ast().get_can_cached()); + pl_func.set_action(addr); + pl_func.set_can_cached(get_ast().get_can_cached()); pl_func.set_is_all_sql_stmt(get_ast().get_is_all_sql_stmt()); pl_func.set_has_parallel_affect_factor(get_ast().has_parallel_affect_factor()); } diff --git a/src/pl/ob_pl_package_manager.cpp b/src/pl/ob_pl_package_manager.cpp index b607d4755..7ced6b2c1 100644 --- a/src/pl/ob_pl_package_manager.cpp +++ b/src/pl/ob_pl_package_manager.cpp @@ -39,14 +39,152 @@ using namespace sql; namespace pl { -int ObPLPackageManager::read_package_sql(FILE* file, char* buf, int64_t buf_len, bool &eof) +// Usage: ObCharStream *s -> s.open() -> s.next().is_eos() ... -> s.close() +class ObCharStream +{ +public: + ObCharStream(const char *name) : eos_flag_(false), name_(name) {} + virtual ~ObCharStream() {} + ObCharStream(const ObCharStream &) = delete; + const ObCharStream &operator=(const ObCharStream &) = delete; + + const char *get_name() { return name_; } + virtual int open() = 0; + virtual const ObCharStream &next(char &c) = 0; + virtual bool is_eos() const { return eos_flag_; } + virtual int close() = 0; + + VIRTUAL_TO_STRING_KV(K_(eos_flag), K_(name)); + +protected: + static const char EOS = '\0'; + bool eos_flag_; + +private: + const char *const name_; +}; + +// read package sql from array embeded in the oberver binary file +class ObCStringStream final : public ObCharStream +{ +public: + ObCStringStream(const char *package_name, const char *data) + : ObCharStream(package_name), data_(data), cursor_(nullptr) {} + ~ObCStringStream() {} + + int open() override { + int ret = OB_SUCCESS; + if (OB_ISNULL(data_)) { + ret = OB_ERR_NULL_VALUE; + LOG_WARN("package sql data is null", K(ret), K(data_)); + } else if (0 == strlen(data_)) { + LOG_INFO("package sql file is empty or not exists", K(ret)); + } else { + cursor_ = data_; + } + return ret; + } + const ObCharStream &next(char &c) override { + if (OB_NOT_NULL(cursor_) && !eos_flag_) { + c = *cursor_++; + eos_flag_ = (c == EOS); + } else { + c = EOS; + } + return *this; + } + int close() override { + int ret = OB_SUCCESS; + if (OB_NOT_NULL(cursor_)) { + if (EOS != *(cursor_ - 1)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("C string stream has not been completely consumed", K(ret), + K(cursor_ - data_), K(data_), K(cursor_)); + } else { + cursor_ = nullptr; + eos_flag_ = false; + } + } + return ret; + } + + INHERIT_TO_STRING_KV("ObCharStream", ObCharStream, + "C string stream bytes comsumed", cursor_ - data_); + +private: + const char *const data_; + const char *cursor_; +}; + +// read package sql from file under admin/ +class ObFileStream final : public ObCharStream +{ +public: + ObFileStream(const char *package_name, const char *file_path) + : ObCharStream(package_name), file_path_(file_path), file_(nullptr) {} + ~ObFileStream() { + // make sure file_ closed + if (file_) { + fclose(file_); + file_ = nullptr; + } + } + + int open() override { + int ret = OB_SUCCESS; + if (OB_ISNULL(file_path_) || 0 != access(file_path_, F_OK)) { + ret = OB_FILE_NOT_EXIST; + LOG_WARN("package sql file not exists", K(ret), K(file_path_)); + } else if (OB_ISNULL(file_ = fopen(file_path_, "rb"))) { + ret = OB_IO_ERROR; + LOG_WARN("package sql file open failed", K(ret), K(file_path_)); + } + return ret; + } + const ObCharStream &next(char &c) override { + int ch; + if (OB_NOT_NULL(file_) && !eos_flag_) { + if (EOF == (ch = fgetc(file_))) { + c = EOS; + eos_flag_ = true; + } else { + c = static_cast(ch); + } + } else { + c = EOS; + } + return *this; + } + int close() override { + int ret = OB_SUCCESS; + if (OB_NOT_NULL(file_)) { + if (0 == feof(file_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("file content has not been completely consumed", K(ret), K(errno)); + } else if (0 != fclose(file_)) { + ret = OB_IO_ERROR; + LOG_WARN("close file failed", K(ret), K(file_path_), K(file_)); + } else { + file_ = nullptr; + eos_flag_ = false; + } + } + return ret; + } + + INHERIT_TO_STRING_KV("ObCharStream", ObCharStream, K_(file_path), + "file stream bytes comsumed", file_ ? ftell(file_) : -1); + +private: + const char *const file_path_; + FILE *file_; +}; + +int ObPLPackageManager::read_package_sql(ObCharStream &stream, char* buf, int64_t buf_len, bool &eos) { int ret = OB_SUCCESS; enum {S_LINE_START, S_NORMAL, S_COMMENT, S_TERMINATE} state = S_LINE_START; - if (OB_ISNULL(file)) { - ret = OB_ERR_NULL_VALUE; - LOG_WARN("package sql file is null", K(ret)); - } else if (OB_ISNULL(buf)) { + if (OB_ISNULL(buf)) { ret = OB_ERR_NULL_VALUE; LOG_WARN("sql buffer is null", K(ret)); } else if (buf_len <= 0) { @@ -56,13 +194,13 @@ int ObPLPackageManager::read_package_sql(FILE* file, char* buf, int64_t buf_len, char *p = buf; char *p_start = p; char *p_end = p + buf_len - 1; - int c; + char c; // clear buffer *p = '\0'; *p_end = '\0'; while (OB_SUCC(ret) && state != S_TERMINATE) { - if (EOF == (c = fgetc(file))) { - ret = OB_IO_ERROR; + if (stream.next(c).is_eos()) { + ret = OB_ITER_END; } else { if (p >= p_end) { ret = OB_INVALID_ARGUMENT; @@ -77,18 +215,17 @@ int ObPLPackageManager::read_package_sql(FILE* file, char* buf, int64_t buf_len, } else if ('#' == c) { state = S_COMMENT; } else if ('-' == c) { - c = fgetc(file); - if ('-' == c) { + if (stream.next(c).is_eos()) { + ret = OB_ITER_END; + } else if ('-' == c) { state = S_COMMENT; - } else if (EOF == c) { - ret = OB_IO_ERROR; } else { *p++ = '-'; - *p++ = static_cast(c); + *p++ = c; state = S_NORMAL; } } else { - *p++ = static_cast(c); + *p++ = c; state = S_NORMAL; } } @@ -107,11 +244,11 @@ int ObPLPackageManager::read_package_sql(FILE* file, char* buf, int64_t buf_len, *(p - 1) = '\0'; state = S_TERMINATE; } else { - *p++ = static_cast(c); + *p++ = c; state = S_NORMAL; } } else { - *p++ = static_cast(c); + *p++ = c; if ('\n' == c) { state = S_LINE_START; } else { @@ -130,19 +267,20 @@ int ObPLPackageManager::read_package_sql(FILE* file, char* buf, int64_t buf_len, } } if (OB_FAIL(ret)) { - if (feof(file)) { - eof = true; + if (stream.is_eos()) { + eos = true; ret = OB_SUCCESS; } else { - LOG_WARN("read package file error", K(errno), K(ret)); + LOG_WARN("read package file error", K(ret), K(stream)); } } } return ret; } -int ObPLPackageManager::read_and_exec_package_sql( - ObMySQLProxy &sql_proxy, const char* package_full_path, ObCompatibilityMode compa_mode) +int ObPLPackageManager::read_and_exec_package_sql(ObMySQLProxy &sql_proxy, + ObCharStream &stream, + ObCompatibilityMode compa_mode) { int ret = OB_SUCCESS; if (!sql_proxy.is_inited() || !sql_proxy.is_active()) { @@ -150,13 +288,9 @@ int ObPLPackageManager::read_and_exec_package_sql( LOG_WARN("sql_proxy not inited or not active", "sql_proxy inited", sql_proxy.is_inited(), "sql_proxy active", sql_proxy.is_active(), K(ret)); } else { - FILE* file = NULL; int64_t affected_rows = 0; - if (access(package_full_path, F_OK) != 0) { - LOG_INFO("package sql file not exists", K(package_full_path), K(ret)); - } else if (OB_ISNULL(file = fopen(package_full_path, "rb"))) { - ret = OB_IO_ERROR; - LOG_WARN("package sql file open failed", K(package_full_path), K(ret)); + if (OB_FAIL(stream.open())) { + LOG_WARN("failed to open package file data stream", K(ret), K(stream)); } else { // system tenant will run with mysql compatibility mode // but we need to create system packages with oralce compatibility @@ -165,35 +299,35 @@ int ObPLPackageManager::read_and_exec_package_sql( bool create_external_table = false; ObSessionParam param; ObSessionParam *param_ptr = nullptr; - char *last_slash = strrchr(const_cast(package_full_path), '/'); - const char *pacakge_filename = (last_slash != NULL) ? last_slash + 1 : package_full_path; int64_t sql_mode = SMO_STRICT_ALL_TABLES | SMO_NO_ZERO_IN_DATE | SMO_NO_AUTO_CREATE_USER; // allow affected_rows > 0 when exec sql in external_table_alert_log.sql - if (strcmp(pacakge_filename, "external_table_alert_log.sql") == 0) { + if (strcmp(stream.get_name(), "external_table_alert_log") == 0) { create_external_table = true; param.sql_mode_ = &sql_mode; param_ptr = ¶m; } + // do not cache the compilation results of system packages into the PL cache when loading system packages. + param.enable_pl_cache_ = false; SMART_VAR(char[OB_MAX_SQL_LENGTH], sql_buf) { while (OB_SUCC(ret) && !eof) { - if (OB_FAIL(read_package_sql(file, sql_buf, OB_MAX_SQL_LENGTH, eof))) { - LOG_WARN("fail to read package sql file", K(ret)); + if (FAILEDx(read_package_sql(stream, sql_buf, OB_MAX_SQL_LENGTH, eof))) { + LOG_WARN("fail to read package sql data", K(ret)); } else if (strlen(sql_buf) != 0 - && OB_FAIL(sql_proxy.write(OB_SYS_TENANT_ID, + && OB_FAIL(sql_proxy.write(OB_SYS_TENANT_ID, sql_buf, affected_rows, static_cast(compa_mode), - param_ptr))) { + ¶m))) { LOG_WARN("fail to exec package sql", K(sql_buf), K(ret)); } else if (affected_rows != 0 && !create_external_table) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("affected_rows expected to be zero", K(affected_rows), K(ret)); + LOG_WARN("affected_rows expected to be zero", K(ret), K(affected_rows), K(stream.get_name())); } else { OZ (ObSPIService::force_refresh_schema(OB_SYS_TENANT_ID)); } + LOG_INFO("package source data consumed", K(ret), K(stream)); } } - fclose(file); if (create_external_table && OB_SUCC(ret)) { uint64_t data_version = 0; common::ObString alter_table_sql("alter external table sys_external_tbs.__all_external_alert_log_info auto_refresh immediate"); @@ -216,29 +350,80 @@ int ObPLPackageManager::read_and_exec_package_sql( return ret; } -int ObPLPackageManager::load_sys_package( - ObMySQLProxy &sql_proxy, const char *package_spec_name, const char *package_body_name, ObCompatibilityMode compa_mode) +// import source file content array declarations, which is defined in system_package.cpp generated by CMake. +extern const int64_t syspack_source_count; +extern const std::pair syspack_source_contents[]; + +int ObPLPackageManager::get_syspack_source_file_content(const char *file_name, const char *&content) { int ret = OB_SUCCESS; - const int64_t begin_time = ObTimeUtility::current_time(); - LOG_INFO("load sys package begin", - "package name", package_spec_name, "package body name", package_body_name); - - if (OB_FAIL(read_and_exec_package_sql(sql_proxy, package_spec_name, compa_mode))) { - LOG_WARN("fail to read and exec package header sql", K(package_spec_name), K(ret)); - } else if (OB_FAIL(read_and_exec_package_sql(sql_proxy, package_body_name, compa_mode))) { - LOG_WARN("fail to read and exec package body sql", K(package_body_name), K(ret)); + OX (content = nullptr); + if (OB_ISNULL(file_name)) { + // return nullptr as `content` + LOG_INFO("file name c string is null", K(file_name)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < syspack_source_count; i++) { + if (0 == ObString(file_name).case_compare(syspack_source_contents[i].first)) { + content = syspack_source_contents[i].second; + break; + } + } + // `content` not found, report error + OV (OB_NOT_NULL(content), OB_ERR_UNEXPECTED, "system source file not found", ret, file_name); } - - const int64_t now = ObTimeUtility::current_time(); - LOG_INFO("load sys package finish", "total_time_used", now - begin_time); return ret; } -static const char* sys_package_dir = "admin"; -static ObSysPackageFile oracle_sys_package_file_table[] = { +int ObPLPackageManager::load_sys_package(ObMySQLProxy &sql_proxy, + const ObSysPackageFile &pack_file_info, + ObCompatibilityMode compa_mode, + bool from_file) +{ + int ret = OB_SUCCESS; + const char *package_name = pack_file_info.package_name; + const char *spec_file = pack_file_info.package_spec_file_name; + const char *body_file = pack_file_info.package_body_file_name; + + const int64_t begin_time = ObTimeUtility::current_time(); + LOG_INFO("load sys package", K(package_name), K(spec_file), K(body_file), K(begin_time)); + + if (from_file) { + const char *sys_package_dir = "admin"; + char spec_file_path[MAX_PATH_SIZE] = {0}; + char body_file_path[MAX_PATH_SIZE] = {0}; + if (OB_SUCC(ret) && OB_NOT_NULL(spec_file)) { + OZ (databuff_printf(spec_file_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, spec_file)); + ObFileStream spec_stream{package_name, spec_file_path}; + OZ (read_and_exec_package_sql(sql_proxy, spec_stream, compa_mode), spec_stream); + } + if (OB_SUCC(ret) && OB_NOT_NULL(body_file)) { + OZ (databuff_printf(body_file_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, body_file)); + ObFileStream body_stream{package_name, body_file_path}; + OZ (read_and_exec_package_sql(sql_proxy, body_stream, compa_mode), body_stream); + } + } else { + const char *spec_content = nullptr; + const char *body_content = nullptr; + OZ (get_syspack_source_file_content(spec_file, spec_content)); + if (OB_SUCC(ret) && OB_NOT_NULL(spec_content)) { + ObCStringStream spec_stream{package_name, spec_content}; + OZ (read_and_exec_package_sql(sql_proxy, spec_stream, compa_mode), spec_stream); + } + OZ (get_syspack_source_file_content(body_file, body_content)); + if (OB_SUCC(ret) && OB_NOT_NULL(body_content)) { + ObCStringStream body_stream{package_name, body_content}; + OZ (read_and_exec_package_sql(sql_proxy, body_stream, compa_mode), body_stream); + } + } + + const int64_t now = ObTimeUtility::current_time(); + LOG_INFO("load sys package finish", K(ret), K(package_name), "total_time_used", now - begin_time); + return ret; +} + +static const ObSysPackageFile oracle_syspack_file_list[] = { #ifdef OB_BUILD_ORACLE_PL - {"dbms_standard", "dbms_standard.sql", "dbms_standard_body.sql"}, + {"dbms_standard", "dbms_standard.sql", nullptr}, {"dbms_output", "dbms_output.sql", "dbms_output_body.sql"}, {"dbms_metadata", "dbms_metadata.sql", "dbms_metadata_body.sql"}, {"dbms_spm", "dbms_spm.sql", "dbms_spm_body.sql"}, @@ -251,7 +436,7 @@ static ObSysPackageFile oracle_sys_package_file_table[] = { {"sa_sysdba", "sa_sysdba.sql", "sa_sysdba_body.sql"}, {"sa_user_admin", "sa_user_admin.sql", "sa_user_admin_body.sql"}, {"utl_i18n", "utl_i18n.sql", "utl_i18n_body.sql"}, - {"dbms_crypto", "dbms_crypto.sql","dbms_crypto_body.sql"}, + {"dbms_crypto", "dbms_crypto.sql", "dbms_crypto_body.sql"}, {"dbms_random", "dbms_random.sql", "dbms_random_body.sql"}, {"dbms_debug", "dbms_debug.sql", "dbms_debug_body.sql"}, {"utl_inaddr", "utl_inaddr.sql", "utl_inaddr_body.sql"}, @@ -263,16 +448,15 @@ static ObSysPackageFile oracle_sys_package_file_table[] = { {"dbms_xa", "dbms_xa.sql", "dbms_xa_body.sql"}, {"dbms_resource_manager", "dbms_resource_manager.sql", "dbms_resource_manager_body.sql"}, {"dbms_utility", "dbms_utility.sql", "dbms_utility_body.sql"}, - {"odciconst", "odciconst.sql", "odciconst_body.sql"}, + {"odciconst", "odciconst.sql", nullptr}, {"dbms_stats", "dbms_stats.sql", "dbms_stats_body.sql"}, {"dbms_any", "dbms_any.sql", "dbms_any_body.sql"}, {"xml_type", "xml_type.sql", "xml_type_body.sql"}, - {"dbms_crypto", "dbms_crypto.sql", "dbms_crypto_body.sql"}, {"dbms_ijob", "dbms_ijob.sql", "dbms_ijob_body.sql"}, {"dbms_job", "dbms_job.sql", "dbms_job_body.sql"}, {"dbms_ischeduler", "dbms_ischeduler.sql", "dbms_ischeduler_body.sql"}, {"dbms_scheduler", "dbms_scheduler.sql", "dbms_scheduler_body.sql"}, - {"catodci", "catodci.sql", "catodci_body.sql"}, + {"catodci", "catodci.sql", nullptr}, {"dbms_describe", "dbms_describe.sql", "dbms_describe_body.sql"}, {"utl_file", "utl_file.sql", "utl_file_body.sql"}, {"dbms_plan_cache", "dbms_plancache.sql", "dbms_plancache_body.sql"}, @@ -292,7 +476,7 @@ static ObSysPackageFile oracle_sys_package_file_table[] = { {"dbms_mview", "dbms_mview.sql", "dbms_mview_body.sql"}, {"dbms_mview_stats", "dbms_mview_stats.sql", "dbms_mview_stats_body.sql"}, {"json_array_t", "json_array_type.sql", "json_array_type_body.sql"}, - {"xmlsequence", "xml_sequence_type.sql", "xml_sequence_type_body.sql"}, + {"xmlsequence", "xml_sequence_type.sql", nullptr}, {"utl_recomp", "utl_recomp.sql", "utl_recomp_body.sql"}, {"sdo_geometry", "sdo_geometry.sql", "sdo_geometry_body.sql"}, {"sdo_geom", "sdo_geom.sql", "sdo_geom_body.sql"}, @@ -300,8 +484,7 @@ static ObSysPackageFile oracle_sys_package_file_table[] = { {"dbms_profiler", "dbms_profiler.sql", "dbms_profiler_body.sql"}, #endif }; - -static ObSysPackageFile mysql_sys_package_file_table[] = { +static const ObSysPackageFile mysql_syspack_file_list[] = { {"dbms_stats", "dbms_stats_mysql.sql", "dbms_stats_body_mysql.sql"}, {"dbms_scheduler", "dbms_scheduler_mysql.sql", "dbms_scheduler_mysql_body.sql"}, {"dbms_ischeduler", "dbms_ischeduler_mysql.sql", "dbms_ischeduler_mysql_body.sql"}, @@ -320,119 +503,109 @@ static ObSysPackageFile mysql_sys_package_file_table[] = { {"dbms_trusted_certificate_manager", "dbms_trusted_certificate_manager_mysql.sql", "dbms_trusted_certificate_manager_body_mysql.sql"}, {"dbms_ob_limit_calculator", "dbms_ob_limit_calculator_mysql.sql", "dbms_ob_limit_calculator_body_mysql.sql"}, {"dbms_external_table", "dbms_external_table_mysql.sql", "dbms_external_table_body_mysql.sql"}, - {"external_table_alert_log", "external_table_alert_log.sql", "none"} + {"external_table_alert_log", "external_table_alert_log.sql", nullptr} }; -int ObPLPackageManager::load_sys_package(ObMySQLProxy &sql_proxy, ObString &package_name, ObCompatibilityMode compa_mode) +// for now! we only have one special system package "__DBMS_UPGRADE" +static const ObSysPackageFile oracle_special_syspack_file_list[] = { + {"__dbms_upgrade", "__dbms_upgrade.sql", "__dbms_upgrade_body.sql"}, +}; +static const ObSysPackageFile mysql_special_syspack_file_list[] = { + {"__dbms_upgrade", "__dbms_upgrade_mysql.sql", "__dbms_upgrade_body_mysql.sql"}, +}; + +int ObPLPackageManager::load_sys_package(ObMySQLProxy &sql_proxy, + ObString &package_name, + ObCompatibilityMode compa_mode, + bool from_file) { int ret = OB_SUCCESS; - char package_spec_full_path[MAX_PATH_SIZE] = {}; - char package_body_full_path[MAX_PATH_SIZE] = {}; - bool dir_exists = false; - bool package_exists = false; - if (OB_FAIL(FileDirectoryUtils::is_exists(sys_package_dir, dir_exists))) { - LOG_WARN("check sys package dir whether exist failed", K(ret), K(sys_package_dir)); - } else if (!dir_exists) { - ret = OB_ENTRY_NOT_EXIST; - LOG_WARN("sys package dir not exist", K(ret), K(sys_package_dir)); - } + const ObSysPackageFile *pack_file_info = nullptr; + +#define SEARCH_SYSPACK_FILE_BY_NAME(syspack_file_list) \ + do { \ + if (pack_file_info == nullptr) { \ + int sys_package_count = ARRAYSIZEOF(syspack_file_list); \ + for (int64_t i = 0; OB_SUCC(ret) && i < sys_package_count; ++i) { \ + if (0 == package_name.case_compare(ObString(syspack_file_list[i].package_name))) { \ + pack_file_info = &syspack_file_list[i]; \ + break; \ + } \ + } \ + } \ + } while (0) + if (ObCompatibilityMode::ORACLE_MODE == compa_mode) { - int sys_package_count = ARRAYSIZEOF(oracle_sys_package_file_table); - for (int64_t i = 0; OB_SUCC(ret) && i < sys_package_count; ++i) { - if (0 == package_name.case_compare(ObString(oracle_sys_package_file_table[i].package_name))) { - const char *package_spec_name = oracle_sys_package_file_table[i].package_spec_file_name; - const char *package_body_name = oracle_sys_package_file_table[i].package_body_file_name; - OZ (databuff_printf( - package_spec_full_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, package_spec_name)); - OZ (databuff_printf( - package_body_full_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, package_body_name)); - OX (package_exists = true); - break; - } - } + SEARCH_SYSPACK_FILE_BY_NAME(oracle_syspack_file_list); + SEARCH_SYSPACK_FILE_BY_NAME(oracle_special_syspack_file_list); } else if (ObCompatibilityMode::MYSQL_MODE == compa_mode) { - int sys_package_count = ARRAYSIZEOF(mysql_sys_package_file_table); - for (int64_t i = 0; OB_SUCC(ret) && i < sys_package_count; ++i) { - if (0 == package_name.case_compare(ObString(mysql_sys_package_file_table[i].package_name))) { - const char *package_spec_name = mysql_sys_package_file_table[i].package_spec_file_name; - const char *package_body_name = mysql_sys_package_file_table[i].package_body_file_name; - OZ (databuff_printf( - package_spec_full_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, package_spec_name)); - OZ (databuff_printf( - package_body_full_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, package_body_name)); - OX (package_exists = true); - break; - } - } + SEARCH_SYSPACK_FILE_BY_NAME(mysql_syspack_file_list); + SEARCH_SYSPACK_FILE_BY_NAME(mysql_special_syspack_file_list); } - if (OB_SUCC(ret) && !package_exists) { +#undef SEARCH_SYSPACK_FILE_BY_NAME + + if (OB_FAIL(ret)) { + } else if (OB_ISNULL(pack_file_info)) { ret = OB_ERR_PACKAGE_DOSE_NOT_EXIST; - LOG_WARN("package not exists", K(ret), K(package_name)); + LOG_WARN("package not exists", K(ret), K(package_name), K(compa_mode)); LOG_USER_ERROR(OB_ERR_PACKAGE_DOSE_NOT_EXIST, "PACKAGE", ObString("oceanbase").length(), ObString("oceanbase").ptr(), package_name.length(), package_name.ptr()); + } else { + OZ (load_sys_package(sql_proxy, *pack_file_info, compa_mode, from_file)); } - OZ (load_sys_package(sql_proxy, package_spec_full_path, package_body_full_path, compa_mode)); return ret; } -int ObPLPackageManager::load_all_common_sys_package(ObMySQLProxy &sql_proxy, - const ObSysPackageFile *package_file, - int sys_package_count, - ObCompatibilityMode compa_mode) +int ObPLPackageManager::load_sys_package_list(ObMySQLProxy &sql_proxy, + const ObSysPackageFile *sys_package_list, + int sys_package_count, + ObCompatibilityMode compa_mode, + bool from_file) { int ret = OB_SUCCESS; - char package_spec_full_path[MAX_PATH_SIZE] = {}; - char package_body_full_path[MAX_PATH_SIZE] = {}; - CK (OB_NOT_NULL(package_file)); - LOG_INFO("load all sys package begin", "sys package total count", sys_package_count); + CK (OB_NOT_NULL(sys_package_list)); + LOG_INFO("load sys package list begin", "sys package total count", sys_package_count); for (int i = 0; OB_SUCC(ret) && i < sys_package_count; ++i) { - const char *package_spec_name = package_file[i].package_spec_file_name; - const char *package_body_name = package_file[i].package_body_file_name; - OZ (databuff_printf( - package_spec_full_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, package_spec_name)); - OZ (databuff_printf( - package_body_full_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, package_body_name)); - if (OB_SUCC(ret)) { - LOG_INFO("load sys package begin", K(package_spec_name)); - if (OB_FAIL(load_sys_package(sql_proxy, package_spec_full_path, package_body_full_path, compa_mode))) { - LOG_WARN("load sys package failed", - K(package_spec_full_path), K(package_body_full_path), K(compa_mode), K(ret)); - } else { - LOG_INFO("load sys package success", K(ret), K(package_spec_name)); - } - } + OZ (load_sys_package(sql_proxy, sys_package_list[i], compa_mode, from_file)); + } + if (OB_FAIL(ret)) { + LOG_WARN("load sys package list failed", K(ret), K(compa_mode)); + } else { + LOG_INFO("load sys package list success", K(ret), K(compa_mode)); } return ret; } -int ObPLPackageManager::load_all_common_sys_package(ObMySQLProxy &sql_proxy, ObCompatibilityMode need_compa_mode) -{ +int ObPLPackageManager::load_all_common_sys_package( + ObMySQLProxy &sql_proxy, ObCompatibilityMode compa_mode, bool from_file) { int ret = OB_SUCCESS; - bool exist = false; - if (OB_FAIL(FileDirectoryUtils::is_exists(sys_package_dir, exist))) { - LOG_WARN("check sys package dir whether exist failed", K(sys_package_dir), K(ret)); - } else if (!exist) { - LOG_INFO("sys package dir not exist", K(sys_package_dir)); - } else { - if (need_compa_mode == ObCompatibilityMode::OCEANBASE_MODE) { - OZ (load_all_common_sys_package(sql_proxy, oracle_sys_package_file_table, - ARRAYSIZEOF(oracle_sys_package_file_table), ObCompatibilityMode::ORACLE_MODE)); - OZ (load_all_common_sys_package(sql_proxy, mysql_sys_package_file_table, - ARRAYSIZEOF(mysql_sys_package_file_table), ObCompatibilityMode::MYSQL_MODE)); - } else if (need_compa_mode == ObCompatibilityMode::ORACLE_MODE) { - OZ (load_all_common_sys_package(sql_proxy, oracle_sys_package_file_table, - ARRAYSIZEOF(oracle_sys_package_file_table), ObCompatibilityMode::ORACLE_MODE)); - } else if (need_compa_mode == ObCompatibilityMode::MYSQL_MODE) { - OZ (load_all_common_sys_package(sql_proxy, mysql_sys_package_file_table, - ARRAYSIZEOF(mysql_sys_package_file_table), ObCompatibilityMode::MYSQL_MODE)); - } + if (compa_mode == ObCompatibilityMode::OCEANBASE_MODE) { + OZ (load_sys_package_list(sql_proxy, oracle_syspack_file_list, + ARRAYSIZEOF(oracle_syspack_file_list), + ObCompatibilityMode::ORACLE_MODE, + from_file)); + OZ (load_sys_package_list(sql_proxy, mysql_syspack_file_list, + ARRAYSIZEOF(mysql_syspack_file_list), + ObCompatibilityMode::MYSQL_MODE, + from_file)); + } else if (compa_mode == ObCompatibilityMode::ORACLE_MODE) { + OZ (load_sys_package_list(sql_proxy, oracle_syspack_file_list, + ARRAYSIZEOF(oracle_syspack_file_list), + ObCompatibilityMode::ORACLE_MODE, + from_file)); + } else if (compa_mode == ObCompatibilityMode::MYSQL_MODE) { + OZ (load_sys_package_list(sql_proxy, mysql_syspack_file_list, + ARRAYSIZEOF(mysql_syspack_file_list), + ObCompatibilityMode::MYSQL_MODE, + from_file)); } + if (OB_SUCC(ret)) { - LOG_INFO("load all common sys package success!"); + LOG_INFO("load all common sys package success!", K(ret), K(from_file)); } else { - LOG_INFO("load all common sys package failed!"); + LOG_WARN("load all common sys package failed!", K(ret), K(from_file)); } return ret; } @@ -440,39 +613,23 @@ int ObPLPackageManager::load_all_common_sys_package(ObMySQLProxy &sql_proxy, ObC int ObPLPackageManager::load_all_special_sys_package(ObMySQLProxy &sql_proxy) { int ret = OB_SUCCESS; - bool exist = false; - if (OB_FAIL(FileDirectoryUtils::is_exists(sys_package_dir, exist))) { - LOG_WARN("check sys package dir whether exist failed", K(sys_package_dir), K(ret)); - } else if (!exist) { - LOG_INFO("sys package dir not exist", K(sys_package_dir)); - } else { - // for now! we only have one special system package "__DBMS_UPGRADE" - char package_spec_full_path[MAX_PATH_SIZE] = {}; - char package_body_full_path[MAX_PATH_SIZE] = {}; - #ifdef OB_BUILD_ORACLE_PL - OZ (databuff_printf( - package_spec_full_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, "__dbms_upgrade.sql")); - OZ (databuff_printf( - package_body_full_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, "__dbms_upgrade_body.sql")); - OZ (load_sys_package(sql_proxy, package_spec_full_path, package_body_full_path, ObCompatibilityMode::ORACLE_MODE)); + OZ (load_sys_package_list(sql_proxy, oracle_special_syspack_file_list, + ARRAYSIZEOF(oracle_special_syspack_file_list), + ObCompatibilityMode::ORACLE_MODE, + false /* from_file */)); #endif - - memset(package_spec_full_path, 0, sizeof(package_spec_full_path)); - memset(package_body_full_path, 0, sizeof(package_body_full_path)); - OZ (databuff_printf( - package_spec_full_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, "__dbms_upgrade_mysql.sql")); - OZ (databuff_printf( - package_body_full_path, MAX_PATH_SIZE, "%s/%s", sys_package_dir, "__dbms_upgrade_body_mysql.sql")); - OZ (load_sys_package(sql_proxy, package_spec_full_path, package_body_full_path, ObCompatibilityMode::MYSQL_MODE)); - } + OZ (load_sys_package_list(sql_proxy, mysql_special_syspack_file_list, + ARRAYSIZEOF(mysql_special_syspack_file_list), + ObCompatibilityMode::MYSQL_MODE, + false /* from_file */)); return ret; } int ObPLPackageManager::load_all_sys_package(ObMySQLProxy &sql_proxy) { int ret = OB_SUCCESS; - OZ (load_all_common_sys_package(sql_proxy, ObCompatibilityMode::OCEANBASE_MODE)); + OZ (load_all_common_sys_package(sql_proxy, ObCompatibilityMode::OCEANBASE_MODE, false /* from_file */)); OZ (load_all_special_sys_package(sql_proxy)); if (OB_SUCC(ret)) { LOG_INFO("load all sys package success!", K(ret)); diff --git a/src/pl/ob_pl_package_manager.h b/src/pl/ob_pl_package_manager.h index bfa94513a..6ab4605c6 100644 --- a/src/pl/ob_pl_package_manager.h +++ b/src/pl/ob_pl_package_manager.h @@ -63,11 +63,13 @@ class ObPLCursor; class ObPLCacheCtx; struct ObSysPackageFile { - const char *package_name; - const char *package_spec_file_name; - const char *package_body_file_name; + const char *const package_name; + const char *const package_spec_file_name; + const char *const package_body_file_name; }; +class ObCharStream; + class ObPLPackageManager { public: @@ -139,28 +141,24 @@ public: uint64_t package_id, ObPLPackageState *&package_state, bool for_static_member = false); - static int read_package_sql(FILE *file, char* buf, int64_t buf_len, bool &eof); - static int read_and_exec_package_sql( - common::ObMySQLProxy &sql_proxy, const char* package_full_path, ObCompatibilityMode compa_mode); - static int load_sys_package( - common::ObMySQLProxy &sql_proxy, const char *package_spec_name, const char *package_body_name, ObCompatibilityMode compa_mode); - static int load_sys_package(common::ObMySQLProxy &sql_proxy, common::ObString &package_name, ObCompatibilityMode compa_mode); - static int load_all_common_sys_package(common::ObMySQLProxy &sql_proxy, ObCompatibilityMode compa_mode); + + static int load_sys_package(common::ObMySQLProxy &sql_proxy, + common::ObString &package_name, + ObCompatibilityMode compa_mode, + bool from_file); static int load_all_common_sys_package(common::ObMySQLProxy &sql_proxy, - const ObSysPackageFile *package_file, - int sys_package_count, - ObCompatibilityMode compa_mode); - static int load_all_special_sys_package(common::ObMySQLProxy &sql_proxy); + ObCompatibilityMode compa_mode, + bool from_file); static int load_all_sys_package(common::ObMySQLProxy &sql_proxy); + static int add_package_to_plan_cache(const ObPLResolveCtx &resolve_ctx, ObPLPackage *package); static int get_package_from_plan_cache(const ObPLResolveCtx &resolve_ctx, uint64_t package_id, ObPLPackage *&package); - static int get_package_schema_info( - share::schema::ObSchemaGetterGuard &schema_guard, - uint64_t package_id, - const share::schema::ObPackageInfo *&package_spec_info, - const share::schema::ObPackageInfo *&package_body_info); + static int get_package_schema_info(share::schema::ObSchemaGetterGuard &schema_guard, + uint64_t package_id, + const share::schema::ObPackageInfo *&package_spec_info, + const share::schema::ObPackageInfo *&package_body_info); static int destory_package_state(sql::ObSQLSessionInfo &session_info, uint64_t package_id); int check_version(const ObPLResolveCtx &resolve_ctx, uint64_t package_id, const ObPackageStateVersion &state_version, bool &match); @@ -174,6 +172,22 @@ public: private: DISALLOW_COPY_AND_ASSIGN(ObPLPackageManager); + static int read_package_sql(ObCharStream &stream, char* buf, int64_t buf_len, bool &eos); + static int read_and_exec_package_sql(common::ObMySQLProxy &sql_proxy, + ObCharStream &stream, + ObCompatibilityMode compa_mode); + static int get_syspack_source_file_content(const char *file_name, const char *&content); + static int load_sys_package(ObMySQLProxy &sql_proxy, + const ObSysPackageFile &pack_file_info, + ObCompatibilityMode compa_mode, + bool from_file); + static int load_sys_package_list(common::ObMySQLProxy &sql_proxy, + const ObSysPackageFile *sys_package_list, + int sys_package_count, + ObCompatibilityMode compa_mode, + bool from_file); + static int load_all_special_sys_package(common::ObMySQLProxy &sql_proxy); + int get_cached_package_spec(const ObPLResolveCtx &resolve_ctx, uint64_t package_id, ObPLPackage *&package_spec); diff --git a/src/pl/ob_pl_type.cpp b/src/pl/ob_pl_type.cpp index c3e837cb1..74ac9ae96 100644 --- a/src/pl/ob_pl_type.cpp +++ b/src/pl/ob_pl_type.cpp @@ -2389,10 +2389,13 @@ int ObPLCursorInfo::prepare_spi_result(ObPLExecCtx *ctx, ObSPIResultSet *&spi_re } OX (spi_cursor_ = get_allocator()->alloc(sizeof(ObSPIResultSet))); OV (OB_NOT_NULL(spi_cursor_), OB_ALLOCATE_MEMORY_FAILED); + } else { + CK (OB_NOT_NULL(static_cast(spi_cursor_))); + OX ((static_cast(spi_cursor_))->~ObSPIResultSet()); } OX (spi_result = new (spi_cursor_) ObSPIResultSet()); - OZ (spi_result->init(*ctx->exec_ctx_->get_my_session())); OX (last_stream_cursor_ = true); + OZ (spi_result->init(*ctx->exec_ctx_->get_my_session())); return ret; } @@ -2413,6 +2416,14 @@ int ObPLCursorInfo::prepare_spi_cursor(ObSPICursor *&spi_cursor, : sizeof(ObSPICursor); OX (spi_cursor_ = spi_allocator->alloc(alloc_size)); OV (OB_NOT_NULL(spi_cursor_), OB_ALLOCATE_MEMORY_FAILED); + } else { + if (last_stream_cursor_) { + CK (OB_NOT_NULL(static_cast(spi_cursor_))); + OX (static_cast(spi_cursor_)->~ObSPIResultSet()); + } else { + CK (OB_NOT_NULL(static_cast(spi_cursor_))); + OX (static_cast(spi_cursor_)->~ObSPICursor()); + } } OX (spi_cursor = new (spi_cursor_) ObSPICursor(*spi_allocator, session_info)); OX (last_stream_cursor_ = false); diff --git a/src/pl/sys_package/ob_dbms_upgrade.cpp b/src/pl/sys_package/ob_dbms_upgrade.cpp index ee2305a3b..7a3ac37d2 100644 --- a/src/pl/sys_package/ob_dbms_upgrade.cpp +++ b/src/pl/sys_package/ob_dbms_upgrade.cpp @@ -28,15 +28,33 @@ int ObDBMSUpgrade::upgrade_single( { int ret = OB_SUCCESS; ObString package_name; + bool load_from_file = true; ObCompatibilityMode mode = lib::is_oracle_mode() ? ObCompatibilityMode::ORACLE_MODE : ObCompatibilityMode::MYSQL_MODE; UNUSED(result); CK (OB_NOT_NULL(ctx.get_sql_proxy())); - CK (OB_LIKELY(1 == params.count())); - OV (params.at(0).is_varchar(), OB_INVALID_ARGUMENT); - OZ (params.at(0).get_string(package_name)); - OV (!package_name.empty(), OB_INVALID_ARGUMENT); - OZ (ObPLPackageManager::load_sys_package(*ctx.get_sql_proxy(), package_name, mode)); + // OBServer 4.2.4 has added new parameters on the __DBMS_UPGRADE + // interface to control whether to load the system package source code from + // a file or embeded c string. To maintain compatibility during upgarding, + // it is necessary to distinguish the old and new versions of the interface. + // However, the system package does not have version control, so it depends + // on the number of parameters to judge the old and new versions. + if (OB_FAIL(ret)) { + } else if (1 == params.count()) { + OV (params.at(0).is_varchar(), OB_INVALID_ARGUMENT); + OZ (params.at(0).get_string(package_name)); + OV (!package_name.empty(), OB_INVALID_ARGUMENT); + } else if (2 == params.count()) { + OV (params.at(0).is_varchar(), OB_INVALID_ARGUMENT); + OZ (params.at(0).get_string(package_name)); + OV (!package_name.empty(), OB_INVALID_ARGUMENT); + OV (params.at(1).is_tinyint(), OB_INVALID_ARGUMENT); + OZ (params.at(1).get_bool(load_from_file)); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("__DBMS_UPGRADE.UPGRADE_SINGLE require one or two arguments", K(ret), K(params)); + } + OZ (ObPLPackageManager::load_sys_package(*ctx.get_sql_proxy(), package_name, mode, load_from_file)); return ret; } @@ -44,12 +62,22 @@ int ObDBMSUpgrade::upgrade_all( sql::ObExecContext &ctx, sql::ParamStore ¶ms, common::ObObj &result) { int ret = OB_SUCCESS; + bool load_from_file = true; ObCompatibilityMode mode = lib::is_oracle_mode() ? ObCompatibilityMode::ORACLE_MODE : ObCompatibilityMode::MYSQL_MODE; UNUSED(result); CK (OB_NOT_NULL(ctx.get_sql_proxy())); - CK (OB_LIKELY(0 == params.count())); - OZ (ObPLPackageManager::load_all_common_sys_package(*ctx.get_sql_proxy(), mode)); + if (OB_FAIL(ret)) { + } else if (0 == params.count()) { + // do nothing + } else if (1 == params.count()) { + OV (params.at(0).is_tinyint(), OB_INVALID_ARGUMENT); + OZ (params.at(0).get_bool(load_from_file)); + } else { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("__DBMS_UPGRADE.UPGRADE_ALL require none or one arguments", K(ret), K(params)); + } + OZ (ObPLPackageManager::load_all_common_sys_package(*ctx.get_sql_proxy(), mode, load_from_file)); return ret; } diff --git a/src/rootserver/ob_upgrade_executor.cpp b/src/rootserver/ob_upgrade_executor.cpp index 0f6588aaa..5b0b487b7 100644 --- a/src/rootserver/ob_upgrade_executor.cpp +++ b/src/rootserver/ob_upgrade_executor.cpp @@ -915,17 +915,18 @@ int ObUpgradeExecutor::upgrade_mysql_system_package_job_() int64_t timeout = GCONF._ob_ddl_timeout; const char *create_package_sql = "CREATE OR REPLACE PACKAGE __DBMS_UPGRADE \ - PROCEDURE UPGRADE(package_name VARCHAR(1024)); \ - PROCEDURE UPGRADE_ALL(); \ + PROCEDURE UPGRADE(package_name VARCHAR(1024), \ + load_from_file BOOLEAN DEFAULT TRUE); \ + PROCEDURE UPGRADE_ALL(load_from_file BOOLEAN DEFAULT TRUE); \ END;"; const char *create_package_body_sql = "CREATE OR REPLACE PACKAGE BODY __DBMS_UPGRADE \ - PROCEDURE UPGRADE(package_name VARCHAR(1024)); \ + PROCEDURE UPGRADE(package_name VARCHAR(1024), load_from_file BOOLEAN); \ PRAGMA INTERFACE(c, UPGRADE_SINGLE); \ - PROCEDURE UPGRADE_ALL(); \ + PROCEDURE UPGRADE_ALL(load_from_file BOOLEAN); \ PRAGMA INTERFACE(c, UPGRADE_ALL); \ END;"; - const char *upgrade_sql = "CALL __DBMS_UPGRADE.UPGRADE_ALL();"; + const char *upgrade_sql = "CALL __DBMS_UPGRADE.UPGRADE_ALL(FALSE);"; ObTimeoutCtx ctx; int64_t affected_rows = 0; if (OB_FAIL(check_inner_stat_())) { @@ -974,17 +975,18 @@ int ObUpgradeExecutor::upgrade_oracle_system_package_job_() int64_t timeout = GCONF._ob_ddl_timeout; const char *create_package_sql = "CREATE OR REPLACE PACKAGE \"__DBMS_UPGRADE\" IS \ - PROCEDURE UPGRADE(package_name VARCHAR2); \ - PROCEDURE UPGRADE_ALL; \ + PROCEDURE UPGRADE(package_name VARCHAR2, \ + load_from_file BOOLEAN DEFAULT TRUE); \ + PROCEDURE UPGRADE_ALL(load_from_file BOOLEAN DEFAULT TRUE); \ END;"; const char *create_package_body_sql = "CREATE OR REPLACE PACKAGE BODY \"__DBMS_UPGRADE\" IS \ - PROCEDURE UPGRADE(package_name VARCHAR2); \ + PROCEDURE UPGRADE(package_name VARCHAR2, load_from_file BOOLEAN); \ PRAGMA INTERFACE(c, UPGRADE_SINGLE); \ - PROCEDURE UPGRADE_ALL; \ + PROCEDURE UPGRADE_ALL(load_from_file BOOLEAN); \ PRAGMA INTERFACE(c, UPGRADE_ALL); \ END;"; - const char *upgrade_sql = "CALL \"__DBMS_UPGRADE\".UPGRADE_ALL();"; + const char *upgrade_sql = "BEGIN \"__DBMS_UPGRADE\".UPGRADE_ALL(FALSE); END;"; ObTimeoutCtx ctx; int64_t affected_rows = 0; if (OB_FAIL(check_inner_stat_())) { @@ -1017,9 +1019,9 @@ int ObUpgradeExecutor::upgrade_oracle_system_package_job_() OB_SYS_TENANT_ID, upgrade_sql, affected_rows, static_cast(mode)))) { LOG_WARN("fail to execute sql", KR(ret), "sql", upgrade_sql); - } else if (0 != affected_rows) { + } else if (1 != affected_rows) { // default value of oracle anonymous block affected_rows is 1 ret = OB_ERR_UNEXPECTED; - LOG_WARN("affected_rows expected to be zero", KR(ret), K(affected_rows)); + LOG_WARN("affected_rows expected to be 1", KR(ret), K(affected_rows)); } FLOG_INFO("[UPGRADE] finish run upgrade oracle system package job", KR(ret), "cost", ObTimeUtility::current_time() - start_ts); diff --git a/src/share/inner_table/sys_package/__dbms_upgrade_body_mysql.sql b/src/share/inner_table/sys_package/__dbms_upgrade_body_mysql.sql index 2a63d7487..fe1eab6ca 100644 --- a/src/share/inner_table/sys_package/__dbms_upgrade_body_mysql.sql +++ b/src/share/inner_table/sys_package/__dbms_upgrade_body_mysql.sql @@ -3,9 +3,9 @@ CREATE OR REPLACE PACKAGE BODY __DBMS_UPGRADE - PROCEDURE UPGRADE(package_name VARCHAR(1024)); + PROCEDURE UPGRADE(package_name VARCHAR(1024), load_from_file BOOLEAN); PRAGMA INTERFACE(c, UPGRADE_SINGLE); - PROCEDURE UPGRADE_ALL(); + PROCEDURE UPGRADE_ALL(load_from_file BOOLEAN); PRAGMA INTERFACE(c, UPGRADE_ALL); END; // diff --git a/src/share/inner_table/sys_package/__dbms_upgrade_mysql.sql b/src/share/inner_table/sys_package/__dbms_upgrade_mysql.sql index 45b7221bc..6787c0c8e 100644 --- a/src/share/inner_table/sys_package/__dbms_upgrade_mysql.sql +++ b/src/share/inner_table/sys_package/__dbms_upgrade_mysql.sql @@ -3,7 +3,8 @@ CREATE OR REPLACE PACKAGE __DBMS_UPGRADE - PROCEDURE UPGRADE(package_name VARCHAR(1024)); - PROCEDURE UPGRADE_ALL(); + PROCEDURE UPGRADE(package_name VARCHAR(1024), + load_from_file BOOLEAN DEFAULT TRUE); + PROCEDURE UPGRADE_ALL(load_from_file BOOLEAN DEFAULT TRUE); END; // diff --git a/src/sql/engine/basic/ob_ra_row_store.cpp b/src/sql/engine/basic/ob_ra_row_store.cpp index b7e06aa80..0b3469459 100644 --- a/src/sql/engine/basic/ob_ra_row_store.cpp +++ b/src/sql/engine/basic/ob_ra_row_store.cpp @@ -546,6 +546,15 @@ int ObRARowStore::switch_idx_block(bool finish_add /* = false */) if (!is_inited()) { ret = OB_NOT_INIT; LOG_WARN("not init", K(ret)); + } else if (finish_add && NULL == idx_blk_) { + if (OB_FAIL(build_idx_block())) { + LOG_WARN("build index block failed", + K(ret), K(finish_add), KPC(idx_blk_)); + } + } + + if (OB_FAIL(ret)) { + // do nothing } else if (NULL == idx_blk_) { ret = OB_ERR_UNEXPECTED; LOG_WARN("index block should not be null"); diff --git a/src/sql/ob_spi.cpp b/src/sql/ob_spi.cpp index 5520011d8..6008f054b 100644 --- a/src/sql/ob_spi.cpp +++ b/src/sql/ob_spi.cpp @@ -46,6 +46,7 @@ #include "pl/ob_pl_profiler.h" #endif #include "pl/ob_pl_allocator.h" +#include "pl/diagnosis/ob_pl_sql_audit_guard.h" namespace oceanbase { using namespace sqlclient; @@ -1589,50 +1590,6 @@ int ObSPIService::spi_end_trans(ObPLExecCtx *ctx, const char *sql, bool is_rollb return ret; } -class ObSPITimeRecord : public ObITimeRecord -{ -public: - ObSPITimeRecord() - : send_timestamp_(0), - receive_timestamp_(0), - enqueue_timestamp_(0), - run_timestamp_(0), - process_timestamp_(0), - single_process_timestamp_(0), - exec_start_timestamp_(0), - exec_end_timestamp_(0) {} - virtual ~ObSPITimeRecord() {} - - void set_send_timestamp(int64_t send_timestamp) { send_timestamp_ = send_timestamp; } - void set_receive_timestamp(int64_t receive_timestamp) { receive_timestamp_ = receive_timestamp; } - void set_enqueue_timestamp(int64_t enqueue_timestamp) { enqueue_timestamp_ = enqueue_timestamp; } - void set_run_timestamp(int64_t run_timestamp) { run_timestamp_ = run_timestamp; } - void set_process_timestamp(int64_t process_timestamp) { process_timestamp_ = process_timestamp; } - void set_single_process_timestamp(int64_t single_process_timestamp) { single_process_timestamp_ = single_process_timestamp; } - void set_exec_start_timestamp(int64_t exec_start_timestamp) { exec_start_timestamp_ = exec_start_timestamp; } - void set_exec_end_timestamp(int64_t exec_end_timestamp) { exec_end_timestamp_ = exec_end_timestamp; } - - int64_t get_send_timestamp() const { return send_timestamp_; } - int64_t get_receive_timestamp() const { return get_send_timestamp(); } - int64_t get_enqueue_timestamp() const { return get_send_timestamp(); } - int64_t get_run_timestamp() const { return get_send_timestamp(); } - int64_t get_process_timestamp() const { return get_send_timestamp(); } - int64_t get_single_process_timestamp() const { return get_send_timestamp(); } - int64_t get_exec_start_timestamp() const { return get_send_timestamp(); } - int64_t get_exec_end_timestamp() const { return exec_end_timestamp_; } - -public: - int64_t send_timestamp_; - int64_t receive_timestamp_; - int64_t enqueue_timestamp_; - int64_t run_timestamp_; - int64_t process_timestamp_; - int64_t single_process_timestamp_; - int64_t exec_start_timestamp_; - int64_t exec_end_timestamp_; - -}; - ObPLSPITraceIdGuard::ObPLSPITraceIdGuard(const ObString &sql, const ObString &ps_sql, ObSQLSessionInfo &session, @@ -1684,12 +1641,12 @@ ObPLSPITraceIdGuard::~ObPLSPITraceIdGuard() } } -//todo:@hr351303 确认sql 和 ps sql是否可以合一 +// common sql execute interface (static-sql & dynamic-sql & dbms-sql-execute) int ObSPIService::spi_inner_execute(ObPLExecCtx *ctx, const char *sql, const char *ps_sql, int64_t type, - const ObSqlExpression **param_exprs, + void *params, int64_t param_count, const ObSqlExpression **into_exprs, int64_t into_count, @@ -1700,246 +1657,167 @@ int ObSPIService::spi_inner_execute(ObPLExecCtx *ctx, int64_t is_bulk, bool is_forall, bool is_type_record, - bool for_update) + bool for_update, + bool is_dynamic_sql, + ObIArray *using_out_params, + bool is_dbms_sql) { int ret = OB_SUCCESS; - ObWarningBuffer* wb = NULL; - FLTSpanGuard(pl_spi_query); + CK (OB_NOT_NULL(ctx)); CK (OB_NOT_NULL(ctx->allocator_)); CK (OB_NOT_NULL(ctx->exec_ctx_)); CK (OB_NOT_NULL(ctx->exec_ctx_->get_my_session())); CK (OB_NOT_NULL(ctx->exec_ctx_->get_my_session()->get_pl_sqlcode_info())); - CK (OB_NOT_NULL(ctx->func_)); - OX (wb = common::ob_get_tsi_warning_buffer()); - ObExecRecord exec_record; - ObExecTimestamp exec_timestamp; - ObSPITimeRecord time_record; - exec_timestamp.exec_type_ = sql::PLSql; + if (OB_SUCC(ret)) { + HEAP_VAR(ObSPIResultSet, spi_result) { - ObSQLSessionInfo *session = ctx->exec_ctx_->get_my_session(); - ObPLSPITraceIdGuard trace_id_guard(sql, ps_sql, *session, ret); - ObPLSubPLSqlTimeGuard guard(ctx); - if (OB_SUCC(ret) && is_forall && !session->is_enable_batched_multi_statement()) { - /* forall need rollback to for loop */ - ret = OB_BATCHED_MULTI_STMT_ROLLBACK; - LOG_TRACE("cannot batch execute", K(ret), K(sql), K(type)); - } + ObSQLSessionInfo *session = ctx->exec_ctx_->get_my_session(); + stmt::StmtType stmt_type = static_cast(type); + ObString sqlstr(sql); - HEAP_VAR(ObSPIResultSet, spi_result) { - stmt::StmtType stmt_type = stmt::T_NONE; - bool is_diagnostics_stmt = false; - ObString sqlstr(sql); - OZ (spi_result.init(*session)); - OZ (spi_result.start_nested_stmt_if_need(ctx, sqlstr, static_cast(type), for_update)); + if (is_forall && !session->is_enable_batched_multi_statement()) { + /* forall need rollback to for loop */ + ret = OB_BATCHED_MULTI_STMT_ROLLBACK; + LOG_TRACE("cannot batch execute", K(ret), K(sql), K(type)); + } - if (OB_SUCC(ret)) { - int64_t row_count = 0; - ObQueryRetryCtrl retry_ctrl; - int64_t tenant_version = 0; - int64_t sys_version = 0; + OZ (spi_result.init(*session)); + OZ (spi_result.start_nested_stmt_if_need(ctx, sqlstr, stmt_type, for_update)); + OX (spi_result.get_sql_ctx().is_dbms_sql_ = is_dbms_sql); + + if (OB_SUCC(ret)) { + + ObPLSqlAuditRecord audit_record(sql::PLSql); + ObQueryRetryCtrl retry_ctrl; + ObSPIExecEnvGuard env_guard(*session, spi_result); + int save_sqlcode = session->get_pl_sqlcode_info()->get_sqlcode(); + ObString save_sqlmsg = session->get_pl_sqlcode_info()->get_sqlmsg(); - ObSPIOutParams out_params; - int64_t old_query_start_time = session->get_query_start_time(); - HEAP_VAR(ObPLSqlCodeInfo, saved_sqlcode_info) { - session->set_query_start_time(ObTimeUtility::current_time()); - session->init_use_rich_format(); - saved_sqlcode_info = *(ctx->exec_ctx_->get_my_session()->get_pl_sqlcode_info()); - bool is_retry = false; do { - // SQL_AUDIT_START - ObWaitEventDesc max_wait_desc; - ObWaitEventStat total_wait_desc; + + bool can_retry = true; + int64_t row_count = 0; ObArenaAllocator allocator(GET_PL_MOD_STRING(PL_MOD_IDX::OB_PL_STATIC_SQL_EXEC), OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); - const bool enable_perf_event = lib::is_diagnose_info_enabled(); - const bool enable_sql_audit = - GCONF.enable_sql_audit && ctx->exec_ctx_->get_my_session()->get_local_ob_enable_sql_audit(); { - ObMaxWaitGuard max_wait_guard(enable_perf_event ? &max_wait_desc : NULL); - ObTotalWaitGuard total_wait_guard(enable_perf_event ? &total_wait_desc : NULL); - if (enable_perf_event) { - exec_record.record_start(); + ObPLSPITraceIdGuard trace_id_guard(sql, ps_sql, *session, ret); + ObPLSubPLSqlTimeGuard guard(ctx); + ObPLSqlAuditGuard audit_guard( + *(ctx->exec_ctx_), *(session), spi_result, audit_record, ret, (sql != NULL ? sql : ps_sql), retry_ctrl, trace_id_guard, static_cast(type)); + ObSPIRetryCtrlGuard retry_guard(retry_ctrl, spi_result, *session, ret); + + OX (session->get_pl_sqlcode_info()->set_sqlcode(OB_SUCCESS)); + + if (OB_SUCC(ret) && !ObStmt::is_diagnostic_stmt(stmt_type) && lib::is_mysql_mode()) { + ob_reset_tsi_warning_buffer(); } - //监控项统计开始 - time_record.set_send_timestamp(ObTimeUtility::current_time()); - ctx->exec_ctx_->get_my_session()->get_pl_sqlcode_info()->set_sqlcode(OB_SUCCESS); - if (!ObStmt::is_diagnostic_stmt(static_cast(type)) - && lib::is_mysql_mode() - && OB_NOT_NULL(wb)) { - wb->reset(); + + OZ (inner_open(ctx, + allocator, + sql, + ps_sql, + type, + params, + param_count, + into_exprs, + into_count, + spi_result, + spi_result.get_out_params(), + is_forall, + is_dynamic_sql, + is_dbms_sql), K(sql), K(ps_sql)); + OZ (inner_fetch(ctx, + can_retry, + spi_result, + into_exprs, + into_count, + column_types, + type_count, + exprs_not_null_flag, + pl_integer_ranges, + OB_NOT_NULL(using_out_params) && !using_out_params->empty() ? using_out_params : NULL, + row_count, + is_bulk, + is_forall, + is_dynamic_sql, + NULL, + false, + false, + INT64_MAX, + NULL, + 0, + is_type_record), K(ps_sql), K(sql), K(type)); + + if (OB_SUCC(ret) && (is_dynamic_sql || is_dbms_sql) && !is_bulk) { + // if it is bulk into, not allow using out param, so no need deep copy + OZ (dynamic_out_params( + *(ctx->allocator_), spi_result.get_result_set(), params, param_count, is_dbms_sql)); } - row_count = 0; - out_params.reset(); - if (is_retry) { - spi_result.reset_member_for_retry(*session); + if (spi_result.get_out_params().has_out_param()) { + OZ (process_function_out_result(ctx, *spi_result.get_result_set(), spi_result.get_out_params().get_out_params())); } - retry_ctrl.clear_state_before_each_retry(session->get_retry_info_for_update()); - if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(session->get_effective_tenant_id(), spi_result.get_scheme_guard()))) { - LOG_WARN("get schema guard failed", K(ret)); - } else if (OB_FAIL(spi_result.get_scheme_guard().get_schema_version(session->get_effective_tenant_id(), tenant_version))) { - LOG_WARN("fail get schema version", K(ret)); - } else if (OB_FAIL(spi_result.get_scheme_guard().get_schema_version(OB_SYS_TENANT_ID, sys_version))) { - LOG_WARN("fail get sys schema version", K(ret)); - } else { - retry_ctrl.set_tenant_local_schema_version(tenant_version); - retry_ctrl.set_sys_local_schema_version(sys_version); - spi_result.get_sql_ctx().schema_guard_ = &spi_result.get_scheme_guard(); - if (OB_FAIL(inner_open(ctx, - allocator, - sql, - ps_sql, - type, - param_exprs, - param_count, - into_exprs, - into_count, - spi_result, - out_params, - &retry_ctrl, - is_forall))) { - LOG_WARN("failed to open", K(ret), K(sql), K(ps_sql)); - } else if (OB_FAIL(inner_fetch(ctx, - retry_ctrl, - spi_result, - into_exprs, - into_count, - column_types, - type_count, - exprs_not_null_flag, - pl_integer_ranges, - NULL, - row_count, - is_bulk, - is_forall, - false, - NULL, - false, - false, - INT64_MAX, - NULL, - 0, - is_type_record))) { - LOG_WARN("failed to execute inner_fetch for pl/sql", K(ret), K(ps_sql), K(sql), K(type)); - } else if (out_params.has_out_param()) { - OZ (process_function_out_result( - ctx, *spi_result.get_result_set(), out_params.get_out_params()), out_params); - } - if (OB_SUCC(ret)) { // 如果成功记录下stmttype,用于DDL语句的schema刷新 - OX (stmt_type = spi_result.get_result_set()->get_stmt_type()); - OX (is_diagnostics_stmt = ObStmt::is_diagnostic_stmt(spi_result.get_result_set()->get_literal_stmt_type())); - } - // 无论成功或者失败都在这里close result set - // 原因是close函数里面会设置audit的sched info信息, audit会在ObInnerSQLConnection::process_record中处理audit信息 - int close_ret = spi_result.close_result_set(); - if (OB_SUCCESS != close_ret) { - LOG_WARN("close spi result failed", K(ret), K(close_ret)); - } - ret = OB_SUCCESS == ret ? close_ret : ret; - spi_result.destruct_exec_params(*session); + + ObPLSqlCodeInfo *sqlcode_info = session->get_pl_sqlcode_info(); + if (sqlcode_info->get_sqlcode() == OB_SUCCESS) { + sqlcode_info->set_sqlcode(save_sqlcode, save_sqlmsg); } - //if (OB_SUCCESS == ret) { - if (session->get_pl_sqlcode_info()->get_sqlcode() == OB_SUCCESS) { - session->get_pl_sqlcode_info()->set_sqlcode( - saved_sqlcode_info.get_sqlcode(), saved_sqlcode_info.get_sqlmsg()); - } - //} else { - // session->get_pl_sqlcode_info()->set_sqlcode(ret); - //} - //监控项统计结束 - time_record.set_exec_end_timestamp(ObTimeUtility::current_time()); - if (enable_perf_event) { - exec_record.record_end(); + + if (can_retry) { + retry_guard.test(); } } - LOG_DEBUG("start process record", K(ret), K(ps_sql), K(sql), K(type), K(enable_sql_audit)); - // 处理监控统计项 - if (OB_NOT_NULL(spi_result.get_result_set())) { - if (spi_result.get_result_set()->is_inited()) { - ObSQLSessionInfo *session_info = ctx->exec_ctx_->get_my_session(); - ObString record_sql(sql != NULL ? sql : ps_sql); - if (ObStmt::is_execute_stmt(static_cast(type))) { - record_sql = session_info->get_current_query_string(); - } - int64_t try_cnt = session_info->get_raw_audit_record().try_cnt_; - ObExecRecord record_bk = session_info->get_raw_audit_record().exec_record_; - session_info->get_raw_audit_record().try_cnt_ = retry_ctrl.get_retry_times(); - session_info->get_raw_audit_record().pl_trace_id_.set(trace_id_guard.origin_trace_id_); - ObInnerSQLConnection::process_record(*spi_result.get_result_set(), - spi_result.get_sql_ctx(), - *session_info, - time_record, - ret, - session_info->get_current_execution_id(), // sql execute id - OB_INVALID_ID, - max_wait_desc, - total_wait_desc, - exec_record, - exec_timestamp, - true, - record_sql, - true, - spi_result.get_exec_params_str_ptr()); - session_info->get_raw_audit_record().exec_record_ = record_bk; - session_info->get_raw_audit_record().try_cnt_ = try_cnt; - session_info->get_raw_audit_record().pl_trace_id_.reset(); - } else { - LOG_DEBUG("result set is not inited, do not process record", - K(ret), K(ps_sql), K(sql), K(type)); - } - } else { - if (OB_SUCC(ret)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, result_set is null", K(ret), K(ps_sql), K(sql), K(type)); - } else { - LOG_WARN("result_set is null", K(ret), K(ps_sql), K(sql), K(type)); - } + + int close_ret = spi_result.close_result_set(); + if (OB_SUCCESS != close_ret) { + LOG_WARN("close spi result failed", K(ret), K(close_ret)); } - is_retry = true; + ret = OB_SUCCESS == ret ? close_ret : ret; + + spi_result.destruct_exec_params(*session); + } while (RETRY_TYPE_NONE != retry_ctrl.get_retry_type()); //SPI只做LOCAL重试 } - session->get_retry_info_for_update().clear(); - session->set_query_start_time(old_query_start_time); - session->init_use_rich_format(); - } - //自动提交,禁掉PL整体重试 - if (OB_SUCC(ret) - && (ObStmt::is_ddl_stmt(stmt_type, true) - || ObStmt::is_tcl_stmt(stmt_type) - || (lib::is_mysql_mode() && session->get_local_autocommit()))) { - OX (session->set_pl_can_retry(false)); - } - if (OB_SUCC(ret) - && OB_ISNULL(param_exprs)) { - if (ObStmt::is_ddl_stmt(stmt_type, true) || ObStmt::is_savepoint_stmt(stmt_type)) { - if (ObStmt::is_ddl_stmt(stmt_type, true)) { - OZ (force_refresh_schema(session->get_effective_tenant_id()), sql); - } - recreate_implicit_savapoint_if_need(ctx, ret); - } - } - // 记录第一条SQL执行PartitionHit信息, 并对PartitionHit进行Freeze, 防止后续的SQL冲掉 - if (OB_SUCC(ret)) { - if (OB_NOT_NULL(spi_result.get_result_set()->get_physical_plan())) { - session->partition_hit().freeze(); - } - } - spi_result.end_nested_stmt_if_need(ctx, ret); - if (OB_FAIL(ret) && !is_diagnostics_stmt) { - // support `SHOW WARNINGS` in mysql PL - session->set_show_warnings_buf(ret); + if (OB_SUCC(ret) + && (ObStmt::is_ddl_stmt(stmt_type, true) + || ObStmt::is_tcl_stmt(stmt_type) + || (lib::is_mysql_mode() && session->get_local_autocommit()))) { + OX (session->set_pl_can_retry(false)); + } + + if (OB_SUCC(ret) && 0 == param_count) { + if (ObStmt::is_ddl_stmt(stmt_type, true) + || ObStmt::is_tcl_stmt(stmt_type) + || ObStmt::is_savepoint_stmt(stmt_type)) { + if (ObStmt::is_ddl_stmt(stmt_type, true)) { + OZ (force_refresh_schema(session->get_effective_tenant_id()), sql); + } + recreate_implicit_savapoint_if_need(ctx, ret); + } + } + + // 记录第一条SQL执行PartitionHit信息, 并对PartitionHit进行Freeze, 防止后续的SQL冲掉 + if (OB_SUCC(ret)) { + if (OB_NOT_NULL(spi_result.get_result_set()->get_physical_plan())) { + session->partition_hit().freeze(); + } + } + + if (OB_FAIL(ret) && !ObStmt::is_diagnostic_stmt(stmt_type)) { + // support `SHOW WARNINGS` in mysql PL + session->set_show_warnings_buf(ret); + } + + spi_result.end_nested_stmt_if_need(ctx, ret); + + SET_FORALL_BULK_EXCEPTION; + SET_SPI_STATUS; } - SET_FORALL_BULK_EXCEPTION; - SET_SPI_STATUS; } return ret; } -/* - * 本接口是在spi_inner_execute基础上去掉部分代码实现的，因为去掉的代码包括inner_fetch这个关键接口， - * 所以暂时还不方便将两个接口整理重构为一个接口，计划在dbms_sql支持读流程接口和returning语句后再重构。 - */ int ObSPIService::dbms_cursor_execute(ObPLExecCtx *ctx, const ObString ps_sql, stmt::StmtType stmt_type, @@ -1947,213 +1825,25 @@ int ObSPIService::dbms_cursor_execute(ObPLExecCtx *ctx, bool is_dbms_sql) { int ret = OB_SUCCESS; - ObSQLSessionInfo *session = NULL; - OV (OB_NOT_NULL(ctx)); - OV (OB_NOT_NULL(ctx->allocator_)); - OV (OB_NOT_NULL(ctx->exec_ctx_)); - OV (OB_NOT_NULL(ctx->exec_ctx_->get_my_session())); - OX (session = ctx->exec_ctx_->get_my_session()); - OV (OB_NOT_NULL(session->get_pl_sqlcode_info())); - // dbms_sql场景暂时不检查这个 - // OV (OB_NOT_NULL(pl_ctx->func_)); - - ObExecRecord exec_record; - ObExecTimestamp exec_timestamp; - ObSPITimeRecord time_record; - ObString &sql_stmt = cursor.get_sql_stmt(); - ParamStore &exec_params = cursor.get_exec_params(); - exec_timestamp.exec_type_ = sql::PLSql; - - HEAP_VAR(ObSPIResultSet, spi_result) { - OZ (spi_result.init(*session)); - OZ (spi_result.start_nested_stmt_if_need(ctx, sql_stmt, static_cast(stmt_type), cursor.is_for_update())); - OX (spi_result.get_sql_ctx().is_dbms_sql_ = is_dbms_sql); - if (OB_SUCC(ret)) { - int64_t row_count = 0; - ObQueryRetryCtrl retry_ctrl; - int64_t tenant_version = 0; - int64_t sys_version = 0; - ObSPIOutParams out_params; - ObPLSqlCodeInfo saved_sqlcode_info; - uint64_t eff_tenant_id = session->get_effective_tenant_id(); - int64_t old_query_start_time = session->get_query_start_time(); - session->set_query_start_time(ObTimeUtility::current_time()); - bool is_retry = false; - do { - // SQL_AUDIT_START] - ObWaitEventDesc max_wait_desc; - ObWaitEventStat total_wait_desc; - const bool enable_perf_event = lib::is_diagnose_info_enabled(); - const bool enable_sql_audit = - GCONF.enable_sql_audit && session->get_local_ob_enable_sql_audit(); - { - ObMaxWaitGuard max_wait_guard(enable_perf_event ? &max_wait_desc : NULL); - ObTotalWaitGuard total_wait_guard(enable_perf_event ? &total_wait_desc : NULL); - if (enable_perf_event) { - exec_record.record_start(); - } - // 监控项统计开始 - time_record.set_send_timestamp(ObTimeUtility::current_time()); - saved_sqlcode_info = *(session->get_pl_sqlcode_info()); - session->get_pl_sqlcode_info()->set_sqlcode(OB_SUCCESS); - row_count = 0; - out_params.reset(); - if (is_retry) { - spi_result.reset_member_for_retry(*session); - } - retry_ctrl.clear_state_before_each_retry(session->get_retry_info_for_update()); - if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(eff_tenant_id, spi_result.get_scheme_guard()))) { - LOG_WARN("get schema guard failed", K(ret)); - } else if (OB_FAIL(spi_result.get_scheme_guard().get_schema_version(eff_tenant_id, tenant_version))) { - LOG_WARN("fail get schema version", K(ret)); - } else if (OB_FAIL(spi_result.get_scheme_guard().get_schema_version(OB_SYS_TENANT_ID, sys_version))) { - LOG_WARN("fail get sys schema version", K(ret)); - } else { - retry_ctrl.set_tenant_local_schema_version(tenant_version); - retry_ctrl.set_sys_local_schema_version(sys_version); - spi_result.get_sql_ctx().schema_guard_ = &spi_result.get_scheme_guard(); - if (OB_FAIL(inner_open(ctx, - exec_params.count() > 0 ? NULL : sql_stmt.ptr(), - ps_sql, - stmt_type, - exec_params, - spi_result, - out_params))) { - LOG_WARN("failed to open", K(ret), K(ps_sql), K(stmt_type), - K(exec_params), K(sql_stmt), K(sql_stmt.ptr()), K(sql_stmt.length())); - if (spi_result.get_result_set() != NULL) { - int cli_ret = OB_SUCCESS; - retry_ctrl.test_and_save_retry_state( - GCTX, - spi_result.get_sql_ctx(), - *spi_result.get_result_set(), - ret, cli_ret, true, true, true); - LOG_WARN("failed to get_result, check if need retry", - K(ret), K(cli_ret), K(retry_ctrl.need_retry())); - ret = cli_ret; - spi_result.get_sql_ctx().clear(); - ctx->exec_ctx_->get_my_session()->set_session_in_retry(retry_ctrl.need_retry()); - } - } /* else if (OB_FAIL(inner_fetch(ctx, - retry_ctrl, - spi_result.get_mysql_result().get_result(), - NULL, - 0, - NULL, - 0, - NULL, - NULL, - NULL, - row_count))) { - LOG_WARN("failed to get result", K(ret)); - } */ else if (exec_params.count() > 0) { // process out parameters - ObInnerSQLResult *inner_result = NULL; - ObPhysicalPlanCtx *plan_ctx = NULL; - const ParamStore *param_store = NULL; - CK (OB_NOT_NULL(plan_ctx = spi_result.get_result_set()->get_exec_context().get_physical_plan_ctx())); - CK (OB_NOT_NULL(param_store = &(plan_ctx->get_param_store()))); - OV (param_store->count() >= exec_params.count(), OB_ERR_UNEXPECTED, param_store->count()); - for (int64_t i = 0; OB_SUCC(ret) && i < exec_params.count(); ++i) { - OZ (deep_copy_obj(*(ctx->allocator_), param_store->at(i), exec_params.at(i))); - } - } - if (OB_SUCC(ret) && out_params.has_out_param()) { - OZ (process_function_out_result( - ctx, *spi_result.get_result_set(), out_params.get_out_params()), out_params); - } -// TODO: 先注掉，因为目前看参数传进来的stmt_type已经是有效的，不需要再生成。 -// if (OB_SUCC(ret)) { // 如果成功记录下stmttype,用于DDL语句的schema刷新 -// ObInnerSQLResult *inner_result = static_cast(spi_result.get_mysql_result().get_result()); -// CK (OB_NOT_NULL(inner_result)); -// OX (stmt_type = inner_result->result_set().get_stmt_type()); -// } - // 无论成功或者失败都在这里close result set - // 原因是close函数里面会设置audit的sched info信息, audit会在ObInnerSQLConnection::process_record中处理audit信息 - int close_ret = spi_result.close_result_set(); - if (OB_SUCCESS != close_ret) { - LOG_WARN("close spi result failed", K(ret), K(close_ret)); - } - ret = OB_SUCCESS == ret ? close_ret : ret; - } - if (session->get_pl_sqlcode_info()->get_sqlcode() == OB_SUCCESS) { - session->get_pl_sqlcode_info()->set_sqlcode( - saved_sqlcode_info.get_sqlcode(), saved_sqlcode_info.get_sqlmsg()); - } - //} else { - // session->get_pl_sqlcode_info()->set_sqlcode(ret); - //} - //监控项统计结束 - time_record.set_exec_end_timestamp(ObTimeUtility::current_time()); - if (enable_perf_event) { - exec_record.record_end(); - } - } - LOG_DEBUG("start process record", - K(ret), K(sql_stmt), K(stmt_type), K(enable_sql_audit)); - // 处理监控统计项 - if (OB_NOT_NULL(spi_result.get_result_set())) { - if (spi_result.get_result_set()->is_inited()) { - ObInnerSQLConnection::process_record(*spi_result.get_result_set(), - spi_result.get_sql_ctx(), - *session, - time_record, - ret, - session->get_current_execution_id(), - OB_INVALID_ID, //FIXME@hr351303 - max_wait_desc, - total_wait_desc, - exec_record, - exec_timestamp, - true, - ps_sql, - true, - spi_result.get_exec_params_str_ptr()); - } else { - LOG_DEBUG("result set is not inited, do not process record", - K(ret), K(ps_sql), K(sql_stmt), K(stmt_type)); - } - } else { - if (OB_SUCC(ret)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, result_set is null", - K(ret), K(ps_sql), K(sql_stmt), K(stmt_type)); - } else { - LOG_WARN("result_set is null", - K(ret), K(ps_sql), K(sql_stmt), K(stmt_type)); - } - } - is_retry = true; - } while (RETRY_TYPE_NONE != retry_ctrl.get_retry_type()); //SPI只做LOCAL重试 - session->get_retry_info_for_update().clear(); - session->set_query_start_time(old_query_start_time); - } - //自动提交,禁掉PL整体重试 - if (OB_SUCC(ret) - && (ObStmt::is_ddl_stmt(stmt_type, true) - || ObStmt::is_tcl_stmt(stmt_type) - || (lib::is_mysql_mode() && session->get_local_autocommit()))) { - OX (session->set_pl_can_retry(false)); - } - if (OB_SUCC(ret) && exec_params.empty()) { - if (ObStmt::is_ddl_stmt(stmt_type, true) || ObStmt::is_savepoint_stmt(stmt_type)) { - if (ObStmt::is_ddl_stmt(stmt_type, true)) { - OZ (force_refresh_schema(session->get_effective_tenant_id()), sql_stmt); - } - recreate_implicit_savapoint_if_need(ctx, ret); - } - } - // 记录第一条SQL执行PartitionHit信息, 并对PartitionHit进行Freeze, 防止后续的SQL冲掉 - if (OB_SUCC(ret)) { - if (OB_NOT_NULL(spi_result.get_result_set()->get_physical_plan())) { - session->partition_hit().freeze(); - } - } - - spi_result.end_nested_stmt_if_need(ctx, ret); - - SET_FORALL_BULK_EXCEPTION; - SET_SPI_STATUS; - } + OZ (spi_inner_execute(ctx, + cursor.get_sql_stmt().ptr(), + ps_sql.ptr(), + stmt_type, + &cursor.get_exec_params(), + cursor.get_exec_params().count(), + nullptr, /*into_exprs*/ + 0, /*into_count*/ + nullptr, /*column_types*/ + 0,/*type_count*/ + nullptr, /*exprs_not_null_flag*/ + nullptr, /*pl_integer_ranges*/ + false, /*is_bulk*/ + false, /*is_forall*/ + false, /*is_type_record*/ + false, /*for_update*/ + false, /*is_dynamic_sql*/ + nullptr, /*using_out_params*/ + true/*is_dbms_sql*/)); return ret; } @@ -2798,7 +2488,7 @@ int ObSPIService::prepare_dynamic(ObPLExecCtx *ctx, } else { remove_into = true; } - } else if (stmt::T_SELECT == stmt_type) { + } else if (stmt::T_SELECT == stmt_type || pl_prepare_result.result_set_->is_returning()) { /* * 动态语句如果是select into，INTO子句会被忽略掉，INTO子句里面占位符的也不需要绑定实参 * 例如： @@ -2834,8 +2524,7 @@ int ObSPIService::prepare_dynamic(ObPLExecCtx *ctx, } int ObSPIService::dynamic_out_params( - common::ObIAllocator &allocator, - ObResultSet *result_set, common::ObObjParam **params, int64_t param_count) + common::ObIAllocator &allocator, ObResultSet *result_set, void* params, int64_t param_count, bool is_dbms_sql) { int ret = OB_SUCCESS; ObPhysicalPlanCtx *plan_ctx = NULL; @@ -2849,20 +2538,172 @@ int ObSPIService::dynamic_out_params( OV (param_store->count() >= param_count, OB_ERR_UNEXPECTED, param_store->count(), param_count); for (int64_t i = 0; OB_SUCC(ret) && i < param_count; ++i) { - // param_store`memory from result set will released by result close. + // memory of params_store from result set will released by result close. // so we need to deep copy it. + ObObjParam &obj = is_dbms_sql ? reinterpret_cast(params)->at(i) + : *(reinterpret_cast(params)[i]); if (param_store->at(i).is_pl_extend() && param_store->at(i).get_meta().get_extend_type() != PL_CURSOR_TYPE && param_store->at(i).get_meta().get_extend_type() != PL_REF_CURSOR_TYPE) { - OZ (pl::ObUserDefinedType::deep_copy_obj(allocator, param_store->at(i), *params[i], true)); + OZ (pl::ObUserDefinedType::deep_copy_obj(allocator, param_store->at(i), obj, true)); } else { - OZ (deep_copy_obj(allocator, param_store->at(i), *params[i])); + OZ (deep_copy_obj(allocator, param_store->at(i), obj)); } } } return ret; } + +int ObSPIService::check_dynamic_sql_legal(ObPLExecCtx *ctx, + ObSqlString &sql_str, + stmt::StmtType stmt_type, + int64_t into_count, + int64_t inner_into_cnt, + common::ObObjParam **params, + int64_t param_count, + const int64_t *params_mode, + bool is_returning, + bool for_update, + int64_t &exec_param_cnt, + ObIArray &out_using_params) +{ + int ret = OB_SUCCESS; + if (OB_SUCC(ret)) { + if (ObStmt::is_ddl_stmt(stmt_type, false) + && (into_count > 0 || param_count > 0 || is_returning)) { + ret = OB_ERR_DDL_IN_ILLEGAL_CONTEXT; + LOG_WARN("DDL statement is executed in an illegal context", + K(ret), K(into_count), K(param_count), K(is_returning)); + } else if (ObStmt::is_dml_write_stmt(stmt_type) && inner_into_cnt > 0 && 0 == into_count) { + /* + * 处理 + * 仅当dml语句含returning变量，并且外部没有INTO变量时才允许使用USING OUT接收参数 + */ + CK (param_count >= inner_into_cnt); + OX (exec_param_cnt = param_count - inner_into_cnt); + for (int64_t i = exec_param_cnt; OB_SUCC(ret) && i < param_count; ++i) { + ObPLRoutineParamMode pm = static_cast(params_mode[i]); + if (PL_PARAM_INOUT == pm || PL_PARAM_OUT == pm) { + OZ (out_using_params.push_back(params[i])); + } else { + ret = OB_ERR_INOUT_PARAM_PLACEMENT_NOT_PROPERLY; + LOG_WARN("OBE-06536: IN bind variable bound to an OUT position", K(ret)); + } + } + } else if (ObStmt::is_dml_write_stmt(stmt_type) && inner_into_cnt > 0 && into_count > 0 && !is_returning) { + CK (param_count >= inner_into_cnt); + for (int64_t i = param_count - inner_into_cnt; OB_SUCC(ret) && i < param_count; ++i) { + ObPLRoutineParamMode pm = static_cast(params_mode[i]); + if (PL_PARAM_IN == pm) { + ret = OB_ERR_INOUT_PARAM_PLACEMENT_NOT_PROPERLY; + LOG_WARN("OBE-06536: IN bind variable bound to an OUT position", K(ret)); + } + } + } else { /*do nothing*/ } + } + + for (int i = 0; OB_SUCC(ret) && i < param_count; ++i) { + ObPLRoutineParamMode pm = static_cast(params_mode[i]); + if (ObStmt::is_select_stmt(stmt_type) && (PL_PARAM_INOUT == pm || PL_PARAM_OUT == pm)) { + ret = OB_ERR_INOUT_PARAM_PLACEMENT_NOT_PROPERLY; + LOG_WARN("select stmt with using out/inout param mode is not allowed", K(ret)); + } + if (ObStmt::is_dml_write_stmt(stmt_type) + && i < param_count - inner_into_cnt + && PL_PARAM_OUT == pm) { + ret = OB_ERR_INOUT_PARAM_PLACEMENT_NOT_PROPERLY; + LOG_WARN("using out/inout param mode is not allowed", K(ret)); + } else if (ObStmt::is_dml_write_stmt(stmt_type) && + into_count > 0 && !is_returning && + (PL_PARAM_INOUT == pm || PL_PARAM_OUT == pm)) { + ret = OB_ERR_INOUT_PARAM_PLACEMENT_NOT_PROPERLY; + LOG_WARN("using out/inout param mode is not allowed", K(ret)); + } + if (OB_SUCC(ret) && ObStmt::is_dml_stmt(stmt_type)) { + if (PL_PARAM_IN == pm && + NULL != params[i] && + params[i]->is_pl_extend() && + PL_RECORD_TYPE == params[i]->get_meta().get_extend_type()) { + const ObUserDefinedType *user_type = NULL; + ObPLComposite *composite = reinterpret_cast(params[i]->get_ext()); + CK (OB_NOT_NULL(composite)); + OZ (ctx->get_user_type(composite->get_id(), user_type)); + CK (OB_NOT_NULL(user_type)); + if (OB_SUCC(ret) && user_type->is_type_record()) { + ret = OB_ERR_EXPR_SQL_TYPE; + LOG_WARN("expressions have to be of SQL types", K(ret)); + } + } + } + } + + if (OB_SUCC(ret)) { + ObParser parser(*ctx->allocator_, STD_MODE); + ObMPParseStat parse_stat; + ObSEArray queries; + OZ (parser.split_multiple_stmt(sql_str.string(), queries, parse_stat)); + if (OB_SUCC(ret) && queries.count() > 1) { + ret = OB_ERR_CMD_NOT_PROPERLY_ENDED; + LOG_WARN("execute immdeidate only support one stmt", K(ret)); + } + } + return ret; +} + +int ObSPIService::prepare_dynamic_sql_params(ObPLExecCtx *ctx, + ObSPIResultSet &spi_result, + ObIAllocator &allocator, + int64_t exec_param_cnt, + ObObjParam **params, + ParamStore *&exec_params) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(exec_params = reinterpret_cast(allocator.alloc(sizeof(ParamStore))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for exec params", K(ret)); + } else { + new (exec_params) ParamStore((ObWrapperAllocator(allocator))); + } + for (int64_t i = 0; OB_SUCC(ret) && i < exec_param_cnt; ++i) { + CK (OB_NOT_NULL(params[i])); + if (OB_SUCC(ret)) { + ObObjParam new_param = *params[i]; + if (params[i]->is_pl_extend()) { + if (params[i]->get_meta().get_extend_type() != PL_REF_CURSOR_TYPE) { + new_param.set_int_value(0); + if (!ob_is_xml_pl_type(params[i]->get_type(), params[i]->get_udt_id())) { + OZ (pl::ObUserDefinedType::deep_copy_obj(allocator, *params[i], new_param, true)); + } else { + uint64_t udt_id = params[i]->get_udt_id(); + const ObDataTypeCastParams dtc_params = sql::ObBasicSessionInfo::create_dtc_params(ctx->exec_ctx_->get_my_session()); + ObCastCtx cast_ctx(ctx->allocator_, &dtc_params, CM_NONE, ObCharset::get_system_collation()); + cast_ctx.exec_ctx_ = ctx->exec_ctx_; + uint16_t subschema_id = ObInvalidSqlType; + if (OB_FAIL(cast_ctx.exec_ctx_->get_subschema_id_by_udt_id(udt_id, + subschema_id, + &spi_result.get_scheme_guard()))) { + LOG_WARN("failed to get subschema id by udt_id", K(ret), K(params[i])); + } else if (FALSE_IT(new_param.set_subschema_id(subschema_id))) { + } else if (OB_FAIL(ObObjCaster::to_type(ObUserDefinedSQLType, cast_ctx, *params[i], new_param))) { + LOG_WARN("failed to_type", K(ret), K(new_param)); + } else { // sql udt params need original udt id for plan choose + new_param.set_udt_id(udt_id); + new_param.set_param_meta(); // param meta also changed + } + } + } + } else { + OZ (deep_copy_obj(allocator, *params[i], new_param)); + } + OX (new_param.set_need_to_check_type(true)); + OZ (exec_params->push_back(new_param), K(new_param)); + } + } + OZ (store_params_string(ctx, spi_result, exec_params)); + return ret; +} + int ObSPIService::spi_execute_immediate(ObPLExecCtx *ctx, const int64_t sql_idx, common::ObObjParam **params, @@ -2879,392 +2720,100 @@ int ObSPIService::spi_execute_immediate(ObPLExecCtx *ctx, bool is_type_record) { int ret = OB_SUCCESS; - // HEAP_VAR(char[OB_MAX_SQL_LENGTH], sql_buffer) { - bool need_execute_sql = true; - ObSQLSessionInfo *session = NULL; - ObMySQLProxy *sql_proxy = NULL; - const ObSqlExpression *sql = nullptr; - const ObSqlExpression **into_exprs = nullptr; - ObSqlString sql_str; - ObArenaAllocator allocator(GET_PL_MOD_STRING(PL_MOD_IDX::OB_PL_DYNAMIC_SQL_EXEC), OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); - ObPLSubPLSqlTimeGuard guard(ctx); - HEAP_VAR(ObSPIResultSet, spi_result) { - stmt::StmtType stmt_type = stmt::T_NONE; - ObString ps_sql; - bool for_update = false; - bool hidden_rowid = false; - bool skip_locked = false; - ObQueryRetryCtrl retry_ctrl; - int64_t tenant_version = 0; - int64_t sys_version = 0; - share::schema::ObSchemaGetterGuard schema_guard; - int64_t inner_into_cnt = 0; //动态语句里into子句的变量个数 - ObArray out_using_params; - int64_t exec_param_cnt = param_count; + ObSQLSessionInfo *session = NULL; + ObSqlString sql_str; + const ObSqlExpression *sql = nullptr; + const ObSqlExpression **into_exprs = nullptr; - stmt::StmtType saved_stmt_type = stmt::T_NONE; - CK (OB_NOT_NULL(ctx), ctx->valid()); - CK (OB_NOT_NULL(ctx->func_)); - CK ((OB_NOT_NULL(params) && param_count > 0) || (OB_ISNULL(params) && 0 == param_count)); - CK (OB_NOT_NULL(session = ctx->exec_ctx_->get_my_session())); - CK (OB_NOT_NULL(sql_proxy = ctx->exec_ctx_->get_sql_proxy())); - CK (sql_idx != OB_INVALID_ID); - CK (OB_LIKELY(0 <= sql_idx && sql_idx < ctx->func_->get_expressions().count())); - OX (sql = ctx->func_->get_expressions().at(sql_idx)); - CK (OB_NOT_NULL(sql)); - CK (OB_NOT_NULL(GCTX.sql_engine_)); - OZ(spi_result.init(*session)); - OX(saved_stmt_type = session->get_stmt_type()); - MAKE_EXPR_BUFFER(allocator, into_exprs_idx, into_count, into_exprs); - // Step1: Prepare dynamic SQL! Only prepare once! - OZ (prepare_dynamic(ctx, - sql, - allocator, - is_returning, - param_count, - sql_str, - ps_sql, - stmt_type, - for_update, - hidden_rowid, - inner_into_cnt, - skip_locked)); - if (OB_SUCC(ret)) { - if (ObStmt::is_ddl_stmt(stmt_type, false) - && (into_count > 0 || param_count > 0 || is_returning)) { - ret = OB_ERR_DDL_IN_ILLEGAL_CONTEXT; - LOG_WARN("DDL statement is executed in an illegal context", - K(ret), K(into_count), K(param_count), K(is_returning)); - } else if (ObStmt::is_select_stmt(stmt_type) && !for_update && 0 >= into_count) { - /* If dynamic_sql_statement is a SELECT statement, and you omit both - * into_clause and bulk_collect_into_clause, then - * execute_immediate_statement never executes. - * For example, this statement never increments the sequence: - * EXECUTE IMMEDIATE 'SELECT S.NEXTVAL FROM DUAL' - */ - need_execute_sql = false; - } else if (ObStmt::is_dml_write_stmt(stmt_type) && inner_into_cnt > 0 && 0 == into_count) { - /* - * 处理 - * 仅当dml语句含returning变量，并且外部没有INTO变量时才允许使用USING OUT接收参数 - */ - CK (param_count >= inner_into_cnt); - OX (exec_param_cnt = param_count - inner_into_cnt); - for (int64_t i = exec_param_cnt; OB_SUCC(ret) && i < param_count; ++i) { - ObPLRoutineParamMode pm = static_cast(params_mode[i]); - if (PL_PARAM_INOUT == pm || PL_PARAM_OUT == pm) { - OZ (out_using_params.push_back(params[i])); - } else { - ret = OB_ERR_INOUT_PARAM_PLACEMENT_NOT_PROPERLY; - LOG_WARN("OBE-06536: IN bind variable bound to an OUT position", K(ret)); - } - } - } else if (ObStmt::is_dml_write_stmt(stmt_type) && inner_into_cnt > 0 && into_count > 0 && !is_returning) { - CK (param_count >= inner_into_cnt); - for (int64_t i = param_count - inner_into_cnt; OB_SUCC(ret) && i < param_count; ++i) { - ObPLRoutineParamMode pm = static_cast(params_mode[i]); - if (PL_PARAM_IN == pm) { - ret = OB_ERR_INOUT_PARAM_PLACEMENT_NOT_PROPERLY; - LOG_WARN("OBE-06536: IN bind variable bound to an OUT position", K(ret)); - } - } - } else { /*do nothing*/ } - } - for (int i = 0; OB_SUCC(ret) && i < param_count; ++i) { - ObPLRoutineParamMode pm = static_cast(params_mode[i]); - if (ObStmt::is_select_stmt(stmt_type) && (PL_PARAM_INOUT == pm || PL_PARAM_OUT == pm)) { - ret = OB_ERR_INOUT_PARAM_PLACEMENT_NOT_PROPERLY; - LOG_WARN("select stmt with using out/inout param mode is not allowed", K(ret)); - } - if (ObStmt::is_dml_write_stmt(stmt_type) - && i < param_count - inner_into_cnt - && PL_PARAM_OUT == pm) { - ret = OB_ERR_INOUT_PARAM_PLACEMENT_NOT_PROPERLY; - LOG_WARN("using out/inout param mode is not allowed", K(ret)); - } else if (ObStmt::is_dml_write_stmt(stmt_type) && - into_count > 0 && !is_returning && - (PL_PARAM_INOUT == pm || PL_PARAM_OUT == pm)) { - ret = OB_ERR_INOUT_PARAM_PLACEMENT_NOT_PROPERLY; - LOG_WARN("using out/inout param mode is not allowed", K(ret)); - } - if (OB_SUCC(ret) && ObStmt::is_dml_stmt(stmt_type)) { - if (PL_PARAM_IN == pm && - NULL != params[i] && - params[i]->is_pl_extend() && - PL_RECORD_TYPE == params[i]->get_meta().get_extend_type()) { - const ObUserDefinedType *user_type = NULL; - ObPLComposite *composite = reinterpret_cast(params[i]->get_ext()); - CK (OB_NOT_NULL(composite)); - OZ (ctx->get_user_type(composite->get_id(), user_type)); - CK (OB_NOT_NULL(user_type)); - if (OB_SUCC(ret) && user_type->is_type_record()) { - ret = OB_ERR_EXPR_SQL_TYPE; - LOG_WARN("expressions have to be of SQL types", K(ret)); - } - } - } - } + ObArenaAllocator allocator(GET_PL_MOD_STRING(PL_MOD_IDX::OB_PL_DYNAMIC_SQL_EXEC), OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); - if (OB_SUCC(ret)) { - ObParser parser(*ctx->allocator_, STD_MODE); - ObMPParseStat parse_stat; - ObSEArray queries; - OZ (parser.split_multiple_stmt(sql_str.string(), queries, parse_stat)); - if (OB_SUCC(ret) && queries.count() > 1) { - ret = OB_ERR_CMD_NOT_PROPERLY_ENDED; - LOG_WARN("execute immdeidate only support one stmt", K(ret)); - } - } - OX (session->set_stmt_type(saved_stmt_type)); - OZ (spi_result.start_nested_stmt_if_need(ctx, sql_str.string(), stmt_type, for_update)); + stmt::StmtType stmt_type = stmt::T_NONE; + ObString ps_sql; + bool for_update = false; + bool hidden_rowid = false; + bool skip_locked = false; + int64_t inner_into_cnt = 0; //动态语句里into子句的变量个数 + ObArray out_using_params; + int64_t exec_param_cnt = param_count; - // Step2: execute dynamic SQL now! - ObPLSPITraceIdGuard trace_id_guard(sql_str.string(), ps_sql, *session, ret); - if (OB_FAIL(ret)) { - } else if (need_execute_sql) { - ObExecRecord exec_record; - ObExecTimestamp exec_timestamp; - ObSPITimeRecord time_record; - exec_timestamp.exec_type_ = sql::PLSql; + stmt::StmtType saved_stmt_type = stmt::T_NONE; + CK (OB_NOT_NULL(ctx), ctx->valid()); + CK (OB_NOT_NULL(ctx->func_)); + CK ((OB_NOT_NULL(params) && param_count > 0) || (OB_ISNULL(params) && 0 == param_count)); + CK (OB_NOT_NULL(session = ctx->exec_ctx_->get_my_session())); + CK (sql_idx != OB_INVALID_ID); + CK (OB_LIKELY(0 <= sql_idx && sql_idx < ctx->func_->get_expressions().count())); + OX (sql = ctx->func_->get_expressions().at(sql_idx)); + CK (OB_NOT_NULL(sql)); + CK (OB_NOT_NULL(GCTX.sql_engine_)); + OX (saved_stmt_type = session->get_stmt_type()); - int64_t old_query_start_time = session->get_query_start_time(); - session->set_query_start_time(ObTimeUtility::current_time()); - bool is_retry = false; - do { - ObArenaAllocator allocator(GET_PL_MOD_STRING(PL_MOD_IDX::OB_PL_DYNAMIC_SQL_EXEC), OB_MALLOC_NORMAL_BLOCK_SIZE, MTL_ID()); - ParamStore exec_params( (ObWrapperAllocator(allocator)) ); - ObWaitEventDesc max_wait_desc; - ObWaitEventStat total_wait_desc; - const bool enable_perf_event = lib::is_diagnose_info_enabled(); - const bool enable_sql_audit = - GCONF.enable_sql_audit && session->get_local_ob_enable_sql_audit(); - { - ObMaxWaitGuard max_wait_guard(enable_perf_event ? &max_wait_desc : NULL); - ObTotalWaitGuard total_wait_guard(enable_perf_event ? &total_wait_desc : NULL); - if (enable_perf_event) { - exec_record.record_start(); - } - //监控项统计开始 - time_record.set_send_timestamp(ObTimeUtility::current_time()); + MAKE_EXPR_BUFFER(allocator, into_exprs_idx, into_count, into_exprs); - ret = OB_SUCCESS; - if (is_retry) { - spi_result.reset_member_for_retry(*session); - } - retry_ctrl.clear_state_before_each_retry(session->get_retry_info_for_update()); - OZ (GCTX.schema_service_->get_tenant_schema_guard(session->get_effective_tenant_id(), - spi_result.get_scheme_guard())); - OZ (spi_result.get_scheme_guard().get_schema_version(session->get_effective_tenant_id(), tenant_version)); - OZ (spi_result.get_scheme_guard().get_schema_version(OB_SYS_TENANT_ID, sys_version)); + // Step: Prepare dynamic SQL! Only prepare once! + OZ (prepare_dynamic(ctx, + sql, + allocator, + is_returning, + param_count, + sql_str, + ps_sql, + stmt_type, + for_update, + hidden_rowid, + inner_into_cnt, + skip_locked)); - OX (retry_ctrl.set_tenant_local_schema_version(tenant_version)); - OX (retry_ctrl.set_sys_local_schema_version(sys_version)); - OX (spi_result.get_sql_ctx().schema_guard_ = &spi_result.get_scheme_guard()); + OZ (check_dynamic_sql_legal(ctx, + sql_str, + stmt_type, + into_count, + inner_into_cnt, + params, + param_count, + params_mode, + is_returning, + for_update, + exec_param_cnt, + out_using_params)); - bool old_client_return_rowid = session->is_client_return_rowid(); - bool is_inner_session = session->is_inner(); - ObSQLSessionInfo::SessionType old_session_type = session->get_session_type(); - !is_inner_session ? session->set_inner_session() : (void)NULL; - session->set_session_type(ObSQLSessionInfo::USER_SESSION); - if (NULL != ctx->pl_ctx_) { - session->set_client_return_rowid(false); - } - if (OB_SUCC(ret)) { - WITH_CONTEXT(spi_result.get_memory_ctx()) { - if (OB_FAIL(ret)) { - } else if (0 == param_count) { - spi_result.get_result_set()->set_user_sql(true); - OZ (GCTX.sql_engine_->handle_pl_execute( - sql_str.string(), *session, exec_params, *spi_result.get_result_set(), spi_result.get_sql_ctx(), - false /* is_prepare_protocol */, false /* is_dynamic_sql*/)); - } else { - for (int64_t i = 0; OB_SUCC(ret) && i < exec_param_cnt; ++i) { - CK (OB_NOT_NULL(params[i])); - if (OB_SUCC(ret)) { - ObObjParam new_param = *params[i]; - if (params[i]->is_pl_extend()) { - if (params[i]->get_meta().get_extend_type() != PL_REF_CURSOR_TYPE) { - new_param.set_int_value(0); - if (!ob_is_xml_pl_type(params[i]->get_type(), params[i]->get_udt_id())) { - OZ (pl::ObUserDefinedType::deep_copy_obj(allocator, *params[i], new_param, true)); - } else { - uint64_t udt_id = params[i]->get_udt_id(); - const ObDataTypeCastParams dtc_params = sql::ObBasicSessionInfo::create_dtc_params(ctx->exec_ctx_->get_my_session()); - ObCastCtx cast_ctx(ctx->allocator_, &dtc_params, CM_NONE, ObCharset::get_system_collation()); - cast_ctx.exec_ctx_ = ctx->exec_ctx_; - uint16_t subschema_id = ObInvalidSqlType; - if (OB_FAIL(cast_ctx.exec_ctx_->get_subschema_id_by_udt_id(udt_id, - subschema_id, - &spi_result.get_scheme_guard()))) { - LOG_WARN("failed to get subschema id by udt_id", K(ret), K(params[i])); - } else if (FALSE_IT(new_param.set_subschema_id(subschema_id))) { - } else if (OB_FAIL(ObObjCaster::to_type(ObUserDefinedSQLType, cast_ctx, *params[i], new_param))) { - LOG_WARN("failed to_type", K(ret), K(new_param)); - } else { // sql udt params need original udt id for plan choose - new_param.set_udt_id(udt_id); - new_param.set_param_meta(); // param meta also changed - } - } - } - } else { - OZ (deep_copy_obj(allocator, *params[i], new_param)); - } - OX (new_param.set_need_to_check_type(true)); - OZ (exec_params.push_back(new_param), new_param); - } - } - if (OB_SUCC(ret)) { - char *tmp_ptr = NULL; - int64_t tmp_len = 0; - OZ (ObMPStmtExecute::store_params_value_to_str(spi_result.get_memory_ctx()->get_arena_allocator(), - *ctx->exec_ctx_->get_my_session(), - &exec_params, - tmp_ptr, - tmp_len)); - OX (spi_result.get_exec_params_str_ptr()->assign(tmp_ptr, tmp_len)); - } - LOG_INFO("execute dynamic sql using", K(ps_sql), K(exec_params)); - OZ (GCTX.sql_engine_->handle_pl_execute( - ps_sql, *session, exec_params, *spi_result.get_result_set(), spi_result.get_sql_ctx(), - true /* is_prepare_protocol */, true /* is_dynamic_sql*/)); - } - } - } - if (OB_FAIL(ret) && spi_result.get_result_set() != NULL) { - int cli_ret = OB_SUCCESS; - retry_ctrl.test_and_save_retry_state( - GCTX, - spi_result.get_sql_ctx(), - *spi_result.get_result_set(), - ret, cli_ret, true, true, true); - LOG_WARN("failed to get_result, check if need retry", - K(ret), K(cli_ret), K(retry_ctrl.need_retry())); - ret = cli_ret; - spi_result.get_sql_ctx().clear(); - ctx->exec_ctx_->get_my_session()->set_session_in_retry(retry_ctrl.need_retry()); - } - // todo:@hr351303 确认session标记是否还需要 - !is_inner_session ? session->set_user_session() : (void)NULL; - session->set_session_type(old_session_type); - session->set_client_return_rowid(old_client_return_rowid); - LOG_TRACE("execute dynamic sql", K(ret), K(sql), K(sql_str), K(ps_sql)); + OX (session->set_stmt_type(saved_stmt_type)); - int64_t row_count = 0; - OZ (inner_fetch(ctx, - retry_ctrl, - spi_result, - into_exprs, - into_count, - column_types, - type_count, - exprs_not_null_flag, - pl_integer_ranges, - out_using_params.empty() ? NULL : &out_using_params, - row_count, - is_bulk, - false, - true/*is_dynamic_sql*/, - NULL, - false, - false, - INT64_MAX, - NULL, - 0, - is_type_record)); - - //此处仅需要处理非DML RETURNING返回的USING OUT参数 - // if it is bulk into, not allow using out param, so no need deep copy - if (OB_SUCC(ret) && !is_bulk) { - OZ (dynamic_out_params(*(ctx->allocator_), spi_result.get_result_set(), params, exec_param_cnt)); - } - //监控项统计结束 - time_record.set_exec_end_timestamp(ObTimeUtility::current_time()); - if (enable_perf_event) { - exec_record.record_end(); - } - } - // 处理监控统计项 - if (OB_NOT_NULL(spi_result.get_result_set())) { - if (spi_result.get_result_set()->is_inited()) { - int64_t try_cnt = session->get_raw_audit_record().try_cnt_; - ObExecRecord record_bk = session->get_raw_audit_record().exec_record_; - session->get_raw_audit_record().try_cnt_ = retry_ctrl.get_retry_times(); - session->get_raw_audit_record().pl_trace_id_ = trace_id_guard.origin_trace_id_; - ObInnerSQLConnection::process_record(*spi_result.get_result_set(), - spi_result.get_sql_ctx(), - *session, - time_record, - ret, - session->get_current_execution_id(), // sql execute id - OB_INVALID_ID, // ps stmt id FIXME@hr351303 - max_wait_desc, - total_wait_desc, - exec_record, - exec_timestamp, - true, - 0 == param_count ? sql_str.string() : ps_sql, - true, - spi_result.get_exec_params_str_ptr()); - session->get_raw_audit_record().exec_record_ = record_bk; - session->get_raw_audit_record().try_cnt_ = try_cnt; - session->get_raw_audit_record().pl_trace_id_.reset(); - } else { - LOG_DEBUG("result set is not inited, do not process record", - K(ret), K(ps_sql), K(sql_str), K(stmt_type)); - } - } else { - if (OB_SUCC(ret)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected error, result_set is null", - K(ret), K(ps_sql), K(sql_str), K(stmt_type)); - } else { - LOG_WARN("result_set is null", - K(ret), K(ps_sql), K(sql_str), K(stmt_type)); - } - } - // 无论成功或者失败都在这里close result set - int close_ret = spi_result.close_result_set(); - if (OB_SUCCESS != close_ret) { - LOG_WARN("close result set failed", K(ret), K(close_ret)); - } - ret = OB_SUCCESS == ret ? close_ret : ret; - spi_result.destruct_exec_params(*session); - is_retry = true; - } while (RETRY_TYPE_NONE != retry_ctrl.get_retry_type()); //SPI只做LOCAL重试 - session->get_retry_info_for_update().clear(); - session->set_query_start_time(old_query_start_time); - } else { - // 不需要执行, 但是要维护隐式游标 - ObPLCursorInfo *implicit_cursor = session->get_pl_implicit_cursor(); - CK (OB_NOT_NULL(implicit_cursor)); - OX (implicit_cursor->set_rowcount(0)); - } - //自动提交,禁掉PL整体重试 - if (OB_SUCC(ret) - && (ObStmt::is_ddl_stmt(stmt_type, true) - || ObStmt::is_tcl_stmt(stmt_type) - || (lib::is_mysql_mode() && session->get_local_autocommit()))) { - OX (session->set_pl_can_retry(false)); - } - if (OB_SUCC(ret) - && (ObStmt::is_ddl_stmt(stmt_type, true) - || ObStmt::is_tcl_stmt(stmt_type) - || ObStmt::is_savepoint_stmt(stmt_type))) { - if (ObStmt::is_ddl_stmt(stmt_type, true)) { - //DDL语句,需要强制刷新schema - OZ (force_refresh_schema(session->get_effective_tenant_id())); - } - recreate_implicit_savapoint_if_need(ctx, ret); - } - - spi_result.end_nested_stmt_if_need(ctx, ret); - - SET_FORALL_BULK_EXCEPTION; - SET_SPI_STATUS; - } - // } + // Step: execute dynamic SQL now! + if (OB_FAIL(ret)) { + } else if (ObStmt::is_select_stmt(stmt_type) && !for_update && into_count <= 0) { + /*! + * If dynamic_sql_statement is a SELECT statement, and you omit both + * into_clause and bulk_collect_into_clause, then + * execute_immediate_statement never executes. + * For example, this statement never increments the sequence: + * EXECUTE IMMEDIATE 'SELECT S.NEXTVAL FROM DUAL' + */ + ObPLCursorInfo *implicit_cursor = session->get_pl_implicit_cursor(); + CK (OB_NOT_NULL(implicit_cursor)); + OX (implicit_cursor->set_rowcount(0)); + } else { + OZ (SMART_CALL(spi_inner_execute(ctx, + sql_str.ptr(), + ps_sql.ptr(), + stmt_type, + params, + exec_param_cnt, + into_exprs, + into_count, + column_types, + type_count, + exprs_not_null_flag, + pl_integer_ranges, + is_bulk, + false, /*is_forall*/ + is_type_record, + for_update, + true, /*is_dynamic_sql*/ + &out_using_params))); + } return ret; } @@ -3692,23 +3241,37 @@ int ObSPIService::spi_dynamic_open(ObPLExecCtx *ctx, int ObSPIService::dbms_dynamic_open(ObPLExecCtx *pl_ctx, ObDbmsCursorInfo &cursor, - bool is_dbms_sql) + bool is_dbms_sql, + int64_t orc_max_ret_rows) { int ret = OB_SUCCESS; - // ObString &sql_stmt = cursor.get_sql_stmt(); - // ParamStore &exec_params = cursor.get_exec_params(); + stmt::StmtType stmt_type = cursor.get_stmt_type(); const ObString ps_sql = cursor.get_ps_sql(); bool for_update = cursor.is_for_update(); bool hidden_rowid = cursor.has_hidden_rowid(); - // int64_t into_cnt = 0; OV (OB_NOT_NULL(pl_ctx->exec_ctx_->get_my_session())); if (ObStmt::is_select_stmt(stmt_type) || cursor.get_into_names().count() > 0) { // NOTICE: DML Returning also use cursor impl. - OZ (dbms_cursor_open(pl_ctx, cursor, ps_sql, stmt_type, for_update, hidden_rowid), cursor); + OZ (dbms_cursor_open( + pl_ctx, cursor, ps_sql, stmt_type, for_update, hidden_rowid, orc_max_ret_rows), + cursor); } else { OZ (dbms_cursor_execute(pl_ctx, ps_sql, stmt_type, cursor, is_dbms_sql), cursor); + if (OB_SUCC(ret) && cursor.get_exec_params().count() > 0) { + ObIAllocator &alloc = cursor.get_dbms_entity()->get_arena_allocator(); + for (int64_t i = 0; i < cursor.get_exec_params().count(); ++i) { + ObObjParam &obj = cursor.get_exec_params().at(i); + if (obj.is_pl_extend() + && obj.get_meta().get_extend_type() != PL_CURSOR_TYPE + && obj.get_meta().get_extend_type() != PL_REF_CURSOR_TYPE) { + OZ (pl::ObUserDefinedType::deep_copy_obj(alloc, obj, obj, true)); + } else { + OZ (deep_copy_obj(alloc, obj, obj)); + } + } + } OX (cursor.set_affected_rows(pl_ctx->exec_ctx_->get_my_session()->get_affected_rows())); } return ret; @@ -3793,6 +3356,233 @@ int ObSPIService::spi_cursor_open_with_param_idx(ObPLExecCtx *ctx, return ret; } +int ObSPIService::streaming_cursor_open(ObPLExecCtx *ctx, + ObPLCursorInfo &cursor, + ObSQLSessionInfo &session_info, + const ObString &sql, + const ObString &ps_sql, + int64_t type, + void *params, + int64_t sql_param_count, + bool is_server_cursor, + bool for_update, + bool has_hidden_rowid, + bool is_dbms_cursor) +{ + int ret = OB_SUCCESS; + + cursor.set_streaming(); + ObSPIResultSet *spi_result = NULL; + ObString ps_sql_copy; + + OZ (cursor.prepare_spi_result(ctx, spi_result)); + CK (OB_NOT_NULL(spi_result)); + CK (OB_NOT_NULL(spi_result->get_memory_ctx())); + OZ (spi_result->start_cursor_stmt(ctx, static_cast(type), for_update)); + + // in a streaming cursor, lifetime of ps_sql may be shorter than the cursor itself, + // so we need to open it with a deep copy. + OZ (ob_write_string(spi_result->get_allocator(), ps_sql, ps_sql_copy)); + + if (OB_SUCC(ret)) { + + ObPLSqlAuditRecord audit_record(sql::PLSql); + ObQueryRetryCtrl retry_ctrl; + ObSPIExecEnvGuard env_guard(session_info, *spi_result); + + do { + { + ObPLSubPLSqlTimeGuard guard(ctx); + ObPLSPITraceIdGuard trace_id_guard(sql, ps_sql, session_info, ret); + ObPLSqlAuditGuard audit_guard( + *(ctx->exec_ctx_), session_info, *spi_result, audit_record, ret, (sql != NULL ? sql : ps_sql), retry_ctrl, trace_id_guard, static_cast(type)); + ObSPIRetryCtrlGuard retry_guard(retry_ctrl, *spi_result, session_info, ret); + + if (OB_FAIL(ret)) { + } else if (is_server_cursor) { + WITH_CONTEXT(cursor.get_cursor_entity()) { + lib::ContextTLOptGuard guard(false); + OZ (inner_open(ctx, + spi_result->get_memory_ctx()->get_arena_allocator(), + sql, + ps_sql_copy, + type, + params, + sql_param_count, + nullptr, + 0, + *spi_result, + spi_result->get_out_params(), + false, /*is_forall*/ + false, /*is_dynamic_sql*/ + is_dbms_cursor /*is_dbms_sql*/)); + } + } else { + OZ (inner_open(ctx, + spi_result->get_memory_ctx()->get_arena_allocator(), + sql, + ps_sql_copy, + type, + params, + sql_param_count, + nullptr, + 0, + *spi_result, + spi_result->get_out_params(), + false, /*is_forall*/ + false, /*is_dynamic_sql*/ + is_dbms_cursor /*is_dbms_sql*/)); + } + CK (OB_NOT_NULL(spi_result->get_result_set()->get_field_columns())); + if (OB_SUCC(ret) && is_dbms_cursor) { + ObDbmsCursorInfo &dbms_cursor = static_cast(cursor); + if (dbms_cursor.get_field_columns().empty()) { + const common::ColumnsFieldArray* field_column = + dynamic_cast + (spi_result->get_result_set()->get_field_columns()); + OX (dbms_cursor.get_field_columns().set_allocator(&dbms_cursor.get_dbms_entity()->get_arena_allocator())); + OZ (dbms_cursor.get_field_columns().assign(*field_column)); + } + } + if (OB_SUCC(ret) && OB_INVALID_ID != cursor.get_id()) { + //如果是客户端游标，设置结果集为二进制模式 + OX (spi_result->get_result_set()->set_ps_protocol()); + } + OX (cursor.open(spi_result)); + OX (for_update ? cursor.set_for_update() : (void)NULL); + OX (for_update ? cursor.set_trans_id(session_info.get_tx_id()) : (void)NULL); + OX (has_hidden_rowid ? cursor.set_hidden_rowid() : (void)NULL); + OZ (setup_cursor_snapshot_verify_(&cursor, spi_result)); + if (!cursor.is_ps_cursor()) { + retry_guard.test(); + } + } + if (OB_FAIL(ret)) { + int close_ret = spi_result->close_result_set(); + if (OB_SUCCESS != close_ret) { + LOG_WARN("close mysql result set failed", K(ret), K(close_ret)); + } + } + } while (RETRY_TYPE_NONE != retry_ctrl.get_retry_type()); + spi_result->end_cursor_stmt(ctx, ret); + cursor.set_last_execute_time(ObTimeUtility::current_time()); + // only care about end_cursor_stmt failed. + // if cursor already open, spi_result will released by cursor close. + // there is not a situation like '!cursor.is_open && spi_result not close'. + // so do not need call spi_result.close_result_set. + if (OB_FAIL(ret) && !cursor.isopen()) { + spi_result->~ObSPIResultSet(); + } + } + return ret; +} + +int ObSPIService::unstreaming_cursor_open(ObPLExecCtx *ctx, + ObPLCursorInfo &cursor, + ObSQLSessionInfo &session_info, + const ObString &sql, + const ObString &ps_sql, + int64_t type, + void *params, + int64_t sql_param_count, + bool is_server_cursor, + bool for_update, + bool has_hidden_rowid, + bool is_dbms_cursor, + int64_t orc_max_ret_rows) +{ + int ret = OB_SUCCESS; + + HEAP_VAR(ObSPIResultSet, spi_result) { + ObSPICursor* spi_cursor = NULL; + OZ (spi_result.init(session_info)); + OZ (spi_result.start_nested_stmt_if_need(ctx, sql, static_cast(type), for_update)); + + if (OB_SUCC(ret)) { + + ObPLSqlAuditRecord audit_record(sql::PLSql); + ObQueryRetryCtrl retry_ctrl; + ObSPIExecEnvGuard env_guard(session_info, spi_result); + + do { + ret = OB_SUCCESS; + uint64_t size = 0; + { + ObPLSubPLSqlTimeGuard guard(ctx); + ObPLSPITraceIdGuard trace_id_guard(sql, ps_sql, session_info, ret); + ObPLSqlAuditGuard audit_guard( + *(ctx->exec_ctx_), session_info, spi_result, audit_record, ret, (sql != NULL ? sql : ps_sql), retry_ctrl, trace_id_guard, static_cast(type)); + ObSPIRetryCtrlGuard retry_guard(retry_ctrl, spi_result, session_info, ret); + + CK (OB_NOT_NULL(spi_result.get_memory_ctx())); + OZ (inner_open(ctx, + spi_result.get_memory_ctx()->get_arena_allocator(), + sql, + ps_sql, + type, + params, + sql_param_count, + NULL, + 0, + spi_result, + spi_result.get_out_params(), + false, /*is_forall*/ + false, /*is_dynamic_sql*/ + is_dbms_cursor /*is_dbms_sql*/)); + OZ (session_info.get_tmp_table_size(size)); + OZ (cursor.prepare_spi_cursor(spi_cursor, + session_info.get_effective_tenant_id(), + size, + (for_update && !is_server_cursor && !is_dbms_cursor), + &session_info), K(size)); + CK (OB_NOT_NULL(spi_result.get_result_set())); + OZ (fill_cursor(*spi_result.get_result_set(), spi_cursor, ObTimeUtility::current_time(), orc_max_ret_rows)); + OZ (spi_cursor->row_store_.finish_add_row()); + if (OB_FAIL(ret)) { + } else if (is_dbms_cursor) { + ObDbmsCursorInfo &dbms_cursor = static_cast(cursor); + OZ (ObDbmsInfo::deep_copy_field_columns( + dbms_cursor.get_dbms_entity()->get_arena_allocator(), + spi_result.get_result_set()->get_field_columns(), + dbms_cursor.get_field_columns())); + } else { + CK (OB_NOT_NULL(cursor.get_allocator())); + OZ (ObDbmsInfo::deep_copy_field_columns( + *cursor.get_allocator(), spi_result.get_result_set()->get_field_columns(), spi_cursor->fields_)); + } + if (OB_SUCC(ret) && OB_NOT_NULL(spi_result.get_result_set()) && OB_NOT_NULL(spi_result.get_result_set()->get_physical_plan())) { + cursor.set_packed(spi_result.get_result_set()->get_physical_plan()->is_packed()); + } + OX (cursor.open(spi_cursor)); + OX (for_update ? cursor.set_for_update() : (void)NULL); + OX (for_update ? cursor.set_trans_id(session_info.get_tx_id()) : (void)NULL); + OX (has_hidden_rowid ? cursor.set_hidden_rowid() : (void)NULL); + if (OB_SUCC(ret) && lib::is_oracle_mode()) { + OZ (setup_cursor_snapshot_verify_(&cursor, &spi_result)); + } + if (!cursor.is_ps_cursor()) { + retry_guard.test(); + } + } + int close_ret = spi_result.close_result_set(); + if (OB_SUCCESS != close_ret) { + LOG_WARN("close mysql result failed", K(ret), K(close_ret)); + } + ret = (OB_SUCCESS == ret ? close_ret : ret); + if (!is_dbms_cursor) { + spi_result.destruct_exec_params(session_info); + } + + } while (RETRY_TYPE_NONE != retry_ctrl.get_retry_type()); + } + if (OB_FAIL(ret) && OB_NOT_NULL(spi_cursor)) { + spi_cursor->~ObSPICursor(); + } + spi_result.end_nested_stmt_if_need(ctx, ret); + } + return ret; +} + int ObSPIService::spi_cursor_open(ObPLExecCtx *ctx, const char *sql, const char *ps_sql, @@ -3824,8 +3614,7 @@ int ObSPIService::spi_cursor_open(ObPLExecCtx *ctx, LOG_WARN("Argument passed in is NULL", K(ctx), K(sql), K(ps_sql), K(type), K(sql_param_exprs), K(sql_param_count), K(ret)); } else if (OB_FAIL(spi_get_cursor_info(ctx, package_id, routine_id, cursor_index, cursor, cursor_var, loc))) { LOG_WARN("failed to get cursor info", K(ret), K(cursor_index)); - } else if (OB_FAIL(cursor_open_check(ctx, package_id, routine_id, - cursor_index, cursor, cursor_var, loc))) { + } else if (OB_FAIL(cursor_open_check(ctx, package_id, routine_id, cursor_index, cursor, cursor_var, loc))) { LOG_WARN("cursor info not init", K(ret), K(cursor)); } else if (cursor->isopen()) { ret = OB_ER_SP_CURSOR_ALREADY_OPEN; @@ -3838,353 +3627,70 @@ int ObSPIService::spi_cursor_open(ObPLExecCtx *ctx, LOG_USER_ERROR(OB_NOT_SUPPORTED, "non-select stmt in cursor"); } else { ParamStore current_params(ObWrapperAllocator(ctx->allocator_)); - ObIAllocator *allocator = cursor->get_allocator(); ObPLSubPLSqlTimeGuard guard(ctx); - if (OB_ISNULL(allocator)) { + if (cursor_param_count > 0 && (NULL == formal_param_idxs || NULL == actual_param_exprs)) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("Argument in pl context is NULL", K(allocator), K(ret)); + LOG_WARN("cursor params in not valid", + K(cursor_param_count), K(formal_param_idxs), K(actual_param_exprs), K(ret)); } else { - //首先准备Cursor的实参 - if (cursor_param_count > 0 && (NULL == formal_param_idxs || NULL == actual_param_exprs)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("cursor params in not valid", - K(cursor_param_count), K(formal_param_idxs), K(actual_param_exprs), K(ret)); - } else { - OZ (prepare_cursor_parameters( - ctx, *session_info, package_id, - routine_id, loc, formal_param_idxs, actual_param_exprs, cursor_param_count)); - } + OZ (prepare_cursor_parameters( + ctx, *session_info, package_id, + routine_id, loc, formal_param_idxs, actual_param_exprs, cursor_param_count)); + } - if (OB_SUCC(ret) && DECL_SUBPROG == loc) { - ParamStore *subprog_params = NULL; - OZ (current_params.assign(*ctx->params_)); - OZ (ObPLContext::get_param_store_from_local(*session_info, package_id, routine_id, subprog_params)); - CK (OB_NOT_NULL(subprog_params)); - OZ (ctx->params_->assign(*subprog_params)); + if (OB_SUCC(ret) && DECL_SUBPROG == loc) { + ParamStore *subprog_params = NULL; + OZ (current_params.assign(*ctx->params_)); + OZ (ObPLContext::get_param_store_from_local(*session_info, package_id, routine_id, subprog_params)); + CK (OB_NOT_NULL(subprog_params)); + OZ (ctx->params_->assign(*subprog_params)); + } + + bool is_server_cursor = false; + bool use_stream = false; + if (OB_SUCC(ret)) { + is_server_cursor = OB_INVALID_ID != cursor->get_id() + || (package_id != OB_INVALID_ID && OB_INVALID_ID == routine_id); + if (is_server_cursor) { + OZ (ObPLCursorInfo::prepare_entity(*session_info, cursor->get_cursor_entity())); + OX (cursor->set_spi_cursor(NULL)); } - bool is_server_cursor = false; - bool use_stream = false; - if (OB_SUCC(ret)) { - is_server_cursor = OB_INVALID_ID != cursor->get_id() - || (package_id != OB_INVALID_ID && OB_INVALID_ID == routine_id); - if (is_server_cursor) { - OZ (ObPLCursorInfo::prepare_entity(*session_info, cursor->get_cursor_entity())); - OX (cursor->set_spi_cursor(NULL)); - } - } - OZ (session_info->ps_use_stream_result_set(use_stream)); - if (OB_FAIL(ret)) { - // do nothing - } else if (lib::is_oracle_mode() - && ((is_server_cursor && use_stream) || !is_server_cursor) - && (!for_update || (for_update && skip_locked))) { - cursor->set_streaming(); - ObSPIResultSet *spi_result = NULL; - ObExecRecord exec_record; - ObExecTimestamp exec_timestamp; - ObSPITimeRecord time_record; - exec_timestamp.exec_type_ = sql::PLSql; - if (OB_FAIL(ret)) { - // do nothing - } else if (OB_ISNULL(cursor->get_allocator())) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("Argument in pl context is NULL", K(cursor->get_allocator()), K(ret)); - } else { - ObQueryRetryCtrl retry_ctrl; - int64_t tenant_version = 0; - int64_t sys_version = 0; - ObPLSPITraceIdGuard trace_id_guard(sql, ps_sql, *session_info, ret); - do { - // SQL_AUDIT_START - ObWaitEventDesc max_wait_desc; - ObWaitEventStat total_wait_desc; - const bool enable_perf_event = lib::is_diagnose_info_enabled(); - const bool enable_sql_audit = - GCONF.enable_sql_audit && ctx->exec_ctx_->get_my_session()->get_local_ob_enable_sql_audit(); - { - ObMaxWaitGuard max_wait_guard(enable_perf_event ? &max_wait_desc : NULL); - ObTotalWaitGuard total_wait_guard(enable_perf_event ? &total_wait_desc : NULL); - if (enable_perf_event) { - exec_record.record_start(); - } - //监控项统计开始 - time_record.set_send_timestamp(ObTimeUtility::current_time()); - ret = OB_SUCCESS; - // 如果当前cursor已经有spi_result则复用,避免内存占用过多 - retry_ctrl.clear_state_before_each_retry(session_info->get_retry_info_for_update()); - OZ (cursor->prepare_spi_result(ctx, spi_result)); - CK (OB_NOT_NULL(spi_result->get_memory_ctx())); - OZ (spi_result->start_cursor_stmt(ctx, static_cast(type), for_update)); - OZ ((GCTX.schema_service_->get_tenant_schema_guard(session_info->get_effective_tenant_id(), spi_result->get_scheme_guard()))); - OX (spi_result->get_sql_ctx().schema_guard_ = &spi_result->get_scheme_guard()); - OZ (spi_result->get_scheme_guard().get_schema_version(session_info->get_effective_tenant_id(), tenant_version)); - OZ (spi_result->get_scheme_guard().get_schema_version(OB_SYS_TENANT_ID, sys_version)); - OX (retry_ctrl.set_tenant_local_schema_version(tenant_version)); - OX (retry_ctrl.set_sys_local_schema_version(sys_version)); - if (OB_FAIL(ret)) { - // do nothing - } else if (is_server_cursor) { - WITH_CONTEXT(cursor->get_cursor_entity()) { - lib::ContextTLOptGuard guard(false); - OZ (inner_open(ctx, - spi_result->get_memory_ctx()->get_arena_allocator(), + } + OZ (session_info->ps_use_stream_result_set(use_stream)); + + if (OB_FAIL(ret)) { + } else if (lib::is_oracle_mode() // streaming cursor + && ((is_server_cursor && use_stream) || !is_server_cursor) + && (!for_update || (for_update && skip_locked))) { + OZ (streaming_cursor_open(ctx, + *cursor, + *session_info, sql, ps_sql, type, sql_param_exprs, sql_param_count, - NULL, - 0, - *spi_result, - spi_result->get_out_params(), - &retry_ctrl)); - } - } else { - ret = inner_open(ctx, - spi_result->get_memory_ctx()->get_arena_allocator(), - sql, - ps_sql, - type, - sql_param_exprs, - sql_param_count, - NULL, - 0, - *spi_result, - spi_result->get_out_params(), - &retry_ctrl); - } - OX (cursor->open(spi_result)); - CK (OB_NOT_NULL(spi_result->get_result_set())); - if (OB_SUCC(ret) && OB_INVALID_ID != cursor->get_id()) { - //如果是客户端游标，设置结果集为二进制模式 - OX (spi_result->get_result_set()->set_ps_protocol()); - } - OX (for_update ? cursor->set_for_update() : (void)NULL); - OX (for_update ? cursor->set_trans_id(session_info->get_tx_id()) : (void)NULL); - OX (has_hidden_rowid ? cursor->set_hidden_rowid() : (void)NULL); - OZ (setup_cursor_snapshot_verify_(cursor, spi_result)); - bool need_destruct = false; - if (OB_FAIL(ret) && OB_NOT_NULL(spi_result)) { - int tmp_ret = ret; - ret = OB_SUCCESS; - if (OB_NOT_NULL(spi_result->get_result_set())) { - // 此分支所有错误码都被吞掉，最终返回最初的错误码 - int close_ret = spi_result->close_result_set(); - if (OB_SUCCESS != close_ret) { - LOG_WARN("close mysql result set failed", K(ret), K(close_ret)); - } - } - ret = tmp_ret; - need_destruct = true; - LOG_WARN("cursor open result failed.", K(ret)); - } - if (OB_NOT_NULL(spi_result)) { - spi_result->end_cursor_stmt(ctx, ret); - if (!need_destruct && OB_SUCCESS != ret) { - need_destruct = true; - if (OB_NOT_NULL(spi_result->get_result_set())) { - // 此分支所有错误码都被吞掉，最终返回最初的错误码 - int close_ret = spi_result->close_result_set(); - if (OB_SUCCESS != close_ret) { - LOG_WARN("close mysql result set failed", K(ret), K(close_ret)); - } - } - } - } - if (need_destruct) { - spi_result->~ObSPIResultSet(); - } - } - //监控项统计结束 - time_record.set_exec_end_timestamp(ObTimeUtility::current_time()); - if (enable_perf_event) { - exec_record.record_end(); - } - // 处理监控统计项 - if (OB_NOT_NULL(spi_result) && OB_NOT_NULL(spi_result->get_result_set())) { - if (spi_result->get_result_set()->is_inited()) { - ObSQLSessionInfo *session_info = ctx->exec_ctx_->get_my_session(); - int64_t try_cnt = session_info->get_raw_audit_record().try_cnt_; - ObExecRecord record_bk = session_info->get_raw_audit_record().exec_record_; - session_info->get_raw_audit_record().try_cnt_ = retry_ctrl.get_retry_times(); - session_info->get_raw_audit_record().pl_trace_id_.set(trace_id_guard.origin_trace_id_); - ObInnerSQLConnection::process_record(*spi_result->get_result_set(), - spi_result->get_sql_ctx(), - *session_info, - time_record, - ret, - session_info->get_current_execution_id(), // sql execute id - OB_INVALID_ID, - max_wait_desc, - total_wait_desc, - exec_record, - exec_timestamp, - true, - sql != NULL ? sql : ps_sql, - true); - session_info->get_raw_audit_record().exec_record_ = record_bk; - session_info->get_raw_audit_record().try_cnt_ = try_cnt; - session_info->get_raw_audit_record().pl_trace_id_.reset(); - } - } - } while (RETRY_TYPE_NONE != retry_ctrl.get_retry_type()); - } - cursor->set_last_execute_time(ObTimeUtility::current_time()); - } else { //MySQL Cursor/Updated Cursor/Server Cursor(REF_CURSOR, PACKAGE CURSOR) - ObPLSPITraceIdGuard trace_id_guard(sql, ps_sql, *session_info, ret); - HEAP_VAR(ObSPIResultSet, spi_result) { - ObString sqlstr(sql); - OZ (spi_result.init(*session_info)); - OZ (spi_result.start_nested_stmt_if_need(ctx, sqlstr, static_cast(type), for_update)); - int64_t old_query_start_time = session_info->get_query_start_time(); - // query_start_time_ set to 0 in begin_nested_session, here we reset it. - int64_t new_query_start_time = ObTimeUtility::current_time(); - session_info->set_query_start_time(new_query_start_time); - if (OB_SUCC(ret)) { - ObExecRecord exec_record; - ObExecTimestamp exec_timestamp; - ObSPITimeRecord time_record; - exec_timestamp.exec_type_ = sql::PLSql; - ObQueryRetryCtrl retry_ctrl; - int64_t tenant_version = 0; - int64_t sys_version = 0; - bool is_retry = false; - do { - // SQL_AUDIT_START - ObWaitEventDesc max_wait_desc; - ObWaitEventStat total_wait_desc; - const bool enable_perf_event = lib::is_diagnose_info_enabled(); - const bool enable_sql_audit = - GCONF.enable_sql_audit && ctx->exec_ctx_->get_my_session()->get_local_ob_enable_sql_audit(); - { - ObMaxWaitGuard max_wait_guard(enable_perf_event ? &max_wait_desc : NULL); - ObTotalWaitGuard total_wait_guard(enable_perf_event ? &total_wait_desc : NULL); - if (enable_perf_event) { - exec_record.record_start(); - } - //监控项统计开始 - time_record.set_send_timestamp(ObTimeUtility::current_time()); - ret = OB_SUCCESS; - if (is_retry) { - spi_result.get_out_params().reset(); - spi_result.reset_member_for_retry(*session_info); - } - CK (OB_NOT_NULL(spi_result.get_memory_ctx())); - OX (retry_ctrl.clear_state_before_each_retry(session_info->get_retry_info_for_update())); - OZ ((GCTX.schema_service_->get_tenant_schema_guard(session_info->get_effective_tenant_id(), spi_result.get_scheme_guard()))); - OX (spi_result.get_sql_ctx().schema_guard_ = &spi_result.get_scheme_guard()); - OZ (spi_result.get_scheme_guard().get_schema_version(session_info->get_effective_tenant_id(), tenant_version)); - OZ (spi_result.get_scheme_guard().get_schema_version(OB_SYS_TENANT_ID, sys_version)); - OX (retry_ctrl.set_tenant_local_schema_version(tenant_version)); - OX (retry_ctrl.set_sys_local_schema_version(sys_version)); - - OZ (inner_open(ctx, - spi_result.get_memory_ctx()->get_arena_allocator(), - sql, - ps_sql, - type, - sql_param_exprs, - sql_param_count, - NULL, - 0, - spi_result, - spi_result.get_out_params(), - &retry_ctrl)); - if (OB_SUCC(ret)) { - ObSPICursor* spi_cursor = cursor->get_spi_cursor(); - uint64_t size = 0; - OZ (session_info->get_tmp_table_size(size)); - OZ (cursor->prepare_spi_cursor(spi_cursor, - session_info->get_effective_tenant_id(), - size, - for_update && !is_server_cursor, - session_info), K(size)); - CK (OB_NOT_NULL(cursor->get_allocator())); - CK (OB_NOT_NULL(spi_result.get_result_set())); - OZ (fill_cursor(*spi_result.get_result_set(), spi_cursor, new_query_start_time)); - OZ (ObDbmsInfo::deep_copy_field_columns( - *cursor->get_allocator(), - spi_result.get_result_set()->get_field_columns(), - spi_cursor->fields_)); - if (OB_FAIL(ret) && OB_NOT_NULL(spi_result.get_result_set())) { - int cli_ret = OB_SUCCESS; - retry_ctrl.test_and_save_retry_state(GCTX, - spi_result.get_sql_ctx(), - *spi_result.get_result_set(), - ret, - cli_ret, - true, - true, - true); - LOG_WARN("failed to do fill_cursor, check if need retry", K(ret), K(cli_ret), K(retry_ctrl.need_retry()), K(sql), K(ps_sql)); - ret = cli_ret; - spi_result.get_sql_ctx().clear(); - ctx->exec_ctx_->get_my_session()->set_session_in_retry(retry_ctrl.need_retry()); - } - OX (spi_cursor->row_store_.finish_add_row()) - OX (cursor->open(spi_cursor)); - if (OB_FAIL(ret)) { - spi_cursor->~ObSPICursor(); - } - } - OX (for_update ? cursor->set_for_update() : (void)NULL); - OX (for_update ? cursor->set_trans_id(session_info->get_tx_id()) : (void)NULL); - OX (has_hidden_rowid ? cursor->set_hidden_rowid() : (void)NULL); - - CK (OB_NOT_NULL(spi_result.get_result_set())); - if (OB_SUCC(ret) && lib::is_oracle_mode()) { - OZ (setup_cursor_snapshot_verify_(cursor, &spi_result)); - } - } - //监控项统计结束 - time_record.set_exec_end_timestamp(ObTimeUtility::current_time()); - if (OB_SUCC(ret)) { - LOG_DEBUG("start process record", K(ret), K(ps_sql), K(sql), K(type), K(enable_sql_audit)); - if (spi_result.get_result_set()->is_inited()) { - ObSQLSessionInfo *session_info = ctx->exec_ctx_->get_my_session(); - int64_t try_cnt = session_info->get_raw_audit_record().try_cnt_; - ObExecRecord record_bk = session_info->get_raw_audit_record().exec_record_; - session_info->get_raw_audit_record().try_cnt_ = retry_ctrl.get_retry_times(); - session_info->get_raw_audit_record().pl_trace_id_.set(trace_id_guard.origin_trace_id_); - ObInnerSQLConnection::process_record(*spi_result.get_result_set(), - spi_result.get_sql_ctx(), - *session_info, - time_record, - ret, - session_info->get_current_execution_id(), // sql execute id - OB_INVALID_ID, - max_wait_desc, - total_wait_desc, - exec_record, - exec_timestamp, - true, - sql != NULL ? sql : ps_sql, - true); - session_info->get_raw_audit_record().exec_record_ = record_bk; - session_info->get_raw_audit_record().try_cnt_ = try_cnt; - session_info->get_raw_audit_record().pl_trace_id_.reset(); - } - } - - int close_ret = spi_result.close_result_set(); - if (OB_SUCCESS != close_ret) { - LOG_WARN("close mysql result failed", K(ret), K(close_ret)); - } - ret = (OB_SUCCESS == ret ? close_ret : ret); - is_retry = true; - } while (RETRY_TYPE_NONE != retry_ctrl.get_retry_type()); - } - spi_result.destruct_exec_params(*session_info); - spi_result.end_nested_stmt_if_need(ctx, ret); - session_info->set_query_start_time(old_query_start_time); - } - } - if (OB_SUCC(ret)) { - if (DECL_SUBPROG == loc) { - OZ (ctx->params_->assign(current_params)); - } else if (DECL_PKG == loc) { - OZ (spi_update_package_change_info(ctx, package_id, cursor_index)); - } + is_server_cursor, + for_update, + has_hidden_rowid)); + } else { //MySQL Cursor/Updated Cursor/Server Cursor(REF_CURSOR, PACKAGE CURSOR), unstreaming cursor + OZ (unstreaming_cursor_open(ctx, + *cursor, + *session_info, + sql, + ps_sql, + type, + sql_param_exprs, + sql_param_count, + is_server_cursor, + for_update, + has_hidden_rowid)); + } + if (OB_SUCC(ret)) { + if (DECL_SUBPROG == loc) { + OZ (ctx->params_->assign(current_params)); + } else if (DECL_PKG == loc) { + OZ (spi_update_package_change_info(ctx, package_id, cursor_index)); } } } @@ -4200,325 +3706,58 @@ int ObSPIService::dbms_cursor_open(ObPLExecCtx *ctx, const ObString &ps_sql, int64_t stmt_type, bool for_update, - bool hidden_rowid) + bool hidden_rowid, + int64_t orc_max_ret_rows) { int ret = OB_SUCCESS; ObSQLSessionInfo *session = NULL; ParamStore &exec_params = cursor.get_exec_params(); - ObString &sql_stmt = cursor.get_sql_stmt(); - ObString sql_str = - (exec_params.count() > 0 || cursor.get_into_names().count() > 0) ? ObString() : sql_stmt; + ObString sql_str = (exec_params.count() > 0 || cursor.get_into_names().count() > 0) ? ObString() : cursor.get_sql_stmt(); bool use_stream = false; - ObExecRecord exec_record; - ObExecTimestamp exec_timestamp; - ObSPITimeRecord time_record; - exec_timestamp.exec_type_ = cursor.is_ps_cursor() ? sql::PSCursor : sql::DbmsCursor; - ObWaitEventDesc max_wait_desc; - ObWaitEventStat total_wait_desc; - ObArenaAllocator exec_param_alloc; - ObString exec_param_str; - const bool enable_perf_event = lib::is_diagnose_info_enabled(); - const bool enable_sql_audit = GCONF.enable_sql_audit - && ctx->exec_ctx_->get_my_session()->get_local_ob_enable_sql_audit(); ObPLSubPLSqlTimeGuard guard(ctx); + OV (OB_NOT_NULL(ctx) && OB_NOT_NULL(ctx->exec_ctx_) && OB_NOT_NULL(ctx->allocator_) && OB_NOT_NULL(session = ctx->exec_ctx_->get_my_session()), - OB_INVALID_ARGUMENT, ctx, ps_sql, stmt_type); + OB_INVALID_ARGUMENT, KP(ctx), K(ps_sql), K(stmt_type)); OZ (session->ps_use_stream_result_set(use_stream)); - if (enable_perf_event) { - exec_record.record_start(); - } - //监控项统计开始 - time_record.set_send_timestamp(ObTimeUtility::current_time()); if (OB_SUCC(ret) && cursor.isopen()) { - if(OB_FAIL(dbms_cursor_close(*ctx->exec_ctx_, cursor))) { - LOG_WARN("close cursor fail.", K(ret), K(cursor.get_id())); - } else { - //为cursor分配内存空间 - if (OB_FAIL(ObPLCursorInfo::prepare_entity(*session, cursor.get_cursor_entity()))) { - // 此处只能处理cursor.get_cursor_entity(), 不能处理cursor.get_dbms_entity(),否则exec_params等的值的allocator被reset - // 会导致core - LOG_WARN("failed to alloc ref cursor entity", K(ret)); - } else { - cursor.set_spi_cursor(NULL); - } - } - } - - if (OB_SUCC(ret) && exec_params.count() > 0) { - char *tmp_ptr = NULL; - int64_t tmp_len = 0; - OZ (ObMPStmtExecute::store_params_value_to_str(exec_param_alloc, - *session, - &exec_params, - tmp_ptr, - tmp_len)); - OX (exec_param_str.assign(tmp_ptr, tmp_len)); + OZ (dbms_cursor_close(*ctx->exec_ctx_, cursor), K(cursor.get_id())); + OZ (ObPLCursorInfo::prepare_entity(*session, cursor.get_cursor_entity())); + OX (cursor.set_spi_cursor(NULL)); } if (OB_FAIL(ret)) { - // do nothing - } else if (!for_update && use_stream) { - ObSPIResultSet *spi_result = NULL; - if (stmt::T_SELECT != static_cast(stmt_type)) { - ret = OB_NOT_SUPPORTED; - LOG_WARN("only supported select stmt in cursor", K(ret), K(stmt_type)); - LOG_USER_ERROR(OB_NOT_SUPPORTED, "non-select stmt in cursor"); - } - OX (cursor.set_streaming()); - OV (OB_NOT_NULL(cursor.get_dbms_entity()), OB_NOT_INIT, sql_stmt, ps_sql, exec_params); - OV (OB_NOT_NULL(cursor.get_cursor_entity()), OB_NOT_INIT, sql_stmt, ps_sql, exec_params); - ObQueryRetryCtrl retry_ctrl; - int64_t tenant_version = 0; - int64_t sys_version = 0; - int64_t retry_cnt = 0; - ObPLSPITraceIdGuard trace_id_guard(sql_stmt, ps_sql, *session, ret); - do { - ret = OB_SUCCESS; - retry_ctrl.clear_state_before_each_retry(session->get_retry_info_for_update()); - OZ (cursor.prepare_spi_result(ctx, spi_result), sql_stmt, ps_sql, exec_params); - OV (OB_NOT_NULL(spi_result), OB_ERR_UNEXPECTED, sql_stmt, ps_sql, exec_params); - OZ (spi_result->start_cursor_stmt(ctx, static_cast(stmt_type)), - sql_stmt, ps_sql, exec_params); - OZ ((GCTX.schema_service_->get_tenant_schema_guard(session->get_effective_tenant_id(), spi_result->get_scheme_guard()))); - OZ (spi_result->get_scheme_guard().get_schema_version(session->get_effective_tenant_id(), tenant_version)); - OZ (spi_result->get_scheme_guard().get_schema_version(OB_SYS_TENANT_ID, sys_version)); - OX (retry_ctrl.set_tenant_local_schema_version(tenant_version)); - OX (retry_ctrl.set_sys_local_schema_version(sys_version)); - OX (spi_result->get_sql_ctx().schema_guard_ = &spi_result->get_scheme_guard()); - if (OB_SUCC(ret)) { - WITH_CONTEXT(cursor.get_cursor_entity()) { - lib::ContextTLOptGuard guard(false); - if (OB_FAIL(inner_open(ctx, sql_str, ps_sql, stmt_type, exec_params, - *spi_result, spi_result->get_out_params()))) { - if (spi_result->get_result_set() != NULL) { - int cli_ret = OB_SUCCESS; - retry_ctrl.test_and_save_retry_state(GCTX, - spi_result->get_sql_ctx(), - *spi_result->get_result_set(), - ret, cli_ret, true, true, true); - LOG_WARN("fail to open, check if need retry", K(ret), K(cli_ret), - K(retry_ctrl.need_retry()), K(sql_str), K(ps_sql), K(exec_params)); - if (!cursor.is_ps_cursor()) { - ret = cli_ret; - spi_result->get_sql_ctx().clear(); - ctx->exec_ctx_->get_my_session()->set_session_in_retry(retry_ctrl.need_retry()); - } - } - } - } - } - OV (OB_NOT_NULL(spi_result->get_result_set()), - OB_ERR_UNEXPECTED, sql_stmt, ps_sql, exec_params); - CK (OB_NOT_NULL(spi_result->get_result_set()->get_field_columns())); - if (OB_SUCC(ret) && cursor.get_field_columns().empty()) { - const common::ColumnsFieldArray* field_column = - dynamic_cast - (spi_result->get_result_set()->get_field_columns()); - OX (cursor.get_field_columns().set_allocator(&cursor.get_dbms_entity()->get_arena_allocator())); - OZ (cursor.get_field_columns().assign(*field_column)); - } - OX (spi_result->get_result_set()->set_ps_protocol()); - if (OB_SUCCESS != ret && OB_NOT_NULL(spi_result)) { - int tmp_ret = ret; - ret = OB_SUCCESS; - if (OB_NOT_NULL(spi_result->get_result_set())) { - // 此分支所有错误码都被吞掉，最终返回最初的错误码 - int close_ret = spi_result->close_result_set(); - if (OB_SUCCESS != close_ret) { - LOG_WARN("close mysql result set failed", K(ret), K(close_ret)); - } - } - ret = tmp_ret; - spi_result->~ObSPIResultSet(); - LOG_WARN("cursor open result failed.", K(ret), K(sql_stmt), K(ps_sql), K(exec_params)); - } - OX (cursor.open(spi_result)); - if (OB_NOT_NULL(spi_result)) { - // no OX - spi_result->end_cursor_stmt(ctx, ret); - } - OZ (setup_cursor_snapshot_verify_(&cursor, spi_result)); - LOG_DEBUG("start process record", K(ret), K(ps_sql), K(sql_str), K(enable_sql_audit)); - //监控项统计结束 - time_record.set_exec_end_timestamp(ObTimeUtility::current_time()); - if (enable_perf_event) { - exec_record.record_end(); - } - - if (OB_NOT_NULL(spi_result) && - OB_NOT_NULL(spi_result->get_result_set()) && - spi_result->get_result_set()->is_inited()) { - ObSQLSessionInfo *session_info = ctx->exec_ctx_->get_my_session(); - int64_t try_cnt = session_info->get_raw_audit_record().try_cnt_; - ObExecRecord record_bk = session_info->get_raw_audit_record().exec_record_; - session_info->get_raw_audit_record().pl_trace_id_.set(trace_id_guard.origin_trace_id_); - // 会在inner_open的时候被改成了 inner ，所以这个地方需要重新设置一下 - exec_timestamp.exec_type_ = cursor.is_ps_cursor() ? sql::PSCursor : sql::DbmsCursor; - ObInnerSQLConnection::process_record(*spi_result->get_result_set(), - spi_result->get_sql_ctx(), - *session_info, - time_record, - ret, - session_info->get_current_execution_id(), - cursor.is_ps_cursor() ? cursor.get_id() : OB_INVALID_ID, - max_wait_desc, - total_wait_desc, - exec_record, - exec_timestamp, - true, - (exec_params.count() > 0 || cursor.is_ps_cursor()) ? ps_sql : sql_str, - true, - &exec_param_str); - session_info->get_raw_audit_record().exec_record_ = record_bk; - session_info->get_raw_audit_record().try_cnt_ = try_cnt; - session_info->get_raw_audit_record().pl_trace_id_.reset(); - } - } while (RETRY_TYPE_NONE != retry_ctrl.get_retry_type() && !cursor.is_ps_cursor()); - } else { - SMART_VAR(ObSPIResultSet, spi_result) { - ObSPICursor *spi_cursor = NULL; - uint64_t size = 0; - OZ (session->get_tmp_table_size(size)); - OZ (spi_result.init(*ctx->exec_ctx_->get_my_session())); - OZ (spi_result.start_nested_stmt_if_need(ctx, sql_stmt, static_cast(stmt_type), for_update), - sql_stmt, ps_sql, exec_params); - - if (OB_SUCC(ret)) { - ObQueryRetryCtrl retry_ctrl; - int64_t tenant_version = 0; - int64_t sys_version = 0; - int64_t retry_cnt = 0; - ObPLSPITraceIdGuard trace_id_guard(sql_stmt, ps_sql, *session, ret); - int64_t old_query_start_time = session->get_query_start_time(); - int64_t new_query_start_time = ObTimeUtility::current_time(); - if (!cursor.is_ps_cursor()) { - session->set_query_start_time(new_query_start_time); - } - do { - ret = OB_SUCCESS; - if (retry_cnt > 0) { - spi_result.get_out_params().reset(); - spi_result.reset_member_for_retry(*session); - } - retry_ctrl.clear_state_before_each_retry(session->get_retry_info_for_update()); - OZ (cursor.prepare_spi_cursor(spi_cursor, - session->get_effective_tenant_id(), - size, - false, - session)); - OZ (GCTX.schema_service_->get_tenant_schema_guard(session->get_effective_tenant_id(), spi_result.get_scheme_guard())); - OZ (spi_result.get_scheme_guard().get_schema_version(session->get_effective_tenant_id(), tenant_version)); - OZ (spi_result.get_scheme_guard().get_schema_version(OB_SYS_TENANT_ID, sys_version)); - OX (retry_ctrl.set_tenant_local_schema_version(tenant_version)); - OX (retry_ctrl.set_sys_local_schema_version(sys_version)); - OX (spi_result.get_sql_ctx().schema_guard_ = &spi_result.get_scheme_guard()); - if (OB_SUCC(ret)) { - OZ (inner_open(ctx, sql_str, ps_sql, stmt_type, exec_params, - spi_result, spi_result.get_out_params()), - sql_stmt, ps_sql, exec_params); - OZ (ObDbmsInfo::deep_copy_field_columns( - cursor.get_dbms_entity()->get_arena_allocator(), - spi_result.get_result_set()->get_field_columns(), - cursor.get_field_columns())); - if (OB_SUCC(ret)) { - if (OB_NOT_NULL(spi_result.get_result_set()) && OB_NOT_NULL(spi_result.get_result_set()->get_physical_plan())) { - ObPhysicalPlan *plan = spi_result.get_result_set()->get_physical_plan(); - cursor.set_packed(plan->is_packed()); - } - } - OZ (fill_cursor(*spi_result.get_result_set(), spi_cursor, new_query_start_time)); - if (OB_FAIL(ret) && OB_NOT_NULL(spi_result.get_result_set())) { - int cli_ret = OB_SUCCESS; - retry_ctrl.test_and_save_retry_state(GCTX, - spi_result.get_sql_ctx(), - *spi_result.get_result_set(), - ret, - cli_ret, - true, - true, - true); - LOG_WARN("failed to fill_cursor, check if need retry", - K(ret), K(cli_ret), K(retry_ctrl.need_retry()), - K(sql_stmt), K(ps_sql), K(exec_params)); - if (!cursor.is_ps_cursor()) { - ret = cli_ret; - spi_result.get_sql_ctx().clear(); - ctx->exec_ctx_->get_my_session()->set_session_in_retry(retry_ctrl.need_retry()); - } - } - } - OX (spi_cursor->row_store_.finish_add_row()); - LOG_DEBUG("start process record", K(ret), K(ps_sql), K(sql_str), K(enable_sql_audit)); - //监控项统计结束 - time_record.set_exec_end_timestamp(ObTimeUtility::current_time()); - if (enable_perf_event) { - exec_record.record_end(); - } - ObResultSet* result_set = spi_result.get_result_set(); - if (OB_NOT_NULL(result_set) && result_set->is_inited()) { - ObSQLSessionInfo *session_info = ctx->exec_ctx_->get_my_session(); - int64_t try_cnt = session_info->get_raw_audit_record().try_cnt_; - ObExecRecord record_bk = session_info->get_raw_audit_record().exec_record_; - session_info->get_raw_audit_record().try_cnt_ = retry_ctrl.get_retry_times(); - session_info->get_raw_audit_record().pl_trace_id_.set(trace_id_guard.origin_trace_id_); - // 会在inner_open的时候被改成了 inner ，所以这个地方需要重新设置一下 - exec_timestamp.exec_type_ = cursor.is_ps_cursor() ? sql::PSCursor : sql::DbmsCursor; - ObInnerSQLConnection::process_record(*result_set, - spi_result.get_sql_ctx(), - *session_info, - time_record, - ret, - session_info->get_current_execution_id(), - cursor.is_ps_cursor() ? cursor.get_id() : OB_INVALID_ID, - max_wait_desc, - total_wait_desc, - exec_record, - exec_timestamp, - true, - (exec_params.count() > 0 || cursor.is_ps_cursor()) ? ps_sql : sql_str, - true, - &exec_param_str); - session_info->get_raw_audit_record().exec_record_ = record_bk; - session_info->get_raw_audit_record().try_cnt_ = retry_cnt; - session_info->get_raw_audit_record().pl_trace_id_.reset(); - } - int close_ret = spi_result.close_result_set(); - if (OB_SUCCESS != close_ret) { - LOG_WARN("close mysql result failed", K(ret), K(close_ret)); - } - ret = OB_SUCCESS == ret ? close_ret : ret; - if (OB_SUCCESS != ret && OB_NOT_NULL(spi_cursor)) { - spi_cursor->~ObSPICursor(); - LOG_WARN("fill cursor failed.", K(ret), K(cursor.get_id()), K(sql_stmt), K(ps_sql), K(session->get_sessid())); - } - - retry_cnt++; - } while (!cursor.is_ps_cursor() && RETRY_TYPE_NONE != retry_ctrl.get_retry_type()); - if (!cursor.is_ps_cursor()) { - session->set_query_start_time(old_query_start_time); - } - } - - if (OB_SUCC(ret)) { - cursor.open(spi_cursor); - if (for_update) { - OX (cursor.set_for_update()); - OX (cursor.set_trans_id(ctx->exec_ctx_->get_my_session()->get_tx_id())); - } - if (hidden_rowid) { - OX (cursor.set_hidden_rowid()); - } - if (OB_SUCC(ret) && lib::is_oracle_mode()) { - OZ (setup_cursor_snapshot_verify_(&cursor, &spi_result)); - } - } - spi_result.end_nested_stmt_if_need(ctx, ret); - } + } else if (!for_update && use_stream) { // streaming branch + OZ (streaming_cursor_open(ctx, + cursor, + *session, + sql_str, + ps_sql, + stmt_type, + &exec_params, + exec_params.count(), + true, /*is_server_cursor*/ + for_update, + hidden_rowid, + true /*is_dbms_cursor*/)); + } else { // unstreaming branch + OZ (unstreaming_cursor_open(ctx, + cursor, + *session, + sql_str, + ps_sql, + stmt_type, + &exec_params, + exec_params.count(), + true, /*is_server_cursor*/ + for_update, + hidden_rowid, + true, /*is_dbms_cursor*/ + orc_max_ret_rows)); } SET_SPI_STATUS; return ret; @@ -4540,22 +3779,14 @@ int ObSPIService::do_cursor_fetch(ObPLExecCtx *ctx, bool is_type_record) { int ret = OB_SUCCESS; + ObSPIResultSet *spi_result = NULL; - ObSQLSessionInfo *session = ctx->exec_ctx_->get_my_session(); - ObExecRecord exec_record; - ObExecTimestamp exec_timestamp; - ObSPITimeRecord time_record; - exec_timestamp.exec_type_ = sql::CursorFetch; - ObWaitEventDesc max_wait_desc; - ObWaitEventStat total_wait_desc; - const bool enable_perf_event = lib::is_diagnose_info_enabled(); - const bool enable_sql_audit = GCONF.enable_sql_audit - && ctx->exec_ctx_->get_my_session()->get_local_ob_enable_sql_audit(); + ObSQLSessionInfo *session = NULL; + CK (OB_NOT_NULL(ctx)); CK (OB_NOT_NULL(ctx->exec_ctx_)); CK (OB_NOT_NULL(session = ctx->exec_ctx_->get_my_session())); CK (OB_NOT_NULL(cursor)); - ObPLSubPLSqlTimeGuard guard(ctx); // check cursor is valid for fetch if (OB_SUCC(ret) @@ -4572,13 +3803,13 @@ int ObSPIService::do_cursor_fetch(ObPLExecCtx *ctx, can't fetch if transaction committed, because snapshot data has been removed */ if (!cursor->get_snapshot().is_valid()) { - ret = OB_ERR_FETCH_OUT_SEQUENCE; - LOG_WARN("snapshot is invalid", K(cursor->get_snapshot()), K(ret)); + ret = OB_ERR_FETCH_OUT_SEQUENCE; + LOG_WARN("snapshot is invalid", K(cursor->get_snapshot()), K(ret)); } else if (cursor->is_for_update()) { if (cursor->get_snapshot().is_committed()) { - ret = OB_ERR_FETCH_OUT_SEQUENCE; - LOG_WARN("transaction has been committed, for update cursor can not fetch", - K(cursor->get_snapshot()), K(ret)); + ret = OB_ERR_FETCH_OUT_SEQUENCE; + LOG_WARN("transaction has been committed, for update cursor can not fetch", + K(cursor->get_snapshot()), K(ret)); } } else if (cursor->is_streaming()) { if(cursor->get_snapshot().is_committed()) { @@ -4588,11 +3819,6 @@ int ObSPIService::do_cursor_fetch(ObPLExecCtx *ctx, } } } - if (enable_perf_event) { - exec_record.record_start(); - } - //监控项统计开始 - time_record.set_send_timestamp(ObTimeUtility::current_time()); if (OB_FAIL(ret)) { } else if (!is_bulk && INT64_MAX != limit) { //limit子句必须和Bulk Collect合用 ret = OB_INVALID_ARGUMENT; @@ -4605,36 +3831,36 @@ int ObSPIService::do_cursor_fetch(ObPLExecCtx *ctx, LOG_USER_ERROR(OB_ER_SP_CURSOR_NOT_OPEN); LOG_WARN("Cursor is not open", K(cursor), K(ret)); } - if (OB_SUCC(ret)) { - if (cursor->is_streaming()) { - CK (OB_NOT_NULL(spi_result = cursor->get_cursor_handler())); - OZ (spi_result->set_cursor_env(*ctx->exec_ctx_->get_my_session())); - OZ (adjust_out_params(ctx, into_exprs, into_count, spi_result->get_out_params())); - } else if (OB_NOT_NULL(cursor->get_spi_cursor()) - && cursor->get_spi_cursor()->row_store_.get_row_cnt() > 0 - && cursor->get_current_row().is_invalid()) { //有数据才需要做，避免重复做 - //仅缓存在ObRowStore的Cursor需要初始化ObNewRow结构 - CK (OB_NOT_NULL(cursor->get_spi_cursor())); - int64_t column_count = cursor->get_spi_cursor()->row_desc_.count(); - ObIAllocator *spi_allocator = NULL; - OX (spi_allocator = NULL == cursor->get_cursor_entity() - ? cursor->get_allocator() - : &cursor->get_cursor_entity()->get_arena_allocator()); - CK (OB_NOT_NULL(spi_allocator)); - if (OB_SUCC(ret)) { - void *ptr = spi_allocator->alloc(column_count * sizeof(ObObj)); - if (OB_ISNULL(ptr)) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("alloc memory for row failed", "size", column_count * sizeof(ObObj)); - } else { - cursor->get_current_row().cells_ = new(ptr) common::ObObj[column_count]; - cursor->get_current_row().count_ = column_count; - } + + if (OB_FAIL(ret)) { + } else if (cursor->is_streaming()) { + CK (OB_NOT_NULL(spi_result = cursor->get_cursor_handler())); + OZ (spi_result->set_cursor_env(*ctx->exec_ctx_->get_my_session())); + OZ (adjust_out_params(ctx, into_exprs, into_count, spi_result->get_out_params())); + } else if (OB_NOT_NULL(cursor->get_spi_cursor()) + && cursor->get_spi_cursor()->row_store_.get_row_cnt() > 0 + && cursor->get_current_row().is_invalid()) { //有数据才需要做，避免重复做 + //仅缓存在ObRowStore的Cursor需要初始化ObNewRow结构 + CK (OB_NOT_NULL(cursor->get_spi_cursor())); + int64_t column_count = cursor->get_spi_cursor()->row_desc_.count(); + ObIAllocator *spi_allocator = NULL; + OX (spi_allocator = NULL == cursor->get_cursor_entity() + ? cursor->get_allocator() + : &cursor->get_cursor_entity()->get_arena_allocator()); + CK (OB_NOT_NULL(spi_allocator)); + if (OB_SUCC(ret)) { + void *ptr = spi_allocator->alloc(column_count * sizeof(ObObj)); + if (OB_ISNULL(ptr)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("alloc memory for row failed", "size", column_count * sizeof(ObObj)); + } else { + cursor->get_current_row().cells_ = new(ptr) common::ObObj[column_count]; + cursor->get_current_row().count_ = column_count; } } } + if (OB_SUCC(ret)) { - ObCurTraceId::TraceId pl_trace_id; int64_t row_count = 0; #define GET_RESULT \ @@ -4672,13 +3898,16 @@ int ObSPIService::do_cursor_fetch(ObPLExecCtx *ctx, } else { \ sql = spi_result->get_sql_ctx().cur_sql_; \ } \ - ObPLSPITraceIdGuard trace_id_guard( \ - sql, ps_sql, *session, ret, cursor->get_sql_trace_id()); \ + ObPLSqlAuditRecord audit_record(sql::CursorFetch); \ + ObQueryRetryCtrl retry_ctrl; \ + ObPLSPITraceIdGuard trace_id_guard(sql, ps_sql, *session, ret, cursor->get_sql_trace_id()); \ + ObPLSubPLSqlTimeGuard guard(ctx); \ + ObPLSqlAuditGuard audit_guard( \ + *(ctx->exec_ctx_), *session, *spi_result, audit_record, ret, ps_sql, retry_ctrl, trace_id_guard, spi_result->get_sql_ctx().stmt_type_); \ if (cursor->get_sql_trace_id()->is_invalid() \ && OB_NOT_NULL(ObCurTraceId::get_trace_id())) { \ cursor->get_sql_trace_id()->set(*ObCurTraceId::get_trace_id()); \ } \ - pl_trace_id = trace_id_guard.origin_trace_id_; \ ret = inner_fetch_with_retry(ctx, \ *cursor->get_cursor_handler(), \ into_exprs, \ @@ -4701,11 +3930,8 @@ int ObSPIService::do_cursor_fetch(ObPLExecCtx *ctx, } while(0) if (is_server_cursor) { -// FIXME: (yunxing.cyx) comment in streaming disabled, because report 4002, should be revert after enable streaming -// WITH_CONTEXT(cursor->get_cursor_entity()) { - lib::ContextTLOptGuard guard(false); - GET_RESULT; -// } + lib::ContextTLOptGuard guard(false); + GET_RESULT; } else { GET_RESULT; } @@ -4713,49 +3939,10 @@ int ObSPIService::do_cursor_fetch(ObPLExecCtx *ctx, #undef GET_RESULT if (cursor->is_streaming()) { - ObPLSPITraceIdGuard trace_id_guard(nullptr, nullptr, *session, ret, cursor->get_sql_trace_id()); - if (OB_ISNULL(spi_result)) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("spi result must be not null in oracle mode", K(ret), K(spi_result)); - } else if (pl_trace_id.is_invalid() || OB_ISNULL(cursor->get_sql_trace_id()) || cursor->get_sql_trace_id()->is_invalid()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected invalid trace_id", K(ret), K(pl_trace_id), K(trace_id_guard.origin_trace_id_)); - } else { - LOG_DEBUG("start process record", K(ret), K(cursor->get_id()), K(enable_sql_audit)); - //监控项统计结束 - time_record.set_exec_end_timestamp(ObTimeUtility::current_time()); - if (enable_perf_event) { - exec_record.record_end(); - } - ObResultSet* result_set = spi_result->get_result_set(); - if (OB_NOT_NULL(result_set) && result_set->is_inited()) { - ObSQLSessionInfo *session_info = ctx->exec_ctx_->get_my_session(); - int64_t try_cnt = session_info->get_raw_audit_record().try_cnt_; - ObExecRecord record_bk = session_info->get_raw_audit_record().exec_record_; - session_info->get_raw_audit_record().pl_trace_id_.set(pl_trace_id); - ObInnerSQLConnection::process_record(*(result_set), - spi_result->get_sql_ctx(), - *session_info, - time_record, - ret, - session_info->get_current_execution_id(), - cursor->get_id(), // ps stmt id - max_wait_desc, - total_wait_desc, - exec_record, - exec_timestamp, - true, - ObString(), - true, - spi_result->get_exec_params_str_ptr()); - session_info->get_raw_audit_record().exec_record_ = record_bk; - session_info->get_raw_audit_record().try_cnt_ = try_cnt; - session_info->get_raw_audit_record().pl_trace_id_.reset(); - } - spi_result->end_cursor_stmt(ctx, ret); - cursor->set_last_execute_time(ObTimeUtility::current_time()); - } + spi_result->end_cursor_stmt(ctx, ret); + cursor->set_last_execute_time(ObTimeUtility::current_time()); } + // Oracle模式的Cursor会吞掉READ_NOTHING的错误, 为了避免无效日志过多, 只在Mysql模式下打印WARN if (OB_SUCC(ret) || (OB_READ_NOTHING == ret && lib::is_oracle_mode())) { @@ -4770,22 +3957,25 @@ int ObSPIService::do_cursor_fetch(ObPLExecCtx *ctx, return ret; } -int ObSPIService::dbms_cursor_fetch(ObPLExecCtx *ctx, - ObDbmsCursorInfo &cursor, - bool is_server_cursor) +int ObSPIService::dbms_cursor_fetch( + ObPLExecCtx *ctx, ObDbmsCursorInfo &cursor, bool is_server_cursor) { int ret = OB_SUCCESS; - OZ (do_cursor_fetch(ctx, - &cursor, - is_server_cursor,/*ps is server cursor*/ - NULL, - 0, - NULL, - 0, - NULL, - NULL, - false, - INT64_MAX)); + if (OB_FAIL(do_cursor_fetch(ctx, + &cursor, + is_server_cursor,/*ps is server cursor*/ + NULL, + 0, + NULL, + 0, + NULL, + NULL, + false, + INT64_MAX))) { + if (ret != OB_READ_NOTHING) { + LOG_WARN("failed to do cursor fetch", K(ret), K(is_server_cursor), K(cursor)); + } + } return ret; } @@ -4909,10 +4099,7 @@ int ObSPIService::spi_cursor_close(ObPLExecCtx *ctx, return ret; } -// 只 close dbms_cursor 中 cursor 相关内容 -// dbms_cursor 中的 param 信息不做处理 -int ObSPIService::dbms_cursor_close(ObExecContext &exec_ctx, - ObPLCursorInfo &cursor) +int ObSPIService::dbms_cursor_close(ObExecContext &exec_ctx, ObPLCursorInfo &cursor) { int ret = OB_SUCCESS; // dbms cursor与pl cursor不同： @@ -6875,64 +6062,83 @@ int ObSPIService::process_function_out_result(ObPLExecCtx *ctx, return ret; } -int ObSPIService::inner_open(ObPLExecCtx *ctx, - ObIAllocator ¶m_allocator, //用于拷贝执行期参数 - const char *sql, - const char *ps_sql, - int64_t type, - const ObSqlExpression **param_exprs, - int64_t param_count, - const ObSqlExpression **into_exprs, - int64_t into_count, - ObSPIResultSet &spi_result, - ObSPIOutParams &out_params, - observer::ObQueryRetryCtrl *retry_ctrl, - bool is_forall) +int ObSPIService::store_params_string( + ObPLExecCtx *ctx, ObSPIResultSet &spi_result, ParamStore *exec_params) { int ret = OB_SUCCESS; - int64_t query_num = 0; + if (OB_SUCC(ret) + && OB_NOT_NULL(ctx) + && OB_NOT_NULL(ctx->exec_ctx_) + && OB_NOT_NULL(ctx->exec_ctx_->get_my_session())) { + char *tmp_ptr = NULL; + int64_t tmp_len = 0; + OZ (ObMPStmtExecute::store_params_value_to_str(spi_result.get_memory_ctx()->get_arena_allocator(), + *ctx->exec_ctx_->get_my_session(), + exec_params, + tmp_ptr, + tmp_len)); + OX (spi_result.get_exec_params_str_ptr()->assign(tmp_ptr, tmp_len)); + } + return ret; +} + +int ObSPIService::prepare_static_sql_params(ObPLExecCtx *ctx, + ObIAllocator ¶m_allocator, //用于拷贝执行期参数 + const ObString &sql, + const ObString &ps_sql, + int64_t type, + const ObSqlExpression **params, + int64_t param_count, + const ObSqlExpression **into_exprs, + int64_t into_count, + ObSPIResultSet &spi_result, + ObSPIOutParams &out_params, + bool is_forall, + ParamStore *&curr_params) +{ + int ret = OB_SUCCESS; + int64_t array_binding_count = 0; - ParamStore exec_params( (ObWrapperAllocator(param_allocator)) ); - ParamStore *curr_params = &exec_params; + ParamStore *exec_params = NULL; ParamStore *batch_params = NULL; + const ObSqlExpression **param_exprs = reinterpret_cast(params); + + curr_params = NULL; + + if (OB_ISNULL(exec_params = reinterpret_cast(param_allocator.alloc(sizeof(ParamStore))))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to alloc memory for exec params", K(ret)); + } + OX (new (exec_params) ParamStore( (ObWrapperAllocator(param_allocator)) ) ); + OX (curr_params = exec_params); if (NULL == sql) { OZ (construct_exec_params(ctx, param_allocator, param_exprs, param_count, - into_exprs, into_count, exec_params, out_params, is_forall), - K(sql), K(ps_sql), K(type), K(param_count), K(out_params), K(exec_params)); - if (OB_SUCC(ret) - && OB_NOT_NULL(ctx) - && OB_NOT_NULL(ctx->exec_ctx_) - && OB_NOT_NULL(ctx->exec_ctx_->get_my_session())) { - // add exec_param_info for sql_audit - char *tmp_ptr = NULL; - int64_t tmp_len = 0; - OZ (ObMPStmtExecute::store_params_value_to_str(spi_result.get_memory_ctx()->get_arena_allocator(), - *ctx->exec_ctx_->get_my_session(), - &exec_params, - tmp_ptr, - tmp_len)); - OX (spi_result.get_exec_params_str_ptr()->assign(tmp_ptr, tmp_len)); - } + into_exprs, into_count, *exec_params, out_params, is_forall), + K(sql), K(ps_sql), K(type), K(param_count), K(out_params), KPC(exec_params)); + + OZ (store_params_string(ctx, spi_result, exec_params)); } - if (OB_SUCC(ret) && is_forall) { - for (int64_t i = 0; OB_SUCC(ret) && i < exec_params.count(); ++i) { - if (exec_params.at(i).is_ext()) { + if (OB_SUCC(ret) && is_forall && OB_NOT_NULL(exec_params)) { + + for (int64_t i = 0; OB_SUCC(ret) && i < exec_params->count(); ++i) { + if (exec_params->at(i).is_ext()) { pl::ObPLCollection *coll = NULL; - CK (OB_NOT_NULL(coll = reinterpret_cast(exec_params.at(i).get_ext()))); + CK (OB_NOT_NULL(coll = reinterpret_cast(exec_params->at(i).get_ext()))); if (OB_SUCC(ret)) { array_binding_count = coll->get_actual_count(); break; } } } + if (OB_SUCC(ret)) { if (array_binding_count <= 0) { ret = OB_ERR_UNEXPECTED; LOG_WARN("array_binding_count is wrong", K(array_binding_count), K(ret)); } else if (OB_FAIL(ObSQLUtils::transform_pl_ext_type( - exec_params, array_binding_count, param_allocator, batch_params, true))) { + *exec_params, array_binding_count, param_allocator, batch_params, true))) { LOG_WARN("transform failed", K(ret)); } else if (OB_ISNULL(batch_params)) { ret = OB_ERR_UNEXPECTED; @@ -6944,36 +6150,64 @@ int ObSPIService::inner_open(ObPLExecCtx *ctx, } } } - if (OB_FAIL(ret)) { - } else if (OB_FAIL(inner_open(ctx, sql, ps_sql, type, - *curr_params, spi_result, - out_params))) { - if (retry_ctrl != nullptr/*can_retry*/) { - int cli_ret = OB_SUCCESS; - retry_ctrl->test_and_save_retry_state( - GCTX, - spi_result.get_sql_ctx(), - *spi_result.get_result_set(), - ret, cli_ret, true, true, true); - LOG_WARN("failed to get_result, check if need retry", - K(ret), K(cli_ret), K(retry_ctrl->need_retry()), K(sql), K(ps_sql), K(type)); - ret = cli_ret; - spi_result.get_sql_ctx().clear(); - ctx->exec_ctx_->get_my_session()->set_session_in_retry(retry_ctrl->need_retry()); + + return ret; +} + +int ObSPIService::inner_open(ObPLExecCtx *ctx, + ObIAllocator ¶m_allocator, + const ObString &sql, + const ObString &ps_sql, + int64_t type, + void *params, + int64_t param_count, + const ObSqlExpression **into_exprs, + int64_t into_count, + ObSPIResultSet &spi_result, + ObSPIOutParams &out_params, + bool is_forall, + bool is_dynamic_sql, + bool is_dbms_sql) +{ + int ret = OB_SUCCESS; + + ParamStore *curr_params = NULL; + + if (is_dynamic_sql) { + ObObjParam **dynamic_params = reinterpret_cast(params); + OZ (prepare_dynamic_sql_params(ctx, spi_result, param_allocator, param_count, dynamic_params, curr_params)); + } else if (is_dbms_sql) { + OX (curr_params = reinterpret_cast(params)); + OZ (store_params_string(ctx, spi_result, curr_params)); + } else { + const ObSqlExpression **static_params = reinterpret_cast(params); + OZ (prepare_static_sql_params(ctx, + param_allocator, + sql, + ps_sql, + type, + static_params, + param_count, + into_exprs, + into_count, + spi_result, + out_params, + is_forall, + curr_params)); + } + + CK (OB_NOT_NULL(curr_params)); + OZ (inner_open(ctx, sql, ps_sql, type, *curr_params, spi_result, out_params, is_dynamic_sql)); + + // if failed, we need release complex parameter memory in here + if (OB_FAIL(ret) && OB_NOT_NULL(curr_params) && !is_dbms_sql) { + int ret = OB_SUCCESS; // ignore destruct obj error + for (int64_t i = 0; i < curr_params->count(); ++i) { + OZ (ObUserDefinedType::destruct_obj(curr_params->at(i), ctx->exec_ctx_->get_my_session())); + ret = OB_SUCCESS; } } - // if failed, we need release complex parameter memory in here - if (OB_FAIL(ret) - && NULL == sql - && OB_NOT_NULL(ctx) - && OB_NOT_NULL(ctx->exec_ctx_) - && OB_NOT_NULL(ctx->exec_ctx_->get_my_session())) { - int ret = OB_SUCCESS; // ignore destruct obj error - for (int64_t i = 0; OB_SUCC(ret) && i < exec_params.count(); ++i) { - OZ (ObUserDefinedType::destruct_obj(exec_params.at(i), ctx->exec_ctx_->get_my_session())); - } - } return ret; } @@ -6983,12 +6217,13 @@ int ObSPIService::inner_open(ObPLExecCtx *ctx, int64_t type, ParamStore &exec_params, ObSPIResultSet &spi_result, - ObSPIOutParams &out_params) + ObSPIOutParams &out_params, + bool is_dynamic_sql) { int ret = OB_SUCCESS; - // unconditional adjustment OZ (adjust_out_params(*spi_result.get_result_set(), out_params)); + if (OB_ISNULL(ctx) || OB_ISNULL(ctx->exec_ctx_) || (NULL == ctx->allocator_)) { @@ -7016,19 +6251,24 @@ int ObSPIService::inner_open(ObPLExecCtx *ctx, } if (OB_SUCC(ret)) { WITH_CONTEXT(spi_result.get_memory_ctx()) { - if (NULL != sql.ptr()) { + if (exec_params.count() <= 0 && !sql.empty()) { spi_result.get_result_set()->set_user_sql(true); - if (OB_FAIL(GCTX.sql_engine_->handle_pl_execute( - sql, *session, exec_params, *spi_result.get_result_set(), spi_result.get_sql_ctx(), - false /* is_prepare_protocol */, false /* is_dynamic_sql*/))) { - LOG_WARN("query failed", K(ret), K(sql)); - } + OZ (GCTX.sql_engine_->handle_pl_execute(sql, + *session, + exec_params, + *spi_result.get_result_set(), + spi_result.get_sql_ctx(), + false /* is_prepare_protocol */, + false /* is_dynamic_sql*/), K(sql), K(ps_sql), K(exec_params)); } else { spi_result.get_result_set()->set_stmt_type(static_cast(type)); - OZ (GCTX.sql_engine_->handle_pl_execute( - ps_sql, *session, exec_params, *spi_result.get_result_set(), spi_result.get_sql_ctx(), - true /* is_prepare_protocol */, false /* is_dynamic_sql */), - K(ps_sql), K(exec_params)); + OZ (GCTX.sql_engine_->handle_pl_execute(ps_sql, + *session, + exec_params, + *spi_result.get_result_set(), + spi_result.get_sql_ctx(), + true /* is_prepare_protocol */, + is_dynamic_sql /* is_dynamic_sql */), K(sql), K(ps_sql), K(exec_params)); OZ (adjust_out_params(*spi_result.get_result_set(), out_params)); } } @@ -7043,7 +6283,7 @@ int ObSPIService::inner_open(ObPLExecCtx *ctx, } int ObSPIService::inner_fetch(ObPLExecCtx *ctx, - ObQueryRetryCtrl &retry_ctrl, + bool &can_retry, ObSPIResultSet &spi_result, const ObSqlExpression **into_exprs, int64_t into_count, @@ -7072,7 +6312,6 @@ int ObSPIService::inner_fetch(ObPLExecCtx *ctx, CK (OB_NOT_NULL(result_set)); if (OB_SUCC(ret)) { ObNewRow dummy_row; - bool can_retry = true; current_row = NULL != current_row ? current_row : &dummy_row; WITH_CONTEXT(spi_result.get_memory_ctx()) { if (OB_FAIL((get_result(ctx, @@ -7097,22 +6336,11 @@ int ObSPIService::inner_fetch(ObPLExecCtx *ctx, return_types, return_type_count, is_type_record)))) { - if (can_retry) { - int cli_ret = OB_SUCCESS; - retry_ctrl.test_and_save_retry_state( - GCTX, - spi_result.get_sql_ctx(), - *result_set, ret, cli_ret, true, true, true); - if (!for_cursor || (for_cursor && ret != OB_READ_NOTHING)) { - LOG_WARN("failed to get_result, check if need retry", - K(ret), K(cli_ret), K(retry_ctrl.need_retry())); - } - ret = cli_ret; + if (!for_cursor || (for_cursor && ret != OB_READ_NOTHING)) { + LOG_WARN("failed to get_result, check if need retry", K(ret)); } } } - spi_result.get_sql_ctx().clear(); - ctx->exec_ctx_->get_my_session()->set_session_in_retry(retry_ctrl.need_retry()); } return ret; } @@ -7138,16 +6366,11 @@ int ObSPIService::inner_fetch_with_retry(ObPLExecCtx *ctx, { int ret = OB_SUCCESS; ObQueryRetryCtrl retry_ctrl; - int64_t tenant_version = 0; - int64_t sys_version = 0; - share::schema::ObSchemaGetterGuard schema_guard; ObSQLSessionInfo *session = NULL; ObResultSet *result_set = spi_result.get_result_set(); - int64_t query_timeout = 0; CK (OB_NOT_NULL(ctx)); CK (OB_NOT_NULL(ctx->exec_ctx_)); CK (OB_NOT_NULL(session = ctx->exec_ctx_->get_my_session())); - OZ (session->get_query_timeout(query_timeout)); CK (OB_NOT_NULL(result_set)); if (OB_SUCC(ret)) { int64_t time_gap = ObTimeUtility::current_time() - last_exec_time; @@ -7165,46 +6388,38 @@ int ObSPIService::inner_fetch_with_retry(ObPLExecCtx *ctx, if (OB_SUCC(ret)) { do { ret = OB_SUCCESS; - retry_ctrl.clear_state_before_each_retry(session->get_retry_info_for_update()); - if (THIS_WORKER.is_timeout()) { - ret = OB_TIMEOUT; - LOG_WARN("inner fetch with retry already timeout!", - K(ret), - K(old_timeout_ts), K(query_timeout), - K(min_timeout_ts), K(time_gap), - K(THIS_WORKER.get_timeout_ts()), K(old_query_start_time)); - break; + bool can_retry = true; + ObSPIRetryCtrlGuard retry_guard(retry_ctrl, spi_result, *session, ret, true); + if (FAILEDx(inner_fetch(ctx, + can_retry, + spi_result, + into_exprs, + into_count, + column_types, + type_count, + exprs_not_null_flag, + pl_integer_ranges, + NULL, /*out_using_params*/ + row_count, + is_bulk, + false, + false, /*is_dynamic_sql*/ + ¤t_row, + has_hidden_rowid, + for_cursor, + limit, + return_types, + return_type_count, + is_type_record))) { + if (!for_cursor || (for_cursor && ret != OB_READ_NOTHING)) { + LOG_WARN("failed to get_result, check if need retry", K(ret)); + } } - OZ (GCTX.schema_service_->get_tenant_schema_guard( - session->get_effective_tenant_id(), schema_guard)); - OZ (schema_guard.get_schema_version(session->get_effective_tenant_id(), tenant_version)); - OZ (schema_guard.get_schema_version(OB_SYS_TENANT_ID, sys_version)); - OX (retry_ctrl.set_tenant_local_schema_version(tenant_version)); - OX (retry_ctrl.set_sys_local_schema_version(sys_version)); - if (OB_SUCC(ret)) { - ret = inner_fetch(ctx, - retry_ctrl, - spi_result, - into_exprs, - into_count, - column_types, - type_count, - exprs_not_null_flag, - pl_integer_ranges, - NULL, /*out_using_params*/ - row_count, - is_bulk, - false, - false, /*is_dynamic_sql*/ - ¤t_row, - has_hidden_rowid, - for_cursor, - limit, - return_types, - return_type_count, - is_type_record); + if (can_retry) { + retry_guard.test(); } // NOTE: cursor fetch failed can not retry, we only use this to refresh location cache. + // actully, this function only called by cursor, so `for_cursor` just in case. } while (RETRY_TYPE_NONE != retry_ctrl.get_retry_type() && !for_cursor); session->get_retry_info_for_update().clear(); session->set_query_start_time(old_query_start_time); @@ -8669,8 +7884,10 @@ int ObSPIService::store_datum(int64_t ¤t_addr, const ObObj &obj, ObSQLSess return ret; } -int ObSPIService::fill_cursor(ObResultSet &result_set, ObSPICursor *cursor, int64_t new_query_start_time) -{ +int ObSPIService::fill_cursor(ObResultSet &result_set, + ObSPICursor *cursor, + int64_t new_query_start_time, + int64_t orc_max_ret_rows) { int ret = OB_SUCCESS; int64_t old_time_out_ts = THIS_WORKER.get_timeout_ts(); if (OB_ISNULL(cursor) || OB_ISNULL(cursor->allocator_)) { @@ -8700,9 +7917,14 @@ int ObSPIService::fill_cursor(ObResultSet &result_set, ObSPICursor *cursor, int6 K(ret)); } } - while (OB_SUCC(ret)) { + for (int64_t i = 0; OB_SUCC(ret) && i < orc_max_ret_rows; i++) { if (OB_FAIL(result_set.get_next_row(row))) { - //break + if (OB_ITER_END == ret) { + ret = OB_SUCCESS; + } else { + LOG_WARN("read result error", K(ret)); + } + break; } else if (OB_ISNULL(row)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("get a invalud row", K(ret)); @@ -8725,11 +7947,6 @@ int ObSPIService::fill_cursor(ObResultSet &result_set, ObSPICursor *cursor, int6 } } } - if (OB_ITER_END == ret) { - ret = OB_SUCCESS; - } else { - LOG_WARN("read result error", K(ret)); - } } THIS_WORKER.set_timeout_ts(old_time_out_ts); return ret; @@ -9688,5 +8905,71 @@ ObPLSubPLSqlTimeGuard::~ObPLSubPLSqlTimeGuard() } } +ObSPIRetryCtrlGuard::ObSPIRetryCtrlGuard( + ObQueryRetryCtrl &retry_ctrl, ObSPIResultSet &spi_result, ObSQLSessionInfo &session_info, int &ret, bool for_fetch) + : retry_ctrl_(retry_ctrl), spi_result_(spi_result), session_info_(session_info), ret_(ret), init_(false) +{ + int64_t tenant_version = 0; + int64_t sys_version = 0; + uint64_t eff_tenant_id = session_info_.get_effective_tenant_id(); + if (!for_fetch) { + spi_result_.get_out_params().reset(); + spi_result_.reset_member_for_retry(session_info_); + } + retry_ctrl_.clear_state_before_each_retry(session_info_.get_retry_info_for_update()); + if (THIS_WORKER.is_timeout()) { + ret = OB_TIMEOUT; + LOG_WARN("already timeout!", K(ret)); + } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(eff_tenant_id, spi_result_.get_scheme_guard()))) { + LOG_WARN("get schema guard failed", K(ret)); + } else if (OB_FAIL(spi_result_.get_scheme_guard().get_schema_version(eff_tenant_id, tenant_version))) { + LOG_WARN("fail get schema version", K(ret)); + } else if (OB_FAIL(spi_result_.get_scheme_guard().get_schema_version(OB_SYS_TENANT_ID, sys_version))) { + LOG_WARN("fail get sys schema version", K(ret)); + } else { + retry_ctrl_.set_tenant_local_schema_version(tenant_version); + retry_ctrl_.set_sys_local_schema_version(sys_version); + spi_result_.get_sql_ctx().schema_guard_ = &spi_result.get_scheme_guard(); + init_ = true; + } +} + +ObSPIRetryCtrlGuard::~ObSPIRetryCtrlGuard() +{ +} + +void ObSPIRetryCtrlGuard::test() +{ + int &ret = ret_; + if (init_ && OB_FAIL(ret) && spi_result_.get_result_set() != NULL) { + int cli_ret = OB_SUCCESS; + retry_ctrl_.test_and_save_retry_state(GCTX, + spi_result_.get_sql_ctx(), + *(spi_result_.get_result_set()), + ret, + cli_ret, + true, /*force_local_retry in SPI, we can only do local retry*/ + false, /*is_inner_sql*/ + true); /*is_part_of_pl_sql*/ + ret = cli_ret; + spi_result_.get_sql_ctx().clear(); + session_info_.set_session_in_retry(retry_ctrl_.need_retry()); + } +} + +ObSPIExecEnvGuard::ObSPIExecEnvGuard(ObSQLSessionInfo &session_info, ObSPIResultSet &spi_result) + : session_info_(session_info), spi_result_(spi_result) +{ + query_start_time_bk_ = session_info.get_query_start_time(); + session_info.set_query_start_time(ObTimeUtility::current_time()); + +} + +ObSPIExecEnvGuard::~ObSPIExecEnvGuard() +{ + session_info_.get_retry_info_for_update().clear(); + session_info_.set_query_start_time(query_start_time_bk_); +} + } } diff --git a/src/sql/ob_spi.h b/src/sql/ob_spi.h index 530f0e248..1bdd9fb76 100644 --- a/src/sql/ob_spi.h +++ b/src/sql/ob_spi.h @@ -35,6 +35,7 @@ using common::ObPsStmtId; namespace pl { class ObDbmsCursorInfo; +class ObPLSqlCodeInfo; } namespace sql @@ -58,6 +59,40 @@ struct ObPLSPITraceIdGuard int &ret_; }; +class ObSPIRetryCtrlGuard +{ +public: + ObSPIRetryCtrlGuard( + observer::ObQueryRetryCtrl &retry_ctrl, + ObSPIResultSet &spi_result, + ObSQLSessionInfo &session_info, + int &ret, + bool for_fetch = false); + ~ObSPIRetryCtrlGuard(); + + void test(); + +private: + observer::ObQueryRetryCtrl &retry_ctrl_; + ObSPIResultSet &spi_result_; + ObSQLSessionInfo &session_info_; + pl::ObPLSqlCodeInfo save_sqlcode_info_; + int &ret_; + bool init_; +}; + +class ObSPIExecEnvGuard +{ +public: + ObSPIExecEnvGuard(ObSQLSessionInfo &session_info, ObSPIResultSet &spi_result); + ~ObSPIExecEnvGuard(); +private: + ObSQLSessionInfo &session_info_; + ObSPIResultSet &spi_result_; + int64_t query_start_time_bk_; + pl::ObPLSqlCodeInfo *sqlcode_info_bk_; +}; + struct ObSPICursor { ObSPICursor(ObIAllocator &allocator, sql::ObSQLSessionInfo* session_info) : @@ -201,7 +236,7 @@ public: sql::ObSqlCtx &get_sql_ctx() { return sql_ctx_; } ObResultSet *get_result_set() { return result_set_; } ObSPIOutParams &get_out_params() { return out_params_; } - ObIAllocator &get_allocaor() { return allocator_; } + ObIAllocator &get_allocator() { return allocator_; } int destruct_exec_params(ObSQLSessionInfo &session); private: enum EndStmtType @@ -512,6 +547,55 @@ public: bool is_returning = false, bool is_type_record = false); + static int check_dynamic_sql_legal(pl::ObPLExecCtx *ctx, + ObSqlString &sql_str, + stmt::StmtType stmt_type, + int64_t into_count, + int64_t inner_into_count, + common::ObObjParam **params, + int64_t param_count, + const int64_t *params_mode, + bool is_returning, + bool for_update, + int64_t &exec_param_cnt, + common::ObIArray &out_using_params); + + static int prepare_dynamic_sql_params(pl::ObPLExecCtx *ctx, + ObSPIResultSet &spi_result, + common::ObIAllocator &allocator, + int64_t exec_param_cnt, + ObObjParam **params, + ParamStore *&exec_params); + + static int inner_open(pl::ObPLExecCtx *ctx, + ObIAllocator ¶m_allocator, //用于拷贝执行期参数 + const ObString &sql, + const ObString &ps_sql, + int64_t type, + void *params, + int64_t param_count, + const ObSqlExpression **into_exprs, + int64_t into_count, + ObSPIResultSet &spi_result, + ObSPIOutParams &out_params, + bool is_forall, + bool is_dynamic_sql, + bool is_dbms_sql); + + static int prepare_static_sql_params(pl::ObPLExecCtx *ctx, + ObIAllocator ¶m_allocator, + const ObString &sql, + const ObString &ps_sql, + int64_t type, + const ObSqlExpression **params, + int64_t param_count, + const ObSqlExpression **into_exprs, + int64_t into_count, + ObSPIResultSet &spi_result, + ObSPIOutParams &out_params, + bool is_forall, + ParamStore *&curr_params); + static int spi_get_subprogram_cursor_info(pl::ObPLExecCtx *ctx, uint64_t package_id, uint64_t routine_id, @@ -581,7 +665,8 @@ public: const ObString &ps_sql, int64_t stmt_type, bool for_update, - bool has_hidden_rowid); + bool has_hidden_rowid, + int64_t orc_max_ret_rows = INT64_MAX); static int spi_dynamic_open(pl::ObPLExecCtx *ctx, const int64_t sql_idx, const int64_t *sql_param_exprs_idx, @@ -591,7 +676,8 @@ public: int64_t cursor_index); static int dbms_dynamic_open(pl::ObPLExecCtx *ctx, pl::ObDbmsCursorInfo &cursor, - bool is_dbms_sql = false); + bool is_dbms_sql = false, + int64_t orc_max_ret_rows = INT64_MAX); static int dbms_cursor_fetch(pl::ObPLExecCtx *ctx, pl::ObDbmsCursorInfo &cursor, bool is_server_cursor = false); @@ -807,10 +893,14 @@ public: int64_t stmt_type, ParamStore &exec_params, ObSPIResultSet &spi_result, - ObSPIOutParams &out_params); + ObSPIOutParams &out_params, + bool is_dynamic_sql = false); static void adjust_pl_status_for_xa(sql::ObExecContext &ctx, int &result); - static int fill_cursor(ObResultSet &result_set, ObSPICursor *cursor, int64_t new_query_start_time); + static int fill_cursor(ObResultSet &result_set, + ObSPICursor *cursor, + int64_t new_query_start_time, + int64_t orc_max_ret_rows = INT64_MAX); static int spi_opaque_assign_null(int64_t opaque_ptr); @@ -897,7 +987,7 @@ private: const char *sql, const char *ps_sql, int64_t type, - const ObSqlExpression **param_exprs, + void *params, int64_t param_count, const ObSqlExpression **into_exprs, int64_t into_count, @@ -908,7 +998,10 @@ private: int64_t is_bulk, bool is_forall = false, bool is_type_record = false, - bool for_update = false); + bool for_update = false, + bool is_dynamic_sql = false, + ObIArray *using_out_params = nullptr, + bool is_dbms_sql = false); static int dbms_cursor_execute(pl::ObPLExecCtx *ctx, const ObString ps_sql, @@ -940,22 +1033,8 @@ private: ObSPIOutParams &out_params, bool is_forall = false); - static int inner_open(pl::ObPLExecCtx *ctx, - ObIAllocator ¶m_allocator, //用于拷贝执行期参数 - const char* sql, - const char* ps_sql, - int64_t type, - const ObSqlExpression **param_exprs, - int64_t param_count, - const ObSqlExpression **into_exprs, - int64_t into_count, - ObSPIResultSet &spi_result, - ObSPIOutParams &out_params, - observer::ObQueryRetryCtrl *retry_ctrl = nullptr, - bool is_forall = false); - static int inner_fetch(pl::ObPLExecCtx *ctx, - observer::ObQueryRetryCtrl &retry_ctrl, + bool &can_retry, ObSPIResultSet &spi_result, const ObSqlExpression **into_exprs, int64_t into_count, @@ -1136,9 +1215,11 @@ private: static int calc_dynamic_sqlstr( pl::ObPLExecCtx *ctx, const ObSqlExpression *sql, ObSqlString &sqlstr); - static int dynamic_out_params( - common::ObIAllocator &allocator, - ObResultSet *result, common::ObObjParam **params, int64_t param_count); + static int dynamic_out_params(common::ObIAllocator &allocator, + ObResultSet *result, + void *params, + int64_t param_count, + bool is_dbms_sql); static int cursor_close_impl(pl::ObPLExecCtx *ctx, pl::ObPLCursorInfo *cursor, @@ -1177,7 +1258,35 @@ private: const ObSqlExpression **actual_param_exprs, int64_t cursor_param_count); static bool is_sql_type_into_pl(ObObj &dest_addr, ObIArray &obj_array); -private: + + static int streaming_cursor_open(pl::ObPLExecCtx *ctx, + pl::ObPLCursorInfo &cursor, + ObSQLSessionInfo &session_info, + const ObString &sql, + const ObString &ps_sql, + int64_t type, + void *params, + int64_t sql_param_count, + bool is_server_cursor, + bool is_for_update, + bool has_hidden_rowid, + bool is_dbms_cursor = false); + + static int unstreaming_cursor_open(pl::ObPLExecCtx *ctx, + pl::ObPLCursorInfo &cursor, + ObSQLSessionInfo &session_info, + const ObString &sql, + const ObString &ps_sql, + int64_t type, + void *params, + int64_t sql_param_count, + bool is_server_cursor, + bool for_update, + bool has_hidden_rowid, + bool is_dbms_cursor = false, + int64_t orc_max_ret_rows = INT64_MAX); + static int store_params_string(pl::ObPLExecCtx *ctx, ObSPIResultSet &spi_result, ParamStore *exec_params); + static int setup_cursor_snapshot_verify_(pl::ObPLCursorInfo *cursor, ObSPIResultSet *spi_result); }; diff --git a/src/sql/session/ob_basic_session_info.cpp b/src/sql/session/ob_basic_session_info.cpp index 7aab1c45d..00c995d8b 100644 --- a/src/sql/session/ob_basic_session_info.cpp +++ b/src/sql/session/ob_basic_session_info.cpp @@ -2404,6 +2404,12 @@ OB_INLINE int ObBasicSessionInfo::process_session_variable(ObSysVarClassType var } break; } + case SYS_VAR__ORACLE_SQL_SELECT_LIMIT: { + int64_t int_val = 0; + OZ (val.get_int(int_val), val); + OX (sys_vars_cache_.set_oracle_sql_select_limit(int_val)); + break; + } case SYS_VAR_AUTO_INCREMENT_OFFSET: { uint64_t uint_val = 0; OZ (val.get_uint64(uint_val), val); @@ -2969,6 +2975,12 @@ int ObBasicSessionInfo::fill_sys_vars_cache_base_value( OX (sys_vars_cache.set_base_sql_select_limit(int_val)); break; } + case SYS_VAR__ORACLE_SQL_SELECT_LIMIT: { + int64_t int_val = 0; + OZ (val.get_int(int_val), val); + OX (sys_vars_cache.set_base_oracle_sql_select_limit(int_val)); + break; + } case SYS_VAR_AUTO_INCREMENT_OFFSET: { uint64_t uint_val = 0; OZ (val.get_uint64(uint_val), val); diff --git a/src/sql/session/ob_basic_session_info.h b/src/sql/session/ob_basic_session_info.h index 26e96d940..11ef06b2a 100644 --- a/src/sql/session/ob_basic_session_info.h +++ b/src/sql/session/ob_basic_session_info.h @@ -480,6 +480,7 @@ public: uint64_t get_local_auto_increment_increment() const; uint64_t get_local_auto_increment_offset() const; uint64_t get_local_last_insert_id() const; + void set_local_ob_enable_pl_cache(bool v) { sys_vars_cache_.set_ob_enable_pl_cache(v); } bool get_local_ob_enable_pl_cache() const; bool get_local_ob_enable_plan_cache() const; bool get_local_ob_enable_sql_audit() const; @@ -651,6 +652,11 @@ public: sql_select_limit = sys_vars_cache_.get_sql_select_limit(); return common::OB_SUCCESS; } + int get_oracle_sql_select_limit(int64_t &oracle_sql_select_limit) const + { + oracle_sql_select_limit = sys_vars_cache_.get_oracle_sql_select_limit(); + return common::OB_SUCCESS; + } // session保留compatible mode，主要用于传递mode，方便后续进行guard切换，如inner sql connection等 // 其他需要用mode地方请尽量使用线程上的is_oracle|mysql_mode // 同时可以使用check_compatibility_mode来检查线程与session上的mode是否一致 @@ -1667,6 +1673,7 @@ public: sql_throttle_current_priority_(100), ob_last_schema_version_(0), sql_select_limit_(0), + oracle_sql_select_limit_(0), auto_increment_offset_(0), last_insert_id_(0), binlog_row_image_(2), @@ -1729,6 +1736,7 @@ public: sql_throttle_current_priority_ = 100; ob_last_schema_version_ = 0; sql_select_limit_ = 0; + oracle_sql_select_limit_ = 0; auto_increment_offset_ = 0; last_insert_id_ = 0; binlog_row_image_ = 2; @@ -1789,6 +1797,7 @@ public: sql_throttle_current_priority_ == other.sql_throttle_current_priority_ && ob_last_schema_version_ == other.ob_last_schema_version_ && sql_select_limit_ == other.sql_select_limit_ && + oracle_sql_select_limit_ == other.oracle_sql_select_limit_ && auto_increment_offset_ == other.auto_increment_offset_ && last_insert_id_ == other.last_insert_id_ && binlog_row_image_ == other.binlog_row_image_ && @@ -1945,7 +1954,7 @@ public: K(ob_org_cluster_id_), K(ob_query_timeout_), K(ob_trx_timeout_), K(collation_connection_), K(sql_mode_), K(nls_formats_[0]), K(nls_formats_[1]), K(nls_formats_[2]), K(ob_trx_idle_timeout_), K(ob_trx_lock_timeout_), K(nls_collation_), K(nls_nation_collation_), - K_(sql_throttle_current_priority), K_(ob_last_schema_version), K_(sql_select_limit), + K_(sql_throttle_current_priority), K_(ob_last_schema_version), K_(sql_select_limit), K_(oracle_sql_select_limit), K_(optimizer_use_sql_plan_baselines), K_(optimizer_capture_sql_plan_baselines), K_(is_result_accurate), K_(character_set_results), K_(character_set_connection), K_(ob_pl_block_timeout), K_(ob_plsql_ccflags), @@ -1958,6 +1967,7 @@ public: int64_t sql_throttle_current_priority_; int64_t ob_last_schema_version_; int64_t sql_select_limit_; + int64_t oracle_sql_select_limit_; uint64_t auto_increment_offset_; uint64_t last_insert_id_; int64_t binlog_row_image_; @@ -2085,6 +2095,7 @@ private: DEF_SYS_VAR_CACHE_FUNCS(int64_t, sql_throttle_current_priority); DEF_SYS_VAR_CACHE_FUNCS(int64_t, ob_last_schema_version); DEF_SYS_VAR_CACHE_FUNCS(int64_t, sql_select_limit); + DEF_SYS_VAR_CACHE_FUNCS(int64_t, oracle_sql_select_limit); DEF_SYS_VAR_CACHE_FUNCS(uint64_t, auto_increment_offset); DEF_SYS_VAR_CACHE_FUNCS(uint64_t, last_insert_id); DEF_SYS_VAR_CACHE_FUNCS(int64_t, binlog_row_image); @@ -2160,6 +2171,7 @@ private: bool inc_sql_throttle_current_priority_:1; bool inc_ob_last_schema_version_:1; bool inc_sql_select_limit_:1; + bool inc_oracle_sql_select_limit_:1; bool inc_auto_increment_offset_:1; bool inc_last_insert_id_:1; bool inc_binlog_row_image_:1;