From f9862f9eed5848b35fc15a1ecec7dbe452eff1b3 Mon Sep 17 00:00:00 2001 From: Vinoth Veeraraghavan Date: Thu, 15 Oct 2020 12:18:08 +0800 Subject: [PATCH 1/4] Fix BPCHAR behavior in MOT, added space padding for search key --- .../storage/mot/fdw_adapter/src/mot_internal.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/gausskernel/storage/mot/fdw_adapter/src/mot_internal.cpp b/src/gausskernel/storage/mot/fdw_adapter/src/mot_internal.cpp index e4b2af853..29c3730fa 100644 --- a/src/gausskernel/storage/mot/fdw_adapter/src/mot_internal.cpp +++ b/src/gausskernel/storage/mot/fdw_adapter/src/mot_internal.cpp @@ -2476,10 +2476,18 @@ void MOTAdaptor::DatumToMOTKey( size -= VARHDRSZ; if (oper == KEY_OPER::READ_KEY_LIKE) { - if (src[size - 1] == '%') + if (src[size - 1] == '%') { size -= 1; - else - fill = 0x00; // switch to equal + } else { + // switch to equal + if (type == BPCHAROID) { + fill = 0x20; // space ' ' == 0x20 + } else { + fill = 0x00; + } + } + } else if (type == BPCHAROID) { // handle padding for blank-padded type + fill = 0x20; } col->PackKey(data, (uintptr_t)src, size, fill); From c0f130a6bac958033f408098de9c5895c429d14f Mon Sep 17 00:00:00 2001 From: Vinoth Veeraraghavan Date: Thu, 15 Oct 2020 19:02:27 +0800 Subject: [PATCH 2/4] 1. Prohibit user from setting unexposed configs in mot.conf. 2. Fix PG memory context usage in JIT. --- src/gausskernel/storage/mot/core/src/mot.conf | 4 +- .../mot/core/src/system/mot_configuration.cpp | 100 ++++++++------ .../mot/core/src/system/mot_configuration.h | 9 ++ .../storage/mot/jit_exec/src/jit_context.cpp | 123 +++++++----------- .../storage/mot/jit_exec/src/jit_exec.cpp | 9 +- .../storage/mot/jit_exec/src/jit_helpers.cpp | 8 +- 6 files changed, 129 insertions(+), 124 deletions(-) mode change 100755 => 100644 src/gausskernel/storage/mot/jit_exec/src/jit_context.cpp diff --git a/src/gausskernel/storage/mot/core/src/mot.conf b/src/gausskernel/storage/mot/core/src/mot.conf index 34a379a2b..75f334c08 100644 --- a/src/gausskernel/storage/mot/core/src/mot.conf +++ b/src/gausskernel/storage/mot/core/src/mot.conf @@ -145,11 +145,11 @@ # Configure specific loggers according to the following syntax: # -# Log.COMPONENT.LOGGER=LOG_LEVEL +# Log.COMPONENT.LOGGER.log_level=LOG_LEVEL # # For instance, configure TRACE log level for the ThreadIdPool logger in System component as follows: # -# Log.System.ThreadIdPool=TRACE +# Log.System.ThreadIdPool.log_level=TRACE # # In order the configure log level for all loggers under some component, use the following syntax: # diff --git a/src/gausskernel/storage/mot/core/src/system/mot_configuration.cpp b/src/gausskernel/storage/mot/core/src/system/mot_configuration.cpp index 1a81bf5cc..61c7fbbaf 100644 --- a/src/gausskernel/storage/mot/core/src/system/mot_configuration.cpp +++ b/src/gausskernel/storage/mot/core/src/system/mot_configuration.cpp @@ -508,7 +508,8 @@ MOTConfiguration::MOTConfiguration() m_configMonitorPeriodSeconds(DEFAULT_CFG_MONITOR_PERIOD_SECONDS), m_runInternalConsistencyValidation(DEFAULT_RUN_INTERNAL_CONSISTENCY_VALIDATION), m_totalMemoryMb(DEFAULT_TOTAL_MEMORY_MB), - m_suppressLog(0) + m_suppressLog(0), + m_loadExtraParams(false) {} void MOTConfiguration::Initialize() @@ -691,8 +692,14 @@ void MOTConfiguration::LoadConfig() const LayeredConfigTree* cfg = ConfigManager::GetInstance().GetLayeredConfigTree(); // logger configuration - UPDATE_BOOL_CFG(m_enableRedoLog, "enable_redo_log", DEFAULT_ENABLE_REDO_LOG); - UPDATE_USER_CFG(m_loggerType, "logger_type", DEFAULT_LOGGER_TYPE); + if (m_loadExtraParams) { + UPDATE_BOOL_CFG(m_enableRedoLog, "enable_redo_log", DEFAULT_ENABLE_REDO_LOG); + UPDATE_USER_CFG(m_loggerType, "logger_type", DEFAULT_LOGGER_TYPE); + } + + // Even though we allow loading this unexposed parameter (redo_log_handler_type), in reality it is always + // overridden by the external configuration loader GaussdbConfigLoader (so in effect whatever is defined in + // mot.conf is discarded). See GaussdbConfigLoader::ConfigureRedoLogHandler() for more details. UPDATE_USER_CFG(m_redoLogHandlerType, "redo_log_handler_type", DEFAULT_REDO_LOG_HANDLER_TYPE); UPDATE_INT_CFG(m_asyncRedoLogBufferArrayCount, "async_log_buffer_count", @@ -715,7 +722,9 @@ void MOTConfiguration::LoadConfig() MAX_GROUP_COMMIT_TIMEOUT_USEC); // Checkpoint configuration - UPDATE_BOOL_CFG(m_enableCheckpoint, "enable_checkpoint", DEFAULT_ENABLE_CHECKPOINT); + if (m_loadExtraParams) { + UPDATE_BOOL_CFG(m_enableCheckpoint, "enable_checkpoint", DEFAULT_ENABLE_CHECKPOINT); + } if (!m_enableCheckpoint && m_enableRedoLog) { if (m_suppressLog == 0) { @@ -745,9 +754,11 @@ void MOTConfiguration::LoadConfig() MAX_CHECKPOINT_RECOVERY_WORKERS); // Tx configuration - not configurable yet - UPDATE_BOOL_CFG(m_abortBufferEnable, "tx_abort_buffers_enable", true); - UPDATE_BOOL_CFG(m_preAbort, "tx_pre_abort", true); - m_validationLock = TxnValidation::TXN_VALIDATION_NO_WAIT; + if (m_loadExtraParams) { + UPDATE_BOOL_CFG(m_abortBufferEnable, "tx_abort_buffers_enable", true); + UPDATE_BOOL_CFG(m_preAbort, "tx_pre_abort", true); + m_validationLock = TxnValidation::TXN_VALIDATION_NO_WAIT; + } // statistics configuration UPDATE_BOOL_CFG(m_enableStats, "enable_stats", DEFAULT_ENABLE_STATS); @@ -777,9 +788,11 @@ void MOTConfiguration::LoadConfig() // log configuration UPDATE_USER_CFG(m_logLevel, "log_level", DEFAULT_LOG_LEVEL); SetGlobalLogLevel(m_logLevel); - UPDATE_USER_CFG(m_numaErrorsLogLevel, "numa_errors_log_level", DEFAULT_NUMA_ERRORS_LOG_LEVEL); - UPDATE_USER_CFG(m_numaWarningsLogLevel, "numa_warnings_log_level", DEFAULT_NUMA_WARNINGS_LOG_LEVEL); - UPDATE_USER_CFG(m_cfgStartupLogLevel, "cfg_startup_log_level", DEFAULT_CFG_STARTUP_LOG_LEVEL); + if (m_loadExtraParams) { + UPDATE_USER_CFG(m_numaErrorsLogLevel, "numa_errors_log_level", DEFAULT_NUMA_ERRORS_LOG_LEVEL); + UPDATE_USER_CFG(m_numaWarningsLogLevel, "numa_warnings_log_level", DEFAULT_NUMA_WARNINGS_LOG_LEVEL); + UPDATE_USER_CFG(m_cfgStartupLogLevel, "cfg_startup_log_level", DEFAULT_CFG_STARTUP_LOG_LEVEL); + } // memory configuration UPDATE_BOOL_CFG(m_enableNuma, "enable_numa", DEFAULT_ENABLE_NUMA); @@ -935,11 +948,13 @@ void MOTConfiguration::LoadConfig() DEFAULT_CHUNK_PREALLOC_WORKER_COUNT, MIN_CHUNK_PREALLOC_WORKER_COUNT, MAX_CHUNK_PREALLOC_WORKER_COUNT); - UPDATE_INT_CFG(m_highRedMarkPercent, - "high_red_mark_percent", - DEFAULT_HIGH_RED_MARK_PERCENT, - MIN_HIGH_RED_MARK_PERCENT, - MAX_HIGH_RED_MARK_PERCENT); + if (m_loadExtraParams) { + UPDATE_INT_CFG(m_highRedMarkPercent, + "high_red_mark_percent", + DEFAULT_HIGH_RED_MARK_PERCENT, + MIN_HIGH_RED_MARK_PERCENT, + MAX_HIGH_RED_MARK_PERCENT); + } UPDATE_ABS_MEM_CFG(m_sessionLargeBufferStoreSizeMB, "session_large_buffer_store_size", DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE, @@ -987,20 +1002,24 @@ void MOTConfiguration::LoadConfig() m_codegenLimit, "mot_codegen_limit", DEFAULT_MOT_CODEGEN_LIMIT, MIN_MOT_CODEGEN_LIMIT, MAX_MOT_CODEGEN_LIMIT); // storage configuration - UPDATE_BOOL_CFG( - m_allowIndexOnNullableColumn, "allow_index_on_nullable_column", DEFAULT_ALLOW_INDEX_ON_NULLABLE_COLUMN); - UPDATE_USER_CFG(m_indexTreeFlavor, "index_tree_flavor", DEFAULT_INDEX_TREE_FLAVOR); + if (m_loadExtraParams) { + UPDATE_BOOL_CFG( + m_allowIndexOnNullableColumn, "allow_index_on_nullable_column", DEFAULT_ALLOW_INDEX_ON_NULLABLE_COLUMN); + UPDATE_USER_CFG(m_indexTreeFlavor, "index_tree_flavor", DEFAULT_INDEX_TREE_FLAVOR); + } // general configuration - UPDATE_TIME_CFG(m_configMonitorPeriodSeconds, - "config_update_period", - DEFAULT_CFG_MONITOR_PERIOD, - SCALE_SECONDS, - MIN_CFG_MONITOR_PERIOD_SECONDS, - MAX_CFG_MONITOR_PERIOD_SECONDS); - UPDATE_BOOL_CFG(m_runInternalConsistencyValidation, - "internal_consistency_validation", - DEFAULT_RUN_INTERNAL_CONSISTENCY_VALIDATION); + if (m_loadExtraParams) { + UPDATE_TIME_CFG(m_configMonitorPeriodSeconds, + "config_update_period", + DEFAULT_CFG_MONITOR_PERIOD, + SCALE_SECONDS, + MIN_CFG_MONITOR_PERIOD_SECONDS, + MAX_CFG_MONITOR_PERIOD_SECONDS); + UPDATE_BOOL_CFG(m_runInternalConsistencyValidation, + "internal_consistency_validation", + DEFAULT_RUN_INTERNAL_CONSISTENCY_VALIDATION); + } // load component log levels UpdateComponentLogLevel(); @@ -1134,8 +1153,8 @@ void MOTConfiguration::UpdateComponentLogLevel() // configure component log level first then override log level specific loggers in the component LogLevel componentLevel = componentCfg->GetUserConfigValue("log_level", globalLogLevel); if (componentLevel != globalLogLevel) { - mot_string logLevelStr; if (m_suppressLog == 0) { + mot_string logLevelStr; MOT_LOG_INFO("Updating the log level of component %s to: %s", componentName.c_str(), TypeFormatter::ToString(componentLevel, logLevelStr)); @@ -1145,7 +1164,7 @@ void MOTConfiguration::UpdateComponentLogLevel() // all configuration values are logger configuration pairs (loggerName=log_level) mot_string_list loggerNames; - if (!componentCfg->GetConfigValueNames(loggerNames)) { + if (!componentCfg->GetConfigSectionNames(loggerNames)) { MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, "Load Configuration", "Failed to retrieve logger names for section %s", @@ -1155,21 +1174,18 @@ void MOTConfiguration::UpdateComponentLogLevel() mot_string_list::const_iterator loggerItr = loggerNames.cbegin(); while (loggerItr != loggerNames.cend()) { const mot_string& loggerName = *loggerItr; - if (loggerName.compare("log_level") != 0) { // skip special value for entire component log level - MOT_LOG_DEBUG( - "Loading component/logger %s/%s log level", componentName.c_str(), loggerName.c_str()); - LogLevel loggerLevel = - componentCfg->GetUserConfigValue(loggerName.c_str(), LogLevel::LL_INFO); - if (loggerLevel != LogLevel::LL_INFO) { + MOT_LOG_DEBUG("Loading component/logger %s/%s log level", componentName.c_str(), loggerName.c_str()); + const ConfigSection* loggerCfg = componentCfg->GetConfigSection(loggerName.c_str()); + LogLevel loggerLevel = loggerCfg->GetUserConfigValue("log_level", componentLevel); + if (loggerLevel != componentLevel) { + if (m_suppressLog == 0) { mot_string logLevelStr; - if (m_suppressLog == 0) { - MOT_LOG_INFO("Updating the log level of logger %s in component %s to: %s", - loggerName.c_str(), - componentName.c_str(), - TypeFormatter::ToString(loggerLevel, logLevelStr)); - } - SetLoggerLogLevel(componentName.c_str(), loggerName.c_str(), loggerLevel); + MOT_LOG_INFO("Updating the log level of logger %s in component %s to: %s", + loggerName.c_str(), + componentName.c_str(), + TypeFormatter::ToString(loggerLevel, logLevelStr)); } + SetLoggerLogLevel(componentName.c_str(), loggerName.c_str(), loggerLevel); } ++loggerItr; } diff --git a/src/gausskernel/storage/mot/core/src/system/mot_configuration.h b/src/gausskernel/storage/mot/core/src/system/mot_configuration.h index 6a9e7ae73..60d4b1b3b 100644 --- a/src/gausskernel/storage/mot/core/src/system/mot_configuration.h +++ b/src/gausskernel/storage/mot/core/src/system/mot_configuration.h @@ -85,6 +85,12 @@ public: --m_suppressLog; } + /** @brief Enables loading extra configuration parameters (by default disabled, used for testing). */ + inline void EnableLoadExtraParams() + { + m_loadExtraParams = true; + } + /** * @brief Validates configuration. Call this function after loading configuration to validate * the loaded values are valid, not out of bounds, and not self-contradicting. @@ -702,6 +708,9 @@ private: /** @var Controls suppressing of log messages during configuration loading. */ int m_suppressLog; + /** @var Controls loading of extra configuration parameters. */ + bool m_loadExtraParams; + /** @brief Loads configuration from main configuration. */ void LoadConfig(); diff --git a/src/gausskernel/storage/mot/jit_exec/src/jit_context.cpp b/src/gausskernel/storage/mot/jit_exec/src/jit_context.cpp old mode 100755 new mode 100644 index 5430f1802..6e449dcdb --- a/src/gausskernel/storage/mot/jit_exec/src/jit_context.cpp +++ b/src/gausskernel/storage/mot/jit_exec/src/jit_context.cpp @@ -48,24 +48,6 @@ static void CleanupJitContextInner(JitContext* jitContext); static void CleanupJitContextSubQueryDataArray(JitContext* jitContext); static void CleanupJitContextSubQueryData(JitContext::SubQueryData* subQueryData); -// Helpers to allocate/free from top memory context -inline void* palloc_top(size_t size_bytes) -{ - MemoryContext oldCtx = CurrentMemoryContext; - CurrentMemoryContext = u_sess->top_mem_cxt; - void* res = palloc(size_bytes); - CurrentMemoryContext = oldCtx; - return res; -} - -inline void pfree_top(void* obj) -{ - MemoryContext oldCtx = CurrentMemoryContext; - CurrentMemoryContext = u_sess->top_mem_cxt; - pfree(obj); - CurrentMemoryContext = oldCtx; -} - extern bool InitGlobalJitContextPool() { return InitJitContextPool(&g_globalJitCtxPool, JIT_CONTEXT_GLOBAL, GetMotCodegenLimit()); @@ -78,16 +60,16 @@ extern void DestroyGlobalJitContextPool() extern JitContext* AllocJitContext(JitContextUsage usage) { - JitContext* result = NULL; + JitContext* result = nullptr; if (usage == JIT_CONTEXT_GLOBAL) { // allocate from global pool result = AllocPooledJitContext(&g_globalJitCtxPool); } else { // allocate from session local pool (create pool on demand and schedule cleanup during end of session) - if (u_sess->mot_cxt.jit_session_context_pool == NULL) { + if (u_sess->mot_cxt.jit_session_context_pool == nullptr) { u_sess->mot_cxt.jit_session_context_pool = AllocSessionJitContextPool(); - if (u_sess->mot_cxt.jit_session_context_pool == NULL) { - return NULL; + if (u_sess->mot_cxt.jit_session_context_pool == nullptr) { + return nullptr; } } result = AllocPooledJitContext(u_sess->mot_cxt.jit_session_context_pool); @@ -250,12 +232,12 @@ static List* GetSubQueryTargetList(const char* queryString, int subQueryIndex) extern bool ReFetchIndices(JitContext* jitContext) { + if (jitContext->m_commandType != JIT_COMMAND_INSERT) { + return true; + } + // re-fetch main index - if ((jitContext->m_commandType != JIT_COMMAND_INSERT) && (jitContext->m_index == nullptr)) { - if (jitContext->m_indexId == 0) { - MOT_LOG_TRACE("Cannot re-fetch index: missing index identifier"); - return false; - } + if (jitContext->m_index == nullptr) { jitContext->m_index = jitContext->m_table->GetIndexByExtId(jitContext->m_indexId); if (jitContext->m_index == nullptr) { MOT_LOG_TRACE("Failed to fetch index by extern id %" PRIu64, jitContext->m_indexId); @@ -266,10 +248,6 @@ extern bool ReFetchIndices(JitContext* jitContext) // re-fetch inner index (JOIN commands only) if (IsJoinCommand(jitContext->m_commandType)) { if (jitContext->m_innerIndex == nullptr) { - if (jitContext->m_innerIndexId == 0) { - MOT_LOG_TRACE("Cannot re-fetch inner index: missing index identifier"); - return false; - } jitContext->m_innerIndex = jitContext->m_innerTable->GetIndexByExtId(jitContext->m_innerIndexId); if (jitContext->m_innerIndex == nullptr) { MOT_LOG_TRACE("Failed to fetch inner index by extern id %" PRIu64, jitContext->m_innerIndexId); @@ -283,10 +261,6 @@ extern bool ReFetchIndices(JitContext* jitContext) for (uint32_t i = 0; i < jitContext->m_subQueryCount; ++i) { JitContext::SubQueryData* subQueryData = &jitContext->m_subQueryData[i]; if (subQueryData->m_index == nullptr) { - if (subQueryData->m_indexId == 0) { - MOT_LOG_TRACE("Cannot re-fetch sub-query %u index: missing index identifier", i); - return false; - } subQueryData->m_index = subQueryData->m_table->GetIndexByExtId(subQueryData->m_indexId); if (subQueryData->m_index == nullptr) { MOT_LOG_TRACE( @@ -303,10 +277,10 @@ extern bool ReFetchIndices(JitContext* jitContext) extern bool PrepareJitContext(JitContext* jitContext) { // allocate argument-is-null array - if (jitContext->m_argIsNull == NULL) { + if (jitContext->m_argIsNull == nullptr) { MOT_LOG_TRACE("Allocating null argument array with %u slots", (unsigned)jitContext->m_argCount); - jitContext->m_argIsNull = (int*)palloc_top(sizeof(int) * jitContext->m_argCount); - if (jitContext->m_argIsNull == NULL) { + jitContext->m_argIsNull = (int*)MOT::MemSessionAlloc(sizeof(int) * jitContext->m_argCount); + if (jitContext->m_argIsNull == nullptr) { MOT_LOG_TRACE("Failed to allocate null argument array in size of %d slots", jitContext->m_argCount); return false; } @@ -323,7 +297,7 @@ extern bool PrepareJitContext(JitContext* jitContext) if ((jitContext->m_searchKey == nullptr) && (jitContext->m_commandType != JIT_COMMAND_INSERT)) { MOT_LOG_TRACE("Preparing search key from index %s", jitContext->m_index->GetName().c_str()); jitContext->m_searchKey = PrepareJitSearchKey(jitContext, jitContext->m_index); - if (jitContext->m_searchKey == NULL) { + if (jitContext->m_searchKey == nullptr) { MOT_LOG_TRACE("Failed to allocate reusable search key for JIT context, aborting jitted code execution"); return false; // safe cleanup during destroy } else { @@ -335,34 +309,34 @@ extern bool PrepareJitContext(JitContext* jitContext) } // allocate bitmap-set object for incremental-redo when executing UPDATE command - if ((jitContext->m_bitmapSet == NULL) && ((jitContext->m_commandType == JIT_COMMAND_UPDATE) || - (jitContext->m_commandType == JIT_COMMAND_RANGE_UPDATE))) { + if ((jitContext->m_bitmapSet == nullptr) && ((jitContext->m_commandType == JIT_COMMAND_UPDATE) || + (jitContext->m_commandType == JIT_COMMAND_RANGE_UPDATE))) { int fieldCount = (int)jitContext->m_table->GetFieldCount(); MOT_LOG_TRACE( "Initializing reusable bitmap set according to %d fields (including null-bits column 0) in table %s", fieldCount, jitContext->m_table->GetLongTableName().c_str()); - void* buf = palloc_top(sizeof(MOT::BitmapSet)); - if (buf == NULL) { + void* buf = MOT::MemSessionAlloc(sizeof(MOT::BitmapSet)); + if (buf == nullptr) { MOT_LOG_TRACE("Failed to allocate reusable bitmap set for JIT context, aborting jitted code execution"); return false; // safe cleanup during destroy } - uint8_t* bitmapData = (uint8_t*)palloc_top(MOT::BitmapSet::GetLength(fieldCount)); - if (bitmapData == NULL) { + uint8_t* bitmapData = (uint8_t*)MOT::MemSessionAlloc(MOT::BitmapSet::GetLength(fieldCount)); + if (bitmapData == nullptr) { MOT_LOG_TRACE("Failed to allocate reusable bitmap set for JIT context, aborting jitted code execution"); - pfree_top(buf); + MOT::MemSessionFree(buf); return false; // safe cleanup during destroy } jitContext->m_bitmapSet = new (buf) MOT::BitmapSet(bitmapData, fieldCount); } // allocate end-iterator key object when executing range UPDATE command or special SELECT commands - if ((jitContext->m_endIteratorKey == NULL) && IsRangeCommand(jitContext->m_commandType)) { + if ((jitContext->m_endIteratorKey == nullptr) && IsRangeCommand(jitContext->m_commandType)) { MOT_LOG_TRACE("Preparing end iterator key for range update/select command from index %s", jitContext->m_index->GetName().c_str()); jitContext->m_endIteratorKey = PrepareJitSearchKey(jitContext, jitContext->m_index); - if (jitContext->m_endIteratorKey == NULL) { + if (jitContext->m_endIteratorKey == nullptr) { MOT_LOG_TRACE( "Failed to allocate reusable end iterator key for JIT context, aborting jitted code execution"); return false; // safe cleanup during destroy @@ -375,11 +349,11 @@ extern bool PrepareJitContext(JitContext* jitContext) } // allocate inner loop search key for JOIN commands - if ((jitContext->m_innerSearchKey == NULL) && IsJoinCommand(jitContext->m_commandType)) { + if ((jitContext->m_innerSearchKey == nullptr) && IsJoinCommand(jitContext->m_commandType)) { MOT_LOG_TRACE( "Preparing inner search key for JOIN command from index %s", jitContext->m_innerIndex->GetName().c_str()); jitContext->m_innerSearchKey = PrepareJitSearchKey(jitContext, jitContext->m_innerIndex); - if (jitContext->m_innerSearchKey == NULL) { + if (jitContext->m_innerSearchKey == nullptr) { MOT_LOG_TRACE( "Failed to allocate reusable inner search key for JIT context, aborting jitted code execution"); return false; // safe cleanup during destroy @@ -392,11 +366,11 @@ extern bool PrepareJitContext(JitContext* jitContext) } // allocate inner loop end-iterator search key for JOIN commands - if ((jitContext->m_innerEndIteratorKey == NULL) && IsJoinCommand(jitContext->m_commandType)) { + if ((jitContext->m_innerEndIteratorKey == nullptr) && IsJoinCommand(jitContext->m_commandType)) { MOT_LOG_TRACE("Preparing inner end iterator key for JOIN command from index %s", jitContext->m_innerIndex->GetName().c_str()); jitContext->m_innerEndIteratorKey = PrepareJitSearchKey(jitContext, jitContext->m_innerIndex); - if (jitContext->m_innerEndIteratorKey == NULL) { + if (jitContext->m_innerEndIteratorKey == nullptr) { MOT_LOG_TRACE( "Failed to allocate reusable inner end iterator key for JIT context, aborting jitted code execution"); return false; // safe cleanup during destroy @@ -409,10 +383,10 @@ extern bool PrepareJitContext(JitContext* jitContext) } // preparing outer row copy for JOIN commands - if ((jitContext->m_outerRowCopy == NULL) && IsJoinCommand(jitContext->m_commandType)) { + if ((jitContext->m_outerRowCopy == nullptr) && IsJoinCommand(jitContext->m_commandType)) { MOT_LOG_TRACE("Preparing outer row copy for JOIN command"); jitContext->m_outerRowCopy = jitContext->m_table->CreateNewRow(); - if (jitContext->m_outerRowCopy == NULL) { + if (jitContext->m_outerRowCopy == nullptr) { MOT_LOG_TRACE("Failed to allocate reusable outer row copy for JIT context, aborting jitted code execution"); return false; // safe cleanup during destroy } @@ -511,35 +485,35 @@ extern void DestroyJitContext(JitContext* jitContext) CleanupJitContextInner(jitContext); // cleanup bitmap set - if (jitContext->m_bitmapSet != NULL) { - pfree_top(jitContext->m_bitmapSet->GetData()); + if (jitContext->m_bitmapSet != nullptr) { + MOT::MemSessionFree(jitContext->m_bitmapSet->GetData()); jitContext->m_bitmapSet->MOT::BitmapSet::~BitmapSet(); - pfree_top(jitContext->m_bitmapSet); - jitContext->m_bitmapSet = NULL; + MOT::MemSessionFree(jitContext->m_bitmapSet); + jitContext->m_bitmapSet = nullptr; } // cleanup code generator (only in global-usage) if (jitContext->m_codeGen && (jitContext->m_usage == JIT_CONTEXT_GLOBAL)) { FreeGsCodeGen(jitContext->m_codeGen); - jitContext->m_codeGen = NULL; + jitContext->m_codeGen = nullptr; } // cleanup null argument array - if (jitContext->m_argIsNull != NULL) { - pfree_top(jitContext->m_argIsNull); - jitContext->m_argIsNull = NULL; + if (jitContext->m_argIsNull != nullptr) { + MOT::MemSessionFree(jitContext->m_argIsNull); + jitContext->m_argIsNull = nullptr; } // cleanup TVM function (only in global-usage) if (jitContext->m_tvmFunction && (jitContext->m_usage == JIT_CONTEXT_GLOBAL)) { delete jitContext->m_tvmFunction; - jitContext->m_tvmFunction = NULL; + jitContext->m_tvmFunction = nullptr; } // cleanup TVM execution context - if (jitContext->m_execContext != NULL) { + if (jitContext->m_execContext != nullptr) { tvm::freeExecContext(jitContext->m_execContext); - jitContext->m_execContext = NULL; + jitContext->m_execContext = nullptr; } FreeJitContext(jitContext); @@ -593,12 +567,12 @@ static void CleanupJitContextPrimary(JitContext* jitContext) if (jitContext->m_index) { if (jitContext->m_searchKey) { jitContext->m_index->DestroyKey(jitContext->m_searchKey); - jitContext->m_searchKey = NULL; + jitContext->m_searchKey = nullptr; } if (jitContext->m_endIteratorKey != nullptr) { jitContext->m_index->DestroyKey(jitContext->m_endIteratorKey); - jitContext->m_endIteratorKey = NULL; + jitContext->m_endIteratorKey = nullptr; } jitContext->m_index = nullptr; } @@ -606,7 +580,7 @@ static void CleanupJitContextPrimary(JitContext* jitContext) // cleanup JOIN outer row copy if (jitContext->m_outerRowCopy != nullptr) { jitContext->m_table->DestroyRow(jitContext->m_outerRowCopy); - jitContext->m_outerRowCopy = NULL; + jitContext->m_outerRowCopy = nullptr; } } } @@ -616,12 +590,12 @@ static void CleanupJitContextInner(JitContext* jitContext) if (jitContext->m_innerIndex != nullptr) { if (jitContext->m_innerSearchKey != nullptr) { jitContext->m_innerIndex->DestroyKey(jitContext->m_innerSearchKey); - jitContext->m_innerSearchKey = NULL; + jitContext->m_innerSearchKey = nullptr; } if (jitContext->m_innerEndIteratorKey != nullptr) { jitContext->m_innerIndex->DestroyKey(jitContext->m_innerEndIteratorKey); - jitContext->m_innerEndIteratorKey = NULL; + jitContext->m_innerEndIteratorKey = nullptr; } jitContext->m_innerIndex = nullptr; } @@ -646,6 +620,8 @@ static void CleanupJitContextSubQueryDataArray(JitContext* jitContext) static void CleanupJitContextSubQueryData(JitContext::SubQueryData* subQueryData) { + MemoryContext oldCtx = CurrentMemoryContext; + CurrentMemoryContext = u_sess->top_mem_cxt; if (subQueryData->m_slot != nullptr) { ExecDropSingleTupleTableSlot(subQueryData->m_slot); subQueryData->m_slot = nullptr; @@ -665,13 +641,14 @@ static void CleanupJitContextSubQueryData(JitContext::SubQueryData* subQueryData } subQueryData->m_index = nullptr; } + CurrentMemoryContext = oldCtx; } static JitContextPool* AllocSessionJitContextPool() { size_t allocSize = sizeof(JitContextPool); JitContextPool* jitContextPool = (JitContextPool*)MOT::MemGlobalAllocAligned(allocSize, L1_CACHE_LINE); - if (jitContextPool == NULL) { + if (jitContextPool == nullptr) { MOT_REPORT_ERROR(MOT_ERROR_OOM, "Allocate JIT Context", "Failed to allocate %u bytes for JIT context pool", @@ -682,7 +659,7 @@ static JitContextPool* AllocSessionJitContextPool() if (!InitJitContextPool(jitContextPool, JIT_CONTEXT_LOCAL, GetMotCodegenLimit())) { MOT::MemGlobalFree(jitContextPool); - jitContextPool = NULL; + jitContextPool = nullptr; } } return jitContextPool; @@ -697,7 +674,7 @@ extern void FreeSessionJitContextPool(JitContextPool* jitContextPool) static MOT::Key* PrepareJitSearchKey(JitContext* jitContext, MOT::Index* index) { MOT::Key* key = index->CreateNewKey(); - if (key == NULL) { + if (key == nullptr) { MOT_LOG_TRACE("Failed to prepare for executing jitted code: Failed to create reusable search key"); } else { key->InitKey((uint16_t)index->GetKeyLength()); diff --git a/src/gausskernel/storage/mot/jit_exec/src/jit_exec.cpp b/src/gausskernel/storage/mot/jit_exec/src/jit_exec.cpp index 9bf167252..667877136 100644 --- a/src/gausskernel/storage/mot/jit_exec/src/jit_exec.cpp +++ b/src/gausskernel/storage/mot/jit_exec/src/jit_exec.cpp @@ -467,16 +467,19 @@ extern void PurgeJitSourceCache(uint64_t relationId, bool purgeOnly) extern bool JitInitialize() { + if (!IsMotCodegenEnabled()) { + MOT_LOG_INFO("MOT JIT execution is disabled by user configuration"); + return true; + } + if (JitCanInitThreadCodeGen()) { if (IsMotPseudoCodegenForced()) { MOT_LOG_INFO("Forcing TVM on LLVM natively supported platform"); } else { PrintNativeLlvmStartupInfo(); } - } else if (IsMotCodegenEnabled()) { - MOT_LOG_INFO("Using TVM on LLVM natively unsupported platform"); } else { - MOT_LOG_INFO("MOT JIT execution is disabled by user configuration"); + MOT_LOG_INFO("Using TVM on LLVM natively unsupported platform"); } enum InitState { diff --git a/src/gausskernel/storage/mot/jit_exec/src/jit_helpers.cpp b/src/gausskernel/storage/mot/jit_exec/src/jit_helpers.cpp index cf9eeed92..67aeacf0c 100644 --- a/src/gausskernel/storage/mot/jit_exec/src/jit_helpers.cpp +++ b/src/gausskernel/storage/mot/jit_exec/src/jit_helpers.cpp @@ -55,14 +55,14 @@ static double numericToDouble(Datum numeric_value) /** @brief Allocates and initializes a distinct set of integers. */ static void* prepareDistinctIntSet() { - void* buf = palloc(sizeof(DistinctIntSetType)); + void* buf = MOT::MemSessionAlloc(sizeof(DistinctIntSetType)); return new (buf) DistinctIntSetType(); } /** @brief Allocates and initializes a distinct set of double-precision values. */ static void* prepareDistinctDoubleSet() { - void* buf = palloc(sizeof(DistinctDoubleSetType)); + void* buf = MOT::MemSessionAlloc(sizeof(DistinctDoubleSetType)); return new (buf) DistinctDoubleSetType(); } @@ -93,7 +93,7 @@ static void destroyDistinctIntSet(void* distinct_set) { DistinctIntSetType* int_set = (DistinctIntSetType*)distinct_set; int_set->~DistinctIntSetType(); - pfree(distinct_set); + MOT::MemSessionFree(distinct_set); } /** @brief Destroys and frees a distinct set of double-precision values. */ @@ -101,7 +101,7 @@ static void destroyDistinctDoubleSet(void* distinct_set) { DistinctDoubleSetType* double_set = (DistinctDoubleSetType*)distinct_set; double_set->~DistinctDoubleSetType(); - pfree(distinct_set); + MOT::MemSessionFree(distinct_set); } /*--------------------------- DEBUG Print Helpers ---------------------------*/ From d505a7fc6088da4278890b95aaec4b5a878eb4d9 Mon Sep 17 00:00:00 2001 From: Vinoth Veeraraghavan Date: Thu, 15 Oct 2020 19:03:55 +0800 Subject: [PATCH 3/4] Fix MOT table metadata recovery from checkpoint --- .../src/system/recovery/recovery_manager.cpp | 9 +---- .../src/system/recovery/recovery_manager.h | 5 +-- .../core/src/system/recovery/recovery_ops.cpp | 35 ++++++++++++++----- 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/src/gausskernel/storage/mot/core/src/system/recovery/recovery_manager.cpp b/src/gausskernel/storage/mot/core/src/system/recovery/recovery_manager.cpp index 5b94904fd..ae5a380aa 100644 --- a/src/gausskernel/storage/mot/core/src/system/recovery/recovery_manager.cpp +++ b/src/gausskernel/storage/mot/core/src/system/recovery/recovery_manager.cpp @@ -229,7 +229,7 @@ bool RecoveryManager::RecoverTableMetadata(uint32_t tableId) CheckpointUtils::CloseFile(fd); Table* table = nullptr; - CreateTable(dataBuf, status, table, true); + CreateTable(dataBuf, status, table, ADD_TO_ENGINE); delete[] dataBuf; return (status == RC_OK); @@ -482,7 +482,6 @@ bool RecoveryManager::RecoverFromCheckpoint() m_tableIds.size(), m_checkpointId); - BeginTransaction(); for (auto it = m_tableIds.begin(); it != m_tableIds.end(); ++it) { if (IsRecoveryMemoryLimitReached(NUM_REDO_RECOVERY_THREADS)) { MOT_LOG_ERROR("Memory hard limit reached. Cannot recover datanode"); @@ -497,12 +496,6 @@ bool RecoveryManager::RecoverFromCheckpoint() return false; } } - RC status = CommitTransaction(MOT_RECOVERED_TABLE_CSN); - if (status != RC_OK) { - MOT_LOG_ERROR("Failed to commit table recovery: %s (error code: %d)", RcToString(status), (int)status); - OnError(RecoveryManager::ErrCodes::CP_TABLE_COMMIT, "Failed to commit table recovery from checkpoint"); - return false; - } std::vector recoveryThreadPool; for (uint32_t i = 0; i < m_numWorkers; ++i) { diff --git a/src/gausskernel/storage/mot/core/src/system/recovery/recovery_manager.h b/src/gausskernel/storage/mot/core/src/system/recovery/recovery_manager.h index a1371dc6e..cbc9981ff 100644 --- a/src/gausskernel/storage/mot/core/src/system/recovery/recovery_manager.h +++ b/src/gausskernel/storage/mot/core/src/system/recovery/recovery_manager.h @@ -60,6 +60,7 @@ public: }; enum RecoveryOpState { COMMIT = 1, ABORT = 2, TPC_APPLY = 3, TPC_COMMIT = 4, TPC_ABORT = 5 }; + enum CreateTableMethod { TRANSACTIONAL = 1, ADD_TO_ENGINE = 2, DONT_ADD_TO_ENGINE = 3 }; private: /** @@ -870,9 +871,9 @@ private: * @param data the table's data * @param status the returned status of the operation * @param table the returned table object. - * @param addToEngine should the table be added to the engine's metadata + * @param method controls whether the table is added to the engine or not. */ - static void CreateTable(char* data, RC& status, Table*& table, bool addToEngine); + static void CreateTable(char* data, RC& status, Table*& table, CreateTableMethod method); /** * @brief performs the actual table deletion. diff --git a/src/gausskernel/storage/mot/core/src/system/recovery/recovery_ops.cpp b/src/gausskernel/storage/mot/core/src/system/recovery/recovery_ops.cpp index 60ccc783b..71f3b1a8b 100644 --- a/src/gausskernel/storage/mot/core/src/system/recovery/recovery_ops.cpp +++ b/src/gausskernel/storage/mot/core/src/system/recovery/recovery_ops.cpp @@ -102,11 +102,11 @@ uint32_t RecoveryManager::RecoverLogOperationCreateTable( switch (state) { case COMMIT: MOT_LOG_DEBUG("RecoverLogOperationCreateTable: COMMIT"); - CreateTable((char*)data, status, table, true); + CreateTable((char*)data, status, table, TRANSACTIONAL); break; case TPC_APPLY: - CreateTable((char*)data, status, table, false /* don't add to engine yet */); + CreateTable((char*)data, status, table, DONT_ADD_TO_ENGINE); if (status == RC_OK && table != nullptr) { tableInfo = new (std::nothrow) TableInfo(table, transactionId); if (tableInfo != nullptr) { @@ -674,7 +674,7 @@ void RecoveryManager::UpdateRow(uint64_t tableId, uint64_t exId, char* keyData, MOTCurrTxn->DestroyTxnKey(key); } -void RecoveryManager::CreateTable(char* data, RC& status, Table*& table, bool addToEngine) +void RecoveryManager::CreateTable(char* data, RC& status, Table*& table, CreateTableMethod method) { /* first verify that the table does not exists */ string name; @@ -699,25 +699,44 @@ void RecoveryManager::CreateTable(char* data, RC& status, Table*& table, bool ad do { if (!table->IsDeserialized()) { MOT_LOG_ERROR("RecoveryManager::CreateTable: failed to de-serialize table"); + status = RC_ERROR; break; } - if (addToEngine && ((status = MOTCurrTxn->CreateTable(table)) != RC_OK)) { - MOT_LOG_ERROR("RecoveryManager::CreateTable: failed to add table to engine"); + switch (method) { + case TRANSACTIONAL: + status = MOTCurrTxn->CreateTable(table); + break; + + case ADD_TO_ENGINE: + status = GetTableManager()->AddTable(table) ? RC_OK : RC_ERROR; + break; + + case DONT_ADD_TO_ENGINE: + default: + status = RC_OK; + break; + } + + if (status != RC_OK) { + MOT_LOG_ERROR("RecoveryManager::CreateTable: failed to add table %s (id: %u) to engine (method %u)", + table->GetLongTableName().c_str(), + table->GetTableId(), + method); break; } - MOT_LOG_DEBUG("RecoveryManager::CreateTable: table %s [internal id %u] created (%s to engine)", + MOT_LOG_DEBUG("RecoveryManager::CreateTable: table %s (id %u) created (method %u)", table->GetLongTableName().c_str(), table->GetTableId(), - addToEngine ? "added" : "not added"); - status = RC_OK; + method); return; } while (0); MOT_LOG_ERROR("RecoveryManager::CreateTable: failed to recover table"); delete table; + if (status == RC_OK) { status = RC_ERROR; } From 17e320d32263beec20e50a4d3ec66d965f729536 Mon Sep 17 00:00:00 2001 From: Vinoth Veeraraghavan Date: Thu, 15 Oct 2020 20:21:38 +0800 Subject: [PATCH 4/4] Fix ForreignPath generation for parameterized IndexPath --- .../storage/mot/fdw_adapter/src/mot_fdw.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/gausskernel/storage/mot/fdw_adapter/src/mot_fdw.cpp b/src/gausskernel/storage/mot/fdw_adapter/src/mot_fdw.cpp index 04dc3d8f3..3cebfd558 100644 --- a/src/gausskernel/storage/mot/fdw_adapter/src/mot_fdw.cpp +++ b/src/gausskernel/storage/mot/fdw_adapter/src/mot_fdw.cpp @@ -696,12 +696,14 @@ static void MOTGetForeignPaths(PlannerInfo* root, RelOptInfo* baserel, Oid forei set_cheapest(baserel); if (!IS_PGXC_COORDINATOR && list_length(baserel->cheapest_parameterized_paths) > 0) { - bestPath = (Path*)linitial(baserel->cheapest_parameterized_paths); - if (IsA(bestPath, IndexPath) && bestPath->param_info) { - IndexPath* ip = (IndexPath*)bestPath; - bestClause = ip->indexclauses; + foreach (lc, baserel->cheapest_parameterized_paths) { + bestPath = (Path*)lfirst(lc); + if (IsA(bestPath, IndexPath) && bestPath->param_info) { + IndexPath* ip = (IndexPath*)bestPath; + bestClause = ip->indexclauses; + break; + } } - usablePathkeys = nullptr; }