diff --git a/src/gausskernel/storage/mot/core/src/infra/config/config_manager.cpp b/src/gausskernel/storage/mot/core/src/infra/config/config_manager.cpp index 4199bb397..575c13794 100644 --- a/src/gausskernel/storage/mot/core/src/infra/config/config_manager.cpp +++ b/src/gausskernel/storage/mot/core/src/infra/config/config_manager.cpp @@ -37,20 +37,6 @@ namespace MOT { DECLARE_LOGGER(ConfigManager, Configuration) -// Initialization helper macro (for system calls) -#define CHECK_SYS_INIT_STATUS(rc, syscall, format, ...) \ - if (rc != 0) { \ - MOT_REPORT_SYSTEM_PANIC_CODE(rc, syscall, "Configuration Manager Initialization", format, ##__VA_ARGS__); \ - break; \ - } - -// Initialization helper macro (for sub-service initialization) -#define CHECK_INIT_STATUS(result, format, ...) \ - if (!result) { \ - MOT_REPORT_PANIC(MOT_ERROR_INTERNAL, "Configuration Manager Initialization", format, ##__VA_ARGS__); \ - break; \ - } - ConfigManager* ConfigManager::m_manager = nullptr; ConfigManager::ConfigManager() : m_configLoaderCount(0), m_listenerCount(0) @@ -301,26 +287,22 @@ const ConfigTree* ConfigManager::GetConfigTree(const char* configName) const bool ConfigManager::Initialize(char** argv, int argc) { - bool result = false; + bool result = true; char* envvar = getenv("MOT_DEBUG_CFG_LOAD"); if (envvar && (strcmp(envvar, "TRUE") == 0)) { SetLogComponentLogLevel("Configuration", LogLevel::LL_DEBUG); } - do { - // add command line configuration loader - if (argv != nullptr && argc != 0) { - result = AddCmdLineConfigLoader(argv, argc); - CHECK_INIT_STATUS(result, "Failed to create command line configuration loader"); + // add command line configuration loader + if (argv != nullptr && argc != 0) { + result = AddCmdLineConfigLoader(argv, argc); + if (!result) { + MOT_REPORT_PANIC(MOT_ERROR_INTERNAL, + "Configuration Manager Initialization", + "Failed to create command line configuration loader"); } - - // register special listener (workaround until full integration of configuration module) - // must be first listener, other listeners rely on it to be updated with latest configuration - MOTConfiguration& motCfg = GetGlobalConfiguration(); - result = AddConfigChangeListener(&motCfg); - CHECK_INIT_STATUS(result, "Failed to register main configuration listener"); - } while (0); + } return result; } diff --git a/src/gausskernel/storage/mot/core/src/infra/config/config_manager.h b/src/gausskernel/storage/mot/core/src/infra/config/config_manager.h index 03e0ac5e9..1c9f1c97f 100644 --- a/src/gausskernel/storage/mot/core/src/infra/config/config_manager.h +++ b/src/gausskernel/storage/mot/core/src/infra/config/config_manager.h @@ -78,8 +78,8 @@ public: /** * @brief Creates singleton instance using command line arguments as configuration source. Must be called once * during engine startup. - * @param argv Command line argument array. - * @param argc Command line argument array size. + * @param[opt] argc Command line argument count (excluding program name first argument). + * @param[opt] argv Command line argument array (excluding program name first argument). * @return True if instance creation succeeded. */ static bool CreateInstance(char** argv = nullptr, int argc = 0); diff --git a/src/gausskernel/storage/mot/core/src/mot.conf b/src/gausskernel/storage/mot/core/src/mot.conf index 36a349b01..a29765d54 100644 --- a/src/gausskernel/storage/mot/core/src/mot.conf +++ b/src/gausskernel/storage/mot/core/src/mot.conf @@ -20,6 +20,8 @@ # GB = gigabytes # TB = terabytes # +# If no memory units are specified, then bytes are assumed. +# # Some memory units can be also given in the form of percentage from max_process_memory configured # in postgresql.conf as follows: # @@ -31,6 +33,8 @@ # min = minutes (can also specify mins or minutes) # h = hours (can also specify hours) # d = days (can also specify days) +# +# If no time units are specified, then microseconds are assumed. #------------------------------------------------------------------------------ # REDO LOG @@ -296,8 +300,6 @@ #min_mot_local_memory = 0 MB # Configures the maximum memory limit for a single session in MOT engine. -# Specifying percentage value relates to the total defined by max_process_memory configured in -# postgresql.conf. # Usually sessions in MOT engine are free to allocate local memory as much as needed, as long as # the local memory limit is not breached. Nevertheless, in oder to prevent one session from taking # too much memory, and thus denying memory from other sessions, this configuration item is used to @@ -307,24 +309,17 @@ # A value of zero denotes no restriction on any session-local small allocations per session, except # for the restriction arising from the local memory allocation limit configured by # max_mot_local_memory. +# Note: Percentage values cannot be set for this configuration item. # #max_mot_session_memory = 0 MB # Configures the minimum memory reservation for a single session in MOT engine. -# Specifying percentage value relates to the total defined by max_process_memory configured in -# postgresql.conf. # This value is used both for pre-allocation of memory during session creation, as well as for # ensuring a minimum amount of memory is available for the session to perform its normal operation. +# Note: Percentage values cannot be set for this configuration item. # #min_mot_session_memory = 0 MB -# Configures the high red mark for memory allocations. -# This is taken as percentage from the maximum MOT engine configured by max_mot_memory. When total -# memory usage by MOT reaches this red-mark, only destructive operations are allowed, and all other -# operations will result in error being reported to the user. -# -#high_red_mark_percent = 90 - # Configures the large buffer store for sessions. # When a user session executes a query that requires a lot of memory (e.g. when using many rows), # the large buffer store is used both for increasing the certainty level that such memory is @@ -332,6 +327,7 @@ # allocation for a session that is above 1022 KB is considered as a large memory allocation. If the # large buffer store is not used or depleted, such allocations are treated as huge allocations that # are served directly from kernel. +# Note: Percentage values cannot be set for this configuration item. # #session_large_buffer_store_size = 0 MB @@ -341,12 +337,14 @@ # determining the internal division of the buffer store into objects of various size. # This size cannot exceed 1/8 of the session_large_buffer_store_size, and if it does it is # rectified to the possible maximum. +# Note: Percentage values cannot be set for this configuration item. # #session_large_buffer_store_max_object_size = 0 MB # Configures the maximum size of a single huge memory allocation made by a session. # Huge allocations are served directly from kernel, and therefore are not guaranteed to succeed. # This value also pertains to global (i.e. not session-related) memory allocations. +# Note: Percentage values cannot be set for this configuration item. # #session_max_huge_object_size = 1 GB @@ -367,6 +365,7 @@ # garbage collection that might affect performance. Setting a high value will trigger garbage # collection less frequently, but will result in higher levels of un-reclaimed memory. This is very # much dependent upon overall workload. +# Note: Percentage values cannot be set for this configuration item. # #reclaim_threshold = 512 KB @@ -384,6 +383,7 @@ # in the same pass, even though that batch size limit is reached, until the total size of still # waiting to-be-reclaimed objects will be less than this threshold, or there are no more objects # eligible for reclamation. +# Note: Percentage values cannot be set for this configuration item. # #high_reclaim_threshold = 8 MB @@ -413,11 +413,3 @@ # Limits the amount of JIT queries allowed per user session. # #mot_codegen_limit = 100 - -#------------------------------------------------------------------------------ -# STORAGE -#------------------------------------------------------------------------------ - -# Specifies whether it is allowed to define an index over a null-able column. -# -#allow_index_on_nullable_column = true diff --git a/src/gausskernel/storage/mot/core/src/system/global.h b/src/gausskernel/storage/mot/core/src/system/global.h index 437a52c7b..e69400b2d 100644 --- a/src/gausskernel/storage/mot/core/src/system/global.h +++ b/src/gausskernel/storage/mot/core/src/system/global.h @@ -300,10 +300,6 @@ inline void Prefetch(const void* ptr) /** @define Constant denoting indentation used for MOT printouts. */ #define PRINT_REPORT_INDENT 2 - -/** @define Min and Max values for asynchronous redo log buffer array count. */ -#define MIN_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT 8 -#define MAX_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT 128 } // namespace MOT #endif // MOT_GLOBAL_H diff --git a/src/gausskernel/storage/mot/core/src/system/mot_configuration.cpp b/src/gausskernel/storage/mot/core/src/system/mot_configuration.cpp index 802709502..df81fd1f8 100644 --- a/src/gausskernel/storage/mot/core/src/system/mot_configuration.cpp +++ b/src/gausskernel/storage/mot/core/src/system/mot_configuration.cpp @@ -35,24 +35,44 @@ IMPLEMENT_CLASS_LOGGER(MOTConfiguration, Configuration) MOTConfiguration MOTConfiguration::motGlobalConfiguration; // static member definitions +// Memory scaling constants +constexpr uint64_t MOTConfiguration::SCALE_BYTES; +constexpr uint64_t MOTConfiguration::SCALE_KILO_BYTES; +constexpr uint64_t MOTConfiguration::SCALE_MEGA_BYTES; +// Time scaling constants +constexpr uint64_t MOTConfiguration::SCALE_MICROS; +constexpr uint64_t MOTConfiguration::SCALE_MILLIS; +constexpr uint64_t MOTConfiguration::SCALE_SECONDS; // redo-log configuration members constexpr bool MOTConfiguration::DEFAULT_ENABLE_REDO_LOG; constexpr LoggerType MOTConfiguration::DEFAULT_LOGGER_TYPE; constexpr RedoLogHandlerType MOTConfiguration::DEFAULT_REDO_LOG_HANDLER_TYPE; constexpr uint32_t MOTConfiguration::DEFAULT_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT; +constexpr uint32_t MOTConfiguration::MIN_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT; +constexpr uint32_t MOTConfiguration::MAX_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT; constexpr bool MOTConfiguration::DEFAULT_ENABLE_GROUP_COMMIT; constexpr uint64_t MOTConfiguration::DEFAULT_GROUP_COMMIT_SIZE; +constexpr uint64_t MOTConfiguration::MIN_GROUP_COMMIT_SIZE; +constexpr uint64_t MOTConfiguration::MAX_GROUP_COMMIT_SIZE; constexpr const char* MOTConfiguration::DEFAULT_GROUP_COMMIT_TIMEOUT; constexpr uint64_t MOTConfiguration::DEFAULT_GROUP_COMMIT_TIMEOUT_USEC; +constexpr uint64_t MOTConfiguration::MIN_GROUP_COMMIT_TIMEOUT_USEC; +constexpr uint64_t MOTConfiguration::MAX_GROUP_COMMIT_TIMEOUT_USEC; // checkpoint configuration members constexpr bool MOTConfiguration::DEFAULT_ENABLE_CHECKPOINT; constexpr bool MOTConfiguration::DEFAULT_ENABLE_INCREMENTAL_CHECKPOINT; constexpr const char* MOTConfiguration::DEFAULT_CHECKPOINT_DIR; constexpr const char* MOTConfiguration::DEFAULT_CHECKPOINT_SEGSIZE; constexpr uint32_t MOTConfiguration::DEFAULT_CHECKPOINT_SEGSIZE_BYTES; +constexpr uint32_t MOTConfiguration::MIN_CHECKPOINT_SEGSIZE_BYTES; +constexpr uint32_t MOTConfiguration::MAX_CHECKPOINT_SEGSIZE_BYTES; constexpr uint32_t MOTConfiguration::DEFAULT_CHECKPOINT_WORKERS; +constexpr uint32_t MOTConfiguration::MIN_CHECKPOINT_WORKERS; +constexpr uint32_t MOTConfiguration::MAX_CHECKPOINT_WORKERS; // recovery configuration members constexpr uint32_t MOTConfiguration::DEFAULT_CHECKPOINT_RECOVERY_WORKERS; +constexpr uint32_t MOTConfiguration::MIN_CHECKPOINT_RECOVERY_WORKERS; +constexpr uint32_t MOTConfiguration::MAX_CHECKPOINT_RECOVERY_WORKERS; constexpr bool MOTConfiguration::DEFAULT_ENABLE_LOG_RECOVERY_STATS; // machine configuration members constexpr uint16_t MOTConfiguration::DEFAULT_NUMA_NODES; @@ -63,6 +83,8 @@ constexpr uint16_t MOTConfiguration::DEFAULT_MAX_DATA_NODES; constexpr bool MOTConfiguration::DEFAULT_ENABLE_STATS; constexpr const char* MOTConfiguration::DEFAULT_STATS_PRINT_PERIOD; constexpr uint32_t MOTConfiguration::DEFAULT_STATS_PRINT_PERIOD_SECONDS; +constexpr uint32_t MOTConfiguration::MIN_STATS_PRINT_PERIOD_SECONDS; +constexpr uint32_t MOTConfiguration::MAX_STATS_PRINT_PERIOD_SECONDS; constexpr const char* MOTConfiguration::DEFAULT_FULL_STATS_PRINT_PERIOD; constexpr uint32_t MOTConfiguration::DEFAULT_FULL_STATS_PRINT_PERIOD_SECONDS; constexpr bool MOTConfiguration::DEFAULT_ENABLE_DB_SESSION_STAT_PRINT; @@ -81,50 +103,86 @@ constexpr LogLevel MOTConfiguration::DEFAULT_CFG_STARTUP_LOG_LEVEL; // memory configuration members constexpr bool MOTConfiguration::DEFAULT_ENABLE_NUMA; constexpr uint16_t MOTConfiguration::DEFAULT_MAX_THREADS; +constexpr uint16_t MOTConfiguration::MIN_MAX_THREADS; +constexpr uint16_t MOTConfiguration::MAX_MAX_THREADS; constexpr uint32_t MOTConfiguration::DEFAULT_MAX_CONNECTIONS; +constexpr uint32_t MOTConfiguration::MIN_MAX_CONNECTIONS; +constexpr uint32_t MOTConfiguration::MAX_MAX_CONNECTIONS; constexpr AffinityMode MOTConfiguration::DEFAULT_AFFINITY_MODE; constexpr bool MOTConfiguration::DEFAULT_LAZY_LOAD_CHUNK_DIRECTORY; constexpr const char* MOTConfiguration::DEFAULT_MAX_MOT_GLOBAL_MEMORY; constexpr uint32_t MOTConfiguration::DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB; +constexpr uint32_t MOTConfiguration::MIN_MAX_MOT_GLOBAL_MEMORY_MB; +constexpr uint32_t MOTConfiguration::MAX_MAX_MOT_GLOBAL_MEMORY_MB; constexpr const char* MOTConfiguration::DEFAULT_MAX_MOT_LOCAL_MEMORY; constexpr uint32_t MOTConfiguration::DEFAULT_MAX_MOT_LOCAL_MEMORY_MB; +constexpr uint32_t MOTConfiguration::MIN_MAX_MOT_LOCAL_MEMORY_MB; +constexpr uint32_t MOTConfiguration::MAX_MAX_MOT_LOCAL_MEMORY_MB; constexpr const char* MOTConfiguration::DEFAULT_MIN_MOT_GLOBAL_MEMORY; constexpr uint32_t MOTConfiguration::DEFAULT_MIN_MOT_GLOBAL_MEMORY_MB; +constexpr uint32_t MOTConfiguration::MIN_MIN_MOT_GLOBAL_MEMORY_MB; +constexpr uint32_t MOTConfiguration::MAX_MIN_MOT_GLOBAL_MEMORY_MB; constexpr const char* MOTConfiguration::DEFAULT_MIN_MOT_LOCAL_MEMORY; constexpr uint32_t MOTConfiguration::DEFAULT_MIN_MOT_LOCAL_MEMORY_MB; +constexpr uint32_t MOTConfiguration::MIN_MIN_MOT_LOCAL_MEMORY_MB; +constexpr uint32_t MOTConfiguration::MAX_MIN_MOT_LOCAL_MEMORY_MB; constexpr const char* MOTConfiguration::DEFAULT_MAX_MOT_SESSION_MEMORY; constexpr uint32_t MOTConfiguration::DEFAULT_MAX_MOT_SESSION_MEMORY_KB; +constexpr uint32_t MOTConfiguration::MIN_MAX_MOT_SESSION_MEMORY_KB; +constexpr uint32_t MOTConfiguration::MAX_MAX_MOT_SESSION_MEMORY_KB; constexpr const char* MOTConfiguration::DEFAULT_MIN_MOT_SESSION_MEMORY; constexpr uint32_t MOTConfiguration::DEFAULT_MIN_MOT_SESSION_MEMORY_KB; +constexpr uint32_t MOTConfiguration::MIN_MIN_MOT_SESSION_MEMORY_KB; +constexpr uint32_t MOTConfiguration::MAX_MIN_MOT_SESSION_MEMORY_KB; constexpr MemReserveMode MOTConfiguration::DEFAULT_RESERVE_MEMORY_MODE; constexpr MemStorePolicy MOTConfiguration::DEFAULT_STORE_MEMORY_POLICY; constexpr MemAllocPolicy MOTConfiguration::DEFAULT_CHUNK_ALLOC_POLICY; constexpr uint32_t MOTConfiguration::DEFAULT_CHUNK_PREALLOC_WORKER_COUNT; +constexpr uint32_t MOTConfiguration::MIN_CHUNK_PREALLOC_WORKER_COUNT; +constexpr uint32_t MOTConfiguration::MAX_CHUNK_PREALLOC_WORKER_COUNT; constexpr uint32_t MOTConfiguration::DEFAULT_HIGH_RED_MARK_PERCENT; +constexpr uint32_t MOTConfiguration::MIN_HIGH_RED_MARK_PERCENT; +constexpr uint32_t MOTConfiguration::MAX_HIGH_RED_MARK_PERCENT; constexpr const char* MOTConfiguration::DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE; constexpr uint32_t MOTConfiguration::DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE_MB; +constexpr uint32_t MOTConfiguration::MIN_SESSION_LARGE_BUFFER_STORE_SIZE_MB; +constexpr uint32_t MOTConfiguration::MAX_SESSION_LARGE_BUFFER_STORE_SIZE_MB; constexpr const char* MOTConfiguration::DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE; constexpr uint32_t MOTConfiguration::DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB; +constexpr uint32_t MOTConfiguration::MIN_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB; +constexpr uint32_t MOTConfiguration::MAX_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB; constexpr const char* MOTConfiguration::DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE; constexpr uint32_t MOTConfiguration::DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE_MB; +constexpr uint32_t MOTConfiguration::MIN_SESSION_MAX_HUGE_OBJECT_SIZE_MB; +constexpr uint32_t MOTConfiguration::MAX_SESSION_MAX_HUGE_OBJECT_SIZE_MB; // GC configuration members constexpr bool MOTConfiguration::DEFAULT_GC_ENABLE; constexpr const char* MOTConfiguration::DEFAULT_GC_RECLAIM_THRESHOLD; constexpr uint32_t MOTConfiguration::DEFAULT_GC_RECLAIM_THRESHOLD_BYTES; +constexpr uint32_t MOTConfiguration::MIN_GC_RECLAIM_THRESHOLD_BYTES; +constexpr uint32_t MOTConfiguration::MAX_GC_RECLAIM_THRESHOLD_BYTES; constexpr uint32_t MOTConfiguration::DEFAULT_GC_RECLAIM_BATCH_SIZE; +constexpr uint32_t MOTConfiguration::MIN_GC_RECLAIM_BATCH_SIZE; +constexpr uint32_t MOTConfiguration::MAX_GC_RECLAIM_BATCH_SIZE; constexpr const char* MOTConfiguration::DEFAULT_GC_HIGH_RECLAIM_THRESHOLD; constexpr uint32_t MOTConfiguration::DEFAULT_GC_HIGH_RECLAIM_THRESHOLD_BYTES; +constexpr uint32_t MOTConfiguration::MIN_GC_HIGH_RECLAIM_THRESHOLD_BYTES; +constexpr uint32_t MOTConfiguration::MAX_GC_HIGH_RECLAIM_THRESHOLD_BYTES; // JIT configuration members constexpr bool MOTConfiguration::DEFAULT_ENABLE_MOT_CODEGEN; constexpr bool MOTConfiguration::DEFAULT_FORCE_MOT_PSEUDO_CODEGEN; constexpr bool MOTConfiguration::DEFAULT_ENABLE_MOT_CODEGEN_PRINT; constexpr uint32_t MOTConfiguration::DEFAULT_MOT_CODEGEN_LIMIT; +constexpr uint32_t MOTConfiguration::MIN_MOT_CODEGEN_LIMIT; +constexpr uint32_t MOTConfiguration::MAX_MOT_CODEGEN_LIMIT; // storage configuration constexpr bool MOTConfiguration::DEFAULT_ALLOW_INDEX_ON_NULLABLE_COLUMN; constexpr IndexTreeFlavor MOTConfiguration::DEFAULT_INDEX_TREE_FLAVOR; // general configuration members constexpr const char* MOTConfiguration::DEFAULT_CFG_MONITOR_PERIOD; constexpr uint64_t MOTConfiguration::DEFAULT_CFG_MONITOR_PERIOD_SECONDS; +constexpr uint64_t MOTConfiguration::MIN_CFG_MONITOR_PERIOD_SECONDS; +constexpr uint64_t MOTConfiguration::MAX_CFG_MONITOR_PERIOD_SECONDS; constexpr bool MOTConfiguration::DEFAULT_RUN_INTERNAL_CONSISTENCY_VALIDATION; constexpr uint32_t MOTConfiguration::DEFAULT_TOTAL_MEMORY_MB; @@ -448,7 +506,8 @@ MOTConfiguration::MOTConfiguration() m_indexTreeFlavor(DEFAULT_INDEX_TREE_FLAVOR), m_configMonitorPeriodSeconds(DEFAULT_CFG_MONITOR_PERIOD_SECONDS), m_runInternalConsistencyValidation(DEFAULT_RUN_INTERNAL_CONSISTENCY_VALIDATION), - m_totalMemoryMb(DEFAULT_TOTAL_MEMORY_MB) + m_totalMemoryMb(DEFAULT_TOTAL_MEMORY_MB), + m_suppressLog(false) {} void MOTConfiguration::Initialize() @@ -604,40 +663,37 @@ int MOTConfiguration::GetMappedCore(int logicId) const return -1; } -#define UPDATE_CFG(var, cfgPath, defaultValue) \ - UpdateConfigItem(var, cfg->GetConfigValue(cfgPath, defaultValue), cfgPath) +#define UPDATE_BOOL_CFG(var, cfgPath, defaultValue) \ + UpdateBoolConfigItem(var, cfg->GetConfigValue(cfgPath, defaultValue), cfgPath) #define UPDATE_USER_CFG(var, cfgPath, defaultValue) \ UpdateUserConfigItem(var, cfg->GetUserConfigValue(cfgPath, defaultValue), cfgPath) #define UPDATE_STRING_CFG(var, cfgPath, defaultValue) \ - UpdateConfigItem(var, cfg->GetStringConfigValue(cfgPath, defaultValue), cfgPath) + UpdateStringConfigItem(var, cfg->GetStringConfigValue(cfgPath, defaultValue), cfgPath) -#define UPDATE_INT_CFG(var, cfgPath, defaultValue) \ - UpdateConfigItem(var, cfg->GetIntegerConfigValue(cfgPath, defaultValue), cfgPath) +#define UPDATE_INT_CFG(var, cfgPath, defaultValue, lowerBound, upperBound) \ + UpdateIntConfigItem(var, cfg->GetIntegerConfigValue(cfgPath, defaultValue), cfgPath, lowerBound, upperBound) -#define UPDATE_INT_CFG_BOUNDS(var, cfgPath, defaultValue, lowerBound, upperBound) \ - UpdateConfigItem(var, cfg->GetIntegerConfigValue(cfgPath, defaultValue), cfgPath, lowerBound, upperBound) - -#define UPDATE_MEM_CFG(var, cfgPath, defaultValue, scale) \ +#define UPDATE_MEM_CFG(var, cfgPath, defaultValue, scale, lowerBound, upperBound) \ do { \ uint64_t memoryValueBytes = \ ParseMemoryValueBytes(cfg->GetStringConfigValue(cfgPath, defaultValue), (uint64_t)-1, cfgPath); \ - UpdateConfigItem(var, (uint32_t)(memoryValueBytes / scale), cfgPath); \ + UpdateIntConfigItem(var, (uint32_t)(memoryValueBytes / scale), cfgPath, lowerBound, upperBound); \ } while (0); -#define UPDATE_ABS_MEM_CFG(var, cfgPath, defaultValue, scale) \ - do { \ - uint64_t memoryValueBytes = \ - ParseMemoryUnit(cfg->GetStringConfigValue(cfgPath, defaultValue), (uint64_t)-1, cfgPath); \ - UpdateConfigItem(var, (uint32_t)(memoryValueBytes / scale), cfgPath); \ +#define UPDATE_ABS_MEM_CFG(var, cfgPath, defaultValue, scale, lowerBound, upperBound) \ + do { \ + uint64_t memoryValueBytes = \ + ParseMemoryUnit(cfg->GetStringConfigValue(cfgPath, defaultValue), (uint64_t)-1, cfgPath); \ + UpdateIntConfigItem(var, (uint32_t)(memoryValueBytes / scale), cfgPath, lowerBound, upperBound); \ } while (0); -#define UPDATE_TIME_CFG(var, cfgPath, defaultValue, scale) \ +#define UPDATE_TIME_CFG(var, cfgPath, defaultValue, scale, lowerBound, upperBound) \ do { \ uint64_t timeValueMicros = \ ParseTimeValueMicros(cfg->GetStringConfigValue(cfgPath, defaultValue), (uint64_t)-1, cfgPath); \ - UpdateConfigItem(var, (uint64_t)(timeValueMicros / scale), cfgPath); \ + UpdateIntConfigItem(var, (uint64_t)(timeValueMicros / scale), cfgPath, lowerBound, upperBound); \ } while (0); void MOTConfiguration::LoadConfig() @@ -646,48 +702,80 @@ void MOTConfiguration::LoadConfig() const LayeredConfigTree* cfg = ConfigManager::GetInstance().GetLayeredConfigTree(); // logger configuration - UPDATE_CFG(m_enableRedoLog, "enable_redo_log", DEFAULT_ENABLE_REDO_LOG); + UPDATE_BOOL_CFG(m_enableRedoLog, "enable_redo_log", DEFAULT_ENABLE_REDO_LOG); UPDATE_USER_CFG(m_loggerType, "logger_type", DEFAULT_LOGGER_TYPE); UPDATE_USER_CFG(m_redoLogHandlerType, "redo_log_handler_type", DEFAULT_REDO_LOG_HANDLER_TYPE); - UPDATE_INT_CFG_BOUNDS(m_asyncRedoLogBufferArrayCount, + UPDATE_INT_CFG(m_asyncRedoLogBufferArrayCount, "async_log_buffer_count", DEFAULT_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT, MIN_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT, MAX_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT); // commit configuration - UPDATE_CFG(m_enableGroupCommit, "enable_group_commit", DEFAULT_ENABLE_GROUP_COMMIT); - UPDATE_INT_CFG(m_groupCommitSize, "group_commit_size", DEFAULT_GROUP_COMMIT_SIZE); - UPDATE_TIME_CFG(m_groupCommitTimeoutUSec, "group_commit_timeout", DEFAULT_GROUP_COMMIT_TIMEOUT, 1); + UPDATE_BOOL_CFG(m_enableGroupCommit, "enable_group_commit", DEFAULT_ENABLE_GROUP_COMMIT); + UPDATE_INT_CFG(m_groupCommitSize, + "group_commit_size", + DEFAULT_GROUP_COMMIT_SIZE, + MIN_GROUP_COMMIT_SIZE, + MAX_GROUP_COMMIT_SIZE); + UPDATE_TIME_CFG(m_groupCommitTimeoutUSec, + "group_commit_timeout", + DEFAULT_GROUP_COMMIT_TIMEOUT, + SCALE_MICROS, + MIN_GROUP_COMMIT_TIMEOUT_USEC, + MAX_GROUP_COMMIT_TIMEOUT_USEC); // Checkpoint configuration - UPDATE_CFG(m_enableCheckpoint, "enable_checkpoint", DEFAULT_ENABLE_CHECKPOINT); + UPDATE_BOOL_CFG(m_enableCheckpoint, "enable_checkpoint", DEFAULT_ENABLE_CHECKPOINT); UPDATE_STRING_CFG(m_checkpointDir, "checkpoint_dir", DEFAULT_CHECKPOINT_DIR); - UPDATE_MEM_CFG(m_checkpointSegThreshold, "checkpoint_segsize", DEFAULT_CHECKPOINT_SEGSIZE, 1); - UPDATE_INT_CFG(m_checkpointWorkers, "checkpoint_workers", DEFAULT_CHECKPOINT_WORKERS); + UPDATE_MEM_CFG(m_checkpointSegThreshold, + "checkpoint_segsize", + DEFAULT_CHECKPOINT_SEGSIZE, + SCALE_BYTES, + MIN_CHECKPOINT_SEGSIZE_BYTES, + MAX_CHECKPOINT_SEGSIZE_BYTES); + UPDATE_INT_CFG(m_checkpointWorkers, + "checkpoint_workers", + DEFAULT_CHECKPOINT_WORKERS, + MIN_CHECKPOINT_WORKERS, + MAX_CHECKPOINT_WORKERS); // Recovery configuration - UPDATE_INT_CFG(m_checkpointRecoveryWorkers, "checkpoint_recovery_workers", DEFAULT_CHECKPOINT_RECOVERY_WORKERS); + UPDATE_INT_CFG(m_checkpointRecoveryWorkers, + "checkpoint_recovery_workers", + DEFAULT_CHECKPOINT_RECOVERY_WORKERS, + MIN_CHECKPOINT_RECOVERY_WORKERS, + MAX_CHECKPOINT_RECOVERY_WORKERS); // Tx configuration - not configurable yet - UPDATE_CFG(m_abortBufferEnable, "tx_abort_buffers_enable", true); - UPDATE_CFG(m_preAbort, "tx_pre_abort", true); + UPDATE_BOOL_CFG(m_abortBufferEnable, "tx_abort_buffers_enable", true); + UPDATE_BOOL_CFG(m_preAbort, "tx_pre_abort", true); m_validationLock = TxnValidation::TXN_VALIDATION_NO_WAIT; // statistics configuration - UPDATE_CFG(m_enableStats, "enable_stats", DEFAULT_ENABLE_STATS); - UPDATE_TIME_CFG(m_statPrintPeriodSeconds, "print_stats_period", DEFAULT_STATS_PRINT_PERIOD, 1000000); - UPDATE_TIME_CFG(m_statPrintFullPeriodSeconds, "print_full_stats_period", DEFAULT_FULL_STATS_PRINT_PERIOD, 1000000); - UPDATE_CFG(m_enableLogRecoveryStats, "enable_log_recovery_stats", DEFAULT_ENABLE_LOG_RECOVERY_STATS); - UPDATE_CFG(m_enableDbSessionStatistics, "enable_db_session_stats", DEFAULT_ENABLE_DB_SESSION_STAT_PRINT); - UPDATE_CFG(m_enableNetworkStatistics, "enable_network_stats", DEFAULT_ENABLE_NETWORK_STAT_PRINT); - UPDATE_CFG(m_enableLogStatistics, "enable_log_stats", DEFAULT_ENABLE_LOG_STAT_PRINT); - UPDATE_CFG(m_enableMemoryStatistics, "enable_memory_stats", DEFAULT_ENABLE_MEMORY_STAT_PRINT); - UPDATE_CFG( + UPDATE_BOOL_CFG(m_enableStats, "enable_stats", DEFAULT_ENABLE_STATS); + UPDATE_TIME_CFG(m_statPrintPeriodSeconds, + "print_stats_period", + DEFAULT_STATS_PRINT_PERIOD, + SCALE_SECONDS, + MIN_STATS_PRINT_PERIOD_SECONDS, + MAX_STATS_PRINT_PERIOD_SECONDS); + UPDATE_TIME_CFG(m_statPrintFullPeriodSeconds, + "print_full_stats_period", + DEFAULT_FULL_STATS_PRINT_PERIOD, + SCALE_SECONDS, + MIN_STATS_PRINT_PERIOD_SECONDS, + MAX_STATS_PRINT_PERIOD_SECONDS); + UPDATE_BOOL_CFG(m_enableLogRecoveryStats, "enable_log_recovery_stats", DEFAULT_ENABLE_LOG_RECOVERY_STATS); + UPDATE_BOOL_CFG(m_enableDbSessionStatistics, "enable_db_session_stats", DEFAULT_ENABLE_DB_SESSION_STAT_PRINT); + UPDATE_BOOL_CFG(m_enableNetworkStatistics, "enable_network_stats", DEFAULT_ENABLE_NETWORK_STAT_PRINT); + UPDATE_BOOL_CFG(m_enableLogStatistics, "enable_log_stats", DEFAULT_ENABLE_LOG_STAT_PRINT); + UPDATE_BOOL_CFG(m_enableMemoryStatistics, "enable_memory_stats", DEFAULT_ENABLE_MEMORY_STAT_PRINT); + UPDATE_BOOL_CFG( m_enableDetailedMemoryStatistics, "enable_detailed_memory_stats", DEFAULT_ENABLE_DETAILED_MEMORY_STAT_PRINT); - UPDATE_CFG(m_enableProcessStatistics, "enable_process_stats", DEFAULT_ENABLE_PROCESS_STAT_PRINT); - UPDATE_CFG(m_enableSystemStatistics, "enable_system_stats", DEFAULT_ENABLE_SYSTEM_STAT_PRINT); - UPDATE_CFG(m_enableJitStatistics, "enable_jit_stats", DEFAULT_ENABLE_JIT_STAT_PRINT); + UPDATE_BOOL_CFG(m_enableProcessStatistics, "enable_process_stats", DEFAULT_ENABLE_PROCESS_STAT_PRINT); + UPDATE_BOOL_CFG(m_enableSystemStatistics, "enable_system_stats", DEFAULT_ENABLE_SYSTEM_STAT_PRINT); + UPDATE_BOOL_CFG(m_enableJitStatistics, "enable_jit_stats", DEFAULT_ENABLE_JIT_STAT_PRINT); // log configuration UPDATE_USER_CFG(m_logLevel, "log_level", DEFAULT_LOG_LEVEL); @@ -697,56 +785,163 @@ void MOTConfiguration::LoadConfig() UPDATE_USER_CFG(m_cfgStartupLogLevel, "cfg_startup_log_level", DEFAULT_CFG_STARTUP_LOG_LEVEL); // memory configuration - UPDATE_CFG(m_enableNuma, "enable_numa", DEFAULT_ENABLE_NUMA); - UPDATE_INT_CFG(m_maxThreads, "max_threads", DEFAULT_MAX_THREADS); - UPDATE_INT_CFG(m_maxConnections, "max_connections", DEFAULT_MAX_CONNECTIONS); + UPDATE_BOOL_CFG(m_enableNuma, "enable_numa", DEFAULT_ENABLE_NUMA); + UPDATE_INT_CFG(m_maxThreads, "max_threads", DEFAULT_MAX_THREADS, MIN_MAX_THREADS, MAX_MAX_THREADS); + UPDATE_INT_CFG( + m_maxConnections, "max_connections", DEFAULT_MAX_CONNECTIONS, MIN_MAX_CONNECTIONS, MAX_MAX_CONNECTIONS); UPDATE_USER_CFG(m_sessionAffinityMode, "affinity_mode", DEFAULT_AFFINITY_MODE); // we save copies because main affinity mode may be overridden when thread pool is used if (IS_AFFINITY_ACTIVE(m_sessionAffinityMode)) { // update only if active m_taskAffinityMode = m_sessionAffinityMode; } - UPDATE_CFG(m_lazyLoadChunkDirectory, "lazy_load_chunk_directory", DEFAULT_LAZY_LOAD_CHUNK_DIRECTORY); - UPDATE_MEM_CFG(m_globalMemoryMaxLimitMB, "max_mot_global_memory", DEFAULT_MAX_MOT_GLOBAL_MEMORY, MEGA_BYTE); - UPDATE_MEM_CFG(m_globalMemoryMinLimitMB, "min_mot_global_memory", DEFAULT_MIN_MOT_GLOBAL_MEMORY, MEGA_BYTE); - UPDATE_MEM_CFG(m_localMemoryMaxLimitMB, "max_mot_local_memory", DEFAULT_MAX_MOT_LOCAL_MEMORY, MEGA_BYTE); - UPDATE_MEM_CFG(m_localMemoryMinLimitMB, "min_mot_local_memory", DEFAULT_MIN_MOT_LOCAL_MEMORY, MEGA_BYTE); - UPDATE_ABS_MEM_CFG(m_sessionMemoryMaxLimitKB, "max_mot_session_memory", DEFAULT_MAX_MOT_SESSION_MEMORY, KILO_BYTE); - UPDATE_ABS_MEM_CFG(m_sessionMemoryMinLimitKB, "min_mot_session_memory", DEFAULT_MIN_MOT_SESSION_MEMORY, KILO_BYTE); + UPDATE_BOOL_CFG(m_lazyLoadChunkDirectory, "lazy_load_chunk_directory", DEFAULT_LAZY_LOAD_CHUNK_DIRECTORY); + UPDATE_MEM_CFG(m_globalMemoryMaxLimitMB, + "max_mot_global_memory", + DEFAULT_MAX_MOT_GLOBAL_MEMORY, + SCALE_MEGA_BYTES, + MIN_MAX_MOT_GLOBAL_MEMORY_MB, + MAX_MAX_MOT_GLOBAL_MEMORY_MB); + UPDATE_MEM_CFG(m_globalMemoryMinLimitMB, + "min_mot_global_memory", + DEFAULT_MIN_MOT_GLOBAL_MEMORY, + SCALE_MEGA_BYTES, + MIN_MIN_MOT_GLOBAL_MEMORY_MB, + MAX_MIN_MOT_GLOBAL_MEMORY_MB); + // validate that min <= max + if (m_globalMemoryMinLimitMB > m_globalMemoryMaxLimitMB) { + if (!m_suppressLog) { + MOT_LOG_WARN("Invalid global memory configuration: minimum (%u MB) is greater than maximum (%u MB), using " + "defaults (%u MB, %u MB)", + m_globalMemoryMinLimitMB, + m_globalMemoryMaxLimitMB, + DEFAULT_MIN_MOT_GLOBAL_MEMORY_MB, + DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB); + } + UpdateIntConfigItem(m_globalMemoryMaxLimitMB, DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB, "max_mot_global_memory"); + UpdateIntConfigItem(m_globalMemoryMinLimitMB, DEFAULT_MIN_MOT_GLOBAL_MEMORY_MB, "min_mot_global_memory"); + } + UPDATE_MEM_CFG(m_localMemoryMaxLimitMB, + "max_mot_local_memory", + DEFAULT_MAX_MOT_LOCAL_MEMORY, + SCALE_MEGA_BYTES, + MIN_MAX_MOT_LOCAL_MEMORY_MB, + MAX_MAX_MOT_LOCAL_MEMORY_MB); + UPDATE_MEM_CFG(m_localMemoryMinLimitMB, + "min_mot_local_memory", + DEFAULT_MIN_MOT_LOCAL_MEMORY, + SCALE_MEGA_BYTES, + MIN_MIN_MOT_LOCAL_MEMORY_MB, + MAX_MIN_MOT_LOCAL_MEMORY_MB); + // validate that min <= max + if (m_localMemoryMinLimitMB > m_localMemoryMaxLimitMB) { + if (!m_suppressLog) { + MOT_LOG_WARN("Invalid local memory configuration: minimum (%u MB) is greater than maximum (%u MB), using " + "defaults (%u MB, %u MB)", + m_localMemoryMinLimitMB, + m_localMemoryMaxLimitMB, + DEFAULT_MIN_MOT_LOCAL_MEMORY_MB, + DEFAULT_MAX_MOT_LOCAL_MEMORY_MB); + } + UpdateIntConfigItem(m_localMemoryMaxLimitMB, DEFAULT_MAX_MOT_LOCAL_MEMORY_MB, "max_mot_local_memory"); + UpdateIntConfigItem(m_localMemoryMinLimitMB, DEFAULT_MIN_MOT_LOCAL_MEMORY_MB, "min_mot_local_memory"); + } + UPDATE_ABS_MEM_CFG(m_sessionMemoryMaxLimitKB, + "max_mot_session_memory", + DEFAULT_MAX_MOT_SESSION_MEMORY, + SCALE_KILO_BYTES, + MIN_MAX_MOT_SESSION_MEMORY_KB, + MAX_MAX_MOT_SESSION_MEMORY_KB); + UPDATE_ABS_MEM_CFG(m_sessionMemoryMinLimitKB, + "min_mot_session_memory", + DEFAULT_MIN_MOT_SESSION_MEMORY, + SCALE_KILO_BYTES, + MIN_MIN_MOT_SESSION_MEMORY_KB, + MAX_MIN_MOT_SESSION_MEMORY_KB); + // validate that min <= max + if ((m_sessionMemoryMaxLimitKB > 0) && (m_sessionMemoryMinLimitKB > m_sessionMemoryMaxLimitKB)) { + if (!m_suppressLog) { + MOT_LOG_WARN("Invalid session memory configuration: minimum (%u KB) is greater than maximum (%u KB), using " + "defaults (%u KB, %u KB)", + m_sessionMemoryMinLimitKB, + m_sessionMemoryMaxLimitKB, + DEFAULT_MIN_MOT_SESSION_MEMORY_KB, + DEFAULT_MAX_MOT_SESSION_MEMORY_KB); + } + UpdateIntConfigItem(m_sessionMemoryMaxLimitKB, DEFAULT_MAX_MOT_SESSION_MEMORY_KB, "max_mot_session_memory"); + UpdateIntConfigItem(m_sessionMemoryMinLimitKB, DEFAULT_MIN_MOT_SESSION_MEMORY_KB, "min_mot_session_memory"); + } UPDATE_USER_CFG(m_reserveMemoryMode, "reserve_memory_mode", DEFAULT_RESERVE_MEMORY_MODE); UPDATE_USER_CFG(m_storeMemoryPolicy, "store_memory_policy", DEFAULT_STORE_MEMORY_POLICY); UPDATE_USER_CFG(m_chunkAllocPolicy, "chunk_alloc_policy", DEFAULT_CHUNK_ALLOC_POLICY); - UPDATE_INT_CFG(m_chunkPreallocWorkerCount, "chunk_prealloc_worker_count", DEFAULT_CHUNK_PREALLOC_WORKER_COUNT); - UPDATE_INT_CFG(m_highRedMarkPercent, "high_red_mark_percent", DEFAULT_HIGH_RED_MARK_PERCENT); + UPDATE_INT_CFG(m_chunkPreallocWorkerCount, + "chunk_prealloc_worker_count", + DEFAULT_CHUNK_PREALLOC_WORKER_COUNT, + MIN_CHUNK_PREALLOC_WORKER_COUNT, + MAX_CHUNK_PREALLOC_WORKER_COUNT); + UPDATE_INT_CFG(m_highRedMarkPercent, + "high_red_mark_percent", + DEFAULT_HIGH_RED_MARK_PERCENT, + MIN_HIGH_RED_MARK_PERCENT, + MAX_HIGH_RED_MARK_PERCENT); UPDATE_ABS_MEM_CFG(m_sessionLargeBufferStoreSizeMB, "session_large_buffer_store_size", DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE, - MEGA_BYTE); + SCALE_MEGA_BYTES, + MIN_SESSION_LARGE_BUFFER_STORE_SIZE_MB, + MAX_SESSION_LARGE_BUFFER_STORE_SIZE_MB); UPDATE_ABS_MEM_CFG(m_sessionLargeBufferStoreMaxObjectSizeMB, "session_large_buffer_store_max_object_size", DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE, - MEGA_BYTE); - UPDATE_ABS_MEM_CFG( - m_sessionMaxHugeObjectSizeMB, "session_max_huge_object_size", DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE, MEGA_BYTE); + SCALE_MEGA_BYTES, + MIN_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB, + MAX_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB); + UPDATE_ABS_MEM_CFG(m_sessionMaxHugeObjectSizeMB, + "session_max_huge_object_size", + DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE, + SCALE_MEGA_BYTES, + MIN_SESSION_MAX_HUGE_OBJECT_SIZE_MB, + MAX_SESSION_MAX_HUGE_OBJECT_SIZE_MB); // GC configuration - UPDATE_CFG(m_gcEnable, "enable_gc", DEFAULT_GC_ENABLE); - UPDATE_MEM_CFG(m_gcReclaimThresholdBytes, "reclaim_threshold", DEFAULT_GC_RECLAIM_THRESHOLD, 1); - UPDATE_INT_CFG(m_gcReclaimBatchSize, "reclaim_batch_size", DEFAULT_GC_RECLAIM_BATCH_SIZE); - UPDATE_MEM_CFG(m_gcHighReclaimThresholdBytes, "high_reclaim_threshold", DEFAULT_GC_HIGH_RECLAIM_THRESHOLD, 1); + UPDATE_BOOL_CFG(m_gcEnable, "enable_gc", DEFAULT_GC_ENABLE); + UPDATE_ABS_MEM_CFG(m_gcReclaimThresholdBytes, + "reclaim_threshold", + DEFAULT_GC_RECLAIM_THRESHOLD, + SCALE_BYTES, + MIN_GC_RECLAIM_THRESHOLD_BYTES, + MAX_GC_RECLAIM_THRESHOLD_BYTES); + UPDATE_INT_CFG(m_gcReclaimBatchSize, + "reclaim_batch_size", + DEFAULT_GC_RECLAIM_BATCH_SIZE, + MIN_GC_RECLAIM_BATCH_SIZE, + MAX_GC_RECLAIM_BATCH_SIZE); + UPDATE_ABS_MEM_CFG(m_gcHighReclaimThresholdBytes, + "high_reclaim_threshold", + DEFAULT_GC_HIGH_RECLAIM_THRESHOLD, + SCALE_BYTES, + MIN_GC_HIGH_RECLAIM_THRESHOLD_BYTES, + MAX_GC_HIGH_RECLAIM_THRESHOLD_BYTES); // JIT configuration - UPDATE_CFG(m_enableCodegen, "enable_mot_codegen", DEFAULT_ENABLE_MOT_CODEGEN); - UPDATE_CFG(m_forcePseudoCodegen, "force_mot_pseudo_codegen", DEFAULT_FORCE_MOT_PSEUDO_CODEGEN); - UPDATE_CFG(m_enableCodegenPrint, "enable_mot_codegen_print", DEFAULT_ENABLE_MOT_CODEGEN_PRINT); - UPDATE_INT_CFG(m_codegenLimit, "mot_codegen_limit", DEFAULT_MOT_CODEGEN_LIMIT); + UPDATE_BOOL_CFG(m_enableCodegen, "enable_mot_codegen", DEFAULT_ENABLE_MOT_CODEGEN); + UPDATE_BOOL_CFG(m_forcePseudoCodegen, "force_mot_pseudo_codegen", DEFAULT_FORCE_MOT_PSEUDO_CODEGEN); + UPDATE_BOOL_CFG(m_enableCodegenPrint, "enable_mot_codegen_print", DEFAULT_ENABLE_MOT_CODEGEN_PRINT); + UPDATE_INT_CFG( + m_codegenLimit, "mot_codegen_limit", DEFAULT_MOT_CODEGEN_LIMIT, MIN_MOT_CODEGEN_LIMIT, MAX_MOT_CODEGEN_LIMIT); // storage configuration - UPDATE_CFG(m_allowIndexOnNullableColumn, "allow_index_on_nullable_column", DEFAULT_ALLOW_INDEX_ON_NULLABLE_COLUMN); + UPDATE_BOOL_CFG( + m_allowIndexOnNullableColumn, "allow_index_on_nullable_column", DEFAULT_ALLOW_INDEX_ON_NULLABLE_COLUMN); UPDATE_USER_CFG(m_indexTreeFlavor, "index_tree_flavor", DEFAULT_INDEX_TREE_FLAVOR); // general configuration - UPDATE_TIME_CFG(m_configMonitorPeriodSeconds, "config_update_period", DEFAULT_CFG_MONITOR_PERIOD, 1000000); - UPDATE_CFG(m_runInternalConsistencyValidation, + UPDATE_TIME_CFG(m_configMonitorPeriodSeconds, + "config_update_period", + DEFAULT_CFG_MONITOR_PERIOD, + SCALE_SECONDS, + MIN_CFG_MONITOR_PERIOD_SECONDS, + MAX_CFG_MONITOR_PERIOD_SECONDS); + UPDATE_BOOL_CFG(m_runInternalConsistencyValidation, "internal_consistency_validation", DEFAULT_RUN_INTERNAL_CONSISTENCY_VALIDATION); @@ -781,9 +976,11 @@ void MOTConfiguration::UpdateComponentLogLevel() LogLevel componentLevel = componentCfg->GetUserConfigValue("log_level", globalLogLevel); if (componentLevel != globalLogLevel) { mot_string logLevelStr; - MOT_LOG_INFO("Updating the log level of component %s to: %s", - componentName.c_str(), - TypeFormatter::ToString(componentLevel, logLevelStr)); + if (!m_suppressLog) { + MOT_LOG_INFO("Updating the log level of component %s to: %s", + componentName.c_str(), + TypeFormatter::ToString(componentLevel, logLevelStr)); + } SetLogComponentLogLevel(componentName.c_str(), componentLevel); } @@ -806,10 +1003,12 @@ void MOTConfiguration::UpdateComponentLogLevel() componentCfg->GetUserConfigValue(loggerName.c_str(), LogLevel::LL_INFO); if (loggerLevel != LogLevel::LL_INFO) { mot_string logLevelStr; - MOT_LOG_INFO("Updating the log level of logger %s in component %s to: %s", - loggerName.c_str(), - componentName.c_str(), - TypeFormatter::ToString(loggerLevel, logLevelStr)); + if (!m_suppressLog) { + MOT_LOG_INFO("Updating the log level of logger %s in component %s to: %s", + loggerName.c_str(), + componentName.c_str(), + TypeFormatter::ToString(loggerLevel, logLevelStr)); + } SetLoggerLogLevel(componentName.c_str(), loggerName.c_str(), loggerLevel); } } @@ -821,7 +1020,7 @@ void MOTConfiguration::UpdateComponentLogLevel() } } -void MOTConfiguration::UpdateConfigItem(bool& oldValue, bool newValue, const char* name) +void MOTConfiguration::UpdateBoolConfigItem(bool& oldValue, bool newValue, const char* name) { if (oldValue != newValue) { MOT_LOG_TRACE( @@ -830,7 +1029,7 @@ void MOTConfiguration::UpdateConfigItem(bool& oldValue, bool newValue, const cha } } -void MOTConfiguration::UpdateConfigItem(std::string& oldValue, const char* newValue, const char* name) +void MOTConfiguration::UpdateStringConfigItem(std::string& oldValue, const char* newValue, const char* name) { if (oldValue != newValue) { MOT_LOG_TRACE("Configuration of %s changed: %s --> %s", name, oldValue.c_str(), newValue); @@ -885,8 +1084,10 @@ uint64_t MOTConfiguration::ParseMemoryPercentTotal(const char* memoryValue, uint int percent = ParseMemoryPercent(memoryValue); if (percent >= 0) { memoryValueBytes = ((uint64_t)m_totalMemoryMb) * MEGA_BYTE * percent / 100; - MOT_LOG_INFO( - "Loaded %s: %d%% from total = %" PRIu64 " MB", cfgPath, percent, memoryValueBytes / 1024ul / 1024ul); + if (!m_suppressLog) { + MOT_LOG_INFO( + "Loaded %s: %d%% from total = %" PRIu64 " MB", cfgPath, percent, memoryValueBytes / 1024ul / 1024ul); + } } else { MOT_LOG_WARN("Invalid %s memory format: illegal percent specification", cfgPath); } diff --git a/src/gausskernel/storage/mot/core/src/system/mot_configuration.h b/src/gausskernel/storage/mot/core/src/system/mot_configuration.h index 31b5eb2b0..828483b2a 100644 --- a/src/gausskernel/storage/mot/core/src/system/mot_configuration.h +++ b/src/gausskernel/storage/mot/core/src/system/mot_configuration.h @@ -27,6 +27,7 @@ #include #include +#include #include "affinity.h" #include "global.h" @@ -72,6 +73,18 @@ public: LoadConfig(); } + /** + * @brief Load partial configuration (i.e. not all loaders are fully loaded yet). In such case we suppress all + * messages. + */ + inline void LoadPartial() + { + MOT_LOG_TRACE("Loading partial configuration after change"); + m_suppressLog = true; + LoadConfig(); + m_suppressLog = false; + } + /** * @brief Validates configuration. Call this function after loading configuration to validate * the loaded values are valid, not out of bounds, and not self-contradicting. @@ -382,11 +395,15 @@ public: MOTConfiguration& operator=(const MOTConfiguration&& orig) = delete; /** @endcond */ + /** @var Asynchronous redo-log buffer array bounds (exposed as public for external use). */ + static constexpr uint32_t MIN_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT = 8; + static constexpr uint32_t MAX_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT = 128; + private: /** @var A singleton instance available for static initializers. */ static MOTConfiguration motGlobalConfiguration; - /** @var Map of CPUs to numa nodes */ + /** @var Map of CPUs to NUMA nodes */ CpuNodeMap m_cpuNodeMapper; CpuMap m_osCpuMap; @@ -396,7 +413,20 @@ private: /** @var The total memory value to be used when loading memory percent values. By default uses system total. */ uint32_t m_totalMemoryMb; - // Default logger configuration + /** @var Controls suppressing of log messages during configuration loading. */ + bool m_suppressLog; + + /** @var Memory scaling constants (from bytes). */ + static constexpr uint64_t SCALE_BYTES = 1; + static constexpr uint64_t SCALE_KILO_BYTES = KILO_BYTE; + static constexpr uint64_t SCALE_MEGA_BYTES = MEGA_BYTE; + + /** @var Time scaling constants (from micros). */ + static constexpr uint64_t SCALE_MICROS = 1; + static constexpr uint64_t SCALE_MILLIS = 1000; + static constexpr uint64_t SCALE_SECONDS = 1000000; + + /** ------------------ Default Redo-Log Configuration ------------ */ /** @var Default enable file logger. */ static constexpr bool DEFAULT_ENABLE_REDO_LOG = true; @@ -409,20 +439,23 @@ private: /** @var Default asynchronous redo log buffer array count. */ static constexpr uint32_t DEFAULT_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT = 24; - // default commit configuration /** @var Default enable group commit. */ static constexpr bool DEFAULT_ENABLE_GROUP_COMMIT = false; /** @var Default group commit size. */ static constexpr uint64_t DEFAULT_GROUP_COMMIT_SIZE = 16; + static constexpr uint64_t MIN_GROUP_COMMIT_SIZE = 2; + static constexpr uint64_t MAX_GROUP_COMMIT_SIZE = 1000; /** @var Default group commit timeout. */ static constexpr const char* DEFAULT_GROUP_COMMIT_TIMEOUT = "10 ms"; /** @var Default group commit timeout in micro-seconds. */ static constexpr uint64_t DEFAULT_GROUP_COMMIT_TIMEOUT_USEC = 10000; + static constexpr uint64_t MIN_GROUP_COMMIT_TIMEOUT_USEC = 100; + static constexpr uint64_t MAX_GROUP_COMMIT_TIMEOUT_USEC = 200000; // 200 ms - // default checkpoint configuration + /** ------------------ Default Checkpoint Configuration ------------ */ /** @var Default enable checkpoint. */ static constexpr bool DEFAULT_ENABLE_CHECKPOINT = true; @@ -434,19 +467,25 @@ private: /** @var Default checkpoint segments size */ static constexpr const char* DEFAULT_CHECKPOINT_SEGSIZE = "16 MB"; - static constexpr uint32_t DEFAULT_CHECKPOINT_SEGSIZE_BYTES = (16 * 1024 * 1024); + static constexpr uint32_t DEFAULT_CHECKPOINT_SEGSIZE_BYTES = 16 * MEGA_BYTE; + static constexpr uint32_t MIN_CHECKPOINT_SEGSIZE_BYTES = 16 * MEGA_BYTE; + static constexpr uint32_t MAX_CHECKPOINT_SEGSIZE_BYTES = 512 * MEGA_BYTE; /** @var Default number of worker threads to spawn */ static constexpr uint32_t DEFAULT_CHECKPOINT_WORKERS = 3; + static constexpr uint32_t MIN_CHECKPOINT_WORKERS = 1; + static constexpr uint32_t MAX_CHECKPOINT_WORKERS = 1024; - // default recovery configuration + /** ------------------ Default Recovery Configuration ------------ */ /** @var Default number of workers used in recovery from checkpoint. */ static constexpr uint32_t DEFAULT_CHECKPOINT_RECOVERY_WORKERS = 3; + static constexpr uint32_t MIN_CHECKPOINT_RECOVERY_WORKERS = 1; + static constexpr uint32_t MAX_CHECKPOINT_RECOVERY_WORKERS = 1024; /** @var Default enable log recovery statistics. */ static constexpr bool DEFAULT_ENABLE_LOG_RECOVERY_STATS = false; - // default machine configuration + /** ------------------ Default Machine Configuration ------------ */ /** @var Default number of NUMA nodes of the machine. */ static constexpr uint16_t DEFAULT_NUMA_NODES = 1; @@ -459,13 +498,15 @@ private: /* @var Default upper bound number of pg nodes on the machine. */ static constexpr uint16_t DEFAULT_MAX_DATA_NODES = 1; - // default statistics configuration + /** ------------------ Default Statistics Configuration ------------ */ /** @var Default enable statistics printing. */ static constexpr bool DEFAULT_ENABLE_STATS = false; /** @var Default statistics printing period in seconds. */ static constexpr const char* DEFAULT_STATS_PRINT_PERIOD = "1 minutes"; static constexpr uint32_t DEFAULT_STATS_PRINT_PERIOD_SECONDS = 60; + static constexpr uint32_t MIN_STATS_PRINT_PERIOD_SECONDS = 1; + static constexpr uint32_t MAX_STATS_PRINT_PERIOD_SECONDS = 86400; // 1 day /** @var Default full statistics printing period in seconds. */ static constexpr const char* DEFAULT_FULL_STATS_PRINT_PERIOD = "5 minutes"; @@ -495,7 +536,7 @@ private: /** @var Default enable JIT execution statistics printing. */ static constexpr bool DEFAULT_ENABLE_JIT_STAT_PRINT = false; - // default error log configuration + /** ------------------ Default Error-Log Configuration ------------ */ /** @var Default log level limit. */ static constexpr LogLevel DEFAULT_LOG_LEVEL = LogLevel::LL_INFO; @@ -508,15 +549,19 @@ private: /** @var Default log level for configuration loading on startup messages. */ static constexpr LogLevel DEFAULT_CFG_STARTUP_LOG_LEVEL = LogLevel::LL_TRACE; - // default memory configuration + /** ------------------ Default Memory Configuration ------------ */ /** @var Default enable NUMA. */ static constexpr bool DEFAULT_ENABLE_NUMA = true; /** @var Default maximum number of threads in the system. */ static constexpr uint16_t DEFAULT_MAX_THREADS = 1024; + static constexpr uint16_t MIN_MAX_THREADS = 1; + static constexpr uint16_t MAX_MAX_THREADS = UINT16_MAX; /** @var Default maximum number of connections in the system. */ static constexpr uint32_t DEFAULT_MAX_CONNECTIONS = 1024; + static constexpr uint32_t MIN_MAX_CONNECTIONS = 1; + static constexpr uint32_t MAX_MAX_CONNECTIONS = UINT16_MAX; /** @var Default thread affinity policy. */ static constexpr AffinityMode DEFAULT_AFFINITY_MODE = AffinityMode::FILL_PHYSICAL_FIRST; @@ -526,28 +571,40 @@ private: /** @var Default maximum limit for MOT global memory. */ static constexpr const char* DEFAULT_MAX_MOT_GLOBAL_MEMORY = "80%"; - static constexpr uint32_t DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB = 8 * 1024; + static constexpr uint32_t DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB = 8 * KILO_BYTE; // 8 GB + static constexpr uint32_t MIN_MAX_MOT_GLOBAL_MEMORY_MB = 128; // 128 MB + static constexpr uint32_t MAX_MAX_MOT_GLOBAL_MEMORY_MB = 512 * MEGA_BYTE; // 512 TB /** @var Default minimum (pre-allocated) limit for MOT global memory. */ static constexpr const char* DEFAULT_MIN_MOT_GLOBAL_MEMORY = "0 GB"; static constexpr uint32_t DEFAULT_MIN_MOT_GLOBAL_MEMORY_MB = 0; + static constexpr uint32_t MIN_MIN_MOT_GLOBAL_MEMORY_MB = 0; + static constexpr uint32_t MAX_MIN_MOT_GLOBAL_MEMORY_MB = 512 * MEGA_BYTE; // 512 TB /** @var Default maximum limit for MOT global memory (used to establish new ratio between local and global pools). */ static constexpr const char* DEFAULT_MAX_MOT_LOCAL_MEMORY = "15%"; - static constexpr uint32_t DEFAULT_MAX_MOT_LOCAL_MEMORY_MB = 2 * 1024; + static constexpr uint32_t DEFAULT_MAX_MOT_LOCAL_MEMORY_MB = 2 * KILO_BYTE; // 2 GB + static constexpr uint32_t MIN_MAX_MOT_LOCAL_MEMORY_MB = 64; // 64 MB (about 8 normal sessions) + static constexpr uint32_t MAX_MAX_MOT_LOCAL_MEMORY_MB = 512 * KILO_BYTE; // 512 GB (MANY very heavy sessions) /** @var Default minimum (pre-allocated) limit for MOT local memory. */ static constexpr const char* DEFAULT_MIN_MOT_LOCAL_MEMORY = "0 GB"; - static constexpr uint32_t DEFAULT_MIN_MOT_LOCAL_MEMORY_MB = 0; + static constexpr uint32_t DEFAULT_MIN_MOT_LOCAL_MEMORY_MB = 0; // no pre-allocation + static constexpr uint32_t MIN_MIN_MOT_LOCAL_MEMORY_MB = 0; // no pre-allocation + static constexpr uint32_t MAX_MIN_MOT_LOCAL_MEMORY_MB = 512 * KILO_BYTE; // max pre-allocate 512 GB /** @var Default maximum for single MOT session small memory allocations. */ static constexpr const char* DEFAULT_MAX_MOT_SESSION_MEMORY = "0 MB"; - static constexpr uint32_t DEFAULT_MAX_MOT_SESSION_MEMORY_KB = 0; + static constexpr uint32_t DEFAULT_MAX_MOT_SESSION_MEMORY_KB = 0; // no session-memory limit + static constexpr uint32_t MIN_MAX_MOT_SESSION_MEMORY_KB = 0; + static constexpr uint32_t MAX_MAX_MOT_SESSION_MEMORY_KB = 512 * MEGA_BYTE; // limit single session to 512 GB /** @var Default minimum (pre-allocated) for single MOT session small memory allocations. */ static constexpr const char* DEFAULT_MIN_MOT_SESSION_MEMORY = "0 MB"; - static constexpr uint32_t DEFAULT_MIN_MOT_SESSION_MEMORY_KB = 0; + static constexpr uint32_t DEFAULT_MIN_MOT_SESSION_MEMORY_KB = 0; // no session-memory pre-allocation + static constexpr uint32_t MIN_MIN_MOT_SESSION_MEMORY_KB = 0; // no session-memory pre-allocation + static constexpr uint32_t MAX_MIN_MOT_SESSION_MEMORY_KB = 64 * KILO_BYTE; // up to 64 MB pre-allocation /** @var Default physical or virtual memory reservation. */ static constexpr MemReserveMode DEFAULT_RESERVE_MEMORY_MODE = MEM_RESERVE_VIRTUAL; @@ -560,38 +617,54 @@ private: /** @var Default number of workers used to pre-allocate initial memory. */ static constexpr uint32_t DEFAULT_CHUNK_PREALLOC_WORKER_COUNT = 8; + static constexpr uint32_t MIN_CHUNK_PREALLOC_WORKER_COUNT = 1; + static constexpr uint32_t MAX_CHUNK_PREALLOC_WORKER_COUNT = 1024; /** @var Default chunk store high red mark in percents of maximum. */ - static constexpr uint32_t DEFAULT_HIGH_RED_MARK_PERCENT = 90; + static constexpr uint32_t DEFAULT_HIGH_RED_MARK_PERCENT = 90; // reject constructive ops from 90% memory usage + static constexpr uint32_t MIN_HIGH_RED_MARK_PERCENT = 50; // reject constructive ops from 50% memory usage + static constexpr uint32_t MAX_HIGH_RED_MARK_PERCENT = 95; // Don't allow disabling high red-mark /** @var The default size in megabytes of the session large buffer store. */ static constexpr const char* DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE = " 0 MB"; static constexpr uint32_t DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE_MB = 0; + static constexpr uint32_t MIN_SESSION_LARGE_BUFFER_STORE_SIZE_MB = 0; // disabled + static constexpr uint32_t MAX_SESSION_LARGE_BUFFER_STORE_SIZE_MB = 128 * KILO_BYTE; // 128 GB /** @var The default largest object size in megabytes in the session large buffer store. */ static constexpr const char* DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE = "0 MB"; - static constexpr uint32_t DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB = 0; + static constexpr uint32_t DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB = 0; // use calculation + static constexpr uint32_t MIN_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB = 0; + static constexpr uint32_t MAX_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB = 1024; // 1 GB /** @var The default largest object size in megabytes that can be allocated form kernel for sessions. */ static constexpr const char* DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE = "1 GB"; - static constexpr uint32_t DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE_MB = 1024; + static constexpr uint32_t DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE_MB = 1024; // 1 GB + static constexpr uint32_t MIN_SESSION_MAX_HUGE_OBJECT_SIZE_MB = 8; // 8 MB + static constexpr uint32_t MAX_SESSION_MAX_HUGE_OBJECT_SIZE_MB = 8 * KILO_BYTE; // 8 GB - // default GC configuration + /** ------------------ Default Garbage-Collection Configuration ------------ */ /** @var Enable/disable garbage collection. */ static constexpr bool DEFAULT_GC_ENABLE = true; /** @var The threshold in bytes for reclamation to be triggered (per-thread) */ static constexpr const char* DEFAULT_GC_RECLAIM_THRESHOLD = "512 KB"; - static constexpr uint32_t DEFAULT_GC_RECLAIM_THRESHOLD_BYTES = 512 * KILO_BYTE; + static constexpr uint32_t DEFAULT_GC_RECLAIM_THRESHOLD_BYTES = 512 * KILO_BYTE; // 512 KB + static constexpr uint32_t MIN_GC_RECLAIM_THRESHOLD_BYTES = KILO_BYTE; // 1 KB + static constexpr uint32_t MAX_GC_RECLAIM_THRESHOLD_BYTES = 64 * MEGA_BYTE; // 64 MB /** @var The amount of objects reclaimed in each cleanup round of a limbo group. */ - static constexpr uint32_t DEFAULT_GC_RECLAIM_BATCH_SIZE = 8000; + static constexpr uint32_t DEFAULT_GC_RECLAIM_BATCH_SIZE = 8 * KILO_BYTE; // 8 KB entries + static constexpr uint32_t MIN_GC_RECLAIM_BATCH_SIZE = 128; // 128 entries + static constexpr uint32_t MAX_GC_RECLAIM_BATCH_SIZE = MEGA_BYTE; // 1 MB entries /** @var The high threshold in bytes for reclamation to be triggered (per-thread) */ static constexpr const char* DEFAULT_GC_HIGH_RECLAIM_THRESHOLD = "8 MB"; - static constexpr uint32_t DEFAULT_GC_HIGH_RECLAIM_THRESHOLD_BYTES = 8 * MEGA_BYTE; + static constexpr uint32_t DEFAULT_GC_HIGH_RECLAIM_THRESHOLD_BYTES = 8 * MEGA_BYTE; // 8 MB + static constexpr uint32_t MIN_GC_HIGH_RECLAIM_THRESHOLD_BYTES = 1 * MEGA_BYTE; // 1 MB + static constexpr uint32_t MAX_GC_HIGH_RECLAIM_THRESHOLD_BYTES = 64 * MEGA_BYTE; // 64 MB - // default JIT configuration + /** ------------------ Default JIT Configuration ------------ */ /** @var Default enable JIT compilation and execution. */ static constexpr bool DEFAULT_ENABLE_MOT_CODEGEN = true; @@ -603,24 +676,28 @@ private: /** @vart Default limit for the amount of JIT queries allowed per user session. */ static constexpr uint32_t DEFAULT_MOT_CODEGEN_LIMIT = 100; + static constexpr uint32_t MIN_MOT_CODEGEN_LIMIT = 1; + static constexpr uint32_t MAX_MOT_CODEGEN_LIMIT = 1000; - // default storage configuration + /** ------------------ Default Storage Configuration ------------ */ /** @var The default allow index on null-able column. */ - static constexpr bool DEFAULT_ALLOW_INDEX_ON_NULLABLE_COLUMN = true; + static constexpr bool DEFAULT_ALLOW_INDEX_ON_NULLABLE_COLUMN = false; /** @var The default tree flavor for tree indexes. */ static constexpr IndexTreeFlavor DEFAULT_INDEX_TREE_FLAVOR = IndexTreeFlavor::INDEX_TREE_FLAVOR_MASSTREE; - // default general configuration + /** ------------------ Default General Configuration ------------ */ /** @var Default configuration monitor period in seconds. */ static constexpr const char* DEFAULT_CFG_MONITOR_PERIOD = "5 seconds"; - static constexpr uint64_t DEFAULT_CFG_MONITOR_PERIOD_SECONDS = 5; + static constexpr uint64_t DEFAULT_CFG_MONITOR_PERIOD_SECONDS = 5; // 5 seconds + static constexpr uint64_t MIN_CFG_MONITOR_PERIOD_SECONDS = 1; // 1 seconds + static constexpr uint64_t MAX_CFG_MONITOR_PERIOD_SECONDS = 300; // 5 minutes /** @var The default value for consistency validation tests after benchmark running. */ static constexpr bool DEFAULT_RUN_INTERNAL_CONSISTENCY_VALIDATION = false; /** @var The default total memory reference used for calculating memory percent value. */ - static constexpr uint32_t DEFAULT_TOTAL_MEMORY_MB = 10 * 1024; + static constexpr uint32_t DEFAULT_TOTAL_MEMORY_MB = 10 * KILO_BYTE; // 10 MB /** @brief Loads configuration from main configuration. */ void LoadConfig(); @@ -632,26 +709,42 @@ private: static bool FindNumProcessors(uint16_t* maxCoresPerNode, CpuNodeMap* cpuNodeMapper, CpuMap* cpuOsMapper); static bool CheckHyperThreads(); - static void UpdateConfigItem(bool& oldValue, bool newValue, const char* name); - static void UpdateConfigItem(std::string& oldValue, const char* newValue, const char* name); + static void UpdateBoolConfigItem(bool& oldValue, bool newValue, const char* name); + static void UpdateStringConfigItem(std::string& oldValue, const char* newValue, const char* name); template - static void UpdateConfigItem(uint64_t& oldValue, T newValue, const char* name) + void UpdateIntConfigItem( + uint64_t& oldValue, T newValue, const char* name, uint64_t lowerBound = 0, uint64_t upperBound = UINT64_MAX) { - if (oldValue != newValue) { + if ((newValue > upperBound) || (lowerBound > 0 && newValue < lowerBound)) { + if (!m_suppressLog) { + MOT_LOG_WARN("Configuration of %s=%" PRIu64 " is out of bounds [%" PRIu64 ", %" PRIu64 "]: keeping " + "default value %" PRIu64, + name, + (uint64_t)newValue, + lowerBound, + upperBound, + oldValue); + } + } else if (oldValue != newValue) { MOT_LOG_TRACE("Configuration of %s changed: %" PRIu64 " --> %" PRIu64, name, oldValue, (uint64_t)newValue); oldValue = newValue; } } template - static void UpdateConfigItem( - uint32_t& oldValue, T newValue, const char* name, uint32_t lowerBound = 0, uint32_t upperBound = UINT_MAX) + void UpdateIntConfigItem( + uint32_t& oldValue, T newValue, const char* name, uint32_t lowerBound = 0, uint32_t upperBound = UINT32_MAX) { - if (newValue > upperBound) { - MOT_LOG_WARN("Configuration of %s overflowed: keeping default value %u", name, oldValue); - } else if (lowerBound > 0 && newValue < lowerBound) { - MOT_LOG_WARN("Configuration of %s overflowed: keeping default value %u", name, oldValue); + if ((newValue > UINT32_MAX) || (newValue > upperBound) || (lowerBound > 0 && newValue < lowerBound)) { + if (!m_suppressLog) { + MOT_LOG_WARN("Configuration of %s=%" PRIu64 " is out of bounds [%u, %u]: keeping default value %u", + name, + (uint64_t)newValue, + lowerBound, + upperBound, + oldValue); + } } else if (oldValue != newValue) { MOT_LOG_TRACE("Configuration of %s changed: %u --> %u", name, oldValue, (uint32_t)newValue); oldValue = newValue; @@ -659,10 +752,18 @@ private: } template - static void UpdateConfigItem(uint16_t& oldValue, T newValue, const char* name) + void UpdateIntConfigItem( + uint16_t& oldValue, T newValue, const char* name, uint16_t lowerBound = 0, uint16_t upperBound = UINT16_MAX) { - if (newValue > USHRT_MAX) { - MOT_LOG_WARN("Configuration of %s overflowed: keeping default value %" PRIu16, name, oldValue); + if ((newValue > UINT16_MAX) || (newValue > upperBound) || (lowerBound > 0 && newValue < lowerBound)) { + if (!m_suppressLog) { + MOT_LOG_WARN("Configuration of %s=%" PRIu64 " is out of bounds [%u, %u]: keeping default value %u", + name, + (uint64_t)newValue, + (unsigned)lowerBound, + (unsigned)upperBound, + (unsigned)oldValue); + } } else if (oldValue != newValue) { MOT_LOG_TRACE("Configuration of %s changed: %" PRIu16 " --> %" PRIu16, name, oldValue, (uint16_t)newValue); oldValue = newValue; diff --git a/src/gausskernel/storage/mot/core/src/system/mot_engine.cpp b/src/gausskernel/storage/mot/core/src/system/mot_engine.cpp index fd73d1e16..a8534d54b 100644 --- a/src/gausskernel/storage/mot/core/src/system/mot_engine.cpp +++ b/src/gausskernel/storage/mot/core/src/system/mot_engine.cpp @@ -571,15 +571,24 @@ bool MOTEngine::InitializeConfiguration(const char* configFilePath, int argc, ch } } - // initialize global configuration singleton - GetGlobalConfiguration().Initialize(); + // explicitly initialize global configuration singleton + MOTConfiguration& motCfg = GetGlobalConfiguration(); + motCfg.Initialize(); // create manager, and configure it with configuration file loader - if (!ConfigManager::CreateInstance(argv + 1, argc - 1)) { + if (!ConfigManager::CreateInstance(argv, argc)) { MOT_LOG_ERROR("Failed to create configuration manager instance"); result = false; } else { - if (configFilePath != nullptr) { + // register global configuration as first listener (other listeners rely on it) + result = ConfigManager::GetInstance().AddConfigChangeListener(&motCfg); + if (!result) { + MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, + "Load Configuration", + "Failed to register main configuration listener at the configuration manager"); + ConfigManager::DestroyInstance(); + } else if (configFilePath != nullptr) { + // add configuration file loader result = ConfigManager::GetInstance().AddConfigFile(configFilePath); if (!result) { MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, diff --git a/src/gausskernel/storage/mot/core/src/system/mot_engine.h b/src/gausskernel/storage/mot/core/src/system/mot_engine.h index 5bbe32cc2..8997af755 100644 --- a/src/gausskernel/storage/mot/core/src/system/mot_engine.h +++ b/src/gausskernel/storage/mot/core/src/system/mot_engine.h @@ -77,8 +77,8 @@ public: * @brief Creates the single main memory engine instance. Use this variant when you have no external configuration * loaders involved. * @param[opt] configFilePath The path to the configuration file of the main memory engine. - * @param[opt] argc Command line argument count. - * @param[opt] argv Command line argument array. + * @param[opt] argc Command line argument count (excluding program name first argument). + * @param[opt] argv Command line argument array (excluding program name first argument). * @return A reference to the single main memory engine instance, or null if initialization failed. */ static MOTEngine* CreateInstance(const char* configFilePath = nullptr, int argc = 0, char* argv[] = nullptr); @@ -103,8 +103,8 @@ public: * initialization takes place. In this case initialization has several steps: @ref CreateInstanceNoInit(), @ref * AddConfigLoader(), @ref AddConfigLoader(), ..., @ref LoadConfig(), @ref Initialize(). * @param[opt] configFilePath The path to the configuration file of the main memory engine. - * @param[opt] argc Command line argument count. - * @param[opt] argv Command line argument array. + * @param[opt] argc Command line argument count (excluding program name first argument). + * @param[opt] argv Command line argument array (excluding program name first argument). * @return A reference to the single main memory engine instance if configuration loading * succeeded, otherwise nullptr. */ diff --git a/src/gausskernel/storage/mot/core/src/system/transaction/txn.cpp b/src/gausskernel/storage/mot/core/src/system/transaction/txn.cpp index 47cbc9b16..3fa36fab2 100644 --- a/src/gausskernel/storage/mot/core/src/system/transaction/txn.cpp +++ b/src/gausskernel/storage/mot/core/src/system/transaction/txn.cpp @@ -917,7 +917,7 @@ RC TxnInsertAction::ExecuteOptimisticInsert(Row* row) rc = RC_UNIQUE_VIOLATION; goto end; default: - break; + goto end; } } else if (res == true or pIndexInsertResult->IsCommited() == false) { // tag all the sentinels the insert metadata @@ -934,7 +934,7 @@ RC TxnInsertAction::ExecuteOptimisticInsert(Row* row) } end: - if (rc != RC_OK) { + if ((rc != RC_OK) && (currentItem != EndCursor())) { // Clean current aborted row and clean secondary indexes that were not inserts // Clean first Object! - wither primary or secondary! // Return Local Row to pull for PI diff --git a/src/gausskernel/storage/mot/core/src/system/transaction_logger/asynchronous_redo_log/asynchronous_redo_log_handler.h b/src/gausskernel/storage/mot/core/src/system/transaction_logger/asynchronous_redo_log/asynchronous_redo_log_handler.h index fa04ddf86..11654dcf2 100644 --- a/src/gausskernel/storage/mot/core/src/system/transaction_logger/asynchronous_redo_log/asynchronous_redo_log_handler.h +++ b/src/gausskernel/storage/mot/core/src/system/transaction_logger/asynchronous_redo_log/asynchronous_redo_log_handler.h @@ -30,6 +30,7 @@ #include "redo_log_handler.h" #include "redo_log_buffer_pool.h" #include "rw_lock.h" +#include "mot_configuration.h" namespace MOT { class TxnManager; @@ -88,7 +89,7 @@ private: RedoLogBufferPool m_bufferPool; // array of RedoLogBufferArray for switching in cyclic manner. - RedoLogBufferArray m_redoLogBufferArrayArray[MAX_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT]; + RedoLogBufferArray m_redoLogBufferArrayArray[MOTConfiguration::MAX_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT]; uint32_t m_redoLogBufferArrayCount; volatile int m_activeBuffer; bool m_initialized; diff --git a/src/gausskernel/storage/mot/fdw_adapter/src/mot_internal.cpp b/src/gausskernel/storage/mot/fdw_adapter/src/mot_internal.cpp index 6dc010777..c0361e896 100644 --- a/src/gausskernel/storage/mot/fdw_adapter/src/mot_internal.cpp +++ b/src/gausskernel/storage/mot/fdw_adapter/src/mot_internal.cpp @@ -267,7 +267,7 @@ protected: // the layered configuration tree in the configuration manager MOT_LOG_TRACE("Triggering initial partial configuration loading for external configuration overlaying"); MOT::MOTConfiguration& motCfg = MOT::GetGlobalConfiguration(); - motCfg.OnConfigChange(); // trigger partial load + motCfg.LoadPartial(); // trigger partial load // load any relevant GUC values here bool result = ConfigureMaxConnections(); diff --git a/src/test/regress/expected/mot/single_numeric.out b/src/test/regress/expected/mot/single_numeric.out index 52eda6682..b520e34a8 100644 --- a/src/test/regress/expected/mot/single_numeric.out +++ b/src/test/regress/expected/mot/single_numeric.out @@ -2,14 +2,14 @@ -- NUMERIC -- CREATE FOREIGN TABLE num_data (id int4, val numeric(210,10)) SERVER mot_server ; -CREATE FOREIGN TABLE num_exp_add (id1 int4, id2 int4, expected numeric(210,10)) SERVER mot_server ; -CREATE FOREIGN TABLE num_exp_sub (id1 int4, id2 int4, expected numeric(210,10)) SERVER mot_server ; -CREATE FOREIGN TABLE num_exp_div (id1 int4, id2 int4, expected numeric(210,10)) SERVER mot_server ; -CREATE FOREIGN TABLE num_exp_mul (id1 int4, id2 int4, expected numeric(210,10)) SERVER mot_server ; -CREATE FOREIGN TABLE num_exp_sqrt (id int4, expected numeric(210,10)) SERVER mot_server ; -CREATE FOREIGN TABLE num_exp_ln (id int4, expected numeric(210,10)) SERVER mot_server ; -CREATE FOREIGN TABLE num_exp_log10 (id int4, expected numeric(210,10)) SERVER mot_server ; -CREATE FOREIGN TABLE num_exp_power_10_ln (id int4, expected numeric(210,10)) SERVER mot_server ; +CREATE FOREIGN TABLE num_exp_add (id1 int4 not null, id2 int4 not null, expected numeric(210,10)) SERVER mot_server ; +CREATE FOREIGN TABLE num_exp_sub (id1 int4 not null, id2 int4 not null, expected numeric(210,10)) SERVER mot_server ; +CREATE FOREIGN TABLE num_exp_div (id1 int4 not null, id2 int4 not null, expected numeric(210,10)) SERVER mot_server ; +CREATE FOREIGN TABLE num_exp_mul (id1 int4 not null, id2 int4 not null, expected numeric(210,10)) SERVER mot_server ; +CREATE FOREIGN TABLE num_exp_sqrt (id int4 not null, expected numeric(210,10)) SERVER mot_server ; +CREATE FOREIGN TABLE num_exp_ln (id int4 not null, expected numeric(210,10)) SERVER mot_server ; +CREATE FOREIGN TABLE num_exp_log10 (id int4 not null, expected numeric(210,10)) SERVER mot_server ; +CREATE FOREIGN TABLE num_exp_power_10_ln (id int4 not null, expected numeric(210,10)) SERVER mot_server ; CREATE FOREIGN TABLE num_result (id1 int4, id2 int4, result numeric(210,10)) SERVER mot_server ; -- ****************************** -- * The following EXPECTED results are computed by bc(1) diff --git a/src/test/regress/expected/mot/single_reindex.out b/src/test/regress/expected/mot/single_reindex.out index 9ac6e87c9..3cd005869 100644 --- a/src/test/regress/expected/mot/single_reindex.out +++ b/src/test/regress/expected/mot/single_reindex.out @@ -1,4 +1,4 @@ -create foreign table test1 (i integer, y int); +create foreign table test1 (i integer not null, y int not null); create index text_idx on test1(y,i); insert into test1 values (generate_series(1,1000), generate_series(1,1000)); select * from test1 order by i,y asc limit 10; diff --git a/src/test/regress/expected/mot/single_relation_size.out b/src/test/regress/expected/mot/single_relation_size.out index 9f7ab05ca..b7f099c14 100644 --- a/src/test/regress/expected/mot/single_relation_size.out +++ b/src/test/regress/expected/mot/single_relation_size.out @@ -1,6 +1,6 @@ drop foreign table if exists mm_test; NOTICE: foreign table "mm_test" does not exist, skipping -create foreign table mm_test (id int, name varchar(1000)) server mot_server; +create foreign table mm_test (id int not null, name varchar(1000)) server mot_server; insert into mm_test values (1, 'aaa'), (2, 'bbb'), (3, 'ccc'), (4, 'ddd'), (5, 'eee'), (6, 'fff'); select pg_relation_size('mm_test') > 0; ?column? diff --git a/src/test/regress/expected/mot/single_tpcc_queries.out b/src/test/regress/expected/mot/single_tpcc_queries.out index beb1b1289..59ac1162b 100644 --- a/src/test/regress/expected/mot/single_tpcc_queries.out +++ b/src/test/regress/expected/mot/single_tpcc_queries.out @@ -43,8 +43,8 @@ CREATE FOREIGN TABLE bmsql_customer ( c_id integer not null, c_discount decimal(4,4), c_credit char(2), - c_last varchar(16), - c_first varchar(16), + c_last varchar(16) not null, + c_first varchar(16) not null, c_credit_lim decimal(12,2), c_balance decimal(12,2), c_ytd_payment decimal(12,2), @@ -84,7 +84,7 @@ CREATE FOREIGN TABLE bmsql_oorder ( o_w_id integer not null, o_d_id integer not null, o_id integer not null, - o_c_id integer, + o_c_id integer not null, o_carrier_id integer, o_ol_cnt integer, o_all_local integer, diff --git a/src/test/regress/sql/mot/single_numeric.sql b/src/test/regress/sql/mot/single_numeric.sql index c3ad6cd81..eccfac206 100644 --- a/src/test/regress/sql/mot/single_numeric.sql +++ b/src/test/regress/sql/mot/single_numeric.sql @@ -3,14 +3,14 @@ -- CREATE FOREIGN TABLE num_data (id int4, val numeric(210,10)) SERVER mot_server ; -CREATE FOREIGN TABLE num_exp_add (id1 int4, id2 int4, expected numeric(210,10)) SERVER mot_server ; -CREATE FOREIGN TABLE num_exp_sub (id1 int4, id2 int4, expected numeric(210,10)) SERVER mot_server ; -CREATE FOREIGN TABLE num_exp_div (id1 int4, id2 int4, expected numeric(210,10)) SERVER mot_server ; -CREATE FOREIGN TABLE num_exp_mul (id1 int4, id2 int4, expected numeric(210,10)) SERVER mot_server ; -CREATE FOREIGN TABLE num_exp_sqrt (id int4, expected numeric(210,10)) SERVER mot_server ; -CREATE FOREIGN TABLE num_exp_ln (id int4, expected numeric(210,10)) SERVER mot_server ; -CREATE FOREIGN TABLE num_exp_log10 (id int4, expected numeric(210,10)) SERVER mot_server ; -CREATE FOREIGN TABLE num_exp_power_10_ln (id int4, expected numeric(210,10)) SERVER mot_server ; +CREATE FOREIGN TABLE num_exp_add (id1 int4 not null, id2 int4 not null, expected numeric(210,10)) SERVER mot_server ; +CREATE FOREIGN TABLE num_exp_sub (id1 int4 not null, id2 int4 not null, expected numeric(210,10)) SERVER mot_server ; +CREATE FOREIGN TABLE num_exp_div (id1 int4 not null, id2 int4 not null, expected numeric(210,10)) SERVER mot_server ; +CREATE FOREIGN TABLE num_exp_mul (id1 int4 not null, id2 int4 not null, expected numeric(210,10)) SERVER mot_server ; +CREATE FOREIGN TABLE num_exp_sqrt (id int4 not null, expected numeric(210,10)) SERVER mot_server ; +CREATE FOREIGN TABLE num_exp_ln (id int4 not null, expected numeric(210,10)) SERVER mot_server ; +CREATE FOREIGN TABLE num_exp_log10 (id int4 not null, expected numeric(210,10)) SERVER mot_server ; +CREATE FOREIGN TABLE num_exp_power_10_ln (id int4 not null, expected numeric(210,10)) SERVER mot_server ; CREATE FOREIGN TABLE num_result (id1 int4, id2 int4, result numeric(210,10)) SERVER mot_server ; diff --git a/src/test/regress/sql/mot/single_reindex.sql b/src/test/regress/sql/mot/single_reindex.sql index 08ba89137..e26da82dc 100644 --- a/src/test/regress/sql/mot/single_reindex.sql +++ b/src/test/regress/sql/mot/single_reindex.sql @@ -1,4 +1,4 @@ -create foreign table test1 (i integer, y int); +create foreign table test1 (i integer not null, y int not null); create index text_idx on test1(y,i); insert into test1 values (generate_series(1,1000), generate_series(1,1000)); select * from test1 order by i,y asc limit 10; diff --git a/src/test/regress/sql/mot/single_relation_size.sql b/src/test/regress/sql/mot/single_relation_size.sql index f10d9cfab..6541f6fa5 100644 --- a/src/test/regress/sql/mot/single_relation_size.sql +++ b/src/test/regress/sql/mot/single_relation_size.sql @@ -1,5 +1,5 @@ drop foreign table if exists mm_test; -create foreign table mm_test (id int, name varchar(1000)) server mot_server; +create foreign table mm_test (id int not null, name varchar(1000)) server mot_server; insert into mm_test values (1, 'aaa'), (2, 'bbb'), (3, 'ccc'), (4, 'ddd'), (5, 'eee'), (6, 'fff'); select pg_relation_size('mm_test') > 0; create index mm_test_index on mm_test using btree(id); diff --git a/src/test/regress/sql/mot/single_tpcc_queries.sql b/src/test/regress/sql/mot/single_tpcc_queries.sql index 079cea5d2..29904743d 100644 --- a/src/test/regress/sql/mot/single_tpcc_queries.sql +++ b/src/test/regress/sql/mot/single_tpcc_queries.sql @@ -47,8 +47,8 @@ CREATE FOREIGN TABLE bmsql_customer ( c_id integer not null, c_discount decimal(4,4), c_credit char(2), - c_last varchar(16), - c_first varchar(16), + c_last varchar(16) not null, + c_first varchar(16) not null, c_credit_lim decimal(12,2), c_balance decimal(12,2), c_ytd_payment decimal(12,2), @@ -89,7 +89,7 @@ CREATE FOREIGN TABLE bmsql_oorder ( o_w_id integer not null, o_d_id integer not null, o_id integer not null, - o_c_id integer, + o_c_id integer not null, o_carrier_id integer, o_ol_cnt integer, o_all_local integer,