database/openGauss-server
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

!154 Added MOT config bounds check

Browse Source 
Merge pull request !154 from Vinoth/master
This commit is contained in:
opengauss-bot
2020-08-28 16:10:28 +08:00
committed by Gitee
parent f80d95c2ef 971a0ddf15
commit 339cd59f26
19 changed files with 492 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
src/gausskernel/storage/mot/core/src/infra/config/config_manager.cpp
View File

@ -37,20 +37,6 @@
namespace MOT {
DECLARE_LOGGER(ConfigManager, Configuration)
// Initialization helper macro (for system calls)
#define CHECK_SYS_INIT_STATUS(rc, syscall, format, ...) \
if (rc != 0) { \
MOT_REPORT_SYSTEM_PANIC_CODE(rc, syscall, "Configuration Manager Initialization", format, ##__VA_ARGS__); \
break; \
}
// Initialization helper macro (for sub-service initialization)
#define CHECK_INIT_STATUS(result, format, ...) \
if (!result) { \
MOT_REPORT_PANIC(MOT_ERROR_INTERNAL, "Configuration Manager Initialization", format, ##__VA_ARGS__); \
break; \
}
ConfigManager* ConfigManager::m_manager = nullptr;
ConfigManager::ConfigManager() : m_configLoaderCount(0), m_listenerCount(0)
@ -301,26 +287,22 @@ const ConfigTree* ConfigManager::GetConfigTree(const char* configName) const
bool ConfigManager::Initialize(char** argv, int argc)
{
bool result = false;
bool result = true;
char* envvar = getenv("MOT_DEBUG_CFG_LOAD");
if (envvar && (strcmp(envvar, "TRUE") == 0)) {
SetLogComponentLogLevel("Configuration", LogLevel::LL_DEBUG);
}
do {
// add command line configuration loader
if (argv != nullptr && argc != 0) {
result = AddCmdLineConfigLoader(argv, argc);
CHECK_INIT_STATUS(result, "Failed to create command line configuration loader");
// add command line configuration loader
if (argv != nullptr && argc != 0) {
result = AddCmdLineConfigLoader(argv, argc);
if (!result) {
MOT_REPORT_PANIC(MOT_ERROR_INTERNAL,
"Configuration Manager Initialization",
"Failed to create command line configuration loader");
}
// register special listener (workaround until full integration of configuration module)
// must be first listener, other listeners rely on it to be updated with latest configuration
MOTConfiguration& motCfg = GetGlobalConfiguration();
result = AddConfigChangeListener(&motCfg);
CHECK_INIT_STATUS(result, "Failed to register main configuration listener");
} while (0);
}
return result;
}

4
src/gausskernel/storage/mot/core/src/infra/config/config_manager.h
View File

@ -78,8 +78,8 @@ public:
/**
* @brief Creates singleton instance using command line arguments as configuration source. Must be called once
* during engine startup.
* @param argv Command line argument array.
* @param argc Command line argument array size.
* @param[opt] argc Command line argument count (excluding program name first argument).
* @param[opt] argv Command line argument array (excluding program name first argument).
* @return True if instance creation succeeded.
*/
static bool CreateInstance(char** argv = nullptr, int argc = 0);

30
src/gausskernel/storage/mot/core/src/mot.conf
View File

@ -20,6 +20,8 @@
# GB = gigabytes
# TB = terabytes
#
# If no memory units are specified, then bytes are assumed.
#
# Some memory units can be also given in the form of percentage from max_process_memory configured
# in postgresql.conf as follows:
#
@ -31,6 +33,8 @@
# min = minutes (can also specify mins or minutes)
# h = hours (can also specify hours)
# d = days (can also specify days)
#
# If no time units are specified, then microseconds are assumed.
#------------------------------------------------------------------------------
# REDO LOG
@ -296,8 +300,6 @@
#min_mot_local_memory = 0 MB
# Configures the maximum memory limit for a single session in MOT engine.
# Specifying percentage value relates to the total defined by max_process_memory configured in
# postgresql.conf.
# Usually sessions in MOT engine are free to allocate local memory as much as needed, as long as
# the local memory limit is not breached. Nevertheless, in oder to prevent one session from taking
# too much memory, and thus denying memory from other sessions, this configuration item is used to
@ -307,24 +309,17 @@
# A value of zero denotes no restriction on any session-local small allocations per session, except
# for the restriction arising from the local memory allocation limit configured by
# max_mot_local_memory.
# Note: Percentage values cannot be set for this configuration item.
#
#max_mot_session_memory = 0 MB
# Configures the minimum memory reservation for a single session in MOT engine.
# Specifying percentage value relates to the total defined by max_process_memory configured in
# postgresql.conf.
# This value is used both for pre-allocation of memory during session creation, as well as for
# ensuring a minimum amount of memory is available for the session to perform its normal operation.
# Note: Percentage values cannot be set for this configuration item.
#
#min_mot_session_memory = 0 MB
# Configures the high red mark for memory allocations.
# This is taken as percentage from the maximum MOT engine configured by max_mot_memory. When total
# memory usage by MOT reaches this red-mark, only destructive operations are allowed, and all other
# operations will result in error being reported to the user.
#
#high_red_mark_percent = 90
# Configures the large buffer store for sessions.
# When a user session executes a query that requires a lot of memory (e.g. when using many rows),
# the large buffer store is used both for increasing the certainty level that such memory is
@ -332,6 +327,7 @@
# allocation for a session that is above 1022 KB is considered as a large memory allocation. If the
# large buffer store is not used or depleted, such allocations are treated as huge allocations that
# are served directly from kernel.
# Note: Percentage values cannot be set for this configuration item.
#
#session_large_buffer_store_size = 0 MB
@ -341,12 +337,14 @@
# determining the internal division of the buffer store into objects of various size.
# This size cannot exceed 1/8 of the session_large_buffer_store_size, and if it does it is
# rectified to the possible maximum.
# Note: Percentage values cannot be set for this configuration item.
#
#session_large_buffer_store_max_object_size = 0 MB
# Configures the maximum size of a single huge memory allocation made by a session.
# Huge allocations are served directly from kernel, and therefore are not guaranteed to succeed.
# This value also pertains to global (i.e. not session-related) memory allocations.
# Note: Percentage values cannot be set for this configuration item.
#
#session_max_huge_object_size = 1 GB
@ -367,6 +365,7 @@
# garbage collection that might affect performance. Setting a high value will trigger garbage
# collection less frequently, but will result in higher levels of un-reclaimed memory. This is very
# much dependent upon overall workload.
# Note: Percentage values cannot be set for this configuration item.
#
#reclaim_threshold = 512 KB
@ -384,6 +383,7 @@
# in the same pass, even though that batch size limit is reached, until the total size of still
# waiting to-be-reclaimed objects will be less than this threshold, or there are no more objects
# eligible for reclamation.
# Note: Percentage values cannot be set for this configuration item.
#
#high_reclaim_threshold = 8 MB
@ -413,11 +413,3 @@
# Limits the amount of JIT queries allowed per user session.
#
#mot_codegen_limit = 100
#------------------------------------------------------------------------------
# STORAGE
#------------------------------------------------------------------------------
# Specifies whether it is allowed to define an index over a null-able column.
#
#allow_index_on_nullable_column = true

4
src/gausskernel/storage/mot/core/src/system/global.h
View File

@ -300,10 +300,6 @@ inline void Prefetch(const void* ptr)
/** @define Constant denoting indentation used for MOT printouts. */
#define PRINT_REPORT_INDENT 2
/** @define Min and Max values for asynchronous redo log buffer array count. */
#define MIN_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT 8
#define MAX_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT 128
} // namespace MOT
#endif // MOT_GLOBAL_H

359
src/gausskernel/storage/mot/core/src/system/mot_configuration.cpp
View File

@ -35,24 +35,44 @@ IMPLEMENT_CLASS_LOGGER(MOTConfiguration, Configuration)
MOTConfiguration MOTConfiguration::motGlobalConfiguration;
// static member definitions
// Memory scaling constants
constexpr uint64_t MOTConfiguration::SCALE_BYTES;
constexpr uint64_t MOTConfiguration::SCALE_KILO_BYTES;
constexpr uint64_t MOTConfiguration::SCALE_MEGA_BYTES;
// Time scaling constants
constexpr uint64_t MOTConfiguration::SCALE_MICROS;
constexpr uint64_t MOTConfiguration::SCALE_MILLIS;
constexpr uint64_t MOTConfiguration::SCALE_SECONDS;
// redo-log configuration members
constexpr bool MOTConfiguration::DEFAULT_ENABLE_REDO_LOG;
constexpr LoggerType MOTConfiguration::DEFAULT_LOGGER_TYPE;
constexpr RedoLogHandlerType MOTConfiguration::DEFAULT_REDO_LOG_HANDLER_TYPE;
constexpr uint32_t MOTConfiguration::DEFAULT_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT;
constexpr uint32_t MOTConfiguration::MIN_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT;
constexpr uint32_t MOTConfiguration::MAX_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT;
constexpr bool MOTConfiguration::DEFAULT_ENABLE_GROUP_COMMIT;
constexpr uint64_t MOTConfiguration::DEFAULT_GROUP_COMMIT_SIZE;
constexpr uint64_t MOTConfiguration::MIN_GROUP_COMMIT_SIZE;
constexpr uint64_t MOTConfiguration::MAX_GROUP_COMMIT_SIZE;
constexpr const char* MOTConfiguration::DEFAULT_GROUP_COMMIT_TIMEOUT;
constexpr uint64_t MOTConfiguration::DEFAULT_GROUP_COMMIT_TIMEOUT_USEC;
constexpr uint64_t MOTConfiguration::MIN_GROUP_COMMIT_TIMEOUT_USEC;
constexpr uint64_t MOTConfiguration::MAX_GROUP_COMMIT_TIMEOUT_USEC;
// checkpoint configuration members
constexpr bool MOTConfiguration::DEFAULT_ENABLE_CHECKPOINT;
constexpr bool MOTConfiguration::DEFAULT_ENABLE_INCREMENTAL_CHECKPOINT;
constexpr const char* MOTConfiguration::DEFAULT_CHECKPOINT_DIR;
constexpr const char* MOTConfiguration::DEFAULT_CHECKPOINT_SEGSIZE;
constexpr uint32_t MOTConfiguration::DEFAULT_CHECKPOINT_SEGSIZE_BYTES;
constexpr uint32_t MOTConfiguration::MIN_CHECKPOINT_SEGSIZE_BYTES;
constexpr uint32_t MOTConfiguration::MAX_CHECKPOINT_SEGSIZE_BYTES;
constexpr uint32_t MOTConfiguration::DEFAULT_CHECKPOINT_WORKERS;
constexpr uint32_t MOTConfiguration::MIN_CHECKPOINT_WORKERS;
constexpr uint32_t MOTConfiguration::MAX_CHECKPOINT_WORKERS;
// recovery configuration members
constexpr uint32_t MOTConfiguration::DEFAULT_CHECKPOINT_RECOVERY_WORKERS;
constexpr uint32_t MOTConfiguration::MIN_CHECKPOINT_RECOVERY_WORKERS;
constexpr uint32_t MOTConfiguration::MAX_CHECKPOINT_RECOVERY_WORKERS;
constexpr bool MOTConfiguration::DEFAULT_ENABLE_LOG_RECOVERY_STATS;
// machine configuration members
constexpr uint16_t MOTConfiguration::DEFAULT_NUMA_NODES;
@ -63,6 +83,8 @@ constexpr uint16_t MOTConfiguration::DEFAULT_MAX_DATA_NODES;
constexpr bool MOTConfiguration::DEFAULT_ENABLE_STATS;
constexpr const char* MOTConfiguration::DEFAULT_STATS_PRINT_PERIOD;
constexpr uint32_t MOTConfiguration::DEFAULT_STATS_PRINT_PERIOD_SECONDS;
constexpr uint32_t MOTConfiguration::MIN_STATS_PRINT_PERIOD_SECONDS;
constexpr uint32_t MOTConfiguration::MAX_STATS_PRINT_PERIOD_SECONDS;
constexpr const char* MOTConfiguration::DEFAULT_FULL_STATS_PRINT_PERIOD;
constexpr uint32_t MOTConfiguration::DEFAULT_FULL_STATS_PRINT_PERIOD_SECONDS;
constexpr bool MOTConfiguration::DEFAULT_ENABLE_DB_SESSION_STAT_PRINT;
@ -81,50 +103,86 @@ constexpr LogLevel MOTConfiguration::DEFAULT_CFG_STARTUP_LOG_LEVEL;
// memory configuration members
constexpr bool MOTConfiguration::DEFAULT_ENABLE_NUMA;
constexpr uint16_t MOTConfiguration::DEFAULT_MAX_THREADS;
constexpr uint16_t MOTConfiguration::MIN_MAX_THREADS;
constexpr uint16_t MOTConfiguration::MAX_MAX_THREADS;
constexpr uint32_t MOTConfiguration::DEFAULT_MAX_CONNECTIONS;
constexpr uint32_t MOTConfiguration::MIN_MAX_CONNECTIONS;
constexpr uint32_t MOTConfiguration::MAX_MAX_CONNECTIONS;
constexpr AffinityMode MOTConfiguration::DEFAULT_AFFINITY_MODE;
constexpr bool MOTConfiguration::DEFAULT_LAZY_LOAD_CHUNK_DIRECTORY;
constexpr const char* MOTConfiguration::DEFAULT_MAX_MOT_GLOBAL_MEMORY;
constexpr uint32_t MOTConfiguration::DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB;
constexpr uint32_t MOTConfiguration::MIN_MAX_MOT_GLOBAL_MEMORY_MB;
constexpr uint32_t MOTConfiguration::MAX_MAX_MOT_GLOBAL_MEMORY_MB;
constexpr const char* MOTConfiguration::DEFAULT_MAX_MOT_LOCAL_MEMORY;
constexpr uint32_t MOTConfiguration::DEFAULT_MAX_MOT_LOCAL_MEMORY_MB;
constexpr uint32_t MOTConfiguration::MIN_MAX_MOT_LOCAL_MEMORY_MB;
constexpr uint32_t MOTConfiguration::MAX_MAX_MOT_LOCAL_MEMORY_MB;
constexpr const char* MOTConfiguration::DEFAULT_MIN_MOT_GLOBAL_MEMORY;
constexpr uint32_t MOTConfiguration::DEFAULT_MIN_MOT_GLOBAL_MEMORY_MB;
constexpr uint32_t MOTConfiguration::MIN_MIN_MOT_GLOBAL_MEMORY_MB;
constexpr uint32_t MOTConfiguration::MAX_MIN_MOT_GLOBAL_MEMORY_MB;
constexpr const char* MOTConfiguration::DEFAULT_MIN_MOT_LOCAL_MEMORY;
constexpr uint32_t MOTConfiguration::DEFAULT_MIN_MOT_LOCAL_MEMORY_MB;
constexpr uint32_t MOTConfiguration::MIN_MIN_MOT_LOCAL_MEMORY_MB;
constexpr uint32_t MOTConfiguration::MAX_MIN_MOT_LOCAL_MEMORY_MB;
constexpr const char* MOTConfiguration::DEFAULT_MAX_MOT_SESSION_MEMORY;
constexpr uint32_t MOTConfiguration::DEFAULT_MAX_MOT_SESSION_MEMORY_KB;
constexpr uint32_t MOTConfiguration::MIN_MAX_MOT_SESSION_MEMORY_KB;
constexpr uint32_t MOTConfiguration::MAX_MAX_MOT_SESSION_MEMORY_KB;
constexpr const char* MOTConfiguration::DEFAULT_MIN_MOT_SESSION_MEMORY;
constexpr uint32_t MOTConfiguration::DEFAULT_MIN_MOT_SESSION_MEMORY_KB;
constexpr uint32_t MOTConfiguration::MIN_MIN_MOT_SESSION_MEMORY_KB;
constexpr uint32_t MOTConfiguration::MAX_MIN_MOT_SESSION_MEMORY_KB;
constexpr MemReserveMode MOTConfiguration::DEFAULT_RESERVE_MEMORY_MODE;
constexpr MemStorePolicy MOTConfiguration::DEFAULT_STORE_MEMORY_POLICY;
constexpr MemAllocPolicy MOTConfiguration::DEFAULT_CHUNK_ALLOC_POLICY;
constexpr uint32_t MOTConfiguration::DEFAULT_CHUNK_PREALLOC_WORKER_COUNT;
constexpr uint32_t MOTConfiguration::MIN_CHUNK_PREALLOC_WORKER_COUNT;
constexpr uint32_t MOTConfiguration::MAX_CHUNK_PREALLOC_WORKER_COUNT;
constexpr uint32_t MOTConfiguration::DEFAULT_HIGH_RED_MARK_PERCENT;
constexpr uint32_t MOTConfiguration::MIN_HIGH_RED_MARK_PERCENT;
constexpr uint32_t MOTConfiguration::MAX_HIGH_RED_MARK_PERCENT;
constexpr const char* MOTConfiguration::DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE;
constexpr uint32_t MOTConfiguration::DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE_MB;
constexpr uint32_t MOTConfiguration::MIN_SESSION_LARGE_BUFFER_STORE_SIZE_MB;
constexpr uint32_t MOTConfiguration::MAX_SESSION_LARGE_BUFFER_STORE_SIZE_MB;
constexpr const char* MOTConfiguration::DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE;
constexpr uint32_t MOTConfiguration::DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB;
constexpr uint32_t MOTConfiguration::MIN_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB;
constexpr uint32_t MOTConfiguration::MAX_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB;
constexpr const char* MOTConfiguration::DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE;
constexpr uint32_t MOTConfiguration::DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE_MB;
constexpr uint32_t MOTConfiguration::MIN_SESSION_MAX_HUGE_OBJECT_SIZE_MB;
constexpr uint32_t MOTConfiguration::MAX_SESSION_MAX_HUGE_OBJECT_SIZE_MB;
// GC configuration members
constexpr bool MOTConfiguration::DEFAULT_GC_ENABLE;
constexpr const char* MOTConfiguration::DEFAULT_GC_RECLAIM_THRESHOLD;
constexpr uint32_t MOTConfiguration::DEFAULT_GC_RECLAIM_THRESHOLD_BYTES;
constexpr uint32_t MOTConfiguration::MIN_GC_RECLAIM_THRESHOLD_BYTES;
constexpr uint32_t MOTConfiguration::MAX_GC_RECLAIM_THRESHOLD_BYTES;
constexpr uint32_t MOTConfiguration::DEFAULT_GC_RECLAIM_BATCH_SIZE;
constexpr uint32_t MOTConfiguration::MIN_GC_RECLAIM_BATCH_SIZE;
constexpr uint32_t MOTConfiguration::MAX_GC_RECLAIM_BATCH_SIZE;
constexpr const char* MOTConfiguration::DEFAULT_GC_HIGH_RECLAIM_THRESHOLD;
constexpr uint32_t MOTConfiguration::DEFAULT_GC_HIGH_RECLAIM_THRESHOLD_BYTES;
constexpr uint32_t MOTConfiguration::MIN_GC_HIGH_RECLAIM_THRESHOLD_BYTES;
constexpr uint32_t MOTConfiguration::MAX_GC_HIGH_RECLAIM_THRESHOLD_BYTES;
// JIT configuration members
constexpr bool MOTConfiguration::DEFAULT_ENABLE_MOT_CODEGEN;
constexpr bool MOTConfiguration::DEFAULT_FORCE_MOT_PSEUDO_CODEGEN;
constexpr bool MOTConfiguration::DEFAULT_ENABLE_MOT_CODEGEN_PRINT;
constexpr uint32_t MOTConfiguration::DEFAULT_MOT_CODEGEN_LIMIT;
constexpr uint32_t MOTConfiguration::MIN_MOT_CODEGEN_LIMIT;
constexpr uint32_t MOTConfiguration::MAX_MOT_CODEGEN_LIMIT;
// storage configuration
constexpr bool MOTConfiguration::DEFAULT_ALLOW_INDEX_ON_NULLABLE_COLUMN;
constexpr IndexTreeFlavor MOTConfiguration::DEFAULT_INDEX_TREE_FLAVOR;
// general configuration members
constexpr const char* MOTConfiguration::DEFAULT_CFG_MONITOR_PERIOD;
constexpr uint64_t MOTConfiguration::DEFAULT_CFG_MONITOR_PERIOD_SECONDS;
constexpr uint64_t MOTConfiguration::MIN_CFG_MONITOR_PERIOD_SECONDS;
constexpr uint64_t MOTConfiguration::MAX_CFG_MONITOR_PERIOD_SECONDS;
constexpr bool MOTConfiguration::DEFAULT_RUN_INTERNAL_CONSISTENCY_VALIDATION;
constexpr uint32_t MOTConfiguration::DEFAULT_TOTAL_MEMORY_MB;
@ -448,7 +506,8 @@ MOTConfiguration::MOTConfiguration()
m_indexTreeFlavor(DEFAULT_INDEX_TREE_FLAVOR),
m_configMonitorPeriodSeconds(DEFAULT_CFG_MONITOR_PERIOD_SECONDS),
m_runInternalConsistencyValidation(DEFAULT_RUN_INTERNAL_CONSISTENCY_VALIDATION),
m_totalMemoryMb(DEFAULT_TOTAL_MEMORY_MB)
m_totalMemoryMb(DEFAULT_TOTAL_MEMORY_MB),
m_suppressLog(false)
{}
void MOTConfiguration::Initialize()
@ -604,40 +663,37 @@ int MOTConfiguration::GetMappedCore(int logicId) const
return -1;
}
#define UPDATE_CFG(var, cfgPath, defaultValue) \
UpdateConfigItem(var, cfg->GetConfigValue(cfgPath, defaultValue), cfgPath)
#define UPDATE_BOOL_CFG(var, cfgPath, defaultValue) \
UpdateBoolConfigItem(var, cfg->GetConfigValue(cfgPath, defaultValue), cfgPath)
#define UPDATE_USER_CFG(var, cfgPath, defaultValue) \
UpdateUserConfigItem(var, cfg->GetUserConfigValue(cfgPath, defaultValue), cfgPath)
#define UPDATE_STRING_CFG(var, cfgPath, defaultValue) \
UpdateConfigItem(var, cfg->GetStringConfigValue(cfgPath, defaultValue), cfgPath)
UpdateStringConfigItem(var, cfg->GetStringConfigValue(cfgPath, defaultValue), cfgPath)
#define UPDATE_INT_CFG(var, cfgPath, defaultValue) \
UpdateConfigItem(var, cfg->GetIntegerConfigValue(cfgPath, defaultValue), cfgPath)
#define UPDATE_INT_CFG(var, cfgPath, defaultValue, lowerBound, upperBound) \
UpdateIntConfigItem(var, cfg->GetIntegerConfigValue(cfgPath, defaultValue), cfgPath, lowerBound, upperBound)
#define UPDATE_INT_CFG_BOUNDS(var, cfgPath, defaultValue, lowerBound, upperBound) \
UpdateConfigItem(var, cfg->GetIntegerConfigValue(cfgPath, defaultValue), cfgPath, lowerBound, upperBound)
#define UPDATE_MEM_CFG(var, cfgPath, defaultValue, scale) \
#define UPDATE_MEM_CFG(var, cfgPath, defaultValue, scale, lowerBound, upperBound) \
do { \
uint64_t memoryValueBytes = \
ParseMemoryValueBytes(cfg->GetStringConfigValue(cfgPath, defaultValue), (uint64_t)-1, cfgPath); \
UpdateConfigItem(var, (uint32_t)(memoryValueBytes / scale), cfgPath); \
UpdateIntConfigItem(var, (uint32_t)(memoryValueBytes / scale), cfgPath, lowerBound, upperBound); \
} while (0);
#define UPDATE_ABS_MEM_CFG(var, cfgPath, defaultValue, scale) \
do { \
uint64_t memoryValueBytes = \
ParseMemoryUnit(cfg->GetStringConfigValue(cfgPath, defaultValue), (uint64_t)-1, cfgPath); \
UpdateConfigItem(var, (uint32_t)(memoryValueBytes / scale), cfgPath); \
#define UPDATE_ABS_MEM_CFG(var, cfgPath, defaultValue, scale, lowerBound, upperBound) \
do { \
uint64_t memoryValueBytes = \
ParseMemoryUnit(cfg->GetStringConfigValue(cfgPath, defaultValue), (uint64_t)-1, cfgPath); \
UpdateIntConfigItem(var, (uint32_t)(memoryValueBytes / scale), cfgPath, lowerBound, upperBound); \
} while (0);
#define UPDATE_TIME_CFG(var, cfgPath, defaultValue, scale) \
#define UPDATE_TIME_CFG(var, cfgPath, defaultValue, scale, lowerBound, upperBound) \
do { \
uint64_t timeValueMicros = \
ParseTimeValueMicros(cfg->GetStringConfigValue(cfgPath, defaultValue), (uint64_t)-1, cfgPath); \
UpdateConfigItem(var, (uint64_t)(timeValueMicros / scale), cfgPath); \
UpdateIntConfigItem(var, (uint64_t)(timeValueMicros / scale), cfgPath, lowerBound, upperBound); \
} while (0);
void MOTConfiguration::LoadConfig()
@ -646,48 +702,80 @@ void MOTConfiguration::LoadConfig()
const LayeredConfigTree* cfg = ConfigManager::GetInstance().GetLayeredConfigTree();
// logger configuration
UPDATE_CFG(m_enableRedoLog, "enable_redo_log", DEFAULT_ENABLE_REDO_LOG);
UPDATE_BOOL_CFG(m_enableRedoLog, "enable_redo_log", DEFAULT_ENABLE_REDO_LOG);
UPDATE_USER_CFG(m_loggerType, "logger_type", DEFAULT_LOGGER_TYPE);
UPDATE_USER_CFG(m_redoLogHandlerType, "redo_log_handler_type", DEFAULT_REDO_LOG_HANDLER_TYPE);
UPDATE_INT_CFG_BOUNDS(m_asyncRedoLogBufferArrayCount,
UPDATE_INT_CFG(m_asyncRedoLogBufferArrayCount,
"async_log_buffer_count",
DEFAULT_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT,
MIN_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT,
MAX_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT);
// commit configuration
UPDATE_CFG(m_enableGroupCommit, "enable_group_commit", DEFAULT_ENABLE_GROUP_COMMIT);
UPDATE_INT_CFG(m_groupCommitSize, "group_commit_size", DEFAULT_GROUP_COMMIT_SIZE);
UPDATE_TIME_CFG(m_groupCommitTimeoutUSec, "group_commit_timeout", DEFAULT_GROUP_COMMIT_TIMEOUT, 1);
UPDATE_BOOL_CFG(m_enableGroupCommit, "enable_group_commit", DEFAULT_ENABLE_GROUP_COMMIT);
UPDATE_INT_CFG(m_groupCommitSize,
"group_commit_size",
DEFAULT_GROUP_COMMIT_SIZE,
MIN_GROUP_COMMIT_SIZE,
MAX_GROUP_COMMIT_SIZE);
UPDATE_TIME_CFG(m_groupCommitTimeoutUSec,
"group_commit_timeout",
DEFAULT_GROUP_COMMIT_TIMEOUT,
SCALE_MICROS,
MIN_GROUP_COMMIT_TIMEOUT_USEC,
MAX_GROUP_COMMIT_TIMEOUT_USEC);
// Checkpoint configuration
UPDATE_CFG(m_enableCheckpoint, "enable_checkpoint", DEFAULT_ENABLE_CHECKPOINT);
UPDATE_BOOL_CFG(m_enableCheckpoint, "enable_checkpoint", DEFAULT_ENABLE_CHECKPOINT);
UPDATE_STRING_CFG(m_checkpointDir, "checkpoint_dir", DEFAULT_CHECKPOINT_DIR);
UPDATE_MEM_CFG(m_checkpointSegThreshold, "checkpoint_segsize", DEFAULT_CHECKPOINT_SEGSIZE, 1);
UPDATE_INT_CFG(m_checkpointWorkers, "checkpoint_workers", DEFAULT_CHECKPOINT_WORKERS);
UPDATE_MEM_CFG(m_checkpointSegThreshold,
"checkpoint_segsize",
DEFAULT_CHECKPOINT_SEGSIZE,
SCALE_BYTES,
MIN_CHECKPOINT_SEGSIZE_BYTES,
MAX_CHECKPOINT_SEGSIZE_BYTES);
UPDATE_INT_CFG(m_checkpointWorkers,
"checkpoint_workers",
DEFAULT_CHECKPOINT_WORKERS,
MIN_CHECKPOINT_WORKERS,
MAX_CHECKPOINT_WORKERS);
// Recovery configuration
UPDATE_INT_CFG(m_checkpointRecoveryWorkers, "checkpoint_recovery_workers", DEFAULT_CHECKPOINT_RECOVERY_WORKERS);
UPDATE_INT_CFG(m_checkpointRecoveryWorkers,
"checkpoint_recovery_workers",
DEFAULT_CHECKPOINT_RECOVERY_WORKERS,
MIN_CHECKPOINT_RECOVERY_WORKERS,
MAX_CHECKPOINT_RECOVERY_WORKERS);
// Tx configuration - not configurable yet
UPDATE_CFG(m_abortBufferEnable, "tx_abort_buffers_enable", true);
UPDATE_CFG(m_preAbort, "tx_pre_abort", true);
UPDATE_BOOL_CFG(m_abortBufferEnable, "tx_abort_buffers_enable", true);
UPDATE_BOOL_CFG(m_preAbort, "tx_pre_abort", true);
m_validationLock = TxnValidation::TXN_VALIDATION_NO_WAIT;
// statistics configuration
UPDATE_CFG(m_enableStats, "enable_stats", DEFAULT_ENABLE_STATS);
UPDATE_TIME_CFG(m_statPrintPeriodSeconds, "print_stats_period", DEFAULT_STATS_PRINT_PERIOD, 1000000);
UPDATE_TIME_CFG(m_statPrintFullPeriodSeconds, "print_full_stats_period", DEFAULT_FULL_STATS_PRINT_PERIOD, 1000000);
UPDATE_CFG(m_enableLogRecoveryStats, "enable_log_recovery_stats", DEFAULT_ENABLE_LOG_RECOVERY_STATS);
UPDATE_CFG(m_enableDbSessionStatistics, "enable_db_session_stats", DEFAULT_ENABLE_DB_SESSION_STAT_PRINT);
UPDATE_CFG(m_enableNetworkStatistics, "enable_network_stats", DEFAULT_ENABLE_NETWORK_STAT_PRINT);
UPDATE_CFG(m_enableLogStatistics, "enable_log_stats", DEFAULT_ENABLE_LOG_STAT_PRINT);
UPDATE_CFG(m_enableMemoryStatistics, "enable_memory_stats", DEFAULT_ENABLE_MEMORY_STAT_PRINT);
UPDATE_CFG(
UPDATE_BOOL_CFG(m_enableStats, "enable_stats", DEFAULT_ENABLE_STATS);
UPDATE_TIME_CFG(m_statPrintPeriodSeconds,
"print_stats_period",
DEFAULT_STATS_PRINT_PERIOD,
SCALE_SECONDS,
MIN_STATS_PRINT_PERIOD_SECONDS,
MAX_STATS_PRINT_PERIOD_SECONDS);
UPDATE_TIME_CFG(m_statPrintFullPeriodSeconds,
"print_full_stats_period",
DEFAULT_FULL_STATS_PRINT_PERIOD,
SCALE_SECONDS,
MIN_STATS_PRINT_PERIOD_SECONDS,
MAX_STATS_PRINT_PERIOD_SECONDS);
UPDATE_BOOL_CFG(m_enableLogRecoveryStats, "enable_log_recovery_stats", DEFAULT_ENABLE_LOG_RECOVERY_STATS);
UPDATE_BOOL_CFG(m_enableDbSessionStatistics, "enable_db_session_stats", DEFAULT_ENABLE_DB_SESSION_STAT_PRINT);
UPDATE_BOOL_CFG(m_enableNetworkStatistics, "enable_network_stats", DEFAULT_ENABLE_NETWORK_STAT_PRINT);
UPDATE_BOOL_CFG(m_enableLogStatistics, "enable_log_stats", DEFAULT_ENABLE_LOG_STAT_PRINT);
UPDATE_BOOL_CFG(m_enableMemoryStatistics, "enable_memory_stats", DEFAULT_ENABLE_MEMORY_STAT_PRINT);
UPDATE_BOOL_CFG(
m_enableDetailedMemoryStatistics, "enable_detailed_memory_stats", DEFAULT_ENABLE_DETAILED_MEMORY_STAT_PRINT);
UPDATE_CFG(m_enableProcessStatistics, "enable_process_stats", DEFAULT_ENABLE_PROCESS_STAT_PRINT);
UPDATE_CFG(m_enableSystemStatistics, "enable_system_stats", DEFAULT_ENABLE_SYSTEM_STAT_PRINT);
UPDATE_CFG(m_enableJitStatistics, "enable_jit_stats", DEFAULT_ENABLE_JIT_STAT_PRINT);
UPDATE_BOOL_CFG(m_enableProcessStatistics, "enable_process_stats", DEFAULT_ENABLE_PROCESS_STAT_PRINT);
UPDATE_BOOL_CFG(m_enableSystemStatistics, "enable_system_stats", DEFAULT_ENABLE_SYSTEM_STAT_PRINT);
UPDATE_BOOL_CFG(m_enableJitStatistics, "enable_jit_stats", DEFAULT_ENABLE_JIT_STAT_PRINT);
// log configuration
UPDATE_USER_CFG(m_logLevel, "log_level", DEFAULT_LOG_LEVEL);
@ -697,56 +785,163 @@ void MOTConfiguration::LoadConfig()
UPDATE_USER_CFG(m_cfgStartupLogLevel, "cfg_startup_log_level", DEFAULT_CFG_STARTUP_LOG_LEVEL);
// memory configuration
UPDATE_CFG(m_enableNuma, "enable_numa", DEFAULT_ENABLE_NUMA);
UPDATE_INT_CFG(m_maxThreads, "max_threads", DEFAULT_MAX_THREADS);
UPDATE_INT_CFG(m_maxConnections, "max_connections", DEFAULT_MAX_CONNECTIONS);
UPDATE_BOOL_CFG(m_enableNuma, "enable_numa", DEFAULT_ENABLE_NUMA);
UPDATE_INT_CFG(m_maxThreads, "max_threads", DEFAULT_MAX_THREADS, MIN_MAX_THREADS, MAX_MAX_THREADS);
UPDATE_INT_CFG(
m_maxConnections, "max_connections", DEFAULT_MAX_CONNECTIONS, MIN_MAX_CONNECTIONS, MAX_MAX_CONNECTIONS);
UPDATE_USER_CFG(m_sessionAffinityMode, "affinity_mode", DEFAULT_AFFINITY_MODE);
// we save copies because main affinity mode may be overridden when thread pool is used
if (IS_AFFINITY_ACTIVE(m_sessionAffinityMode)) { // update only if active
m_taskAffinityMode = m_sessionAffinityMode;
}
UPDATE_CFG(m_lazyLoadChunkDirectory, "lazy_load_chunk_directory", DEFAULT_LAZY_LOAD_CHUNK_DIRECTORY);
UPDATE_MEM_CFG(m_globalMemoryMaxLimitMB, "max_mot_global_memory", DEFAULT_MAX_MOT_GLOBAL_MEMORY, MEGA_BYTE);
UPDATE_MEM_CFG(m_globalMemoryMinLimitMB, "min_mot_global_memory", DEFAULT_MIN_MOT_GLOBAL_MEMORY, MEGA_BYTE);
UPDATE_MEM_CFG(m_localMemoryMaxLimitMB, "max_mot_local_memory", DEFAULT_MAX_MOT_LOCAL_MEMORY, MEGA_BYTE);
UPDATE_MEM_CFG(m_localMemoryMinLimitMB, "min_mot_local_memory", DEFAULT_MIN_MOT_LOCAL_MEMORY, MEGA_BYTE);
UPDATE_ABS_MEM_CFG(m_sessionMemoryMaxLimitKB, "max_mot_session_memory", DEFAULT_MAX_MOT_SESSION_MEMORY, KILO_BYTE);
UPDATE_ABS_MEM_CFG(m_sessionMemoryMinLimitKB, "min_mot_session_memory", DEFAULT_MIN_MOT_SESSION_MEMORY, KILO_BYTE);
UPDATE_BOOL_CFG(m_lazyLoadChunkDirectory, "lazy_load_chunk_directory", DEFAULT_LAZY_LOAD_CHUNK_DIRECTORY);
UPDATE_MEM_CFG(m_globalMemoryMaxLimitMB,
"max_mot_global_memory",
DEFAULT_MAX_MOT_GLOBAL_MEMORY,
SCALE_MEGA_BYTES,
MIN_MAX_MOT_GLOBAL_MEMORY_MB,
MAX_MAX_MOT_GLOBAL_MEMORY_MB);
UPDATE_MEM_CFG(m_globalMemoryMinLimitMB,
"min_mot_global_memory",
DEFAULT_MIN_MOT_GLOBAL_MEMORY,
SCALE_MEGA_BYTES,
MIN_MIN_MOT_GLOBAL_MEMORY_MB,
MAX_MIN_MOT_GLOBAL_MEMORY_MB);
// validate that min <= max
if (m_globalMemoryMinLimitMB > m_globalMemoryMaxLimitMB) {
if (!m_suppressLog) {
MOT_LOG_WARN("Invalid global memory configuration: minimum (%u MB) is greater than maximum (%u MB), using "
"defaults (%u MB, %u MB)",
m_globalMemoryMinLimitMB,
m_globalMemoryMaxLimitMB,
DEFAULT_MIN_MOT_GLOBAL_MEMORY_MB,
DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB);
}
UpdateIntConfigItem(m_globalMemoryMaxLimitMB, DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB, "max_mot_global_memory");
UpdateIntConfigItem(m_globalMemoryMinLimitMB, DEFAULT_MIN_MOT_GLOBAL_MEMORY_MB, "min_mot_global_memory");
}
UPDATE_MEM_CFG(m_localMemoryMaxLimitMB,
"max_mot_local_memory",
DEFAULT_MAX_MOT_LOCAL_MEMORY,
SCALE_MEGA_BYTES,
MIN_MAX_MOT_LOCAL_MEMORY_MB,
MAX_MAX_MOT_LOCAL_MEMORY_MB);
UPDATE_MEM_CFG(m_localMemoryMinLimitMB,
"min_mot_local_memory",
DEFAULT_MIN_MOT_LOCAL_MEMORY,
SCALE_MEGA_BYTES,
MIN_MIN_MOT_LOCAL_MEMORY_MB,
MAX_MIN_MOT_LOCAL_MEMORY_MB);
// validate that min <= max
if (m_localMemoryMinLimitMB > m_localMemoryMaxLimitMB) {
if (!m_suppressLog) {
MOT_LOG_WARN("Invalid local memory configuration: minimum (%u MB) is greater than maximum (%u MB), using "
"defaults (%u MB, %u MB)",
m_localMemoryMinLimitMB,
m_localMemoryMaxLimitMB,
DEFAULT_MIN_MOT_LOCAL_MEMORY_MB,
DEFAULT_MAX_MOT_LOCAL_MEMORY_MB);
}
UpdateIntConfigItem(m_localMemoryMaxLimitMB, DEFAULT_MAX_MOT_LOCAL_MEMORY_MB, "max_mot_local_memory");
UpdateIntConfigItem(m_localMemoryMinLimitMB, DEFAULT_MIN_MOT_LOCAL_MEMORY_MB, "min_mot_local_memory");
}
UPDATE_ABS_MEM_CFG(m_sessionMemoryMaxLimitKB,
"max_mot_session_memory",
DEFAULT_MAX_MOT_SESSION_MEMORY,
SCALE_KILO_BYTES,
MIN_MAX_MOT_SESSION_MEMORY_KB,
MAX_MAX_MOT_SESSION_MEMORY_KB);
UPDATE_ABS_MEM_CFG(m_sessionMemoryMinLimitKB,
"min_mot_session_memory",
DEFAULT_MIN_MOT_SESSION_MEMORY,
SCALE_KILO_BYTES,
MIN_MIN_MOT_SESSION_MEMORY_KB,
MAX_MIN_MOT_SESSION_MEMORY_KB);
// validate that min <= max
if ((m_sessionMemoryMaxLimitKB > 0) && (m_sessionMemoryMinLimitKB > m_sessionMemoryMaxLimitKB)) {
if (!m_suppressLog) {
MOT_LOG_WARN("Invalid session memory configuration: minimum (%u KB) is greater than maximum (%u KB), using "
"defaults (%u KB, %u KB)",
m_sessionMemoryMinLimitKB,
m_sessionMemoryMaxLimitKB,
DEFAULT_MIN_MOT_SESSION_MEMORY_KB,
DEFAULT_MAX_MOT_SESSION_MEMORY_KB);
}
UpdateIntConfigItem(m_sessionMemoryMaxLimitKB, DEFAULT_MAX_MOT_SESSION_MEMORY_KB, "max_mot_session_memory");
UpdateIntConfigItem(m_sessionMemoryMinLimitKB, DEFAULT_MIN_MOT_SESSION_MEMORY_KB, "min_mot_session_memory");
}
UPDATE_USER_CFG(m_reserveMemoryMode, "reserve_memory_mode", DEFAULT_RESERVE_MEMORY_MODE);
UPDATE_USER_CFG(m_storeMemoryPolicy, "store_memory_policy", DEFAULT_STORE_MEMORY_POLICY);
UPDATE_USER_CFG(m_chunkAllocPolicy, "chunk_alloc_policy", DEFAULT_CHUNK_ALLOC_POLICY);
UPDATE_INT_CFG(m_chunkPreallocWorkerCount, "chunk_prealloc_worker_count", DEFAULT_CHUNK_PREALLOC_WORKER_COUNT);
UPDATE_INT_CFG(m_highRedMarkPercent, "high_red_mark_percent", DEFAULT_HIGH_RED_MARK_PERCENT);
UPDATE_INT_CFG(m_chunkPreallocWorkerCount,
"chunk_prealloc_worker_count",
DEFAULT_CHUNK_PREALLOC_WORKER_COUNT,
MIN_CHUNK_PREALLOC_WORKER_COUNT,
MAX_CHUNK_PREALLOC_WORKER_COUNT);
UPDATE_INT_CFG(m_highRedMarkPercent,
"high_red_mark_percent",
DEFAULT_HIGH_RED_MARK_PERCENT,
MIN_HIGH_RED_MARK_PERCENT,
MAX_HIGH_RED_MARK_PERCENT);
UPDATE_ABS_MEM_CFG(m_sessionLargeBufferStoreSizeMB,
"session_large_buffer_store_size",
DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE,
MEGA_BYTE);
SCALE_MEGA_BYTES,
MIN_SESSION_LARGE_BUFFER_STORE_SIZE_MB,
MAX_SESSION_LARGE_BUFFER_STORE_SIZE_MB);
UPDATE_ABS_MEM_CFG(m_sessionLargeBufferStoreMaxObjectSizeMB,
"session_large_buffer_store_max_object_size",
DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE,
MEGA_BYTE);
UPDATE_ABS_MEM_CFG(
m_sessionMaxHugeObjectSizeMB, "session_max_huge_object_size", DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE, MEGA_BYTE);
SCALE_MEGA_BYTES,
MIN_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB,
MAX_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB);
UPDATE_ABS_MEM_CFG(m_sessionMaxHugeObjectSizeMB,
"session_max_huge_object_size",
DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE,
SCALE_MEGA_BYTES,
MIN_SESSION_MAX_HUGE_OBJECT_SIZE_MB,
MAX_SESSION_MAX_HUGE_OBJECT_SIZE_MB);
// GC configuration
UPDATE_CFG(m_gcEnable, "enable_gc", DEFAULT_GC_ENABLE);
UPDATE_MEM_CFG(m_gcReclaimThresholdBytes, "reclaim_threshold", DEFAULT_GC_RECLAIM_THRESHOLD, 1);
UPDATE_INT_CFG(m_gcReclaimBatchSize, "reclaim_batch_size", DEFAULT_GC_RECLAIM_BATCH_SIZE);
UPDATE_MEM_CFG(m_gcHighReclaimThresholdBytes, "high_reclaim_threshold", DEFAULT_GC_HIGH_RECLAIM_THRESHOLD, 1);
UPDATE_BOOL_CFG(m_gcEnable, "enable_gc", DEFAULT_GC_ENABLE);
UPDATE_ABS_MEM_CFG(m_gcReclaimThresholdBytes,
"reclaim_threshold",
DEFAULT_GC_RECLAIM_THRESHOLD,
SCALE_BYTES,
MIN_GC_RECLAIM_THRESHOLD_BYTES,
MAX_GC_RECLAIM_THRESHOLD_BYTES);
UPDATE_INT_CFG(m_gcReclaimBatchSize,
"reclaim_batch_size",
DEFAULT_GC_RECLAIM_BATCH_SIZE,
MIN_GC_RECLAIM_BATCH_SIZE,
MAX_GC_RECLAIM_BATCH_SIZE);
UPDATE_ABS_MEM_CFG(m_gcHighReclaimThresholdBytes,
"high_reclaim_threshold",
DEFAULT_GC_HIGH_RECLAIM_THRESHOLD,
SCALE_BYTES,
MIN_GC_HIGH_RECLAIM_THRESHOLD_BYTES,
MAX_GC_HIGH_RECLAIM_THRESHOLD_BYTES);
// JIT configuration
UPDATE_CFG(m_enableCodegen, "enable_mot_codegen", DEFAULT_ENABLE_MOT_CODEGEN);
UPDATE_CFG(m_forcePseudoCodegen, "force_mot_pseudo_codegen", DEFAULT_FORCE_MOT_PSEUDO_CODEGEN);
UPDATE_CFG(m_enableCodegenPrint, "enable_mot_codegen_print", DEFAULT_ENABLE_MOT_CODEGEN_PRINT);
UPDATE_INT_CFG(m_codegenLimit, "mot_codegen_limit", DEFAULT_MOT_CODEGEN_LIMIT);
UPDATE_BOOL_CFG(m_enableCodegen, "enable_mot_codegen", DEFAULT_ENABLE_MOT_CODEGEN);
UPDATE_BOOL_CFG(m_forcePseudoCodegen, "force_mot_pseudo_codegen", DEFAULT_FORCE_MOT_PSEUDO_CODEGEN);
UPDATE_BOOL_CFG(m_enableCodegenPrint, "enable_mot_codegen_print", DEFAULT_ENABLE_MOT_CODEGEN_PRINT);
UPDATE_INT_CFG(
m_codegenLimit, "mot_codegen_limit", DEFAULT_MOT_CODEGEN_LIMIT, MIN_MOT_CODEGEN_LIMIT, MAX_MOT_CODEGEN_LIMIT);
// storage configuration
UPDATE_CFG(m_allowIndexOnNullableColumn, "allow_index_on_nullable_column", DEFAULT_ALLOW_INDEX_ON_NULLABLE_COLUMN);
UPDATE_BOOL_CFG(
m_allowIndexOnNullableColumn, "allow_index_on_nullable_column", DEFAULT_ALLOW_INDEX_ON_NULLABLE_COLUMN);
UPDATE_USER_CFG(m_indexTreeFlavor, "index_tree_flavor", DEFAULT_INDEX_TREE_FLAVOR);
// general configuration
UPDATE_TIME_CFG(m_configMonitorPeriodSeconds, "config_update_period", DEFAULT_CFG_MONITOR_PERIOD, 1000000);
UPDATE_CFG(m_runInternalConsistencyValidation,
UPDATE_TIME_CFG(m_configMonitorPeriodSeconds,
"config_update_period",
DEFAULT_CFG_MONITOR_PERIOD,
SCALE_SECONDS,
MIN_CFG_MONITOR_PERIOD_SECONDS,
MAX_CFG_MONITOR_PERIOD_SECONDS);
UPDATE_BOOL_CFG(m_runInternalConsistencyValidation,
"internal_consistency_validation",
DEFAULT_RUN_INTERNAL_CONSISTENCY_VALIDATION);
@ -781,9 +976,11 @@ void MOTConfiguration::UpdateComponentLogLevel()
LogLevel componentLevel = componentCfg->GetUserConfigValue<LogLevel>("log_level", globalLogLevel);
if (componentLevel != globalLogLevel) {
mot_string logLevelStr;
MOT_LOG_INFO("Updating the log level of component %s to: %s",
componentName.c_str(),
TypeFormatter<LogLevel>::ToString(componentLevel, logLevelStr));
if (!m_suppressLog) {
MOT_LOG_INFO("Updating the log level of component %s to: %s",
componentName.c_str(),
TypeFormatter<LogLevel>::ToString(componentLevel, logLevelStr));
}
SetLogComponentLogLevel(componentName.c_str(), componentLevel);
}
@ -806,10 +1003,12 @@ void MOTConfiguration::UpdateComponentLogLevel()
componentCfg->GetUserConfigValue<LogLevel>(loggerName.c_str(), LogLevel::LL_INFO);
if (loggerLevel != LogLevel::LL_INFO) {
mot_string logLevelStr;
MOT_LOG_INFO("Updating the log level of logger %s in component %s to: %s",
loggerName.c_str(),
componentName.c_str(),
TypeFormatter<LogLevel>::ToString(loggerLevel, logLevelStr));
if (!m_suppressLog) {
MOT_LOG_INFO("Updating the log level of logger %s in component %s to: %s",
loggerName.c_str(),
componentName.c_str(),
TypeFormatter<LogLevel>::ToString(loggerLevel, logLevelStr));
}
SetLoggerLogLevel(componentName.c_str(), loggerName.c_str(), loggerLevel);
}
}
@ -821,7 +1020,7 @@ void MOTConfiguration::UpdateComponentLogLevel()
}
}
void MOTConfiguration::UpdateConfigItem(bool& oldValue, bool newValue, const char* name)
void MOTConfiguration::UpdateBoolConfigItem(bool& oldValue, bool newValue, const char* name)
{
if (oldValue != newValue) {
MOT_LOG_TRACE(
@ -830,7 +1029,7 @@ void MOTConfiguration::UpdateConfigItem(bool& oldValue, bool newValue, const cha
}
}
void MOTConfiguration::UpdateConfigItem(std::string& oldValue, const char* newValue, const char* name)
void MOTConfiguration::UpdateStringConfigItem(std::string& oldValue, const char* newValue, const char* name)
{
if (oldValue != newValue) {
MOT_LOG_TRACE("Configuration of %s changed: %s --> %s", name, oldValue.c_str(), newValue);
@ -885,8 +1084,10 @@ uint64_t MOTConfiguration::ParseMemoryPercentTotal(const char* memoryValue, uint
int percent = ParseMemoryPercent(memoryValue);
if (percent >= 0) {
memoryValueBytes = ((uint64_t)m_totalMemoryMb) * MEGA_BYTE * percent / 100;
MOT_LOG_INFO(
"Loaded %s: %d%% from total = %" PRIu64 " MB", cfgPath, percent, memoryValueBytes / 1024ul / 1024ul);
if (!m_suppressLog) {
MOT_LOG_INFO(
"Loaded %s: %d%% from total = %" PRIu64 " MB", cfgPath, percent, memoryValueBytes / 1024ul / 1024ul);
}
} else {
MOT_LOG_WARN("Invalid %s memory format: illegal percent specification", cfgPath);
}

183
src/gausskernel/storage/mot/core/src/system/mot_configuration.h
View File

@ -27,6 +27,7 @@
#include <set>
#include <climits>
#include <cstdint>
#include "affinity.h"
#include "global.h"
@ -72,6 +73,18 @@ public:
LoadConfig();
}
/**
* @brief Load partial configuration (i.e. not all loaders are fully loaded yet). In such case we suppress all
* messages.
*/
inline void LoadPartial()
{
MOT_LOG_TRACE("Loading partial configuration after change");
m_suppressLog = true;
LoadConfig();
m_suppressLog = false;
}
/**
* @brief Validates configuration. Call this function after loading configuration to validate
* the loaded values are valid, not out of bounds, and not self-contradicting.
@ -382,11 +395,15 @@ public:
MOTConfiguration& operator=(const MOTConfiguration&& orig) = delete;
/** @endcond */
/** @var Asynchronous redo-log buffer array bounds (exposed as public for external use). */
static constexpr uint32_t MIN_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT = 8;
static constexpr uint32_t MAX_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT = 128;
private:
/** @var A singleton instance available for static initializers. */
static MOTConfiguration motGlobalConfiguration;
/** @var Map of CPUs to numa nodes */
/** @var Map of CPUs to NUMA nodes */
CpuNodeMap m_cpuNodeMapper;
CpuMap m_osCpuMap;
@ -396,7 +413,20 @@ private:
/** @var The total memory value to be used when loading memory percent values. By default uses system total. */
uint32_t m_totalMemoryMb;
// Default logger configuration
/** @var Controls suppressing of log messages during configuration loading. */
bool m_suppressLog;
/** @var Memory scaling constants (from bytes). */
static constexpr uint64_t SCALE_BYTES = 1;
static constexpr uint64_t SCALE_KILO_BYTES = KILO_BYTE;
static constexpr uint64_t SCALE_MEGA_BYTES = MEGA_BYTE;
/** @var Time scaling constants (from micros). */
static constexpr uint64_t SCALE_MICROS = 1;
static constexpr uint64_t SCALE_MILLIS = 1000;
static constexpr uint64_t SCALE_SECONDS = 1000000;
/** ------------------ Default Redo-Log Configuration ------------ */
/** @var Default enable file logger. */
static constexpr bool DEFAULT_ENABLE_REDO_LOG = true;
@ -409,20 +439,23 @@ private:
/** @var Default asynchronous redo log buffer array count. */
static constexpr uint32_t DEFAULT_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT = 24;
// default commit configuration
/** @var Default enable group commit. */
static constexpr bool DEFAULT_ENABLE_GROUP_COMMIT = false;
/** @var Default group commit size. */
static constexpr uint64_t DEFAULT_GROUP_COMMIT_SIZE = 16;
static constexpr uint64_t MIN_GROUP_COMMIT_SIZE = 2;
static constexpr uint64_t MAX_GROUP_COMMIT_SIZE = 1000;
/** @var Default group commit timeout. */
static constexpr const char* DEFAULT_GROUP_COMMIT_TIMEOUT = "10 ms";
/** @var Default group commit timeout in micro-seconds. */
static constexpr uint64_t DEFAULT_GROUP_COMMIT_TIMEOUT_USEC = 10000;
static constexpr uint64_t MIN_GROUP_COMMIT_TIMEOUT_USEC = 100;
static constexpr uint64_t MAX_GROUP_COMMIT_TIMEOUT_USEC = 200000; // 200 ms
// default checkpoint configuration
/** ------------------ Default Checkpoint Configuration ------------ */
/** @var Default enable checkpoint. */
static constexpr bool DEFAULT_ENABLE_CHECKPOINT = true;
@ -434,19 +467,25 @@ private:
/** @var Default checkpoint segments size */
static constexpr const char* DEFAULT_CHECKPOINT_SEGSIZE = "16 MB";
static constexpr uint32_t DEFAULT_CHECKPOINT_SEGSIZE_BYTES = (16 * 1024 * 1024);
static constexpr uint32_t DEFAULT_CHECKPOINT_SEGSIZE_BYTES = 16 * MEGA_BYTE;
static constexpr uint32_t MIN_CHECKPOINT_SEGSIZE_BYTES = 16 * MEGA_BYTE;
static constexpr uint32_t MAX_CHECKPOINT_SEGSIZE_BYTES = 512 * MEGA_BYTE;
/** @var Default number of worker threads to spawn */
static constexpr uint32_t DEFAULT_CHECKPOINT_WORKERS = 3;
static constexpr uint32_t MIN_CHECKPOINT_WORKERS = 1;
static constexpr uint32_t MAX_CHECKPOINT_WORKERS = 1024;
// default recovery configuration
/** ------------------ Default Recovery Configuration ------------ */
/** @var Default number of workers used in recovery from checkpoint. */
static constexpr uint32_t DEFAULT_CHECKPOINT_RECOVERY_WORKERS = 3;
static constexpr uint32_t MIN_CHECKPOINT_RECOVERY_WORKERS = 1;
static constexpr uint32_t MAX_CHECKPOINT_RECOVERY_WORKERS = 1024;
/** @var Default enable log recovery statistics. */
static constexpr bool DEFAULT_ENABLE_LOG_RECOVERY_STATS = false;
// default machine configuration
/** ------------------ Default Machine Configuration ------------ */
/** @var Default number of NUMA nodes of the machine. */
static constexpr uint16_t DEFAULT_NUMA_NODES = 1;
@ -459,13 +498,15 @@ private:
/* @var Default upper bound number of pg nodes on the machine. */
static constexpr uint16_t DEFAULT_MAX_DATA_NODES = 1;
// default statistics configuration
/** ------------------ Default Statistics Configuration ------------ */
/** @var Default enable statistics printing. */
static constexpr bool DEFAULT_ENABLE_STATS = false;
/** @var Default statistics printing period in seconds. */
static constexpr const char* DEFAULT_STATS_PRINT_PERIOD = "1 minutes";
static constexpr uint32_t DEFAULT_STATS_PRINT_PERIOD_SECONDS = 60;
static constexpr uint32_t MIN_STATS_PRINT_PERIOD_SECONDS = 1;
static constexpr uint32_t MAX_STATS_PRINT_PERIOD_SECONDS = 86400; // 1 day
/** @var Default full statistics printing period in seconds. */
static constexpr const char* DEFAULT_FULL_STATS_PRINT_PERIOD = "5 minutes";
@ -495,7 +536,7 @@ private:
/** @var Default enable JIT execution statistics printing. */
static constexpr bool DEFAULT_ENABLE_JIT_STAT_PRINT = false;
// default error log configuration
/** ------------------ Default Error-Log Configuration ------------ */
/** @var Default log level limit. */
static constexpr LogLevel DEFAULT_LOG_LEVEL = LogLevel::LL_INFO;
@ -508,15 +549,19 @@ private:
/** @var Default log level for configuration loading on startup messages. */
static constexpr LogLevel DEFAULT_CFG_STARTUP_LOG_LEVEL = LogLevel::LL_TRACE;
// default memory configuration
/** ------------------ Default Memory Configuration ------------ */
/** @var Default enable NUMA. */
static constexpr bool DEFAULT_ENABLE_NUMA = true;
/** @var Default maximum number of threads in the system. */
static constexpr uint16_t DEFAULT_MAX_THREADS = 1024;
static constexpr uint16_t MIN_MAX_THREADS = 1;
static constexpr uint16_t MAX_MAX_THREADS = UINT16_MAX;
/** @var Default maximum number of connections in the system. */
static constexpr uint32_t DEFAULT_MAX_CONNECTIONS = 1024;
static constexpr uint32_t MIN_MAX_CONNECTIONS = 1;
static constexpr uint32_t MAX_MAX_CONNECTIONS = UINT16_MAX;
/** @var Default thread affinity policy. */
static constexpr AffinityMode DEFAULT_AFFINITY_MODE = AffinityMode::FILL_PHYSICAL_FIRST;
@ -526,28 +571,40 @@ private:
/** @var Default maximum limit for MOT global memory. */
static constexpr const char* DEFAULT_MAX_MOT_GLOBAL_MEMORY = "80%";
static constexpr uint32_t DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB = 8 * 1024;
static constexpr uint32_t DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB = 8 * KILO_BYTE; // 8 GB
static constexpr uint32_t MIN_MAX_MOT_GLOBAL_MEMORY_MB = 128; // 128 MB
static constexpr uint32_t MAX_MAX_MOT_GLOBAL_MEMORY_MB = 512 * MEGA_BYTE; // 512 TB
/** @var Default minimum (pre-allocated) limit for MOT global memory. */
static constexpr const char* DEFAULT_MIN_MOT_GLOBAL_MEMORY = "0 GB";
static constexpr uint32_t DEFAULT_MIN_MOT_GLOBAL_MEMORY_MB = 0;
static constexpr uint32_t MIN_MIN_MOT_GLOBAL_MEMORY_MB = 0;
static constexpr uint32_t MAX_MIN_MOT_GLOBAL_MEMORY_MB = 512 * MEGA_BYTE; // 512 TB
/** @var Default maximum limit for MOT global memory (used to establish new ratio between local and global pools).
*/
static constexpr const char* DEFAULT_MAX_MOT_LOCAL_MEMORY = "15%";
static constexpr uint32_t DEFAULT_MAX_MOT_LOCAL_MEMORY_MB = 2 * 1024;
static constexpr uint32_t DEFAULT_MAX_MOT_LOCAL_MEMORY_MB = 2 * KILO_BYTE; // 2 GB
static constexpr uint32_t MIN_MAX_MOT_LOCAL_MEMORY_MB = 64; // 64 MB (about 8 normal sessions)
static constexpr uint32_t MAX_MAX_MOT_LOCAL_MEMORY_MB = 512 * KILO_BYTE; // 512 GB (MANY very heavy sessions)
/** @var Default minimum (pre-allocated) limit for MOT local memory. */
static constexpr const char* DEFAULT_MIN_MOT_LOCAL_MEMORY = "0 GB";
static constexpr uint32_t DEFAULT_MIN_MOT_LOCAL_MEMORY_MB = 0;
static constexpr uint32_t DEFAULT_MIN_MOT_LOCAL_MEMORY_MB = 0; // no pre-allocation
static constexpr uint32_t MIN_MIN_MOT_LOCAL_MEMORY_MB = 0; // no pre-allocation
static constexpr uint32_t MAX_MIN_MOT_LOCAL_MEMORY_MB = 512 * KILO_BYTE; // max pre-allocate 512 GB
/** @var Default maximum for single MOT session small memory allocations. */
static constexpr const char* DEFAULT_MAX_MOT_SESSION_MEMORY = "0 MB";
static constexpr uint32_t DEFAULT_MAX_MOT_SESSION_MEMORY_KB = 0;
static constexpr uint32_t DEFAULT_MAX_MOT_SESSION_MEMORY_KB = 0; // no session-memory limit
static constexpr uint32_t MIN_MAX_MOT_SESSION_MEMORY_KB = 0;
static constexpr uint32_t MAX_MAX_MOT_SESSION_MEMORY_KB = 512 * MEGA_BYTE; // limit single session to 512 GB
/** @var Default minimum (pre-allocated) for single MOT session small memory allocations. */
static constexpr const char* DEFAULT_MIN_MOT_SESSION_MEMORY = "0 MB";
static constexpr uint32_t DEFAULT_MIN_MOT_SESSION_MEMORY_KB = 0;
static constexpr uint32_t DEFAULT_MIN_MOT_SESSION_MEMORY_KB = 0; // no session-memory pre-allocation
static constexpr uint32_t MIN_MIN_MOT_SESSION_MEMORY_KB = 0; // no session-memory pre-allocation
static constexpr uint32_t MAX_MIN_MOT_SESSION_MEMORY_KB = 64 * KILO_BYTE; // up to 64 MB pre-allocation
/** @var Default physical or virtual memory reservation. */
static constexpr MemReserveMode DEFAULT_RESERVE_MEMORY_MODE = MEM_RESERVE_VIRTUAL;
@ -560,38 +617,54 @@ private:
/** @var Default number of workers used to pre-allocate initial memory. */
static constexpr uint32_t DEFAULT_CHUNK_PREALLOC_WORKER_COUNT = 8;
static constexpr uint32_t MIN_CHUNK_PREALLOC_WORKER_COUNT = 1;
static constexpr uint32_t MAX_CHUNK_PREALLOC_WORKER_COUNT = 1024;
/** @var Default chunk store high red mark in percents of maximum. */
static constexpr uint32_t DEFAULT_HIGH_RED_MARK_PERCENT = 90;
static constexpr uint32_t DEFAULT_HIGH_RED_MARK_PERCENT = 90; // reject constructive ops from 90% memory usage
static constexpr uint32_t MIN_HIGH_RED_MARK_PERCENT = 50; // reject constructive ops from 50% memory usage
static constexpr uint32_t MAX_HIGH_RED_MARK_PERCENT = 95; // Don't allow disabling high red-mark
/** @var The default size in megabytes of the session large buffer store. */
static constexpr const char* DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE = " 0 MB";
static constexpr uint32_t DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE_MB = 0;
static constexpr uint32_t MIN_SESSION_LARGE_BUFFER_STORE_SIZE_MB = 0; // disabled
static constexpr uint32_t MAX_SESSION_LARGE_BUFFER_STORE_SIZE_MB = 128 * KILO_BYTE; // 128 GB
/** @var The default largest object size in megabytes in the session large buffer store. */
static constexpr const char* DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE = "0 MB";
static constexpr uint32_t DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB = 0;
static constexpr uint32_t DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB = 0; // use calculation
static constexpr uint32_t MIN_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB = 0;
static constexpr uint32_t MAX_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB = 1024; // 1 GB
/** @var The default largest object size in megabytes that can be allocated form kernel for sessions. */
static constexpr const char* DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE = "1 GB";
static constexpr uint32_t DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE_MB = 1024;
static constexpr uint32_t DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE_MB = 1024; // 1 GB
static constexpr uint32_t MIN_SESSION_MAX_HUGE_OBJECT_SIZE_MB = 8; // 8 MB
static constexpr uint32_t MAX_SESSION_MAX_HUGE_OBJECT_SIZE_MB = 8 * KILO_BYTE; // 8 GB
// default GC configuration
/** ------------------ Default Garbage-Collection Configuration ------------ */
/** @var Enable/disable garbage collection. */
static constexpr bool DEFAULT_GC_ENABLE = true;
/** @var The threshold in bytes for reclamation to be triggered (per-thread) */
static constexpr const char* DEFAULT_GC_RECLAIM_THRESHOLD = "512 KB";
static constexpr uint32_t DEFAULT_GC_RECLAIM_THRESHOLD_BYTES = 512 * KILO_BYTE;
static constexpr uint32_t DEFAULT_GC_RECLAIM_THRESHOLD_BYTES = 512 * KILO_BYTE; // 512 KB
static constexpr uint32_t MIN_GC_RECLAIM_THRESHOLD_BYTES = KILO_BYTE; // 1 KB
static constexpr uint32_t MAX_GC_RECLAIM_THRESHOLD_BYTES = 64 * MEGA_BYTE; // 64 MB
/** @var The amount of objects reclaimed in each cleanup round of a limbo group. */
static constexpr uint32_t DEFAULT_GC_RECLAIM_BATCH_SIZE = 8000;
static constexpr uint32_t DEFAULT_GC_RECLAIM_BATCH_SIZE = 8 * KILO_BYTE; // 8 KB entries
static constexpr uint32_t MIN_GC_RECLAIM_BATCH_SIZE = 128; // 128 entries
static constexpr uint32_t MAX_GC_RECLAIM_BATCH_SIZE = MEGA_BYTE; // 1 MB entries
/** @var The high threshold in bytes for reclamation to be triggered (per-thread) */
static constexpr const char* DEFAULT_GC_HIGH_RECLAIM_THRESHOLD = "8 MB";
static constexpr uint32_t DEFAULT_GC_HIGH_RECLAIM_THRESHOLD_BYTES = 8 * MEGA_BYTE;
static constexpr uint32_t DEFAULT_GC_HIGH_RECLAIM_THRESHOLD_BYTES = 8 * MEGA_BYTE; // 8 MB
static constexpr uint32_t MIN_GC_HIGH_RECLAIM_THRESHOLD_BYTES = 1 * MEGA_BYTE; // 1 MB
static constexpr uint32_t MAX_GC_HIGH_RECLAIM_THRESHOLD_BYTES = 64 * MEGA_BYTE; // 64 MB
// default JIT configuration
/** ------------------ Default JIT Configuration ------------ */
/** @var Default enable JIT compilation and execution. */
static constexpr bool DEFAULT_ENABLE_MOT_CODEGEN = true;
@ -603,24 +676,28 @@ private:
/** @vart Default limit for the amount of JIT queries allowed per user session. */
static constexpr uint32_t DEFAULT_MOT_CODEGEN_LIMIT = 100;
static constexpr uint32_t MIN_MOT_CODEGEN_LIMIT = 1;
static constexpr uint32_t MAX_MOT_CODEGEN_LIMIT = 1000;
// default storage configuration
/** ------------------ Default Storage Configuration ------------ */
/** @var The default allow index on null-able column. */
static constexpr bool DEFAULT_ALLOW_INDEX_ON_NULLABLE_COLUMN = true;
static constexpr bool DEFAULT_ALLOW_INDEX_ON_NULLABLE_COLUMN = false;
/** @var The default tree flavor for tree indexes. */
static constexpr IndexTreeFlavor DEFAULT_INDEX_TREE_FLAVOR = IndexTreeFlavor::INDEX_TREE_FLAVOR_MASSTREE;
// default general configuration
/** ------------------ Default General Configuration ------------ */
/** @var Default configuration monitor period in seconds. */
static constexpr const char* DEFAULT_CFG_MONITOR_PERIOD = "5 seconds";
static constexpr uint64_t DEFAULT_CFG_MONITOR_PERIOD_SECONDS = 5;
static constexpr uint64_t DEFAULT_CFG_MONITOR_PERIOD_SECONDS = 5; // 5 seconds
static constexpr uint64_t MIN_CFG_MONITOR_PERIOD_SECONDS = 1; // 1 seconds
static constexpr uint64_t MAX_CFG_MONITOR_PERIOD_SECONDS = 300; // 5 minutes
/** @var The default value for consistency validation tests after benchmark running. */
static constexpr bool DEFAULT_RUN_INTERNAL_CONSISTENCY_VALIDATION = false;
/** @var The default total memory reference used for calculating memory percent value. */
static constexpr uint32_t DEFAULT_TOTAL_MEMORY_MB = 10 * 1024;
static constexpr uint32_t DEFAULT_TOTAL_MEMORY_MB = 10 * KILO_BYTE; // 10 MB
/** @brief Loads configuration from main configuration. */
void LoadConfig();
@ -632,26 +709,42 @@ private:
static bool FindNumProcessors(uint16_t* maxCoresPerNode, CpuNodeMap* cpuNodeMapper, CpuMap* cpuOsMapper);
static bool CheckHyperThreads();
static void UpdateConfigItem(bool& oldValue, bool newValue, const char* name);
static void UpdateConfigItem(std::string& oldValue, const char* newValue, const char* name);
static void UpdateBoolConfigItem(bool& oldValue, bool newValue, const char* name);
static void UpdateStringConfigItem(std::string& oldValue, const char* newValue, const char* name);
template <typename T>
static void UpdateConfigItem(uint64_t& oldValue, T newValue, const char* name)
void UpdateIntConfigItem(
uint64_t& oldValue, T newValue, const char* name, uint64_t lowerBound = 0, uint64_t upperBound = UINT64_MAX)
{
if (oldValue != newValue) {
if ((newValue > upperBound) || (lowerBound > 0 && newValue < lowerBound)) {
if (!m_suppressLog) {
MOT_LOG_WARN("Configuration of %s=%" PRIu64 " is out of bounds [%" PRIu64 ", %" PRIu64 "]: keeping "
"default value %" PRIu64,
name,
(uint64_t)newValue,
lowerBound,
upperBound,
oldValue);
}
} else if (oldValue != newValue) {
MOT_LOG_TRACE("Configuration of %s changed: %" PRIu64 " --> %" PRIu64, name, oldValue, (uint64_t)newValue);
oldValue = newValue;
}
}
template <typename T>
static void UpdateConfigItem(
uint32_t& oldValue, T newValue, const char* name, uint32_t lowerBound = 0, uint32_t upperBound = UINT_MAX)
void UpdateIntConfigItem(
uint32_t& oldValue, T newValue, const char* name, uint32_t lowerBound = 0, uint32_t upperBound = UINT32_MAX)
{
if (newValue > upperBound) {
MOT_LOG_WARN("Configuration of %s overflowed: keeping default value %u", name, oldValue);
} else if (lowerBound > 0 && newValue < lowerBound) {
MOT_LOG_WARN("Configuration of %s overflowed: keeping default value %u", name, oldValue);
if ((newValue > UINT32_MAX) || (newValue > upperBound) || (lowerBound > 0 && newValue < lowerBound)) {
if (!m_suppressLog) {
MOT_LOG_WARN("Configuration of %s=%" PRIu64 " is out of bounds [%u, %u]: keeping default value %u",
name,
(uint64_t)newValue,
lowerBound,
upperBound,
oldValue);
}
} else if (oldValue != newValue) {
MOT_LOG_TRACE("Configuration of %s changed: %u --> %u", name, oldValue, (uint32_t)newValue);
oldValue = newValue;
@ -659,10 +752,18 @@ private:
}
template <typename T>
static void UpdateConfigItem(uint16_t& oldValue, T newValue, const char* name)
void UpdateIntConfigItem(
uint16_t& oldValue, T newValue, const char* name, uint16_t lowerBound = 0, uint16_t upperBound = UINT16_MAX)
{
if (newValue > USHRT_MAX) {
MOT_LOG_WARN("Configuration of %s overflowed: keeping default value %" PRIu16, name, oldValue);
if ((newValue > UINT16_MAX) || (newValue > upperBound) || (lowerBound > 0 && newValue < lowerBound)) {
if (!m_suppressLog) {
MOT_LOG_WARN("Configuration of %s=%" PRIu64 " is out of bounds [%u, %u]: keeping default value %u",
name,
(uint64_t)newValue,
(unsigned)lowerBound,
(unsigned)upperBound,
(unsigned)oldValue);
}
} else if (oldValue != newValue) {
MOT_LOG_TRACE("Configuration of %s changed: %" PRIu16 " --> %" PRIu16, name, oldValue, (uint16_t)newValue);
oldValue = newValue;

17
src/gausskernel/storage/mot/core/src/system/mot_engine.cpp
View File

@ -571,15 +571,24 @@ bool MOTEngine::InitializeConfiguration(const char* configFilePath, int argc, ch
}
}
// initialize global configuration singleton
GetGlobalConfiguration().Initialize();
// explicitly initialize global configuration singleton
MOTConfiguration& motCfg = GetGlobalConfiguration();
motCfg.Initialize();
// create manager, and configure it with configuration file loader
if (!ConfigManager::CreateInstance(argv + 1, argc - 1)) {
if (!ConfigManager::CreateInstance(argv, argc)) {
MOT_LOG_ERROR("Failed to create configuration manager instance");
result = false;
} else {
if (configFilePath != nullptr) {
// register global configuration as first listener (other listeners rely on it)
result = ConfigManager::GetInstance().AddConfigChangeListener(&motCfg);
if (!result) {
MOT_REPORT_ERROR(MOT_ERROR_INTERNAL,
"Load Configuration",
"Failed to register main configuration listener at the configuration manager");
ConfigManager::DestroyInstance();
} else if (configFilePath != nullptr) {
// add configuration file loader
result = ConfigManager::GetInstance().AddConfigFile(configFilePath);
if (!result) {
MOT_REPORT_ERROR(MOT_ERROR_INTERNAL,

8
src/gausskernel/storage/mot/core/src/system/mot_engine.h
View File

@ -77,8 +77,8 @@ public:
* @brief Creates the single main memory engine instance. Use this variant when you have no external configuration
* loaders involved.
* @param[opt] configFilePath The path to the configuration file of the main memory engine.
* @param[opt] argc Command line argument count.
* @param[opt] argv Command line argument array.
* @param[opt] argc Command line argument count (excluding program name first argument).
* @param[opt] argv Command line argument array (excluding program name first argument).
* @return A reference to the single main memory engine instance, or null if initialization failed.
*/
static MOTEngine* CreateInstance(const char* configFilePath = nullptr, int argc = 0, char* argv[] = nullptr);
@ -103,8 +103,8 @@ public:
* initialization takes place. In this case initialization has several steps: @ref CreateInstanceNoInit(), @ref
* AddConfigLoader(), @ref AddConfigLoader(), ..., @ref LoadConfig(), @ref Initialize().
* @param[opt] configFilePath The path to the configuration file of the main memory engine.
* @param[opt] argc Command line argument count.
* @param[opt] argv Command line argument array.
* @param[opt] argc Command line argument count (excluding program name first argument).
* @param[opt] argv Command line argument array (excluding program name first argument).
* @return A reference to the single main memory engine instance if configuration loading
* succeeded, otherwise nullptr.
*/

4
src/gausskernel/storage/mot/core/src/system/transaction/txn.cpp
View File

@ -917,7 +917,7 @@ RC TxnInsertAction::ExecuteOptimisticInsert(Row* row)
rc = RC_UNIQUE_VIOLATION;
goto end;
default:
break;
goto end;
}
} else if (res == true or pIndexInsertResult->IsCommited() == false) {
// tag all the sentinels the insert metadata
@ -934,7 +934,7 @@ RC TxnInsertAction::ExecuteOptimisticInsert(Row* row)
}
end:
if (rc != RC_OK) {
if ((rc != RC_OK) && (currentItem != EndCursor())) {
// Clean current aborted row and clean secondary indexes that were not inserts
// Clean first Object! - wither primary or secondary!
// Return Local Row to pull for PI

3
src/gausskernel/storage/mot/core/src/system/transaction_logger/asynchronous_redo_log/asynchronous_redo_log_handler.h
View File

@ -30,6 +30,7 @@
#include "redo_log_handler.h"
#include "redo_log_buffer_pool.h"
#include "rw_lock.h"
#include "mot_configuration.h"
namespace MOT {
class TxnManager;
@ -88,7 +89,7 @@ private:
RedoLogBufferPool m_bufferPool;
// array of RedoLogBufferArray for switching in cyclic manner.
RedoLogBufferArray m_redoLogBufferArrayArray[MAX_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT];
RedoLogBufferArray m_redoLogBufferArrayArray[MOTConfiguration::MAX_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT];
uint32_t m_redoLogBufferArrayCount;
volatile int m_activeBuffer;
bool m_initialized;

2
src/gausskernel/storage/mot/fdw_adapter/src/mot_internal.cpp
View File

@ -267,7 +267,7 @@ protected:
// the layered configuration tree in the configuration manager
MOT_LOG_TRACE("Triggering initial partial configuration loading for external configuration overlaying");
MOT::MOTConfiguration& motCfg = MOT::GetGlobalConfiguration();
motCfg.OnConfigChange(); // trigger partial load
motCfg.LoadPartial(); // trigger partial load
// load any relevant GUC values here
bool result = ConfigureMaxConnections();

16
src/test/regress/expected/mot/single_numeric.out
View File

@ -2,14 +2,14 @@
-- NUMERIC
--
CREATE FOREIGN TABLE num_data (id int4, val numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_add (id1 int4, id2 int4, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_sub (id1 int4, id2 int4, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_div (id1 int4, id2 int4, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_mul (id1 int4, id2 int4, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_sqrt (id int4, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_ln (id int4, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_log10 (id int4, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_power_10_ln (id int4, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_add (id1 int4 not null, id2 int4 not null, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_sub (id1 int4 not null, id2 int4 not null, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_div (id1 int4 not null, id2 int4 not null, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_mul (id1 int4 not null, id2 int4 not null, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_sqrt (id int4 not null, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_ln (id int4 not null, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_log10 (id int4 not null, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_power_10_ln (id int4 not null, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_result (id1 int4, id2 int4, result numeric(210,10)) SERVER mot_server ;
-- ******************************
-- * The following EXPECTED results are computed by bc(1)

2
src/test/regress/expected/mot/single_reindex.out
View File

@ -1,4 +1,4 @@
create foreign table test1 (i integer, y int);
create foreign table test1 (i integer not null, y int not null);
create index text_idx on test1(y,i);
insert into test1 values (generate_series(1,1000), generate_series(1,1000));
select * from test1 order by i,y asc limit 10;

2
src/test/regress/expected/mot/single_relation_size.out
View File

@ -1,6 +1,6 @@
drop foreign table if exists mm_test;
NOTICE: foreign table "mm_test" does not exist, skipping
create foreign table mm_test (id int, name varchar(1000)) server mot_server;
create foreign table mm_test (id int not null, name varchar(1000)) server mot_server;
insert into mm_test values (1, 'aaa'), (2, 'bbb'), (3, 'ccc'), (4, 'ddd'), (5, 'eee'), (6, 'fff');
select pg_relation_size('mm_test') > 0;
?column?

6
src/test/regress/expected/mot/single_tpcc_queries.out
View File

@ -43,8 +43,8 @@ CREATE FOREIGN TABLE bmsql_customer (
c_id integer not null,
c_discount decimal(4,4),
c_credit char(2),
c_last varchar(16),
c_first varchar(16),
c_last varchar(16) not null,
c_first varchar(16) not null,
c_credit_lim decimal(12,2),
c_balance decimal(12,2),
c_ytd_payment decimal(12,2),
@ -84,7 +84,7 @@ CREATE FOREIGN TABLE bmsql_oorder (
o_w_id integer not null,
o_d_id integer not null,
o_id integer not null,
o_c_id integer,
o_c_id integer not null,
o_carrier_id integer,
o_ol_cnt integer,
o_all_local integer,

16
src/test/regress/sql/mot/single_numeric.sql
View File

@ -3,14 +3,14 @@
--
CREATE FOREIGN TABLE num_data (id int4, val numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_add (id1 int4, id2 int4, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_sub (id1 int4, id2 int4, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_div (id1 int4, id2 int4, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_mul (id1 int4, id2 int4, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_sqrt (id int4, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_ln (id int4, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_log10 (id int4, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_power_10_ln (id int4, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_add (id1 int4 not null, id2 int4 not null, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_sub (id1 int4 not null, id2 int4 not null, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_div (id1 int4 not null, id2 int4 not null, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_mul (id1 int4 not null, id2 int4 not null, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_sqrt (id int4 not null, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_ln (id int4 not null, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_log10 (id int4 not null, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_exp_power_10_ln (id int4 not null, expected numeric(210,10)) SERVER mot_server ;
CREATE FOREIGN TABLE num_result (id1 int4, id2 int4, result numeric(210,10)) SERVER mot_server ;

2
src/test/regress/sql/mot/single_reindex.sql
View File

@ -1,4 +1,4 @@
create foreign table test1 (i integer, y int);
create foreign table test1 (i integer not null, y int not null);
create index text_idx on test1(y,i);
insert into test1 values (generate_series(1,1000), generate_series(1,1000));
select * from test1 order by i,y asc limit 10;

2
src/test/regress/sql/mot/single_relation_size.sql
View File

@ -1,5 +1,5 @@
drop foreign table if exists mm_test;
create foreign table mm_test (id int, name varchar(1000)) server mot_server;
create foreign table mm_test (id int not null, name varchar(1000)) server mot_server;
insert into mm_test values (1, 'aaa'), (2, 'bbb'), (3, 'ccc'), (4, 'ddd'), (5, 'eee'), (6, 'fff');
select pg_relation_size('mm_test') > 0;
create index mm_test_index on mm_test using btree(id);

6
src/test/regress/sql/mot/single_tpcc_queries.sql
View File

@ -47,8 +47,8 @@ CREATE FOREIGN TABLE bmsql_customer (
c_id integer not null,
c_discount decimal(4,4),
c_credit char(2),
c_last varchar(16),
c_first varchar(16),
c_last varchar(16) not null,
c_first varchar(16) not null,
c_credit_lim decimal(12,2),
c_balance decimal(12,2),
c_ytd_payment decimal(12,2),
@ -89,7 +89,7 @@ CREATE FOREIGN TABLE bmsql_oorder (
o_w_id integer not null,
o_d_id integer not null,
o_id integer not null,
o_c_id integer,
o_c_id integer not null,
o_carrier_id integer,
o_ol_cnt integer,
o_all_local integer,