hcq/openGauss-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix MOT memory and time configurations to use default units if none specified

Browse Source
This commit is contained in:
Vinoth
2020-08-31 19:51:44 +08:00
parent f370abbabd
commit 560a969040
19 changed files with 499 additions and 285 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
src/gausskernel/storage/mot/core/src/infra/config/config_section.cpp
View File

@ -130,7 +130,7 @@ bool ConfigSection::GetConfigValueNames(mot_string_list& valueNames) const
return true;
}
bool ConfigSection::AddConfigItem(ConfigItem* configItem)
bool ConfigSection::AddConfigItem(ConfigItem* configItem, bool replaceIfExists /* = false */)
{
bool result = false;
switch (configItem->GetClass()) {
@ -139,7 +139,7 @@ bool ConfigSection::AddConfigItem(ConfigItem* configItem)
break;
case ConfigItemClass::CONFIG_ITEM_VALUE:
result = AddConfigValue(static_cast<ConfigValue*>(configItem));
result = AddConfigValue(static_cast<ConfigValue*>(configItem), replaceIfExists);
break;
case ConfigItemClass::CONFIG_ITEM_ARRAY:
@ -271,15 +271,22 @@ bool ConfigSection::AddConfigSection(ConfigSection* configSection)
return (pairis.second == INSERT_SUCCESS);
}
bool ConfigSection::AddConfigValue(ConfigValue* configValue)
bool ConfigSection::AddConfigValue(ConfigValue* configValue, bool replaceIfExists)
{
ConfigValueMap::pairis pairis = m_valueMap.insert(ConfigValueMap::value_type(configValue->GetName(), configValue));
if (pairis.second == INSERT_EXISTS) {
MOT_REPORT_ERROR(MOT_ERROR_INTERNAL,
"Load Configuration",
"Cannot add configuration value %s to section %s: value already exists",
configValue->GetName(),
GetName());
if (replaceIfExists) {
ConfigValue* prevValue = pairis.first->second;
pairis.first->second = configValue;
delete prevValue;
pairis.second = INSERT_SUCCESS;
} else {
MOT_REPORT_ERROR(MOT_ERROR_INTERNAL,
"Load Configuration",
"Cannot add configuration value %s to section %s: value already exists",
configValue->GetName(),
GetName());
}
} else if (pairis.second == INSERT_FAILED) {
MOT_REPORT_ERROR(MOT_ERROR_INTERNAL,
"Load Configuration",

13
src/gausskernel/storage/mot/core/src/infra/config/config_section.h
View File

@ -86,11 +86,13 @@ public:
/**
* @brief Builds the section by adding a sub-item.
* @param config_item The direct configuration item to add as a child of this section. It could be
* another section or a terminal value.
* @param config_item The direct configuration item to add as a child of this section. It could be another section
* or a terminal value.
* @param[opt] replaceIfExists Specifies whether to replace an existing item if it already exists by the given name.
* This is relevant only for configuration values, not for configuration sections or arrays.
* @return True if the item added or false if a configuration item by that name already exists.
*/
bool AddConfigItem(ConfigItem* config_item);
bool AddConfigItem(ConfigItem* configItem, bool replaceIfExists = false);
/**
* @brief Merges the given section into this section.
@ -225,7 +227,7 @@ public:
* @return The typed sub-value, or null pointer if not found.
*/
template <typename T>
inline TypedConfigValue<T>* MmodifyConfigValue(const char* name)
inline TypedConfigValue<T>* ModifyConfigValue(const char* name)
{
TypedConfigValue<T>* result = nullptr;
ConfigValueMap::iterator itr = m_valueMap.find(name);
@ -286,9 +288,10 @@ private:
/**
* @brief Builds the section by adding a sub-value.
* @param configValue The direct configuration sub-value to add as a child of this section.
* @param replaceIfExists Specifies whether to replace an existing item if it already exists by the given name.
* @return True if the item added or false if a configuration item by that name already exists.
*/
bool AddConfigValue(ConfigValue* configValue);
bool AddConfigValue(ConfigValue* configValue, bool replaceIfExists);
/**
* @brief Builds the section by adding a sub-array.

2
src/gausskernel/storage/mot/core/src/infra/config/config_tree.h
View File

@ -229,7 +229,7 @@ public:
* @param defaultValue
* @return
*/
inline const char* getStringConfigValue(const char* fullPathName, const char* defaultValue) const
inline const char* GetStringConfigValue(const char* fullPathName, const char* defaultValue) const
{
const char* result = defaultValue;
const StringConfigValue* configValue = GetQualifiedConfigItem<StringConfigValue>(fullPathName);

13
src/gausskernel/storage/mot/core/src/infra/config/config_value.h
View File

@ -37,7 +37,9 @@ protected:
* @param valueType The value type of this configuration value.
*/
explicit ConfigValue(ConfigValueType valueType)
: ConfigItem(ConfigItemClass::CONFIG_ITEM_VALUE), m_valueType(valueType)
: ConfigItem(ConfigItemClass::CONFIG_ITEM_VALUE),
m_valueType(valueType),
m_isIntegral(IsConfigValueIntegral(valueType))
{}
public:
@ -54,9 +56,18 @@ public:
return m_valueType;
}
/** @brief Queries whether the value type is integral. */
inline bool IsIntegral() const
{
return m_isIntegral;
}
private:
/** @var The configuration value type. */
ConfigValueType m_valueType;
/** @var Specifies whether this is an integral value type. */
bool m_isIntegral;
};
// specialization

12
src/gausskernel/storage/mot/core/src/infra/config/config_value_type.cpp
View File

@ -90,4 +90,16 @@ extern const char* ConfigValueTypeToString(ConfigValueType valueType)
}
return CONFIG_VALUE_UNDEFINED_STR;
}
extern bool IsConfigValueIntegral(ConfigValueType valueType)
{
bool result = false;
if ((valueType == ConfigValueType::CONFIG_VALUE_INT64) || (valueType == ConfigValueType::CONFIG_VALUE_INT32) ||
(valueType == ConfigValueType::CONFIG_VALUE_INT16) || (valueType == ConfigValueType::CONFIG_VALUE_INT8) ||
(valueType == ConfigValueType::CONFIG_VALUE_UINT64) || (valueType == ConfigValueType::CONFIG_VALUE_UINT32) ||
(valueType == ConfigValueType::CONFIG_VALUE_UINT16) || (valueType == ConfigValueType::CONFIG_VALUE_UINT8)) {
result = true;
}
return result;
}
} // namespace MOT

5
src/gausskernel/storage/mot/core/src/infra/config/config_value_type.h
View File

@ -63,7 +63,7 @@ enum class ConfigValueType : uint32_t {
/** @var The configuration value is of type bool. */
CONFIG_VALUE_BOOL,
/** @var The configuration value is of type std::string. */
/** @var The configuration value is of type MOT::mot_string. */
CONFIG_VALUE_STRING,
/** @var The configuration value is undefined (parse error). */
@ -83,6 +83,9 @@ extern ConfigValueType ConfigValueTypeFromString(const char* valueTypeStr);
* @return The resulting configuration value type string.
*/
extern const char* ConfigValueTypeToString(ConfigValueType valueType);
/** @brief Queries whether a value type is an integral type. */
extern bool IsConfigValueIntegral(ConfigValueType valueType);
} // namespace MOT
#endif /* CONFIG_VALUE_TYPE_H */

68
src/gausskernel/storage/mot/core/src/infra/config/layered_config_tree.h
View File

@ -92,13 +92,13 @@ public:
* @param fullPathName The full path name of the configuration array.
* @return The configuration array, or null pointer if none was found by this name.
*/
inline const ConfigArray* getConfigArray(const char* fullPathName) const
inline const ConfigArray* GetConfigArray(const char* fullPathName) const
{
return GetQualifiedConfigItem<ConfigArray>(fullPathName);
}
/**
* @brief Retrieves a typed configuration value.
* @brief Retrieves a typed configuration value. For string types use @ref GetStringConfigValue().
* @param fullPathName The full path name of the configuration value.
* @param defaultValue The default value to return if not found.
* @return The configuration value, or null pointer if none was found by this name.
@ -106,14 +106,10 @@ public:
template <typename T>
inline const T& GetConfigValue(const char* fullPathName, const T& defaultValue) const
{
const TypedConfigValue<T>* configValue = GetQualifiedConfigItem<TypedConfigValue<T>>(fullPathName);
const TypedConfigValue<T>* configValue = (const TypedConfigValue<T>*)GetQualifiedConfigValue<T>(fullPathName);
if (configValue != nullptr) {
if (configValue->GetConfigValueType() != ConfigValueTypeMapper<T>::CONFIG_VALUE_TYPE) {
MOT_LOG_ERROR("Unexpected item value type: %s (%s instead of %s %d)",
fullPathName,
ConfigValueTypeToString(configValue->GetConfigValueType()),
ConfigValueTypeToString(ConfigValueTypeMapper<T>::CONFIG_VALUE_TYPE),
(int)ConfigValueTypeMapper<T>::CONFIG_VALUE_TYPE);
if (!configValue->IsValid()) {
MOT_LOG_ERROR("Configuration value of %s is invalid (OOM in string allocation?)", fullPathName);
} else {
return configValue->GetValue();
}
@ -131,9 +127,14 @@ public:
inline const char* GetStringConfigValue(const char* fullPathName, const char* defaultValue) const
{
const char* result = defaultValue;
const StringConfigValue* configValue = GetQualifiedConfigItem<StringConfigValue>(fullPathName);
const StringConfigValue* configValue =
(const StringConfigValue*)GetQualifiedConfigValue<mot_string>(fullPathName);
if (configValue != nullptr) {
result = configValue->GetValue().c_str();
if (!configValue->IsValid()) {
MOT_LOG_ERROR("Configuration value of %s is invalid (OOM in string allocation?)", fullPathName);
} else {
result = configValue->GetValue().c_str();
}
}
return result;
}
@ -149,9 +150,16 @@ public:
inline T GetUserConfigValue(const char* fullPathName, const T& defaultValue) const
{
T result = defaultValue;
const StringConfigValue* configValue = GetQualifiedConfigItem<StringConfigValue>(fullPathName);
const StringConfigValue* configValue =
(const StringConfigValue*)GetQualifiedConfigValue<mot_string>(fullPathName);
if (configValue != nullptr) {
if (!TypeFormatter<T>::FromString(configValue->GetValue().c_str(), result)) {
if (!configValue->IsValid()) {
MOT_LOG_ERROR("Configuration value of %s is invalid (OOM in string allocation?)", fullPathName);
} else if (!TypeFormatter<T>::FromString(configValue->GetValue().c_str(), result)) {
mot_string stringValue;
MOT_LOG_WARN("Failed to convert string configuration item %s to user type. Using default %s.",
fullPathName,
TypeFormatter<T>::ToString(defaultValue, stringValue));
result = defaultValue;
}
}
@ -168,7 +176,7 @@ public:
inline const T GetIntegerConfigValue(const char* fullPathName, const T& defaultValue) const
{
const ConfigValue* configValue = GetQualifiedConfigItem<ConfigValue>(fullPathName);
if (configValue) {
if (configValue != nullptr) {
switch (configValue->GetConfigValueType()) {
case ConfigValueType::CONFIG_VALUE_INT64:
return SafeCast<T, int64_t>((TypedConfigValue<int64_t>*)configValue, defaultValue);
@ -235,6 +243,38 @@ private:
return result;
}
/**
* @brief Retrieves a qualified configuration value. Value type is checked according to the template type.
* @param fullPathName The full path name of the configuration array.
* @return The configuration value, or null pointer if none was found by this name, or type does not match.
*/
template <typename T>
inline const ConfigValue* GetQualifiedConfigValue(const char* fullPathName) const
{
const ConfigValue* result = nullptr;
const ConfigItem* configItem = GetConfigItem(fullPathName);
if (configItem == nullptr) {
MOT_LOG_TRACE("Cannot find configuration item: %s", fullPathName);
} else if (configItem->GetClass() != ConfigItemClass::CONFIG_ITEM_VALUE) {
MOT_LOG_ERROR("Unexpected item class: %s (%s instead of %s)",
fullPathName,
ConfigItemClassToString(configItem->GetClass()),
ConfigItemClassToString(ConfigItemClass::CONFIG_ITEM_VALUE));
} else {
ConfigValue* configValue = (ConfigValue*)configItem;
if (configValue->GetConfigValueType() != ConfigValueTypeMapper<T>::CONFIG_VALUE_TYPE) {
MOT_LOG_ERROR("Unexpected item value type: %s (%s instead of %s %d)",
fullPathName,
ConfigValueTypeToString(configValue->GetConfigValueType()),
ConfigValueTypeToString(ConfigValueTypeMapper<T>::CONFIG_VALUE_TYPE),
(int)ConfigValueTypeMapper<T>::CONFIG_VALUE_TYPE);
} else {
result = configValue;
}
}
return result;
}
template <typename T, typename U>
static inline T SafeCast(TypedConfigValue<U>* configValue, T defaultValue)
{

2
src/gausskernel/storage/mot/core/src/infra/config/props_config_file_loader.cpp
View File

@ -148,7 +148,7 @@ ConfigTree* PropsConfigFileLoader::LoadConfigFile(const char* configFilePath)
configFilePath,
reader.GetLineNumber(),
line.c_str());
} else if (!currentSection->AddConfigItem(configItem)) {
} else if (!currentSection->AddConfigItem(configItem, true)) {
REPORT_PARSE_ERROR(MOT_ERROR_OOM, "Failed to add configuration item to parent section");
}
} else {

4
src/gausskernel/storage/mot/core/src/memory/garbage_collector/mm_gc_manager.cpp
View File

@ -112,8 +112,8 @@ GcManager* GcManager::Make(int purpose, int threadId, int rcuMaxFreeCount)
} else {
MOTConfiguration& cfg = GetGlobalConfiguration();
gc->m_isGcEnabled = cfg.m_gcEnable;
gc->m_limboSizeLimit = cfg.m_gcReclaimThresholdBytes;
gc->m_limboSizeLimitHigh = cfg.m_gcHighReclaimThresholdBytes;
gc->m_limboSizeLimit = (uint32_t)cfg.m_gcReclaimThresholdBytes;
gc->m_limboSizeLimitHigh = (uint32_t)cfg.m_gcHighReclaimThresholdBytes;
gc->m_rcuFreeCount = cfg.m_gcReclaimBatchSize;
if (threadId == 0) {
MOT_LOG_INFO(

55
src/gausskernel/storage/mot/core/src/memory/mm_cfg.cpp
View File

@ -41,8 +41,8 @@ DECLARE_LOGGER(MemCfg, Memory)
MemCfg g_memGlobalCfg;
static int memCfgInitOnce = 0; // initialize only once
static uint32_t totalMemoryMb = 0;
static uint32_t availableMemoryMb = 0;
static uint64_t totalMemoryMb = 0;
static uint64_t availableMemoryMb = 0;
static int InitTotalAvailMemory();
static int ValidateCfg();
@ -78,13 +78,13 @@ extern int MemCfgInit()
g_memGlobalCfg.m_minSessionMemoryKb = motCfg.m_sessionMemoryMinLimitKB;
g_memGlobalCfg.m_reserveMemoryMode = motCfg.m_reserveMemoryMode;
g_memGlobalCfg.m_storeMemoryPolicy = motCfg.m_storeMemoryPolicy;
MOT_LOG_INFO("Global Memory Limit is configured to: %u MB --> %u MB",
MOT_LOG_INFO("Global Memory Limit is configured to: %" PRIu64 " MB --> %" PRIu64 " MB",
g_memGlobalCfg.m_minGlobalMemoryMb,
g_memGlobalCfg.m_maxGlobalMemoryMb);
MOT_LOG_INFO("Local Memory Limit is configured to: %u MB --> %u MB",
MOT_LOG_INFO("Local Memory Limit is configured to: %" PRIu64 " MB --> %" PRIu64 " MB",
g_memGlobalCfg.m_minLocalMemoryMb,
g_memGlobalCfg.m_maxLocalMemoryMb);
MOT_LOG_INFO("Session Memory Limit is configured to: %u KB --> %u KB (maximum %u sessions)",
MOT_LOG_INFO("Session Memory Limit is configured to: %" PRIu64 " KB --> %" PRIu64 " KB (maximum %u sessions)",
g_memGlobalCfg.m_minSessionMemoryKb,
g_memGlobalCfg.m_maxSessionMemoryKb,
g_memGlobalCfg.m_maxConnectionCount);
@ -153,12 +153,18 @@ extern void MemCfgToString(int indent, const char* name, StringBuffer* stringBuf
indent,
"",
g_memGlobalCfg.m_lazyLoadChunkDirectory ? "Yes" : "No");
StringBufferAppend(stringBuffer, "%*sMax Global Memory MB: %u\n", indent, "", g_memGlobalCfg.m_maxGlobalMemoryMb);
StringBufferAppend(stringBuffer, "%*sMin Global Memory MB: %u\n", indent, "", g_memGlobalCfg.m_minGlobalMemoryMb);
StringBufferAppend(stringBuffer, "%*sMax Local Memory MB: %u\n", indent, "", g_memGlobalCfg.m_maxLocalMemoryMb);
StringBufferAppend(stringBuffer, "%*sMin Local Memory MB: %u\n", indent, "", g_memGlobalCfg.m_minLocalMemoryMb);
StringBufferAppend(stringBuffer, "%*sMax Session Memory KB: %u\n", indent, "", g_memGlobalCfg.m_maxSessionMemoryKb);
StringBufferAppend(stringBuffer, "%*sMin Session Memory KB: %u\n", indent, "", g_memGlobalCfg.m_minSessionMemoryKb);
StringBufferAppend(
stringBuffer, "%*sMax Global Memory MB: %" PRIu64 "\n", indent, "", g_memGlobalCfg.m_maxGlobalMemoryMb);
StringBufferAppend(
stringBuffer, "%*sMin Global Memory MB: %" PRIu64 "\n", indent, "", g_memGlobalCfg.m_minGlobalMemoryMb);
StringBufferAppend(
stringBuffer, "%*sMax Local Memory MB: %" PRIu64 "\n", indent, "", g_memGlobalCfg.m_maxLocalMemoryMb);
StringBufferAppend(
stringBuffer, "%*sMin Local Memory MB: %" PRIu64 "\n", indent, "", g_memGlobalCfg.m_minLocalMemoryMb);
StringBufferAppend(
stringBuffer, "%*sMax Session Memory KB: %" PRIu64 "\n", indent, "", g_memGlobalCfg.m_maxSessionMemoryKb);
StringBufferAppend(
stringBuffer, "%*sMin Session Memory KB: %" PRIu64 "\n", indent, "", g_memGlobalCfg.m_minSessionMemoryKb);
StringBufferAppend(stringBuffer,
"%*sReserve Memory Mode: %s\n",
indent,
@ -182,25 +188,25 @@ extern void MemCfgToString(int indent, const char* name, StringBuffer* stringBuf
g_memGlobalCfg.m_chunkPreallocWorkerCount);
StringBufferAppend(stringBuffer, "%*sHigh Red Mark: %u%%\n", indent, "", g_memGlobalCfg.m_highRedMarkPercent);
StringBufferAppend(stringBuffer,
"%*sSession Large Buffer Store Size MB: %u\n",
"%*sSession Large Buffer Store Size MB: %" PRIu64 "\n",
indent,
"",
g_memGlobalCfg.m_sessionLargeBufferStoreSizeMb);
StringBufferAppend(stringBuffer,
"%*sSession Large Buffer Store Max Object Size MB: %u\n",
"%*sSession Large Buffer Store Max Object Size MB: %" PRIu64 "\n",
indent,
"",
g_memGlobalCfg.m_sessionLargeBufferStoreMaxObjectSizeMb);
StringBufferAppend(stringBuffer,
"%*sSession Max Huge Object Size MB: %u\n",
"%*sSession Max Huge Object Size MB: %" PRIu64 "\n",
indent,
"",
g_memGlobalCfg.m_sessionMaxHugeObjectSizeMb);
}
extern uint32_t GetTotalSystemMemoryMb()
extern uint64_t GetTotalSystemMemoryMb()
{
uint32_t result = 0;
uint64_t result = 0;
int res = InitTotalAvailMemory();
if (res != 0) {
MOT_REPORT_PANIC(res, "MM Layer Configuration Load", "Failed to load MM Layer configuration");
@ -210,9 +216,9 @@ extern uint32_t GetTotalSystemMemoryMb()
return result;
}
extern uint32_t GetAvailableSystemMemoryMb()
extern uint64_t GetAvailableSystemMemoryMb()
{
uint32_t result = (uint32_t)-1;
uint64_t result = (uint64_t)-1;
int res = InitTotalAvailMemory();
if (res != 0) {
MOT_REPORT_PANIC(res, "MM Layer Configuration Load", "Failed to load MM Layer configuration");
@ -230,8 +236,8 @@ static int InitTotalAvailMemory()
if (sysinfo(&si) == 0) {
totalMemoryMb = si.totalram * si.mem_unit / MEGA_BYTE;
availableMemoryMb = si.freeram * si.mem_unit / MEGA_BYTE;
MOT_LOG_TRACE("Total system memory: %u MB", totalMemoryMb);
MOT_LOG_TRACE("Available system memory: %u MB", availableMemoryMb);
MOT_LOG_TRACE("Total system memory: %" PRIu64 " MB", totalMemoryMb);
MOT_LOG_TRACE("Available system memory: %" PRIu64 " MB", availableMemoryMb);
} else {
MOT_REPORT_SYSTEM_PANIC(
sysinfo, "MM Layer Configuration Load", "Failed to retrieve total/available system memory");
@ -280,7 +286,8 @@ static int ValidateCfg()
if (g_memGlobalCfg.m_minGlobalMemoryMb > g_memGlobalCfg.m_maxGlobalMemoryMb) {
MOT_REPORT_PANIC(MOT_ERROR_INVALID_CFG,
"MM Layer Configuration Validation",
"Invalid memory configuration: Global memory pre-allocation %u MB exceeds the maximum limit %u MB",
"Invalid memory configuration: Global memory pre-allocation %" PRIu64
" MB exceeds the maximum limit %" PRIu64 " MB",
g_memGlobalCfg.m_minGlobalMemoryMb,
g_memGlobalCfg.m_maxGlobalMemoryMb);
return MOT_ERROR_INVALID_CFG;
@ -289,7 +296,8 @@ static int ValidateCfg()
if (g_memGlobalCfg.m_minLocalMemoryMb > g_memGlobalCfg.m_maxLocalMemoryMb) {
MOT_REPORT_PANIC(MOT_ERROR_INVALID_CFG,
"MM Layer Configuration Validation",
"Invalid memory configuration: Local memory pre-allocation %u MB exceeds the maximum limit %u MB",
"Invalid memory configuration: Local memory pre-allocation %" PRIu64
" MB exceeds the maximum limit %" PRIu64 " MB",
g_memGlobalCfg.m_minLocalMemoryMb,
g_memGlobalCfg.m_maxLocalMemoryMb);
return MOT_ERROR_INVALID_CFG;
@ -299,7 +307,8 @@ static int ValidateCfg()
if (g_memGlobalCfg.m_maxGlobalMemoryMb > totalMemoryMb) {
MOT_REPORT_PANIC(MOT_ERROR_INVALID_CFG,
"MM Layer Configuration Validation",
"Invalid memory configuration: Global maximum memory limit %u MB exceeds machine capability %u MB",
"Invalid memory configuration: Global maximum memory limit %" PRIu64
" MB exceeds machine capability %" PRIu64 " MB",
g_memGlobalCfg.m_maxGlobalMemoryMb,
totalMemoryMb);
return MOT_ERROR_INVALID_CFG;

22
src/gausskernel/storage/mot/core/src/memory/mm_cfg.h
View File

@ -42,12 +42,12 @@ struct MemCfg {
uint32_t m_lazyLoadChunkDirectory;
// memory limits
uint32_t m_maxGlobalMemoryMb;
uint32_t m_minGlobalMemoryMb;
uint32_t m_maxLocalMemoryMb;
uint32_t m_minLocalMemoryMb;
uint32_t m_maxSessionMemoryKb;
uint32_t m_minSessionMemoryKb;
uint64_t m_maxGlobalMemoryMb;
uint64_t m_minGlobalMemoryMb;
uint64_t m_maxLocalMemoryMb;
uint64_t m_minLocalMemoryMb;
uint64_t m_maxSessionMemoryKb;
uint64_t m_minSessionMemoryKb;
MemReserveMode m_reserveMemoryMode;
MemStorePolicy m_storeMemoryPolicy;
@ -57,9 +57,9 @@ struct MemCfg {
uint32_t m_highRedMarkPercent;
// session large buffer store
uint32_t m_sessionLargeBufferStoreSizeMb;
uint32_t m_sessionLargeBufferStoreMaxObjectSizeMb;
uint32_t m_sessionMaxHugeObjectSizeMb;
uint64_t m_sessionLargeBufferStoreSizeMb;
uint64_t m_sessionLargeBufferStoreMaxObjectSizeMb;
uint64_t m_sessionMaxHugeObjectSizeMb;
};
/**
@ -95,13 +95,13 @@ extern void MemCfgToString(int indent, const char* name, StringBuffer* stringBuf
* @brief Retrieves the total system memory in megabytes.
* @return The total system memory in megabytes, or zero if failed.
*/
extern uint32_t GetTotalSystemMemoryMb();
extern uint64_t GetTotalSystemMemoryMb();
/**
* @brief Retrieves the available system memory in mega-bytes.
* @return The available system memory in megabytes, or zero if failed.
*/
extern uint32_t GetAvailableSystemMemoryMb();
extern uint64_t GetAvailableSystemMemoryMb();
} // namespace MOT
/**

7
src/gausskernel/storage/mot/core/src/memory/mm_numa.cpp
View File

@ -92,9 +92,10 @@ extern void MemNumaInit()
erc = memset_s(peakLocalMemoryBytes, sizeof(peakLocalMemoryBytes), 0, sizeof(peakLocalMemoryBytes));
securec_check(erc, "\0", "\0");
maxGlobalMemoryBytes = ((uint64_t)g_memGlobalCfg.m_maxGlobalMemoryMb) * MEGABYTE;
maxLocalMemoryBytes = ((uint64_t)g_memGlobalCfg.m_maxLocalMemoryMb) * MEGABYTE;
MOT_LOG_TRACE("Initialized global limit to %u MB (%" PRIu64 " bytes), and local limit to %u MB (%" PRIu64 " bytes)",
maxGlobalMemoryBytes = g_memGlobalCfg.m_maxGlobalMemoryMb * MEGABYTE;
maxLocalMemoryBytes = g_memGlobalCfg.m_maxLocalMemoryMb * MEGABYTE;
MOT_LOG_TRACE("Initialized global limit to %" PRIu64 " MB (%" PRIu64 " bytes), and local limit to %" PRIu64
" MB (%" PRIu64 " bytes)",
g_memGlobalCfg.m_maxGlobalMemoryMb,
maxGlobalMemoryBytes,
g_memGlobalCfg.m_maxLocalMemoryMb,

25
src/gausskernel/storage/mot/core/src/memory/mm_raw_chunk_store.cpp
View File

@ -318,9 +318,9 @@ static int MemRawChunkStoreInitGlobal(MemReserveMode reserveMode, MemStorePolicy
{
int result = 0;
MOT_LOG_TRACE("Initializing global chunk pools");
uint32_t chunkReserveCount = g_memGlobalCfg.m_minGlobalMemoryMb / MEM_CHUNK_SIZE_MB / g_memGlobalCfg.m_nodeCount;
uint32_t highBlackMark = g_memGlobalCfg.m_maxGlobalMemoryMb / MEM_CHUNK_SIZE_MB / g_memGlobalCfg.m_nodeCount;
uint32_t highRedMark = highBlackMark * g_memGlobalCfg.m_highRedMarkPercent / 100;
uint64_t chunkReserveCount = g_memGlobalCfg.m_minGlobalMemoryMb / MEM_CHUNK_SIZE_MB / g_memGlobalCfg.m_nodeCount;
uint64_t highBlackMark = g_memGlobalCfg.m_maxGlobalMemoryMb / MEM_CHUNK_SIZE_MB / g_memGlobalCfg.m_nodeCount;
uint64_t highRedMark = highBlackMark * g_memGlobalCfg.m_highRedMarkPercent / 100;
globalChunkPools = (MemRawChunkPool**)calloc(g_memGlobalCfg.m_nodeCount, sizeof(MemRawChunkPool*));
if (globalChunkPools == NULL) {
@ -334,8 +334,8 @@ static int MemRawChunkStoreInitGlobal(MemReserveMode reserveMode, MemStorePolicy
reserveMode,
storePolicy,
chunkReserveCount,
highBlackMark,
highRedMark,
(uint32_t)highBlackMark,
(uint32_t)highRedMark,
MEM_ALLOC_GLOBAL);
}
@ -365,9 +365,9 @@ static int MemRawChunkStoreInitLocal(MemReserveMode reserveMode, MemStorePolicy
{
int result = 0;
MOT_LOG_TRACE("Initializing local chunk pools");
uint32_t chunkReserveCount = g_memGlobalCfg.m_minLocalMemoryMb / MEM_CHUNK_SIZE_MB / g_memGlobalCfg.m_nodeCount;
uint32_t highBlackMark = g_memGlobalCfg.m_maxLocalMemoryMb / MEM_CHUNK_SIZE_MB / g_memGlobalCfg.m_nodeCount;
uint32_t highRedMark = highBlackMark; // no emergency state for local pools
uint64_t chunkReserveCount = g_memGlobalCfg.m_minLocalMemoryMb / MEM_CHUNK_SIZE_MB / g_memGlobalCfg.m_nodeCount;
uint64_t highBlackMark = g_memGlobalCfg.m_maxLocalMemoryMb / MEM_CHUNK_SIZE_MB / g_memGlobalCfg.m_nodeCount;
uint64_t highRedMark = highBlackMark; // no emergency state for local pools
localChunkPools = (MemRawChunkPool**)calloc(g_memGlobalCfg.m_nodeCount, sizeof(MemRawChunkPool*));
if (localChunkPools == NULL) {
@ -377,8 +377,13 @@ static int MemRawChunkStoreInitLocal(MemReserveMode reserveMode, MemStorePolicy
(unsigned)(g_memGlobalCfg.m_nodeCount * sizeof(MemRawChunkPool*)));
result = MOT_ERROR_OOM;
} else {
result = InitChunkPools(
localChunkPools, reserveMode, storePolicy, chunkReserveCount, highBlackMark, highRedMark, MEM_ALLOC_LOCAL);
result = InitChunkPools(localChunkPools,
reserveMode,
storePolicy,
chunkReserveCount,
(uint32_t)highBlackMark,
(uint32_t)highRedMark,
MEM_ALLOC_LOCAL);
}
if (result != 0) {

2
src/gausskernel/storage/mot/core/src/memory/mm_session_api.cpp
View File

@ -61,7 +61,7 @@ extern int MemSessionApiInit()
{
int result = 0;
MOT_LOG_TRACE("Initializing session API");
memSessionHugeBufferMaxObjectSizeBytes = ((uint64_t)g_memGlobalCfg.m_sessionMaxHugeObjectSizeMb) * MEGA_BYTE;
memSessionHugeBufferMaxObjectSizeBytes = g_memGlobalCfg.m_sessionMaxHugeObjectSizeMb * MEGA_BYTE;
int rc = pthread_mutex_init(&g_sessionLock, nullptr);
if (rc != 0) {

13
src/gausskernel/storage/mot/core/src/mot.conf
View File

@ -81,6 +81,7 @@
# Specifies the segment size used during checkpoint.
# Checkpoint is performed in segments. When a segments is full, it is serialized into disk and a
# new segment is opened for the subsequent checkpoint data.
# Note: Percentage values cannot be set for this configuration item.
#
#checkpoint_segsize = 16 MB
@ -275,7 +276,7 @@
# under which memory is not released back to the kernel, but rather kept in the MOT engine for
# later reuse.
#
#min_mot_global_memory = 0 MB
#min_mot_global_memory = 0
# Configures the maximum memory limit for the local memory of the MOT engine.
# Specifying percentage value relates to the total defined by max_process_memory configured in
@ -297,7 +298,7 @@
# under which memory is not released back to the kernel, but rather kept in the MOT engine for
# later reuse.
#
#min_mot_local_memory = 0 MB
#min_mot_local_memory = 0
# Configures the maximum memory limit for a single session in MOT engine.
# Usually sessions in MOT engine are free to allocate local memory as much as needed, as long as
@ -311,14 +312,14 @@
# max_mot_local_memory.
# Note: Percentage values cannot be set for this configuration item.
#
#max_mot_session_memory = 0 MB
#max_mot_session_memory = 0
# Configures the minimum memory reservation for a single session in MOT engine.
# This value is used both for pre-allocation of memory during session creation, as well as for
# ensuring a minimum amount of memory is available for the session to perform its normal operation.
# Note: Percentage values cannot be set for this configuration item.
#
#min_mot_session_memory = 0 MB
#min_mot_session_memory = 0
# Configures the large buffer store for sessions.
# When a user session executes a query that requires a lot of memory (e.g. when using many rows),
@ -329,7 +330,7 @@
# are served directly from kernel.
# Note: Percentage values cannot be set for this configuration item.
#
#session_large_buffer_store_size = 0 MB
#session_large_buffer_store_size = 0
# Configures the maximum object size in the large allocation buffer store for sessions.
# Internally, the large buffer store is divided into objects of various sizes. This value is used
@ -339,7 +340,7 @@
# rectified to the possible maximum.
# Note: Percentage values cannot be set for this configuration item.
#
#session_large_buffer_store_max_object_size = 0 MB
#session_large_buffer_store_max_object_size = 0
# Configures the maximum size of a single huge memory allocation made by a session.
# Huge allocations are served directly from kernel, and therefore are not guaranteed to succeed.

301
src/gausskernel/storage/mot/core/src/system/mot_configuration.cpp
View File

@ -63,9 +63,9 @@ constexpr bool MOTConfiguration::DEFAULT_ENABLE_CHECKPOINT;
constexpr bool MOTConfiguration::DEFAULT_ENABLE_INCREMENTAL_CHECKPOINT;
constexpr const char* MOTConfiguration::DEFAULT_CHECKPOINT_DIR;
constexpr const char* MOTConfiguration::DEFAULT_CHECKPOINT_SEGSIZE;
constexpr uint32_t MOTConfiguration::DEFAULT_CHECKPOINT_SEGSIZE_BYTES;
constexpr uint32_t MOTConfiguration::MIN_CHECKPOINT_SEGSIZE_BYTES;
constexpr uint32_t MOTConfiguration::MAX_CHECKPOINT_SEGSIZE_BYTES;
constexpr uint64_t MOTConfiguration::DEFAULT_CHECKPOINT_SEGSIZE_BYTES;
constexpr uint64_t MOTConfiguration::MIN_CHECKPOINT_SEGSIZE_BYTES;
constexpr uint64_t MOTConfiguration::MAX_CHECKPOINT_SEGSIZE_BYTES;
constexpr uint32_t MOTConfiguration::DEFAULT_CHECKPOINT_WORKERS;
constexpr uint32_t MOTConfiguration::MIN_CHECKPOINT_WORKERS;
constexpr uint32_t MOTConfiguration::MAX_CHECKPOINT_WORKERS;
@ -82,11 +82,11 @@ constexpr uint16_t MOTConfiguration::DEFAULT_MAX_DATA_NODES;
// statistics configuration members
constexpr bool MOTConfiguration::DEFAULT_ENABLE_STATS;
constexpr const char* MOTConfiguration::DEFAULT_STATS_PRINT_PERIOD;
constexpr uint32_t MOTConfiguration::DEFAULT_STATS_PRINT_PERIOD_SECONDS;
constexpr uint32_t MOTConfiguration::MIN_STATS_PRINT_PERIOD_SECONDS;
constexpr uint32_t MOTConfiguration::MAX_STATS_PRINT_PERIOD_SECONDS;
constexpr uint64_t MOTConfiguration::DEFAULT_STATS_PRINT_PERIOD_SECONDS;
constexpr uint64_t MOTConfiguration::MIN_STATS_PRINT_PERIOD_SECONDS;
constexpr uint64_t MOTConfiguration::MAX_STATS_PRINT_PERIOD_SECONDS;
constexpr const char* MOTConfiguration::DEFAULT_FULL_STATS_PRINT_PERIOD;
constexpr uint32_t MOTConfiguration::DEFAULT_FULL_STATS_PRINT_PERIOD_SECONDS;
constexpr uint64_t MOTConfiguration::DEFAULT_FULL_STATS_PRINT_PERIOD_SECONDS;
constexpr bool MOTConfiguration::DEFAULT_ENABLE_DB_SESSION_STAT_PRINT;
constexpr bool MOTConfiguration::DEFAULT_ENABLE_NETWORK_STAT_PRINT;
constexpr bool MOTConfiguration::DEFAULT_ENABLE_LOG_STAT_PRINT;
@ -111,29 +111,29 @@ constexpr uint32_t MOTConfiguration::MAX_MAX_CONNECTIONS;
constexpr AffinityMode MOTConfiguration::DEFAULT_AFFINITY_MODE;
constexpr bool MOTConfiguration::DEFAULT_LAZY_LOAD_CHUNK_DIRECTORY;
constexpr const char* MOTConfiguration::DEFAULT_MAX_MOT_GLOBAL_MEMORY;
constexpr uint32_t MOTConfiguration::DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB;
constexpr uint32_t MOTConfiguration::MIN_MAX_MOT_GLOBAL_MEMORY_MB;
constexpr uint32_t MOTConfiguration::MAX_MAX_MOT_GLOBAL_MEMORY_MB;
constexpr uint64_t MOTConfiguration::DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB;
constexpr uint64_t MOTConfiguration::MIN_MAX_MOT_GLOBAL_MEMORY_MB;
constexpr uint64_t MOTConfiguration::MAX_MAX_MOT_GLOBAL_MEMORY_MB;
constexpr const char* MOTConfiguration::DEFAULT_MAX_MOT_LOCAL_MEMORY;
constexpr uint32_t MOTConfiguration::DEFAULT_MAX_MOT_LOCAL_MEMORY_MB;
constexpr uint32_t MOTConfiguration::MIN_MAX_MOT_LOCAL_MEMORY_MB;
constexpr uint32_t MOTConfiguration::MAX_MAX_MOT_LOCAL_MEMORY_MB;
constexpr uint64_t MOTConfiguration::DEFAULT_MAX_MOT_LOCAL_MEMORY_MB;
constexpr uint64_t MOTConfiguration::MIN_MAX_MOT_LOCAL_MEMORY_MB;
constexpr uint64_t MOTConfiguration::MAX_MAX_MOT_LOCAL_MEMORY_MB;
constexpr const char* MOTConfiguration::DEFAULT_MIN_MOT_GLOBAL_MEMORY;
constexpr uint32_t MOTConfiguration::DEFAULT_MIN_MOT_GLOBAL_MEMORY_MB;
constexpr uint32_t MOTConfiguration::MIN_MIN_MOT_GLOBAL_MEMORY_MB;
constexpr uint32_t MOTConfiguration::MAX_MIN_MOT_GLOBAL_MEMORY_MB;
constexpr uint64_t MOTConfiguration::DEFAULT_MIN_MOT_GLOBAL_MEMORY_MB;
constexpr uint64_t MOTConfiguration::MIN_MIN_MOT_GLOBAL_MEMORY_MB;
constexpr uint64_t MOTConfiguration::MAX_MIN_MOT_GLOBAL_MEMORY_MB;
constexpr const char* MOTConfiguration::DEFAULT_MIN_MOT_LOCAL_MEMORY;
constexpr uint32_t MOTConfiguration::DEFAULT_MIN_MOT_LOCAL_MEMORY_MB;
constexpr uint32_t MOTConfiguration::MIN_MIN_MOT_LOCAL_MEMORY_MB;
constexpr uint32_t MOTConfiguration::MAX_MIN_MOT_LOCAL_MEMORY_MB;
constexpr uint64_t MOTConfiguration::DEFAULT_MIN_MOT_LOCAL_MEMORY_MB;
constexpr uint64_t MOTConfiguration::MIN_MIN_MOT_LOCAL_MEMORY_MB;
constexpr uint64_t MOTConfiguration::MAX_MIN_MOT_LOCAL_MEMORY_MB;
constexpr const char* MOTConfiguration::DEFAULT_MAX_MOT_SESSION_MEMORY;
constexpr uint32_t MOTConfiguration::DEFAULT_MAX_MOT_SESSION_MEMORY_KB;
constexpr uint32_t MOTConfiguration::MIN_MAX_MOT_SESSION_MEMORY_KB;
constexpr uint32_t MOTConfiguration::MAX_MAX_MOT_SESSION_MEMORY_KB;
constexpr uint64_t MOTConfiguration::DEFAULT_MAX_MOT_SESSION_MEMORY_KB;
constexpr uint64_t MOTConfiguration::MIN_MAX_MOT_SESSION_MEMORY_KB;
constexpr uint64_t MOTConfiguration::MAX_MAX_MOT_SESSION_MEMORY_KB;
constexpr const char* MOTConfiguration::DEFAULT_MIN_MOT_SESSION_MEMORY;
constexpr uint32_t MOTConfiguration::DEFAULT_MIN_MOT_SESSION_MEMORY_KB;
constexpr uint32_t MOTConfiguration::MIN_MIN_MOT_SESSION_MEMORY_KB;
constexpr uint32_t MOTConfiguration::MAX_MIN_MOT_SESSION_MEMORY_KB;
constexpr uint64_t MOTConfiguration::DEFAULT_MIN_MOT_SESSION_MEMORY_KB;
constexpr uint64_t MOTConfiguration::MIN_MIN_MOT_SESSION_MEMORY_KB;
constexpr uint64_t MOTConfiguration::MAX_MIN_MOT_SESSION_MEMORY_KB;
constexpr MemReserveMode MOTConfiguration::DEFAULT_RESERVE_MEMORY_MODE;
constexpr MemStorePolicy MOTConfiguration::DEFAULT_STORE_MEMORY_POLICY;
constexpr MemAllocPolicy MOTConfiguration::DEFAULT_CHUNK_ALLOC_POLICY;
@ -144,30 +144,30 @@ constexpr uint32_t MOTConfiguration::DEFAULT_HIGH_RED_MARK_PERCENT;
constexpr uint32_t MOTConfiguration::MIN_HIGH_RED_MARK_PERCENT;
constexpr uint32_t MOTConfiguration::MAX_HIGH_RED_MARK_PERCENT;
constexpr const char* MOTConfiguration::DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE;
constexpr uint32_t MOTConfiguration::DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE_MB;
constexpr uint32_t MOTConfiguration::MIN_SESSION_LARGE_BUFFER_STORE_SIZE_MB;
constexpr uint32_t MOTConfiguration::MAX_SESSION_LARGE_BUFFER_STORE_SIZE_MB;
constexpr uint64_t MOTConfiguration::DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE_MB;
constexpr uint64_t MOTConfiguration::MIN_SESSION_LARGE_BUFFER_STORE_SIZE_MB;
constexpr uint64_t MOTConfiguration::MAX_SESSION_LARGE_BUFFER_STORE_SIZE_MB;
constexpr const char* MOTConfiguration::DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE;
constexpr uint32_t MOTConfiguration::DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB;
constexpr uint32_t MOTConfiguration::MIN_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB;
constexpr uint32_t MOTConfiguration::MAX_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB;
constexpr uint64_t MOTConfiguration::DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB;
constexpr uint64_t MOTConfiguration::MIN_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB;
constexpr uint64_t MOTConfiguration::MAX_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB;
constexpr const char* MOTConfiguration::DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE;
constexpr uint32_t MOTConfiguration::DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE_MB;
constexpr uint32_t MOTConfiguration::MIN_SESSION_MAX_HUGE_OBJECT_SIZE_MB;
constexpr uint32_t MOTConfiguration::MAX_SESSION_MAX_HUGE_OBJECT_SIZE_MB;
constexpr uint64_t MOTConfiguration::DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE_MB;
constexpr uint64_t MOTConfiguration::MIN_SESSION_MAX_HUGE_OBJECT_SIZE_MB;
constexpr uint64_t MOTConfiguration::MAX_SESSION_MAX_HUGE_OBJECT_SIZE_MB;
// GC configuration members
constexpr bool MOTConfiguration::DEFAULT_GC_ENABLE;
constexpr const char* MOTConfiguration::DEFAULT_GC_RECLAIM_THRESHOLD;
constexpr uint32_t MOTConfiguration::DEFAULT_GC_RECLAIM_THRESHOLD_BYTES;
constexpr uint32_t MOTConfiguration::MIN_GC_RECLAIM_THRESHOLD_BYTES;
constexpr uint32_t MOTConfiguration::MAX_GC_RECLAIM_THRESHOLD_BYTES;
constexpr uint64_t MOTConfiguration::DEFAULT_GC_RECLAIM_THRESHOLD_BYTES;
constexpr uint64_t MOTConfiguration::MIN_GC_RECLAIM_THRESHOLD_BYTES;
constexpr uint64_t MOTConfiguration::MAX_GC_RECLAIM_THRESHOLD_BYTES;
constexpr uint32_t MOTConfiguration::DEFAULT_GC_RECLAIM_BATCH_SIZE;
constexpr uint32_t MOTConfiguration::MIN_GC_RECLAIM_BATCH_SIZE;
constexpr uint32_t MOTConfiguration::MAX_GC_RECLAIM_BATCH_SIZE;
constexpr const char* MOTConfiguration::DEFAULT_GC_HIGH_RECLAIM_THRESHOLD;
constexpr uint32_t MOTConfiguration::DEFAULT_GC_HIGH_RECLAIM_THRESHOLD_BYTES;
constexpr uint32_t MOTConfiguration::MIN_GC_HIGH_RECLAIM_THRESHOLD_BYTES;
constexpr uint32_t MOTConfiguration::MAX_GC_HIGH_RECLAIM_THRESHOLD_BYTES;
constexpr uint64_t MOTConfiguration::DEFAULT_GC_HIGH_RECLAIM_THRESHOLD_BYTES;
constexpr uint64_t MOTConfiguration::MIN_GC_HIGH_RECLAIM_THRESHOLD_BYTES;
constexpr uint64_t MOTConfiguration::MAX_GC_HIGH_RECLAIM_THRESHOLD_BYTES;
// JIT configuration members
constexpr bool MOTConfiguration::DEFAULT_ENABLE_MOT_CODEGEN;
constexpr bool MOTConfiguration::DEFAULT_FORCE_MOT_PSEUDO_CODEGEN;
@ -184,7 +184,7 @@ constexpr uint64_t MOTConfiguration::DEFAULT_CFG_MONITOR_PERIOD_SECONDS;
constexpr uint64_t MOTConfiguration::MIN_CFG_MONITOR_PERIOD_SECONDS;
constexpr uint64_t MOTConfiguration::MAX_CFG_MONITOR_PERIOD_SECONDS;
constexpr bool MOTConfiguration::DEFAULT_RUN_INTERNAL_CONSISTENCY_VALIDATION;
constexpr uint32_t MOTConfiguration::DEFAULT_TOTAL_MEMORY_MB;
constexpr uint64_t MOTConfiguration::DEFAULT_TOTAL_MEMORY_MB;
static constexpr unsigned int MAX_NUMA_NODES = 16u;
#define IS_HYPER_THREAD_CMD "lscpu | grep \"Thread(s) per core:\" | awk '{print $4}'"
@ -552,15 +552,15 @@ bool MOTConfiguration::SetFlag(const std::string& name, const std::string& value
} else if (ParseBool(name, "enable_checkpoint", value, &m_enableCheckpoint)) {
} else if (ParseBool(name, "enable_incremental_checkpoint", value, &m_enableIncrementalCheckpoint)) {
} else if (ParseString(name, "checkpoint_dir", value, &m_checkpointDir)) {
} else if (ParseUint32(name, "checkpoint_segsize", value, &m_checkpointSegThreshold)) {
} else if (ParseUint64(name, "checkpoint_segsize", value, &m_checkpointSegThreshold)) {
} else if (ParseUint32(name, "checkpoint_workers", value, &m_checkpointWorkers)) {
} else if (ParseUint32(name, "checkpoint_recovery_workers", value, &m_checkpointRecoveryWorkers)) {
} else if (ParseBool(name, "abort_buffer_enable", value, &m_abortBufferEnable)) {
} else if (ParseBool(name, "pre_abort", value, &m_preAbort)) {
} else if (ParseValidation(name, "validation_lock", value, &m_validationLock)) {
} else if (ParseBool(name, "enable_stats", value, &m_enableStats)) {
} else if (ParseUint32(name, "stats_period_seconds", value, &m_statPrintPeriodSeconds)) {
} else if (ParseUint32(name, "full_stats_period_seconds", value, &m_statPrintFullPeriodSeconds)) {
} else if (ParseUint64(name, "stats_period_seconds", value, &m_statPrintPeriodSeconds)) {
} else if (ParseUint64(name, "full_stats_period_seconds", value, &m_statPrintFullPeriodSeconds)) {
} else if (ParseBool(name, "enable_log_recovery_stats", value, &m_enableLogRecoveryStats)) {
} else if (ParseBool(name, "enable_db_session_stats", value, &m_enableDbSessionStatistics)) {
} else if (ParseBool(name, "enable_network_stats", value, &m_enableNetworkStatistics)) {
@ -579,23 +579,23 @@ bool MOTConfiguration::SetFlag(const std::string& name, const std::string& value
} else if (ParseUint32(name, "max_connections", value, &m_maxConnections)) {
} else if (ParseAffinity(name, "affinity_mode", value, &m_sessionAffinityMode)) {
} else if (ParseBool(name, "lazy_load_chunk_directory", value, &m_lazyLoadChunkDirectory)) {
} else if (ParseUint32(name, "max_mot_global_memory_mb", value, &m_globalMemoryMaxLimitMB)) {
} else if (ParseUint32(name, "min_mot_global_memory_mb", value, &m_globalMemoryMinLimitMB)) {
} else if (ParseUint32(name, "max_mot_local_memory_mb", value, &m_localMemoryMaxLimitMB)) {
} else if (ParseUint32(name, "min_mot_local_memory_mb", value, &m_localMemoryMinLimitMB)) {
} else if (ParseUint32(name, "max_mot_session_memory_kb", value, &m_sessionMemoryMaxLimitKB)) {
} else if (ParseUint32(name, "min_mot_session_memory_kb", value, &m_sessionMemoryMinLimitKB)) {
} else if (ParseUint64(name, "max_mot_global_memory_mb", value, &m_globalMemoryMaxLimitMB)) {
} else if (ParseUint64(name, "min_mot_global_memory_mb", value, &m_globalMemoryMinLimitMB)) {
} else if (ParseUint64(name, "max_mot_local_memory_mb", value, &m_localMemoryMaxLimitMB)) {
} else if (ParseUint64(name, "min_mot_local_memory_mb", value, &m_localMemoryMinLimitMB)) {
} else if (ParseUint64(name, "max_mot_session_memory_kb", value, &m_sessionMemoryMaxLimitKB)) {
} else if (ParseUint64(name, "min_mot_session_memory_kb", value, &m_sessionMemoryMinLimitKB)) {
} else if (ParseMemoryReserveMode(name, "reserve_memory_mode", value, &m_reserveMemoryMode)) {
} else if (ParseMemoryStorePolicy(name, "store_memory_policy", value, &m_storeMemoryPolicy)) {
} else if (ParseChunkAllocPolicy(name, "chunk_alloc_policy", value, &m_chunkAllocPolicy)) {
} else if (ParseUint32(name, "chunk_prealloc_worker_count", value, &m_chunkPreallocWorkerCount)) {
} else if (ParseUint32(name, "high_red_mark_percent", value, &m_highRedMarkPercent)) {
} else if (ParseUint32(name, "session_large_buffer_store_size_mb", value, &m_sessionLargeBufferStoreSizeMB)) {
} else if (ParseUint32(name,
} else if (ParseUint64(name, "session_large_buffer_store_size_mb", value, &m_sessionLargeBufferStoreSizeMB)) {
} else if (ParseUint64(name,
"session_large_buffer_store_max_object_size_mb",
value,
&m_sessionLargeBufferStoreMaxObjectSizeMB)) {
} else if (ParseUint32(name, "session_max_huge_object_size_mb", value, &m_sessionMaxHugeObjectSizeMB)) {
} else if (ParseUint64(name, "session_max_huge_object_size_mb", value, &m_sessionMaxHugeObjectSizeMB)) {
} else if (ParseBool(name, "enable_mot_codegen", value, &m_enableCodegen)) {
} else if (ParseBool(name, "force_mot_pseudo_codegen", value, &m_forcePseudoCodegen)) {
} else if (ParseBool(name, "enable_mot_codegen_print", value, &m_enableCodegenPrint)) {
@ -675,26 +675,14 @@ int MOTConfiguration::GetMappedCore(int logicId) const
#define UPDATE_INT_CFG(var, cfgPath, defaultValue, lowerBound, upperBound) \
UpdateIntConfigItem(var, cfg->GetIntegerConfigValue(cfgPath, defaultValue), cfgPath, lowerBound, upperBound)
#define UPDATE_MEM_CFG(var, cfgPath, defaultValue, scale, lowerBound, upperBound) \
do { \
uint64_t memoryValueBytes = \
ParseMemoryValueBytes(cfg->GetStringConfigValue(cfgPath, defaultValue), (uint64_t)-1, cfgPath); \
UpdateIntConfigItem(var, (uint32_t)(memoryValueBytes / scale), cfgPath, lowerBound, upperBound); \
} while (0);
#define UPDATE_MEM_CFG(var, cfgPath, defaultValue, scale, lowerBound, upperBound) \
UpdateMemConfigItem(var, cfgPath, defaultValue, scale, lowerBound, upperBound, true)
#define UPDATE_ABS_MEM_CFG(var, cfgPath, defaultValue, scale, lowerBound, upperBound) \
do { \
uint64_t memoryValueBytes = \
ParseMemoryUnit(cfg->GetStringConfigValue(cfgPath, defaultValue), (uint64_t)-1, cfgPath); \
UpdateIntConfigItem(var, (uint32_t)(memoryValueBytes / scale), cfgPath, lowerBound, upperBound); \
} while (0);
#define UPDATE_ABS_MEM_CFG(var, cfgPath, defaultValue, scale, lowerBound, upperBound) \
UpdateMemConfigItem(var, cfgPath, defaultValue, scale, lowerBound, upperBound, false)
#define UPDATE_TIME_CFG(var, cfgPath, defaultValue, scale, lowerBound, upperBound) \
do { \
uint64_t timeValueMicros = \
ParseTimeValueMicros(cfg->GetStringConfigValue(cfgPath, defaultValue), (uint64_t)-1, cfgPath); \
UpdateIntConfigItem(var, (uint64_t)(timeValueMicros / scale), cfgPath, lowerBound, upperBound); \
} while (0);
#define UPDATE_TIME_CFG(var, cfgPath, defaultValue, scale, lowerBound, upperBound) \
UpdateTimeConfigItem(var, cfgPath, defaultValue, scale, lowerBound, upperBound)
void MOTConfiguration::LoadConfig()
{
@ -728,7 +716,7 @@ void MOTConfiguration::LoadConfig()
// Checkpoint configuration
UPDATE_BOOL_CFG(m_enableCheckpoint, "enable_checkpoint", DEFAULT_ENABLE_CHECKPOINT);
UPDATE_STRING_CFG(m_checkpointDir, "checkpoint_dir", DEFAULT_CHECKPOINT_DIR);
UPDATE_MEM_CFG(m_checkpointSegThreshold,
UPDATE_ABS_MEM_CFG(m_checkpointSegThreshold,
"checkpoint_segsize",
DEFAULT_CHECKPOINT_SEGSIZE,
SCALE_BYTES,
@ -810,15 +798,24 @@ void MOTConfiguration::LoadConfig()
// validate that min <= max
if (m_globalMemoryMinLimitMB > m_globalMemoryMaxLimitMB) {
if (!m_suppressLog) {
MOT_LOG_WARN("Invalid global memory configuration: minimum (%u MB) is greater than maximum (%u MB), using "
"defaults (%u MB, %u MB)",
MOT_LOG_WARN("Invalid global memory configuration: minimum (%" PRIu64
" MB) is greater than maximum (%" PRIu64 " MB), using defaults (%" PRIu64 " MB, %" PRIu64
" MB)",
m_globalMemoryMinLimitMB,
m_globalMemoryMaxLimitMB,
DEFAULT_MIN_MOT_GLOBAL_MEMORY_MB,
DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB);
}
UpdateIntConfigItem(m_globalMemoryMaxLimitMB, DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB, "max_mot_global_memory");
UpdateIntConfigItem(m_globalMemoryMinLimitMB, DEFAULT_MIN_MOT_GLOBAL_MEMORY_MB, "min_mot_global_memory");
UpdateIntConfigItem(m_globalMemoryMaxLimitMB,
DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB,
"max_mot_global_memory",
MIN_MAX_MOT_GLOBAL_MEMORY_MB,
MAX_MAX_MOT_GLOBAL_MEMORY_MB);
UpdateIntConfigItem(m_globalMemoryMinLimitMB,
DEFAULT_MIN_MOT_GLOBAL_MEMORY_MB,
"min_mot_global_memory",
MIN_MIN_MOT_GLOBAL_MEMORY_MB,
MAX_MIN_MOT_GLOBAL_MEMORY_MB);
}
UPDATE_MEM_CFG(m_localMemoryMaxLimitMB,
"max_mot_local_memory",
@ -835,15 +832,24 @@ void MOTConfiguration::LoadConfig()
// validate that min <= max
if (m_localMemoryMinLimitMB > m_localMemoryMaxLimitMB) {
if (!m_suppressLog) {
MOT_LOG_WARN("Invalid local memory configuration: minimum (%u MB) is greater than maximum (%u MB), using "
"defaults (%u MB, %u MB)",
MOT_LOG_WARN("Invalid local memory configuration: minimum (%" PRIu64
" MB) is greater than maximum (%" PRIu64 " MB), using defaults (%" PRIu64 " MB, %" PRIu64
" MB)",
m_localMemoryMinLimitMB,
m_localMemoryMaxLimitMB,
DEFAULT_MIN_MOT_LOCAL_MEMORY_MB,
DEFAULT_MAX_MOT_LOCAL_MEMORY_MB);
}
UpdateIntConfigItem(m_localMemoryMaxLimitMB, DEFAULT_MAX_MOT_LOCAL_MEMORY_MB, "max_mot_local_memory");
UpdateIntConfigItem(m_localMemoryMinLimitMB, DEFAULT_MIN_MOT_LOCAL_MEMORY_MB, "min_mot_local_memory");
UpdateIntConfigItem(m_localMemoryMaxLimitMB,
DEFAULT_MAX_MOT_LOCAL_MEMORY_MB,
"max_mot_local_memory",
MIN_MAX_MOT_LOCAL_MEMORY_MB,
MAX_MAX_MOT_LOCAL_MEMORY_MB);
UpdateIntConfigItem(m_localMemoryMinLimitMB,
DEFAULT_MIN_MOT_LOCAL_MEMORY_MB,
"min_mot_local_memory",
MIN_MIN_MOT_LOCAL_MEMORY_MB,
MAX_MIN_MOT_LOCAL_MEMORY_MB);
}
UPDATE_ABS_MEM_CFG(m_sessionMemoryMaxLimitKB,
"max_mot_session_memory",
@ -860,15 +866,24 @@ void MOTConfiguration::LoadConfig()
// validate that min <= max
if ((m_sessionMemoryMaxLimitKB > 0) && (m_sessionMemoryMinLimitKB > m_sessionMemoryMaxLimitKB)) {
if (!m_suppressLog) {
MOT_LOG_WARN("Invalid session memory configuration: minimum (%u KB) is greater than maximum (%u KB), using "
"defaults (%u KB, %u KB)",
MOT_LOG_WARN("Invalid session memory configuration: minimum (%" PRIu64
" KB) is greater than maximum (%" PRIu64 " KB), using defaults (%" PRIu64 " KB, %" PRIu64
" KB)",
m_sessionMemoryMinLimitKB,
m_sessionMemoryMaxLimitKB,
DEFAULT_MIN_MOT_SESSION_MEMORY_KB,
DEFAULT_MAX_MOT_SESSION_MEMORY_KB);
}
UpdateIntConfigItem(m_sessionMemoryMaxLimitKB, DEFAULT_MAX_MOT_SESSION_MEMORY_KB, "max_mot_session_memory");
UpdateIntConfigItem(m_sessionMemoryMinLimitKB, DEFAULT_MIN_MOT_SESSION_MEMORY_KB, "min_mot_session_memory");
UpdateIntConfigItem(m_sessionMemoryMaxLimitKB,
DEFAULT_MAX_MOT_SESSION_MEMORY_KB,
"max_mot_session_memory",
MIN_MAX_MOT_SESSION_MEMORY_KB,
MAX_MAX_MOT_SESSION_MEMORY_KB);
UpdateIntConfigItem(m_sessionMemoryMinLimitKB,
DEFAULT_MIN_MOT_SESSION_MEMORY_KB,
"min_mot_session_memory",
MIN_MIN_MOT_SESSION_MEMORY_KB,
MAX_MIN_MOT_SESSION_MEMORY_KB);
}
UPDATE_USER_CFG(m_reserveMemoryMode, "reserve_memory_mode", DEFAULT_RESERVE_MEMORY_MODE);
UPDATE_USER_CFG(m_storeMemoryPolicy, "store_memory_policy", DEFAULT_STORE_MEMORY_POLICY);
@ -951,6 +966,101 @@ void MOTConfiguration::LoadConfig()
MOT_LOG_TRACE("Main configuration loaded");
}
void MOTConfiguration::UpdateMemConfigItem(uint64_t& oldValue, const char* name, const char* defaultStrValue,
uint64_t scale, uint64_t lowerBound, uint64_t upperBound, bool allowPercentage)
{
// we prepare first a default value from the string default value
uint64_t defaultValueBytes = allowPercentage ? ParseMemoryValueBytes(defaultStrValue, (uint64_t)-1, name)
: ParseMemoryUnit(defaultStrValue, (uint64_t)-1, name);
MOT_LOG_TRACE(
"Converted memory default string value %s to byte value: %" PRIu64, defaultStrValue, defaultValueBytes);
// now we carefully examine the configuration item type
const LayeredConfigTree* cfg = ConfigManager::GetInstance().GetLayeredConfigTree();
const ConfigItem* cfgItem = cfg->GetConfigItem(name);
if (cfgItem == nullptr) {
// just keep default
MOT_LOG_TRACE("Configuration item %s not found, keeping default %" PRIu64 ".", name, defaultValueBytes);
} else if (cfgItem->GetClass() != ConfigItemClass::CONFIG_ITEM_VALUE) {
MOT_LOG_ERROR("Invalid configuration for %s: expected value item, got %s item. Keeping default value %" PRIu64
".",
name,
ConfigItemClassToString(cfgItem->GetClass()),
defaultValueBytes);
UpdateIntConfigItem(oldValue, defaultValueBytes / scale, name, lowerBound, upperBound);
} else {
// now we carefully examine the value type
ConfigValue* cfgValue = (ConfigValue*)cfgItem;
if (cfgValue->IsIntegral()) {
// only a number was specified, so it is interpreted as bytes
uint64_t memoryValueBytes = cfg->GetIntegerConfigValue<uint64_t>(name, defaultValueBytes);
UpdateIntConfigItem(oldValue, memoryValueBytes / scale, name, lowerBound, upperBound);
MOT_LOG_TRACE("Loading integral memory value %s as bytes: %" PRIu64, name, memoryValueBytes);
} else if (cfgValue->GetConfigValueType() == ConfigValueType::CONFIG_VALUE_STRING) {
// value was parsed as string, meaning we have units or percentage specifier
const char* strValue = cfg->GetStringConfigValue(name, defaultStrValue);
if ((strValue != nullptr) && (strValue[0] != 0)) {
uint64_t memoryValueBytes = allowPercentage ? ParseMemoryValueBytes(strValue, (uint64_t)-1, name)
: ParseMemoryUnit(strValue, (uint64_t)-1, name);
UpdateIntConfigItem(oldValue, memoryValueBytes / scale, name, lowerBound, upperBound);
}
} else {
// unexpected value type
MOT_LOG_WARN("Unexpected configuration item %s value type: %s. Keeping default value: %" PRIu64 ".",
name,
ConfigValueTypeToString(cfgValue->GetConfigValueType()),
defaultValueBytes);
UpdateIntConfigItem(oldValue, defaultValueBytes / scale, name, lowerBound, upperBound);
}
}
}
void MOTConfiguration::UpdateTimeConfigItem(uint64_t& oldValue, const char* name, const char* defaultStrValue,
uint64_t scale, uint64_t lowerBound, uint64_t upperBound)
{
// we prepare first a default value from the string default value
uint64_t defaultValueUSecs = ParseTimeValueMicros(defaultStrValue, (uint64_t)-1, name);
MOT_LOG_TRACE("Converted time default string value %s to usec value: %" PRIu64, defaultStrValue, defaultValueUSecs);
// now we carefully examine the configuration item type
const LayeredConfigTree* cfg = ConfigManager::GetInstance().GetLayeredConfigTree();
const ConfigItem* cfgItem = cfg->GetConfigItem(name);
if (cfgItem == nullptr) {
// just keep default
MOT_LOG_TRACE("Configuration item %s not found, keeping default %" PRIu64 ".", name, defaultValueUSecs);
} else if (cfgItem->GetClass() != ConfigItemClass::CONFIG_ITEM_VALUE) {
MOT_LOG_ERROR("Invalid configuration for %s: expected value item, got %s item. Keeping default value %" PRIu64
".",
name,
ConfigItemClassToString(cfgItem->GetClass()),
defaultValueUSecs);
UpdateIntConfigItem(oldValue, defaultValueUSecs / scale, name, lowerBound, upperBound);
} else {
// now we carefully examine the value type
ConfigValue* cfgValue = (ConfigValue*)cfgItem;
if (cfgValue->IsIntegral()) {
// only a number was specified, so it is interpreted as micro-seconds
uint64_t timeValueUSecs = cfg->GetIntegerConfigValue<uint64_t>(name, defaultValueUSecs);
UpdateIntConfigItem(oldValue, timeValueUSecs / scale, name, lowerBound, upperBound);
MOT_LOG_TRACE("Loading integral time value %s as micro-seconds: %" PRIu64, name, timeValueUSecs);
} else if (cfgValue->GetConfigValueType() == ConfigValueType::CONFIG_VALUE_STRING) {
// value was parsed as string, meaning we have units
const char* strValue = cfg->GetStringConfigValue(name, defaultStrValue);
if ((strValue != nullptr) && (strValue[0] != 0)) {
uint64_t timeValueUSecs = ParseTimeValueMicros(strValue, (uint64_t)-1, name);
UpdateIntConfigItem(oldValue, timeValueUSecs / scale, name, lowerBound, upperBound);
}
} else {
// unexpected value type
MOT_LOG_WARN("Unexpected configuration item %s value type: %s. Keeping default value: %" PRIu64 ".",
name,
ConfigValueTypeToString(cfgValue->GetConfigValueType()),
defaultValueUSecs);
UpdateIntConfigItem(oldValue, defaultValueUSecs / scale, name, lowerBound, upperBound);
}
}
}
void MOTConfiguration::UpdateComponentLogLevel()
{
LogLevel globalLogLevel = GetGlobalLogLevel();
@ -1083,7 +1193,7 @@ uint64_t MOTConfiguration::ParseMemoryPercentTotal(const char* memoryValue, uint
// we expect to parse a number between 0-100, and then followed by "%"
int percent = ParseMemoryPercent(memoryValue);
if (percent >= 0) {
memoryValueBytes = ((uint64_t)m_totalMemoryMb) * MEGA_BYTE * percent / 100;
memoryValueBytes = m_totalMemoryMb * MEGA_BYTE * percent / 100;
if (!m_suppressLog) {
MOT_LOG_INFO(
"Loaded %s: %d%% from total = %" PRIu64 " MB", cfgPath, percent, memoryValueBytes / 1024ul / 1024ul);
@ -1102,7 +1212,8 @@ uint64_t MOTConfiguration::ParseMemoryUnit(const char* memoryValue, uint64_t def
if (endptr == memoryValue) {
MOT_LOG_WARN("Invalid %s memory value format: %s (expecting value digits)", cfgPath, memoryValue);
} else if (*endptr == 0) {
MOT_LOG_WARN("Invalid %s memory value format: %s (expecting unit type after value)", cfgPath, memoryValue);
MOT_LOG_TRACE("Missing %s memory value units: bytes assumed", cfgPath, memoryValue);
memoryValueBytes = value;
} else {
// get unit type and convert to mega-bytes
mot_string suffix(endptr);
@ -1146,26 +1257,32 @@ uint64_t MOTConfiguration::ParseTimeValueMicros(const char* timeValue, uint64_t
// get unit type and convert to mega-bytes
mot_string suffix(endptr);
suffix.trim();
if ((suffix.compare_no_case("d") == 0) || (suffix.compare_no_case("days") == 0)) {
if ((suffix.compare_no_case("d") == 0) || (suffix.compare_no_case("days") == 0) ||
(suffix.compare_no_case("day") == 0)) {
MOT_LOG_TRACE("Loaded %s: %u days", cfgPath, value);
timeValueMicors = ((uint64_t)value) * 24ull * 60ull * 60ull * 1000ull * 1000ull;
} else if ((suffix.compare_no_case("h") == 0) || (suffix.compare_no_case("hours") == 0)) {
} else if ((suffix.compare_no_case("h") == 0) || (suffix.compare_no_case("hours") == 0) ||
(suffix.compare_no_case("hour") == 0)) {
MOT_LOG_TRACE("Loaded %s: %u hours", cfgPath, value);
timeValueMicors = ((uint64_t)value) * 60ull * 60ull * 1000ull * 1000ull;
} else if ((suffix.compare_no_case("m") == 0) || (suffix.compare_no_case("mins") == 0) ||
(suffix.compare_no_case("minutes") == 0)) {
(suffix.compare_no_case("minutes") == 0) || (suffix.compare_no_case("min") == 0) ||
(suffix.compare_no_case("minute") == 0)) {
MOT_LOG_TRACE("Loaded %s: %u minutes", cfgPath, value);
timeValueMicors = ((uint64_t)value) * 60ull * 1000ull * 1000ull;
} else if ((suffix.compare_no_case("s") == 0) || (suffix.compare_no_case("secs") == 0) ||
(suffix.compare_no_case("seconds") == 0)) {
(suffix.compare_no_case("seconds") == 0) || (suffix.compare_no_case("sec") == 0) ||
(suffix.compare_no_case("second") == 0)) {
MOT_LOG_TRACE("Loaded %s: %u seconds", cfgPath, value);
timeValueMicors = ((uint64_t)value) * 1000ull * 1000ull;
} else if ((suffix.compare_no_case("ms") == 0) || (suffix.compare_no_case("millis") == 0) ||
(suffix.compare_no_case("milliseocnds") == 0)) {
(suffix.compare_no_case("milliseconds") == 0) || (suffix.compare_no_case("milli") == 0) ||
(suffix.compare_no_case("millisecond") == 0)) {
MOT_LOG_TRACE("Loaded %s: %u milli-seconds", cfgPath, value);
timeValueMicors = ((uint64_t)value) * 1000ull;
} else if ((suffix.compare_no_case("us") == 0) || (suffix.compare_no_case("micros") == 0) ||
(suffix.compare_no_case("microseocnds") == 0)) {
(suffix.compare_no_case("microseconds") == 0) || (suffix.compare_no_case("micro") == 0) ||
(suffix.compare_no_case("microsecond") == 0)) {
MOT_LOG_TRACE("Loaded %s: %u micro-seconds", cfgPath, value);
timeValueMicors = ((uint64_t)value);
} else {

143
src/gausskernel/storage/mot/core/src/system/mot_configuration.h
View File

@ -107,9 +107,9 @@ public:
}
/** @brief Configures the total memory used as reference for computing memory percentage values. */
inline void SetTotalMemoryMb(uint32_t totalMemoryMb)
inline void SetTotalMemoryMb(uint64_t totalMemoryMb)
{
MOT_LOG_INFO("Configuring total memory for relative memory values to: %u MB", totalMemoryMb);
MOT_LOG_INFO("Configuring total memory for relative memory values to: %" PRIu64 " MB", totalMemoryMb);
m_totalMemoryMb = totalMemoryMb;
}
@ -167,7 +167,7 @@ public:
std::string m_checkpointDir;
/** @var checkpoint segments size in bytes. */
uint32_t m_checkpointSegThreshold;
uint64_t m_checkpointSegThreshold;
/** @var number of worker threads to spawn to perform checkpoint. */
uint32_t m_checkpointWorkers;
@ -209,10 +209,10 @@ public:
bool m_enableStats;
/** Statistics printing period seconds. */
uint32_t m_statPrintPeriodSeconds;
uint64_t m_statPrintPeriodSeconds;
/** Full statistics printing period in seconds. */
uint32_t m_statPrintFullPeriodSeconds;
uint64_t m_statPrintFullPeriodSeconds;
/** @var Enable statistics gathering and printing during redo log recovery. */
bool m_enableLogRecoveryStats;
@ -278,22 +278,22 @@ public:
bool m_lazyLoadChunkDirectory;
/** @var Maximum global memory limit in megabytes. */
uint32_t m_globalMemoryMaxLimitMB;
uint64_t m_globalMemoryMaxLimitMB;
/** @var Minimum global memory limit in megabytes (implies pre-allocation and minimum reservation). */
uint32_t m_globalMemoryMinLimitMB;
uint64_t m_globalMemoryMinLimitMB;
/** @var Maximum local (per-node) memory limit in megabytes. */
uint32_t m_localMemoryMaxLimitMB;
uint64_t m_localMemoryMaxLimitMB;
/** @var Minimum local (per-node) memory limit in megabytes (implies pre-allocation and minimum reservation). */
uint32_t m_localMemoryMinLimitMB;
uint64_t m_localMemoryMinLimitMB;
/** @var Maximum for single MOT session small memory allocations. */
uint32_t m_sessionMemoryMaxLimitKB;
uint64_t m_sessionMemoryMaxLimitKB;
/** @var Minimum (pre-allocated) for single MOT session small memory allocations. */
uint32_t m_sessionMemoryMinLimitKB;
uint64_t m_sessionMemoryMinLimitKB;
/* @var Specifies whether to reserve physical memory on startup (or just virtual). */
MemReserveMode m_reserveMemoryMode;
@ -311,13 +311,13 @@ public:
uint32_t m_highRedMarkPercent;
/** @var The size in megabytes of the session large buffer store. */
uint32_t m_sessionLargeBufferStoreSizeMB;
uint64_t m_sessionLargeBufferStoreSizeMB;
/** @var The largest object size in megabytes in the session large buffer store. */
uint32_t m_sessionLargeBufferStoreMaxObjectSizeMB;
uint64_t m_sessionLargeBufferStoreMaxObjectSizeMB;
/** @var The largest single huge object size that can be allocated by any session directly from kernel. */
uint32_t m_sessionMaxHugeObjectSizeMB;
uint64_t m_sessionMaxHugeObjectSizeMB;
/**********************************************************************/
// Garbage Collection configuration
@ -326,13 +326,13 @@ public:
bool m_gcEnable;
/** @var The threshold in bytes for reclamation to be triggered (per-thread). */
uint32_t m_gcReclaimThresholdBytes;
uint64_t m_gcReclaimThresholdBytes;
/** @var The amount of objects reclaimed in each cleanup round of a limbo group. */
uint32_t m_gcReclaimBatchSize;
/** @var The high threshold in bytes for reclamation to be triggered (per-thread). */
uint32_t m_gcHighReclaimThresholdBytes;
uint64_t m_gcHighReclaimThresholdBytes;
/**********************************************************************/
// JIT configuration
@ -411,7 +411,7 @@ private:
bool m_isSystemHyperThreaded = false;
/** @var The total memory value to be used when loading memory percent values. By default uses system total. */
uint32_t m_totalMemoryMb;
uint64_t m_totalMemoryMb;
/** @var Controls suppressing of log messages during configuration loading. */
bool m_suppressLog;
@ -467,9 +467,9 @@ private:
/** @var Default checkpoint segments size */
static constexpr const char* DEFAULT_CHECKPOINT_SEGSIZE = "16 MB";
static constexpr uint32_t DEFAULT_CHECKPOINT_SEGSIZE_BYTES = 16 * MEGA_BYTE;
static constexpr uint32_t MIN_CHECKPOINT_SEGSIZE_BYTES = 16 * MEGA_BYTE;
static constexpr uint32_t MAX_CHECKPOINT_SEGSIZE_BYTES = 512 * MEGA_BYTE;
static constexpr uint64_t DEFAULT_CHECKPOINT_SEGSIZE_BYTES = 16 * MEGA_BYTE;
static constexpr uint64_t MIN_CHECKPOINT_SEGSIZE_BYTES = 16 * MEGA_BYTE;
static constexpr uint64_t MAX_CHECKPOINT_SEGSIZE_BYTES = 512 * MEGA_BYTE;
/** @var Default number of worker threads to spawn */
static constexpr uint32_t DEFAULT_CHECKPOINT_WORKERS = 3;
@ -504,13 +504,13 @@ private:
/** @var Default statistics printing period in seconds. */
static constexpr const char* DEFAULT_STATS_PRINT_PERIOD = "1 minutes";
static constexpr uint32_t DEFAULT_STATS_PRINT_PERIOD_SECONDS = 60;
static constexpr uint32_t MIN_STATS_PRINT_PERIOD_SECONDS = 1;
static constexpr uint32_t MAX_STATS_PRINT_PERIOD_SECONDS = 86400; // 1 day
static constexpr uint64_t DEFAULT_STATS_PRINT_PERIOD_SECONDS = 60;
static constexpr uint64_t MIN_STATS_PRINT_PERIOD_SECONDS = 1;
static constexpr uint64_t MAX_STATS_PRINT_PERIOD_SECONDS = 86400; // 1 day
/** @var Default full statistics printing period in seconds. */
static constexpr const char* DEFAULT_FULL_STATS_PRINT_PERIOD = "5 minutes";
static constexpr uint32_t DEFAULT_FULL_STATS_PRINT_PERIOD_SECONDS = 300;
static constexpr uint64_t DEFAULT_FULL_STATS_PRINT_PERIOD_SECONDS = 300;
/** @var Default enable DB session statistics printing. */
static constexpr bool DEFAULT_ENABLE_DB_SESSION_STAT_PRINT = false;
@ -571,40 +571,40 @@ private:
/** @var Default maximum limit for MOT global memory. */
static constexpr const char* DEFAULT_MAX_MOT_GLOBAL_MEMORY = "80%";
static constexpr uint32_t DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB = 8 * KILO_BYTE; // 8 GB
static constexpr uint32_t MIN_MAX_MOT_GLOBAL_MEMORY_MB = 128; // 128 MB
static constexpr uint32_t MAX_MAX_MOT_GLOBAL_MEMORY_MB = 512 * MEGA_BYTE; // 512 TB
static constexpr uint64_t DEFAULT_MAX_MOT_GLOBAL_MEMORY_MB = 8 * KILO_BYTE; // 8 GB
static constexpr uint64_t MIN_MAX_MOT_GLOBAL_MEMORY_MB = 128; // 128 MB
static constexpr uint64_t MAX_MAX_MOT_GLOBAL_MEMORY_MB = 512 * MEGA_BYTE; // 512 TB
/** @var Default minimum (pre-allocated) limit for MOT global memory. */
static constexpr const char* DEFAULT_MIN_MOT_GLOBAL_MEMORY = "0 GB";
static constexpr uint32_t DEFAULT_MIN_MOT_GLOBAL_MEMORY_MB = 0;
static constexpr uint32_t MIN_MIN_MOT_GLOBAL_MEMORY_MB = 0;
static constexpr uint32_t MAX_MIN_MOT_GLOBAL_MEMORY_MB = 512 * MEGA_BYTE; // 512 TB
static constexpr const char* DEFAULT_MIN_MOT_GLOBAL_MEMORY = "0";
static constexpr uint64_t DEFAULT_MIN_MOT_GLOBAL_MEMORY_MB = 0;
static constexpr uint64_t MIN_MIN_MOT_GLOBAL_MEMORY_MB = 0;
static constexpr uint64_t MAX_MIN_MOT_GLOBAL_MEMORY_MB = 512 * MEGA_BYTE; // 512 TB
/** @var Default maximum limit for MOT global memory (used to establish new ratio between local and global pools).
*/
static constexpr const char* DEFAULT_MAX_MOT_LOCAL_MEMORY = "15%";
static constexpr uint32_t DEFAULT_MAX_MOT_LOCAL_MEMORY_MB = 2 * KILO_BYTE; // 2 GB
static constexpr uint32_t MIN_MAX_MOT_LOCAL_MEMORY_MB = 64; // 64 MB (about 8 normal sessions)
static constexpr uint32_t MAX_MAX_MOT_LOCAL_MEMORY_MB = 512 * KILO_BYTE; // 512 GB (MANY very heavy sessions)
static constexpr uint64_t DEFAULT_MAX_MOT_LOCAL_MEMORY_MB = 2 * KILO_BYTE; // 2 GB
static constexpr uint64_t MIN_MAX_MOT_LOCAL_MEMORY_MB = 64; // 64 MB (about 8 normal sessions)
static constexpr uint64_t MAX_MAX_MOT_LOCAL_MEMORY_MB = 512 * KILO_BYTE; // 512 GB (MANY very heavy sessions)
/** @var Default minimum (pre-allocated) limit for MOT local memory. */
static constexpr const char* DEFAULT_MIN_MOT_LOCAL_MEMORY = "0 GB";
static constexpr uint32_t DEFAULT_MIN_MOT_LOCAL_MEMORY_MB = 0; // no pre-allocation
static constexpr uint32_t MIN_MIN_MOT_LOCAL_MEMORY_MB = 0; // no pre-allocation
static constexpr uint32_t MAX_MIN_MOT_LOCAL_MEMORY_MB = 512 * KILO_BYTE; // max pre-allocate 512 GB
static constexpr const char* DEFAULT_MIN_MOT_LOCAL_MEMORY = "0";
static constexpr uint64_t DEFAULT_MIN_MOT_LOCAL_MEMORY_MB = 0; // no pre-allocation
static constexpr uint64_t MIN_MIN_MOT_LOCAL_MEMORY_MB = 0; // no pre-allocation
static constexpr uint64_t MAX_MIN_MOT_LOCAL_MEMORY_MB = 512 * KILO_BYTE; // max pre-allocate 512 GB
/** @var Default maximum for single MOT session small memory allocations. */
static constexpr const char* DEFAULT_MAX_MOT_SESSION_MEMORY = "0 MB";
static constexpr uint32_t DEFAULT_MAX_MOT_SESSION_MEMORY_KB = 0; // no session-memory limit
static constexpr uint32_t MIN_MAX_MOT_SESSION_MEMORY_KB = 0;
static constexpr uint32_t MAX_MAX_MOT_SESSION_MEMORY_KB = 512 * MEGA_BYTE; // limit single session to 512 GB
static constexpr const char* DEFAULT_MAX_MOT_SESSION_MEMORY = "0";
static constexpr uint64_t DEFAULT_MAX_MOT_SESSION_MEMORY_KB = 0; // no session-memory limit
static constexpr uint64_t MIN_MAX_MOT_SESSION_MEMORY_KB = 0;
static constexpr uint64_t MAX_MAX_MOT_SESSION_MEMORY_KB = 512 * MEGA_BYTE; // limit single session to 512 GB
/** @var Default minimum (pre-allocated) for single MOT session small memory allocations. */
static constexpr const char* DEFAULT_MIN_MOT_SESSION_MEMORY = "0 MB";
static constexpr uint32_t DEFAULT_MIN_MOT_SESSION_MEMORY_KB = 0; // no session-memory pre-allocation
static constexpr uint32_t MIN_MIN_MOT_SESSION_MEMORY_KB = 0; // no session-memory pre-allocation
static constexpr uint32_t MAX_MIN_MOT_SESSION_MEMORY_KB = 64 * KILO_BYTE; // up to 64 MB pre-allocation
static constexpr const char* DEFAULT_MIN_MOT_SESSION_MEMORY = "0";
static constexpr uint64_t DEFAULT_MIN_MOT_SESSION_MEMORY_KB = 0; // no session-memory pre-allocation
static constexpr uint64_t MIN_MIN_MOT_SESSION_MEMORY_KB = 0; // no session-memory pre-allocation
static constexpr uint64_t MAX_MIN_MOT_SESSION_MEMORY_KB = 64 * KILO_BYTE; // up to 64 MB pre-allocation
/** @var Default physical or virtual memory reservation. */
static constexpr MemReserveMode DEFAULT_RESERVE_MEMORY_MODE = MEM_RESERVE_VIRTUAL;
@ -626,22 +626,22 @@ private:
static constexpr uint32_t MAX_HIGH_RED_MARK_PERCENT = 95; // Don't allow disabling high red-mark
/** @var The default size in megabytes of the session large buffer store. */
static constexpr const char* DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE = " 0 MB";
static constexpr uint32_t DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE_MB = 0;
static constexpr uint32_t MIN_SESSION_LARGE_BUFFER_STORE_SIZE_MB = 0; // disabled
static constexpr uint32_t MAX_SESSION_LARGE_BUFFER_STORE_SIZE_MB = 128 * KILO_BYTE; // 128 GB
static constexpr const char* DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE = "0";
static constexpr uint64_t DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE_MB = 0;
static constexpr uint64_t MIN_SESSION_LARGE_BUFFER_STORE_SIZE_MB = 0; // disabled
static constexpr uint64_t MAX_SESSION_LARGE_BUFFER_STORE_SIZE_MB = 128 * KILO_BYTE; // 128 GB
/** @var The default largest object size in megabytes in the session large buffer store. */
static constexpr const char* DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE = "0 MB";
static constexpr uint32_t DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB = 0; // use calculation
static constexpr uint32_t MIN_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB = 0;
static constexpr uint32_t MAX_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB = 1024; // 1 GB
static constexpr const char* DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE = "0";
static constexpr uint64_t DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB = 0; // use calculation
static constexpr uint64_t MIN_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB = 0;
static constexpr uint64_t MAX_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB = 1024; // 1 GB
/** @var The default largest object size in megabytes that can be allocated form kernel for sessions. */
static constexpr const char* DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE = "1 GB";
static constexpr uint32_t DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE_MB = 1024; // 1 GB
static constexpr uint32_t MIN_SESSION_MAX_HUGE_OBJECT_SIZE_MB = 8; // 8 MB
static constexpr uint32_t MAX_SESSION_MAX_HUGE_OBJECT_SIZE_MB = 8 * KILO_BYTE; // 8 GB
static constexpr uint64_t DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE_MB = 1024; // 1 GB
static constexpr uint64_t MIN_SESSION_MAX_HUGE_OBJECT_SIZE_MB = 8; // 8 MB
static constexpr uint64_t MAX_SESSION_MAX_HUGE_OBJECT_SIZE_MB = 8 * KILO_BYTE; // 8 GB
/** ------------------ Default Garbage-Collection Configuration ------------ */
/** @var Enable/disable garbage collection. */
@ -649,9 +649,9 @@ private:
/** @var The threshold in bytes for reclamation to be triggered (per-thread) */
static constexpr const char* DEFAULT_GC_RECLAIM_THRESHOLD = "512 KB";
static constexpr uint32_t DEFAULT_GC_RECLAIM_THRESHOLD_BYTES = 512 * KILO_BYTE; // 512 KB
static constexpr uint32_t MIN_GC_RECLAIM_THRESHOLD_BYTES = KILO_BYTE; // 1 KB
static constexpr uint32_t MAX_GC_RECLAIM_THRESHOLD_BYTES = 64 * MEGA_BYTE; // 64 MB
static constexpr uint64_t DEFAULT_GC_RECLAIM_THRESHOLD_BYTES = 512 * KILO_BYTE; // 512 KB
static constexpr uint64_t MIN_GC_RECLAIM_THRESHOLD_BYTES = KILO_BYTE; // 1 KB
static constexpr uint64_t MAX_GC_RECLAIM_THRESHOLD_BYTES = 64 * MEGA_BYTE; // 64 MB
/** @var The amount of objects reclaimed in each cleanup round of a limbo group. */
static constexpr uint32_t DEFAULT_GC_RECLAIM_BATCH_SIZE = 8 * KILO_BYTE; // 8 KB entries
@ -660,9 +660,9 @@ private:
/** @var The high threshold in bytes for reclamation to be triggered (per-thread) */
static constexpr const char* DEFAULT_GC_HIGH_RECLAIM_THRESHOLD = "8 MB";
static constexpr uint32_t DEFAULT_GC_HIGH_RECLAIM_THRESHOLD_BYTES = 8 * MEGA_BYTE; // 8 MB
static constexpr uint32_t MIN_GC_HIGH_RECLAIM_THRESHOLD_BYTES = 1 * MEGA_BYTE; // 1 MB
static constexpr uint32_t MAX_GC_HIGH_RECLAIM_THRESHOLD_BYTES = 64 * MEGA_BYTE; // 64 MB
static constexpr uint64_t DEFAULT_GC_HIGH_RECLAIM_THRESHOLD_BYTES = 8 * MEGA_BYTE; // 8 MB
static constexpr uint64_t MIN_GC_HIGH_RECLAIM_THRESHOLD_BYTES = 1 * MEGA_BYTE; // 1 MB
static constexpr uint64_t MAX_GC_HIGH_RECLAIM_THRESHOLD_BYTES = 64 * MEGA_BYTE; // 64 MB
/** ------------------ Default JIT Configuration ------------ */
/** @var Default enable JIT compilation and execution. */
@ -697,7 +697,7 @@ private:
static constexpr bool DEFAULT_RUN_INTERNAL_CONSISTENCY_VALIDATION = false;
/** @var The default total memory reference used for calculating memory percent value. */
static constexpr uint32_t DEFAULT_TOTAL_MEMORY_MB = 10 * KILO_BYTE; // 10 MB
static constexpr uint64_t DEFAULT_TOTAL_MEMORY_MB = 10 * KILO_BYTE; // 10 MB
/** @brief Loads configuration from main configuration. */
void LoadConfig();
@ -713,8 +713,7 @@ private:
static void UpdateStringConfigItem(std::string& oldValue, const char* newValue, const char* name);
template <typename T>
void UpdateIntConfigItem(
uint64_t& oldValue, T newValue, const char* name, uint64_t lowerBound = 0, uint64_t upperBound = UINT64_MAX)
void UpdateIntConfigItem(uint64_t& oldValue, T newValue, const char* name, uint64_t lowerBound, uint64_t upperBound)
{
if ((newValue > upperBound) || (lowerBound > 0 && newValue < lowerBound)) {
if (!m_suppressLog) {
@ -733,8 +732,7 @@ private:
}
template <typename T>
void UpdateIntConfigItem(
uint32_t& oldValue, T newValue, const char* name, uint32_t lowerBound = 0, uint32_t upperBound = UINT32_MAX)
void UpdateIntConfigItem(uint32_t& oldValue, T newValue, const char* name, uint32_t lowerBound, uint32_t upperBound)
{
if ((newValue > UINT32_MAX) || (newValue > upperBound) || (lowerBound > 0 && newValue < lowerBound)) {
if (!m_suppressLog) {
@ -752,8 +750,7 @@ private:
}
template <typename T>
void UpdateIntConfigItem(
uint16_t& oldValue, T newValue, const char* name, uint16_t lowerBound = 0, uint16_t upperBound = UINT16_MAX)
void UpdateIntConfigItem(uint16_t& oldValue, T newValue, const char* name, uint16_t lowerBound, uint16_t upperBound)
{
if ((newValue > UINT16_MAX) || (newValue > upperBound) || (lowerBound > 0 && newValue < lowerBound)) {
if (!m_suppressLog) {
@ -784,6 +781,12 @@ private:
}
}
void UpdateMemConfigItem(uint64_t& oldValue, const char* name, const char* strDefaultValue, uint64_t scale,
uint64_t lowerBound, uint64_t upperBound, bool allowPercentage);
void UpdateTimeConfigItem(uint64_t& oldValue, const char* name, const char* strDefaultValue, uint64_t scale,
uint64_t lowerBound, uint64_t upperBound);
void UpdateComponentLogLevel();
static int ParseMemoryPercent(const char* memoryValue);

2
src/gausskernel/storage/mot/core/src/system/mot_engine.cpp
View File

@ -941,7 +941,7 @@ uint64_t MOTEngine::GetCurrentMemoryConsumptionBytes()
uint64_t MOTEngine::GetHardMemoryLimitBytes()
{
return ((uint64_t)g_memGlobalCfg.m_maxGlobalMemoryMb * MEGA_BYTE);
return g_memGlobalCfg.m_maxGlobalMemoryMb * MEGA_BYTE;
}
bool MOTEngine::CheckPolicies()

72
src/gausskernel/storage/mot/fdw_adapter/src/mot_internal.cpp
View File

@ -326,12 +326,12 @@ private:
// get max/min global memory and large allocation store for sessions from configuration
MOT::MOTConfiguration& motCfg = MOT::GetGlobalConfiguration();
uint32_t globalMemoryMb = motCfg.m_globalMemoryMaxLimitMB;
uint32_t localMemoryMb = motCfg.m_localMemoryMaxLimitMB;
uint32_t sessionLargeStoreMb = motCfg.m_sessionLargeBufferStoreSizeMB;
uint64_t globalMemoryMb = motCfg.m_globalMemoryMaxLimitMB;
uint64_t localMemoryMb = motCfg.m_localMemoryMaxLimitMB;
uint64_t sessionLargeStoreMb = motCfg.m_sessionLargeBufferStoreSizeMB;
// compare the sum with max_process_memory and system total
uint32_t maxReserveMemoryMb = globalMemoryMb + localMemoryMb + sessionLargeStoreMb;
uint64_t maxReserveMemoryMb = globalMemoryMb + localMemoryMb + sessionLargeStoreMb;
// if the total memory is less than the required minimum, then issue a warning, fix it and return
if (maxReserveMemoryMb < MOT_MIN_MEMORY_USAGE_MB) {
@ -342,24 +342,25 @@ private:
}
// get system total
uint32_t systemTotalMemoryMb = MOT::GetTotalSystemMemoryMb();
uint64_t systemTotalMemoryMb = MOT::GetTotalSystemMemoryMb();
if (systemTotalMemoryMb == 0) {
MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, "Load Configuration", "Cannot retrieve total system memory");
return false;
}
// get envelope limit
uint32_t processTotalMemoryMb = g_instance.attr.attr_memory.max_process_memory / KILO_BYTE;
uint64_t processTotalMemoryMb = g_instance.attr.attr_memory.max_process_memory / KILO_BYTE;
// compute the real limit
uint32_t upperLimitMb = min(systemTotalMemoryMb, processTotalMemoryMb);
uint64_t upperLimitMb = min(systemTotalMemoryMb, processTotalMemoryMb);
// get dynamic gap we need to preserve between MOT and envelope
uint32_t dynamicGapMb = MIN_DYNAMIC_PROCESS_MEMORY / KILO_BYTE;
uint64_t dynamicGapMb = MIN_DYNAMIC_PROCESS_MEMORY / KILO_BYTE;
MOT_LOG_TRACE("Checking for memory limits: globalMemoryMb=%u, localMemoryMb=%u, sessionLargeStoreMb=%u, "
"systemTotalMemoryMb=%u, processTotalMemoryMb=%u, upperLimitMb=%u, dynamicGapMb=%u, "
"max_process_memory=%u",
MOT_LOG_TRACE("Checking for memory limits: globalMemoryMb=%" PRIu64 ", localMemoryMb=%" PRIu64
", sessionLargeStoreMb=%" PRIu64 ", systemTotalMemoryMb=%" PRIu64
", processTotalMemoryMb=%" PRIu64 ", upperLimitMb=%" PRIu64 ", dynamicGapMb=%" PRIu64
", max_process_memory=%u",
globalMemoryMb,
localMemoryMb,
sessionLargeStoreMb,
@ -372,11 +373,12 @@ private:
// we check that a 2GB gap is preserved
if (upperLimitMb < maxReserveMemoryMb + dynamicGapMb) {
// memory restriction conflict, issue warning and adjust values
MOT_LOG_TRACE(
"MOT engine maximum memory definitions (global: %u MB, local: %u MB, session large store: %u MB, "
"total: "
"%u MB) breach GaussDB maximum process memory restriction (%u MB) and/or total system memory (%u MB). "
"MOT values shall be adjusted accordingly to preserve required gap (%u MB).",
MOT_LOG_TRACE("MOT engine maximum memory definitions (global: %" PRIu64 " MB, local: %" PRIu64
" MB, session large "
"store: %" PRIu64 " MB, total: %" PRIu64
" MB) breach GaussDB maximum process memory restriction (%" PRIu64
" MB) and/or total system memory (%" PRIu64 " MB). "
"MOT values shall be adjusted accordingly to preserve required gap (%" PRIu64 " MB).",
globalMemoryMb,
localMemoryMb,
sessionLargeStoreMb,
@ -386,7 +388,7 @@ private:
dynamicGapMb);
// compute new total memory limit for MOT
uint32_t newTotalMemoryMb = 0;
uint64_t newTotalMemoryMb = 0;
if (upperLimitMb < dynamicGapMb) {
// this can happen only if system memory is less than 2GB, we still allow minor breach
MOT_LOG_WARN("Using minimal memory limits in MOT Engine due to system total memory restrictions");
@ -411,20 +413,20 @@ private:
return result;
}
bool ConfigureMemoryLimits(uint32_t newTotalMemoryMb, uint32_t currentTotalMemoryMb, uint32_t currentGlobalMemoryMb,
uint32_t currentLocalMemoryMb)
bool ConfigureMemoryLimits(uint64_t newTotalMemoryMb, uint64_t currentTotalMemoryMb, uint64_t currentGlobalMemoryMb,
uint64_t currentLocalMemoryMb)
{
uint32_t newGlobalMemoryMb = 0;
uint32_t newLocalMemoryMb = 0;
uint32_t newSessionLargeStoreMemoryMb = 0;
uint64_t newGlobalMemoryMb = 0;
uint64_t newLocalMemoryMb = 0;
uint64_t newSessionLargeStoreMemoryMb = 0;
// compute new configuration values
if (currentTotalMemoryMb > 0) {
// we preserve the existing ratio between global and local memory, but reduce the total sum as required
double ratio = ((double)newTotalMemoryMb) / ((double)currentTotalMemoryMb);
newGlobalMemoryMb = (uint32_t)(currentGlobalMemoryMb * ratio);
newGlobalMemoryMb = (uint64_t)(currentGlobalMemoryMb * ratio);
if (MOT::GetGlobalConfiguration().m_sessionLargeBufferStoreSizeMB > 0) {
newLocalMemoryMb = (uint32_t)(currentLocalMemoryMb * ratio);
newLocalMemoryMb = (uint64_t)(currentLocalMemoryMb * ratio);
newSessionLargeStoreMemoryMb = newTotalMemoryMb - newGlobalMemoryMb - newLocalMemoryMb;
} else {
// if the user configured zero for the session large store, then we want to keep it this way
@ -433,13 +435,13 @@ private:
}
} else {
// when current total memory is zero we split the new total between global and local in ratio of 4:1
newGlobalMemoryMb = (uint32_t)(newTotalMemoryMb * 0.8f); // 80% to global memory
newGlobalMemoryMb = (uint64_t)(newTotalMemoryMb * 0.8f); // 80% to global memory
newLocalMemoryMb = newTotalMemoryMb - newGlobalMemoryMb; // 20% to local memory
// session large store remains zero!
}
MOT_LOG_WARN(
"Adjusting MOT memory limits: global = %u MB, local = %u MB, session large store = %u MB, total = %u MB",
MOT_LOG_WARN("Adjusting MOT memory limits: global = %" PRIu64 " MB, local = %" PRIu64
" MB, session large store = %" PRIu64 " MB, total = %" PRIu64 " MB",
newGlobalMemoryMb,
newLocalMemoryMb,
newSessionLargeStoreMemoryMb,
@ -447,14 +449,14 @@ private:
// stream into MOT new definitions
MOT::mot_string memCfg;
memCfg.format("%u MB", newGlobalMemoryMb);
memCfg.format("%" PRIu64 " MB", newGlobalMemoryMb);
bool result = AddExtStringConfigItem("", "max_mot_global_memory", memCfg.c_str());
if (result) {
memCfg.format("%u MB", newLocalMemoryMb);
memCfg.format("%" PRIu64 " MB", newLocalMemoryMb);
result = AddExtStringConfigItem("", "max_mot_local_memory", memCfg.c_str());
}
if (result) {
memCfg.format("%u MB", newSessionLargeStoreMemoryMb);
memCfg.format("%" PRIu64 " MB", newSessionLargeStoreMemoryMb);
result = AddExtStringConfigItem("", "session_large_buffer_store_size", memCfg.c_str());
}
@ -740,9 +742,9 @@ void MOTAdaptor::Init()
}
// check max process memory here - we do it anyway to protect ourselves from miscalculations
uint32_t globalMemoryKb = MOT::g_memGlobalCfg.m_maxGlobalMemoryMb * KILO_BYTE;
uint32_t localMemoryKb = MOT::g_memGlobalCfg.m_maxLocalMemoryMb * KILO_BYTE;
uint32_t maxReserveMemoryKb = globalMemoryKb + localMemoryKb;
uint64_t globalMemoryKb = MOT::g_memGlobalCfg.m_maxGlobalMemoryMb * KILO_BYTE;
uint64_t localMemoryKb = MOT::g_memGlobalCfg.m_maxLocalMemoryMb * KILO_BYTE;
uint64_t maxReserveMemoryKb = globalMemoryKb + localMemoryKb;
if ((g_instance.attr.attr_memory.max_process_memory < (int32)maxReserveMemoryKb) ||
((g_instance.attr.attr_memory.max_process_memory - maxReserveMemoryKb) < MIN_DYNAMIC_PROCESS_MEMORY)) {
@ -757,8 +759,8 @@ void MOTAdaptor::Init()
MOT::MOTEngine::DestroyInstance();
elog(FATAL,
"The value of pre-reserved memory for MOT engine is not reasonable: "
"Request for a maximum of %u KB global memory, and %u KB session memory (total of %u KB) "
"is invalid since max_process_memory is %u KB",
"Request for a maximum of %" PRIu64 " KB global memory, and %" PRIu64
" KB session memory (total of %" PRIu64 " KB) is invalid since max_process_memory is %u KB",
globalMemoryKb,
localMemoryKb,
maxReserveMemoryKb,