database/doris
2
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

[fix](config) fix memory config enable_query_memroy_overcommit spell problem #19898

Browse Source
This commit is contained in:
ZhangYu0123
2023-05-22 00:32:20 +08:00
committed by GitHub
parent 76c358b3e3
commit 1c950d6930
9 changed files with 12 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
be/src/common/config.cpp
View File

@ -124,7 +124,7 @@ DEFINE_mString(process_full_gc_size, "20%");
// when the process memory exceeds the soft mem limit, the query with the largest ratio between the currently
// used memory and the exec_mem_limit will be canceled.
// If false, cancel query when the memory used exceeds exec_mem_limit, same as before.
DEFINE_mBool(enable_query_memroy_overcommit, "true");
DEFINE_mBool(enable_query_memory_overcommit, "true");
// The maximum time a thread waits for a full GC. Currently only query will wait for full gc.
DEFINE_mInt32(thread_wait_gc_max_milliseconds, "1000");

2
be/src/common/config.h
View File

@ -164,7 +164,7 @@ DECLARE_mString(process_full_gc_size);
// when the process memory exceeds the soft mem limit, the query with the largest ratio between the currently
// used memory and the exec_mem_limit will be canceled.
// If false, cancel query when the memory used exceeds exec_mem_limit, same as before.
DECLARE_mBool(enable_query_memroy_overcommit);
DECLARE_mBool(enable_query_memory_overcommit);
// The maximum time a thread waits for a full GC. Currently only query will wait for full gc.
DECLARE_mInt32(thread_wait_gc_max_milliseconds);

2
be/src/runtime/memory/mem_tracker_limiter.cpp
View File

@ -506,7 +506,7 @@ int64_t MemTrackerLimiter::tg_memory_limit_gc(
MemTracker::print_bytes(mem_consumption), BackendOptions::get_localhost(),
MemTracker::print_bytes(used_memory), MemTracker::print_bytes(memory_limit));
};
if (config::enable_query_memroy_overcommit) {
if (config::enable_query_memory_overcommit) {
freed_mem += MemTrackerLimiter::free_top_overcommit_query(
need_free_mem - freed_mem, query_type, tracker_limiter_groups, cancel_str);
}

2
be/src/runtime/memory/mem_tracker_limiter.h
View File

@ -278,7 +278,7 @@ inline void MemTrackerLimiter::cache_consume(int64_t bytes) {
}
inline Status MemTrackerLimiter::check_limit(int64_t bytes) {
if (bytes <= 0 || (is_overcommit_tracker() && config::enable_query_memroy_overcommit)) {
if (bytes <= 0 || (is_overcommit_tracker() && config::enable_query_memory_overcommit)) {
return Status::OK();
}
if (_limit > 0 && _consumption->current_value() + bytes > _limit) {

2
be/src/runtime/runtime_state.cpp
View File

@ -285,7 +285,7 @@ Status RuntimeState::check_query_state(const std::string& msg) {
// If the thread MemTrackerLimiter exceeds the limit, an error status is returned.
// Usually used after SCOPED_ATTACH_TASK, during query execution.
if (thread_context()->thread_mem_tracker()->limit_exceeded() &&
!config::enable_query_memroy_overcommit) {
!config::enable_query_memory_overcommit) {
auto failed_msg = thread_context()->thread_mem_tracker()->query_tracker_limit_exceeded_str(
thread_context()->thread_mem_tracker()->tracker_limit_exceeded_str(),
thread_context()->thread_mem_tracker_mgr->last_consumer_tracker(), msg);

6
be/src/util/mem_info.cpp
View File

@ -135,7 +135,7 @@ void MemInfo::process_cache_gc(int64_t& freed_mem) {
// step1: free all cache
// step2: free resource groups memory that enable overcommit
// step3: free global top overcommit query, if enable query memroy overcommit
// step3: free global top overcommit query, if enable query memory overcommit
// TODO Now, the meaning is different from java minor gc + full gc, more like small gc + large gc.
bool MemInfo::process_minor_gc() {
MonotonicStopWatch watch;
@ -164,7 +164,7 @@ bool MemInfo::process_minor_gc() {
VLOG_NOTICE << MemTrackerLimiter::type_detail_usage(
"Before free top memory overcommit query in Minor GC", MemTrackerLimiter::Type::QUERY);
if (config::enable_query_memroy_overcommit) {
if (config::enable_query_memory_overcommit) {
freed_mem += MemTrackerLimiter::free_top_overcommit_query(
_s_process_minor_gc_size - freed_mem, vm_rss_str, mem_available_str);
}
@ -221,7 +221,7 @@ bool MemInfo::process_full_gc() {
VLOG_NOTICE << MemTrackerLimiter::type_detail_usage(
"Before free top memory overcommit load in Full GC", MemTrackerLimiter::Type::LOAD);
if (config::enable_query_memroy_overcommit) {
if (config::enable_query_memory_overcommit) {
freed_mem += MemTrackerLimiter::free_top_overcommit_load(
_s_process_full_gc_size - freed_mem, vm_rss_str, mem_available_str);
if (freed_mem > _s_process_full_gc_size) {

4
be/test/util/system_metrics_test.cpp
View File

@ -113,7 +113,7 @@ TEST_F(SystemMetricsTest, normal) {
EXPECT_TRUE(cpu_guest_nice != nullptr);
EXPECT_STREQ("0", cpu_guest_nice->to_string().c_str());
// memroy
// memory
Metric* memory_allocated_bytes = entity->get_metric("memory_allocated_bytes");
EXPECT_TRUE(memory_allocated_bytes != nullptr);
Metric* memory_pgpgin = entity->get_metric("memory_pgpgin");
@ -254,7 +254,7 @@ TEST_F(SystemMetricsTest, no_proc_file) {
auto cpu_entity = registry.get_entity("cpu", {{"device", "cpu"}});
EXPECT_TRUE(cpu_entity == nullptr);
// memroy
// memory
Metric* memory_allocated_bytes = entity->get_metric("memory_allocated_bytes");
EXPECT_TRUE(memory_allocated_bytes != nullptr);
Metric* memory_pgpgin = entity->get_metric("memory_pgpgin");

2
docs/en/docs/admin-manual/maint-monitor/metadata-operation.md
View File

@ -368,7 +368,7 @@ The deployment recommendation of FE is described in the Installation and [Deploy
Master FE generates a mirror file by default for every 50,000 metadata journal. In a frequently used cluster, a new image file is usually generated every half to several days. If you find that the image file has not been updated for a long time (for example, more than a week), you can see the reasons in sequence as follows:
1. Search for `memory is not enough to do checkpoint. Committed memroy XXXX Bytes, used memory XXXX Bytes. ` in the fe.log of Master FE. If found, it indicates that the current FE's JVM memory is insufficient for image generation (usually we need to reserve half of the FE memory for image generation). Then you need to add JVM memory and restart FE before you can observe. Each time Master FE restarts, a new image is generated directly. This restart method can also be used to actively generate new images. Note that if there are multiple FOLLOWER deployments, then when you restart the current Master FE, another FOLLOWER FE will become MASTER, and subsequent image generation will be the responsibility of the new Master. Therefore, you may need to modify the JVM memory configuration of all FOLLOWER FE.
1. Search for `memory is not enough to do checkpoint. Committed memory XXXX Bytes, used memory XXXX Bytes. ` in the fe.log of Master FE. If found, it indicates that the current FE's JVM memory is insufficient for image generation (usually we need to reserve half of the FE memory for image generation). Then you need to add JVM memory and restart FE before you can observe. Each time Master FE restarts, a new image is generated directly. This restart method can also be used to actively generate new images. Note that if there are multiple FOLLOWER deployments, then when you restart the current Master FE, another FOLLOWER FE will become MASTER, and subsequent image generation will be the responsibility of the new Master. Therefore, you may need to modify the JVM memory configuration of all FOLLOWER FE.
2. Search for `begin to generate new image: image.xxxx` in the fe.log of Master FE. If it is found, then the image is generated. Check the subsequent log of this thread, and if `checkpoint finished save image.xxxx` appears, the image is written successfully. If `Exception when generating new image file` occurs, the generation fails and specific error messages need to be viewed.

2
docs/zh-CN/docs/admin-manual/maint-monitor/metadata-operation.md
View File

@ -371,7 +371,7 @@ FE 的部署推荐，在 [安装与部署文档](../../install/standard-deployme
Master FE 会默认每 50000 条元数据 journal，生成一个镜像文件。在一个频繁使用的集群中，通常每隔半天到几天的时间，就会生成一个新的 image 文件。如果你发现 image 文件已经很久没有更新了（比如超过一个星期），则可以顺序的按照如下方法，查看具体原因：
1. 在 Master FE 的 fe.log 中搜索 `memory is not enough to do checkpoint. Committed memroy xxxx Bytes, used memory xxxx Bytes.` 字样。如果找到，则说明当前 FE 的 JVM 内存不足以用于生成镜像（通常我们需要预留一半的 FE 内存用于 image 的生成）。那么需要增加 JVM 的内存并重启 FE 后，再观察。每次 Master FE 重启后，都会直接生成一个新的 image。也可用这种重启方式，主动地生成新的 image。注意，如果是多 FOLLOWER 部署，那么当你重启当前 Master FE 后，另一个 FOLLOWER FE 会变成 MASTER，则后续的 image 生成会由新的 Master 负责。因此，你可能需要修改所有 FOLLOWER FE 的 JVM 内存配置。
1. 在 Master FE 的 fe.log 中搜索 `memory is not enough to do checkpoint. Committed memory xxxx Bytes, used memory xxxx Bytes.` 字样。如果找到，则说明当前 FE 的 JVM 内存不足以用于生成镜像（通常我们需要预留一半的 FE 内存用于 image 的生成）。那么需要增加 JVM 的内存并重启 FE 后，再观察。每次 Master FE 重启后，都会直接生成一个新的 image。也可用这种重启方式，主动地生成新的 image。注意，如果是多 FOLLOWER 部署，那么当你重启当前 Master FE 后，另一个 FOLLOWER FE 会变成 MASTER，则后续的 image 生成会由新的 Master 负责。因此，你可能需要修改所有 FOLLOWER FE 的 JVM 内存配置。
2. 在 Master FE 的 fe.log 中搜索 `begin to generate new image: image.xxxx`。如果找到，则说明开始生成 image 了。检查这个线程的后续日志，如果出现 `checkpoint finished save image.xxxx`，则说明 image 写入成功。如果出现 `Exception when generate new image file`，则生成失败，需要查看具体的错误信息。