hcq/oceanbase
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[CP] fix bug of mem chunk size

Browse Source
This commit is contained in:
obdev
2024-04-23 07:22:53 +00:00
committed by ob-robot
parent 435b05028f
commit 778c43f157
4 changed files with 11 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/observer/table_load/ob_table_load_assigned_memory_manager.cpp
View File

@ -12,10 +12,11 @@
#define USING_LOG_PREFIX SERVER #define USING_LOG_PREFIX SERVER
#include "share/rc/ob_tenant_base.h"
#include "observer/table_load/ob_table_load_assigned_memory_manager.h" #include "observer/table_load/ob_table_load_assigned_memory_manager.h"
#include "observer/omt/ob_multi_tenant.h" #include "observer/omt/ob_multi_tenant.h"
#include "share/rc/ob_tenant_base.h"
#include "observer/omt/ob_tenant.h" #include "observer/omt/ob_tenant.h"
#include "storage/direct_load/ob_direct_load_mem_define.h"
namespace oceanbase namespace oceanbase
{ {
@ -36,7 +37,7 @@ using namespace omt;
ObTableLoadAssignedMemoryManager::ObTableLoadAssignedMemoryManager() ObTableLoadAssignedMemoryManager::ObTableLoadAssignedMemoryManager()
: avail_sort_memory_(0), : avail_sort_memory_(0),
avail_memory_(0), avail_memory_(0),
sort_task_count_(0), chunk_count_(0),
is_inited_(false) is_inited_(false)
{ {
} }
@ -66,9 +67,9 @@ int ObTableLoadAssignedMemoryManager::assign_memory(bool is_sort, int64_t assign
LOG_WARN("ObTableLoadAssignedMemoryManager not init", KR(ret), KP(this)); LOG_WARN("ObTableLoadAssignedMemoryManager not init", KR(ret), KP(this));
} else { } else {
ObMutexGuard guard(mutex_); ObMutexGuard guard(mutex_);
sort_task_count_ += (is_sort ? 1 : 0); chunk_count_ += (is_sort ? assign_memory / ObDirectLoadExternalMultiPartitionRowChunk::MIN_MEMORY_LIMIT : 0);
avail_sort_memory_ -= (is_sort ? 0 : assign_memory); avail_sort_memory_ -= (is_sort ? 0 : assign_memory);
LOG_INFO("ObTableLoadAssignedMemoryManager::assign_memory", K(MTL_ID()), K(is_sort), K(assign_memory), K(sort_task_count_), K(avail_sort_memory_)); LOG_INFO("ObTableLoadAssignedMemoryManager::assign_memory", K(MTL_ID()), K(is_sort), K(chunk_count_), K(assign_memory), K(avail_sort_memory_));
} }
return ret; return ret;
@ -82,9 +83,9 @@ int ObTableLoadAssignedMemoryManager::recycle_memory(bool is_sort, int64_t assig
LOG_WARN("ObTableLoadAssignedMemoryManager not init", KR(ret), KP(this)); LOG_WARN("ObTableLoadAssignedMemoryManager not init", KR(ret), KP(this));
} else { } else {
ObMutexGuard guard(mutex_); ObMutexGuard guard(mutex_);
sort_task_count_ -= (is_sort ? 1 : 0); chunk_count_ -= (is_sort ? assign_memory / ObDirectLoadExternalMultiPartitionRowChunk::MIN_MEMORY_LIMIT : 0);
avail_sort_memory_ += (is_sort ? 0 : assign_memory); avail_sort_memory_ += (is_sort ? 0 : assign_memory);
LOG_INFO("ObTableLoadAssignedMemoryManager::recycle_memory", K(MTL_ID()), K(is_sort), K(assign_memory), K(sort_task_count_), K(avail_sort_memory_)); LOG_INFO("ObTableLoadAssignedMemoryManager::recycle_memory", K(MTL_ID()), K(is_sort), K(chunk_count_), K(assign_memory), K(avail_sort_memory_));
} }
return ret; return ret;
@ -123,11 +124,11 @@ int ObTableLoadAssignedMemoryManager::get_sort_memory(int64_t &sort_memory)
LOG_WARN("ObTableLoadAssignedMemoryManager not init", KR(ret), KP(this)); LOG_WARN("ObTableLoadAssignedMemoryManager not init", KR(ret), KP(this));
} else { } else {
ObMutexGuard guard(mutex_); ObMutexGuard guard(mutex_);
if (OB_UNLIKELY(sort_task_count_ == 0)) { if (OB_UNLIKELY(chunk_count_ == 0)) {
ret = OB_ERR_UNEXPECTED; ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected sort_task_count_ equal to zero", KR(ret)); LOG_WARN("unexpected chunk_count_ equal to zero", KR(ret));
} else { } else {
sort_memory = avail_sort_memory_ / sort_task_count_; sort_memory = avail_sort_memory_ / chunk_count_;
} }
} }

2
src/observer/table_load/ob_table_load_assigned_memory_manager.h
View File

@ -39,7 +39,7 @@ public:
private: private:
int64_t avail_sort_memory_; int64_t avail_sort_memory_;
int64_t avail_memory_; int64_t avail_memory_;
int64_t sort_task_count_; int64_t chunk_count_;
mutable lib::ObMutex mutex_; mutable lib::ObMutex mutex_;
bool is_inited_; bool is_inited_;
}; };

2
src/storage/direct_load/ob_direct_load_mem_loader.cpp
View File

@ -109,8 +109,6 @@ int ObDirectLoadMemLoader::work()
sort_memory = mem_ctx_->table_data_desc_.mem_chunk_size_; sort_memory = mem_ctx_->table_data_desc_.mem_chunk_size_;
} else if (OB_FAIL(ObTableLoadService::get_sort_memory(sort_memory))) { } else if (OB_FAIL(ObTableLoadService::get_sort_memory(sort_memory))) {
LOG_WARN("fail to get sort memory", KR(ret)); LOG_WARN("fail to get sort memory", KR(ret));
} else {
sort_memory /= mem_ctx_->table_data_desc_.max_mem_chunk_count_;
} }
if (OB_SUCC(ret)) { if (OB_SUCC(ret)) {
if (OB_FAIL(chunk->init(MTL_ID(), sort_memory))) { if (OB_FAIL(chunk->init(MTL_ID(), sort_memory))) {

2
src/storage/direct_load/ob_direct_load_multiple_heap_table_sorter.cpp
View File

@ -207,8 +207,6 @@ int ObDirectLoadMultipleHeapTableSorter::work()
sort_memory = mem_ctx_->table_data_desc_.heap_table_mem_chunk_size_; sort_memory = mem_ctx_->table_data_desc_.heap_table_mem_chunk_size_;
} else if (OB_FAIL(ObTableLoadService::get_sort_memory(sort_memory))) { } else if (OB_FAIL(ObTableLoadService::get_sort_memory(sort_memory))) {
LOG_WARN("fail to get sort memory", KR(ret)); LOG_WARN("fail to get sort memory", KR(ret));
} else {
sort_memory /= mem_ctx_->table_data_desc_.session_count_;
} }
if (OB_SUCC(ret)) { if (OB_SUCC(ret)) {
chunk = OB_NEW(ChunkType, ObMemAttr(MTL_ID(), "TLD_MemChunkVal"), sort_memory); chunk = OB_NEW(ChunkType, ObMemAttr(MTL_ID(), "TLD_MemChunkVal"), sort_memory);