[fix](memory) Fix disable_mem_pools to disable cache #12087

be/src/common/config.h

@@ -431,6 +431,8 @@ CONF_Int32(min_buffer_size, "1024"); // 1024, The minimum read buffer size (in b
 // With 1024B through 8MB buffers, this is up to ~2GB of buffers.
 CONF_Int32(max_free_io_buffers, "128");
 
+// Whether to disable the memory cache pool,
+// including MemPool, ChunkAllocator, BufferPool, DiskIO free buffer.
 CONF_Bool(disable_mem_pools, "false");
 
 // Whether to allocate chunk using mmap. If you enable this, you'd better to

be/src/runtime/bufferpool/buffer_allocator.cc

@@ -239,7 +239,7 @@ Status BufferPool::BufferAllocator::AllocateInternal(int64_t len, BufferHandle*
     }
     if (UNLIKELY(len > system_bytes_limit_)) {
         err_stream << "Tried to allocate buffer of " << len << " bytes"
-                   << " > buffer pool limit of  " << MAX_BUFFER_BYTES << " bytes";
+                   << " > buffer pool limit of  " << system_bytes_limit_ << " bytes";
         return Status::InternalError(err_stream.str());
     }
 

be/src/runtime/mem_pool.cpp

@@ -132,6 +132,7 @@ Status MemPool::find_chunk(size_t min_size, bool check_limits) {
     if (config::disable_mem_pools) {
         // Disable pooling by sizing the chunk to fit only this allocation.
         // Make sure the alignment guarantees are respected.
+        // This will generate too many small chunks.
         chunk_size = std::max<size_t>(min_size, alignof(max_align_t));
     } else {
         DCHECK_GE(next_chunk_size_, INITIAL_CHUNK_SIZE);

be/src/runtime/memory/chunk_allocator.cpp

@@ -34,6 +34,11 @@
 
 namespace doris {
 
+// <= MIN_CHUNK_SIZE, A large number of small chunks will waste extra storage and increase lock time.
+static constexpr size_t MIN_CHUNK_SIZE = 4096; // 4K
+// >= MAX_CHUNK_SIZE, Large chunks may not be used for a long time, wasting memory.
+static constexpr size_t MAX_CHUNK_SIZE = 64 * (1ULL << 20); // 64M
+
 ChunkAllocator* ChunkAllocator::_s_instance = nullptr;
 
 DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(chunk_pool_local_core_alloc_count, MetricUnit::NOUNIT);
@@ -199,12 +204,17 @@ Status ChunkAllocator::allocate(size_t size, Chunk* chunk) {
 void ChunkAllocator::free(const Chunk& chunk) {
     DCHECK(chunk.core_id != -1);
     CHECK((chunk.size & (chunk.size - 1)) == 0);
+    if (config::disable_mem_pools) {
+        SystemAllocator::free(chunk.data, chunk.size);
+        return;
+    }
 
     int64_t old_reserved_bytes = _reserved_bytes;
     int64_t new_reserved_bytes = 0;
     do {
         new_reserved_bytes = old_reserved_bytes + chunk.size;
-        if (new_reserved_bytes > _reserve_bytes_limit) {
+        if (chunk.size <= MIN_CHUNK_SIZE || chunk.size >= MAX_CHUNK_SIZE ||
+            new_reserved_bytes > _reserve_bytes_limit) {
             int64_t cost_ns = 0;
             {
                 SCOPED_RAW_TIMER(&cost_ns);