From 2d9cc8fe8f809e5c8d5ee461eb12fd808f50eceb Mon Sep 17 00:00:00 2001 From: zxealous Date: Wed, 17 May 2023 10:23:33 +0800 Subject: [PATCH] [improvement](file cache)Support set min file segment size while use block file cache (#19536) --- be/src/common/config.cpp | 12 ++++++++++-- be/src/common/config.h | 1 + be/src/io/cache/block/cached_remote_file_reader.cpp | 6 +++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 09fcde0d94..879d7cb134 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -939,8 +939,16 @@ DEFINE_Bool(enable_file_cache, "false"); // format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240},{"path":"/path/to/file_cache2","total_size":21474836480,"query_limit":10737418240}] DEFINE_String(file_cache_path, ""); DEFINE_Int64(file_cache_max_file_segment_size, "4194304"); // 4MB -DEFINE_Validator(file_cache_max_file_segment_size, - [](const int64_t config) -> bool { return config >= 4096; }); // 4KB +// 4KB <= file_cache_max_file_segment_size <= 256MB +DEFINE_Validator(file_cache_max_file_segment_size, [](const int64_t config) -> bool { + return config >= 4096 && config <= 268435456; +}); +DEFINE_Int64(file_cache_min_file_segment_size, "1048576"); // 1MB +// 4KB <= file_cache_min_file_segment_size <= 256MB +DEFINE_Validator(file_cache_min_file_segment_size, [](const int64_t config) -> bool { + return config >= 4096 && config <= 268435456 && + config <= config::file_cache_max_file_segment_size; +}); DEFINE_Bool(clear_file_cache, "false"); DEFINE_Bool(enable_file_cache_query_limit, "false"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 82fc2f3a57..cda85c867e 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -962,6 +962,7 @@ DECLARE_Bool(enable_file_cache); // format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240}] // format: [{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240},{"path":"/path/to/file_cache2","total_size":21474836480,"query_limit":10737418240}] DECLARE_String(file_cache_path); +DECLARE_Int64(file_cache_min_file_segment_size); DECLARE_Int64(file_cache_max_file_segment_size); DECLARE_Bool(clear_file_cache); DECLARE_Bool(enable_file_cache_query_limit); diff --git a/be/src/io/cache/block/cached_remote_file_reader.cpp b/be/src/io/cache/block/cached_remote_file_reader.cpp index 81268ec5ad..4ee5b7970d 100644 --- a/be/src/io/cache/block/cached_remote_file_reader.cpp +++ b/be/src/io/cache/block/cached_remote_file_reader.cpp @@ -76,9 +76,9 @@ Status CachedRemoteFileReader::close() { std::pair CachedRemoteFileReader::_align_size(size_t offset, size_t read_size) const { - size_t min_size = 1024 * 1024; // 1MB; - size_t segment_size = std::min(std::max(read_size, min_size), - (size_t)config::file_cache_max_file_segment_size); + size_t segment_size = + std::min(std::max(read_size, (size_t)config::file_cache_min_file_segment_size), + (size_t)config::file_cache_max_file_segment_size); segment_size = BitUtil::next_power_of_two(segment_size); size_t left = offset; size_t right = offset + read_size - 1;